apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-06-28 21:49:40 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: implement background processing for binary attachments in Obsidian plugin

Browse source 
- Added a new Celery task for indexing non-markdown attachments.
- Enhanced the Obsidian plugin schema to support binary attachments.
- Updated routes to enqueue binary attachments for background processing.
- Improved metadata handling for binary attachments during indexing.
- Added tests for binary attachment processing and validation.
This commit is contained in:
Anish Sarkar 2026-04-22 23:00:34 +05:30
parent 5047527b47
commit 6ac5256431
11 changed files with 519 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

4
surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/obsidian-config.tsx
View file

@ -108,7 +108,7 @@ const PluginStats: FC<{ config: Record<string, unknown> }> = ({ config }) => {
const tileRows = useMemo(() => {
const placeholder = statsError ? "—" : stats ? null : "…";
return [
{ label: "Vault", value: (config.vault_name as string) || "—" },
{ label: "Vault name", value: (config.vault_name as string) || "—" },
{
label: "Last sync",
value: placeholder ?? formatTimestamp(stats?.last_sync_at ?? null),
@ -134,7 +134,7 @@ const PluginStats: FC<{ config: Record<string, unknown> }> = ({ config }) => {
</Alert>
<div className="rounded-xl bg-slate-400/5 p-3 sm:p-6 dark:bg-white/5">
<h3 className="mb-3 text-sm font-medium sm:text-base">Vault status</h3>
<h3 className="mb-3 text-sm font-medium sm:text-base">Vault Status</h3>
<dl className="grid grid-cols-1 gap-3 sm:grid-cols-2">
{tileRows.map((stat) => (
<div key={stat.label} className="rounded-lg bg-background/50 p-3">

99
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx
View file

@ -1,7 +1,7 @@
"use client";
import { useAtomValue } from "jotai";
import { ArrowLeft, Info, RefreshCw, Trash2 } from "lucide-react";
import { ArrowLeft, Info, RefreshCw } from "lucide-react";
import { type FC, useCallback, useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
@ -19,6 +19,14 @@ import { VisionLLMConfig } from "../../components/vision-llm-config";
import { getConnectorDisplayName } from "../../tabs/all-connectors-tab";
import { getConnectorConfigComponent } from "../index";
const VISION_LLM_CONNECTOR_TYPES = new Set<SearchSourceConnector["connector_type"]>([
EnumConnectorName.GOOGLE_DRIVE_CONNECTOR,
EnumConnectorName.COMPOSIO_GOOGLE_DRIVE_CONNECTOR,
EnumConnectorName.DROPBOX_CONNECTOR,
EnumConnectorName.ONEDRIVE_CONNECTOR,
EnumConnectorName.OBSIDIAN_CONNECTOR,
]);
const REAUTH_ENDPOINTS: Partial<Record<string, string>> = {
[EnumConnectorName.LINEAR_CONNECTOR]: "/api/v1/auth/linear/connector/reauth",
[EnumConnectorName.NOTION_CONNECTOR]: "/api/v1/auth/notion/connector/reauth",
@ -87,10 +95,9 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
const isAuthExpired = connector.config?.auth_expired === true;
const reauthEndpoint = REAUTH_ENDPOINTS[connector.connector_type];
const [reauthing, setReauthing] = useState(false);
// Obsidian is plugin-driven: name + config are owned by the plugin, so
// the web edit view has nothing the user can persist back. Hide Save
// (and re-auth, which Obsidian never uses) entirely for that type.
const isPluginManagedReadOnly = connector.connector_type === EnumConnectorName.OBSIDIAN_CONNECTOR;
const supportsVisionLlm = VISION_LLM_CONNECTOR_TYPES.has(connector.connector_type);
const showsAiToggles =
connector.is_indexable || connector.connector_type === EnumConnectorName.OBSIDIAN_CONNECTOR;
const handleReauth = useCallback(async () => {
const spaceId = searchSpaceId ?? searchSpaceIdAtom;
@ -275,25 +282,23 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
/>
)}
{/* Summary and sync settings - only shown for indexable connectors */}
{connector.is_indexable && (
{/* Summary + vision toggles (Obsidian is plugin-push, non-indexable by design) */}
{showsAiToggles && (
<>
{/* AI Summary toggle */}
<SummaryConfig enabled={enableSummary} onEnabledChange={onEnableSummaryChange} />
{/* Vision LLM toggle - only for file-based connectors */}
{(connector.connector_type === "GOOGLE_DRIVE_CONNECTOR" ||
connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" ||
connector.connector_type === "DROPBOX_CONNECTOR" ||
connector.connector_type === "ONEDRIVE_CONNECTOR") && (
{/* Vision LLM toggle for file/attachment connectors */}
{supportsVisionLlm && (
<VisionLLMConfig
enabled={enableVisionLlm}
onEnabledChange={onEnableVisionLlmChange}
/>
)}
{/* Date range selector - not shown for file-based connectors (Drive, Dropbox, OneDrive), Webcrawler, GitHub, or Local Folder */}
{connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" &&
{/* Date-range and periodic sync stay indexable-only */}
{connector.is_indexable &&
connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" &&
connector.connector_type !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" &&
connector.connector_type !== "DROPBOX_CONNECTOR" &&
connector.connector_type !== "ONEDRIVE_CONNECTOR" &&
@ -313,37 +318,40 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
/>
)}
{(() => {
const isGoogleDrive = connector.connector_type === "GOOGLE_DRIVE_CONNECTOR";
const isComposioGoogleDrive =
connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR";
const requiresFolderSelection = isGoogleDrive || isComposioGoogleDrive;
const selectedFolders =
(connector.config?.selected_folders as
| Array<{ id: string; name: string }>
| undefined) || [];
const selectedFiles =
(connector.config?.selected_files as
| Array<{ id: string; name: string }>
| undefined) || [];
const hasItemsSelected = selectedFolders.length > 0 || selectedFiles.length > 0;
const isDisabled = requiresFolderSelection && !hasItemsSelected;
{connector.is_indexable &&
(() => {
const isGoogleDrive =
connector.connector_type === "GOOGLE_DRIVE_CONNECTOR";
const isComposioGoogleDrive =
connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR";
const requiresFolderSelection = isGoogleDrive || isComposioGoogleDrive;
const selectedFolders =
(connector.config?.selected_folders as
| Array<{ id: string; name: string }>
| undefined) || [];
const selectedFiles =
(connector.config?.selected_files as
| Array<{ id: string; name: string }>
| undefined) || [];
const hasItemsSelected =
selectedFolders.length > 0 || selectedFiles.length > 0;
const isDisabled = requiresFolderSelection && !hasItemsSelected;
return (
<PeriodicSyncConfig
enabled={periodicEnabled}
frequencyMinutes={frequencyMinutes}
onEnabledChange={onPeriodicEnabledChange}
onFrequencyChange={onFrequencyChange}
disabled={isDisabled}
disabledMessage={
isDisabled
? "Select at least one folder or file above to enable periodic sync"
: undefined
}
/>
);
})()}
return (
<PeriodicSyncConfig
enabled={periodicEnabled}
frequencyMinutes={frequencyMinutes}
onEnabledChange={onPeriodicEnabledChange}
onFrequencyChange={onFrequencyChange}
disabled={isDisabled}
disabledMessage={
isDisabled
? "Select at least one folder or file above to enable periodic sync"
: undefined
}
/>
);
})()}
</>
)}
@ -412,11 +420,10 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
disabled={isSaving || isDisconnecting}
className="text-xs sm:text-sm flex-1 sm:flex-initial h-12 sm:h-auto py-3 sm:py-2"
>
<Trash2 className="mr-2 h-4 w-4" />
Disconnect
</Button>
)}
{isPluginManagedReadOnly ? null : isAuthExpired && reauthEndpoint ? (
{isAuthExpired && reauthEndpoint ? (
<Button
onClick={handleReauth}
disabled={reauthing || isDisconnecting}

31
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx
View file

@ -4,6 +4,7 @@ import { ArrowLeft, Check, Info } from "lucide-react";
import { type FC, useCallback, useEffect, useMemo, useRef, useState } from "react";
import { Button } from "@/components/ui/button";
import { Spinner } from "@/components/ui/spinner";
import { EnumConnectorName } from "@/contracts/enums/connector";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { getConnectorTypeDisplay } from "@/lib/connectors/utils";
import { cn } from "@/lib/utils";
@ -15,6 +16,14 @@ import type { IndexingConfigState } from "../../constants/connector-constants";
import { getConnectorDisplayName } from "../../tabs/all-connectors-tab";
import { getConnectorConfigComponent } from "../index";
const VISION_LLM_CONNECTOR_TYPES = new Set<string>([
"GOOGLE_DRIVE_CONNECTOR",
"COMPOSIO_GOOGLE_DRIVE_CONNECTOR",
"DROPBOX_CONNECTOR",
"ONEDRIVE_CONNECTOR",
"OBSIDIAN_CONNECTOR",
]);
interface IndexingConfigurationViewProps {
config: IndexingConfigState;
connector?: SearchSourceConnector;
@ -63,6 +72,9 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
() => (connector ? getConnectorConfigComponent(connector.connector_type) : null),
[connector]
);
const showsAiToggles =
(connector?.is_indexable ?? false) ||
connector?.connector_type === EnumConnectorName.OBSIDIAN_CONNECTOR;
const [isScrolled, setIsScrolled] = useState(false);
const [hasMoreContent, setHasMoreContent] = useState(false);
const scrollContainerRef = useRef<HTMLDivElement>(null);
@ -157,25 +169,23 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
<ConnectorConfigComponent connector={connector} onConfigChange={onConfigChange} />
)}
{/* Summary and sync settings - only shown for indexable connectors */}
{connector?.is_indexable && (
{/* Summary + vision toggles (Obsidian is plugin-push, non-indexable by design) */}
{showsAiToggles && (
<>
{/* AI Summary toggle */}
<SummaryConfig enabled={enableSummary} onEnabledChange={onEnableSummaryChange} />
{/* Vision LLM toggle - only for file-based connectors */}
{(config.connectorType === "GOOGLE_DRIVE_CONNECTOR" ||
config.connectorType === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" ||
config.connectorType === "DROPBOX_CONNECTOR" ||
config.connectorType === "ONEDRIVE_CONNECTOR") && (
{/* Vision LLM toggle for file/attachment connectors */}
{VISION_LLM_CONNECTOR_TYPES.has(config.connectorType) && (
<VisionLLMConfig
enabled={enableVisionLlm}
onEnabledChange={onEnableVisionLlmChange}
/>
)}
{/* Date range selector - not shown for file-based connectors (Drive, Dropbox, OneDrive), Webcrawler, GitHub, or Local Folder */}
{config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" &&
{/* Date-range and periodic sync stay indexable-only */}
{connector?.is_indexable &&
config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" &&
config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" &&
config.connectorType !== "DROPBOX_CONNECTOR" &&
config.connectorType !== "ONEDRIVE_CONNECTOR" &&
@ -195,7 +205,8 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
/>
)}
{config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" &&
{connector?.is_indexable &&
config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" &&
config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" &&
config.connectorType !== "DROPBOX_CONNECTOR" &&
config.connectorType !== "ONEDRIVE_CONNECTOR" && (