feat: add processing mode support for document uploads and ETL pipeline, improded error handling ux

- Introduced a `ProcessingMode` enum to differentiate between basic and premium processing modes.
- Updated `EtlRequest` to include a `processing_mode` field, defaulting to basic.
- Enhanced ETL pipeline services to utilize the selected processing mode for Azure Document Intelligence and LlamaCloud parsing.
- Modified various routes and services to handle processing mode, affecting document upload and indexing tasks.
- Improved error handling and logging to include processing mode details.
- Added tests to validate processing mode functionality and its impact on ETL operations.

surfsense_web/components/assistant-ui/assistant-message.tsx

@@ -33,11 +33,17 @@ import {
 	useAllCitationMetadata,
 } from "@/components/assistant-ui/citation-metadata-context";
 import { MarkdownText } from "@/components/assistant-ui/markdown-text";
+import { useTokenUsage } from "@/components/assistant-ui/token-usage-context";
 import { ToolFallback } from "@/components/assistant-ui/tool-fallback";
 import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button";
 import { CommentPanelContainer } from "@/components/chat-comments/comment-panel-container/comment-panel-container";
 import { CommentSheet } from "@/components/chat-comments/comment-sheet/comment-sheet";
 import type { SerializableCitation } from "@/components/tool-ui/citation";
+import {
+	openSafeNavigationHref,
+	resolveSafeNavigationHref,
+} from "@/components/tool-ui/shared/media";
+import { Button } from "@/components/ui/button";
 import {
 	Drawer,
 	DrawerContent,
@@ -46,14 +52,11 @@ import {
 	DrawerTitle,
 } from "@/components/ui/drawer";
 import { DropdownMenuLabel } from "@/components/ui/dropdown-menu";
-import { Button } from "@/components/ui/button";
 import { useComments } from "@/hooks/use-comments";
 import { useMediaQuery } from "@/hooks/use-media-query";
 import { useElectronAPI } from "@/hooks/use-platform";
-import { useTokenUsage } from "@/components/assistant-ui/token-usage-context";
 import { getProviderIcon } from "@/lib/provider-icons";
 import { cn } from "@/lib/utils";
-import { openSafeNavigationHref, resolveSafeNavigationHref } from "@/components/tool-ui/shared/media";
 
 // Captured once at module load — survives client-side navigations that strip the query param.
 const IS_QUICK_ASSIST_WINDOW =
@@ -440,7 +443,11 @@ const MessageInfoDropdown: FC = () => {
 							models.map(([model, counts]) => {
 								const { name, icon } = resolveModel(model);
 								return (
-									<ActionBarMorePrimitive.Item key={model} className="focus:bg-neutral-200 dark:focus:bg-neutral-700 relative flex cursor-default flex-col items-start gap-0.5 rounded-sm px-2 py-1.5 text-sm outline-hidden select-none" onSelect={(e) => e.preventDefault()}>
+									<ActionBarMorePrimitive.Item
+										key={model}
+										className="focus:bg-neutral-200 dark:focus:bg-neutral-700 relative flex cursor-default flex-col items-start gap-0.5 rounded-sm px-2 py-1.5 text-sm outline-hidden select-none"
+										onSelect={(e) => e.preventDefault()}
+									>
 										<span className="flex items-center gap-1.5 text-xs font-medium">
 											{icon}
 											{name}
@@ -452,7 +459,10 @@ const MessageInfoDropdown: FC = () => {
 								);
 							})
 						) : (
-							<ActionBarMorePrimitive.Item className="focus:bg-neutral-200 dark:focus:bg-neutral-700 relative flex cursor-default flex-col items-start gap-0.5 rounded-sm px-2 py-1.5 text-sm outline-hidden select-none" onSelect={(e) => e.preventDefault()}>
+							<ActionBarMorePrimitive.Item
+								className="focus:bg-neutral-200 dark:focus:bg-neutral-700 relative flex cursor-default flex-col items-start gap-0.5 rounded-sm px-2 py-1.5 text-sm outline-hidden select-none"
+								onSelect={(e) => e.preventDefault()}
+							>
 								<span className="text-xs text-muted-foreground">
 									{usage.total_tokens.toLocaleString()} tokens
 								</span>

surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx

@@ -282,7 +282,10 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
 									connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" ||
 									connector.connector_type === "DROPBOX_CONNECTOR" ||
 									connector.connector_type === "ONEDRIVE_CONNECTOR") && (
-									<VisionLLMConfig enabled={enableVisionLlm} onEnabledChange={onEnableVisionLlmChange} />
+									<VisionLLMConfig
+										enabled={enableVisionLlm}
+										onEnabledChange={onEnableVisionLlmChange}
+									/>
 								)}
 
 								{/* Date range selector - not shown for file-based connectors (Drive, Dropbox, OneDrive), Webcrawler, GitHub, or Local Folder */}

surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx

@@ -168,7 +168,10 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
 									config.connectorType === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" ||
 									config.connectorType === "DROPBOX_CONNECTOR" ||
 									config.connectorType === "ONEDRIVE_CONNECTOR") && (
-									<VisionLLMConfig enabled={enableVisionLlm} onEnabledChange={onEnableVisionLlmChange} />
+									<VisionLLMConfig
+										enabled={enableVisionLlm}
+										onEnabledChange={onEnableVisionLlmChange}
+									/>
 								)}
 
 								{/* Date range selector - not shown for file-based connectors (Drive, Dropbox, OneDrive), Webcrawler, GitHub, or Local Folder */}

surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts

@@ -10,11 +10,17 @@ import {
 	updateConnectorMutationAtom,
 } from "@/atoms/connectors/connector-mutation.atoms";
 import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
+import {
+	folderWatchDialogOpenAtom,
+	folderWatchInitialFolderAtom,
+} from "@/atoms/folder-sync/folder-sync.atoms";
 import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import type { SearchSourceConnector } from "@/contracts/types/connector.types";
 import { searchSourceConnector } from "@/contracts/types/connector.types";
+import { usePlatform } from "@/hooks/use-platform";
 import { authenticatedFetch } from "@/lib/auth-utils";
+import { isSelfHosted } from "@/lib/env-config";
 import {
 	trackConnectorConnected,
 	trackConnectorDeleted,
@@ -33,12 +39,6 @@ import {
 	OAUTH_CONNECTORS,
 	OTHER_CONNECTORS,
 } from "../constants/connector-constants";
-import { usePlatform } from "@/hooks/use-platform";
-import { isSelfHosted } from "@/lib/env-config";
-import {
-	folderWatchDialogOpenAtom,
-	folderWatchInitialFolderAtom,
-} from "@/atoms/folder-sync/folder-sync.atoms";
 
 import {
 	dateRangeSchema,
@@ -73,7 +73,6 @@ export const useConnectorDialog = () => {
 	const { isDesktop } = usePlatform();
 	const selfHosted = isSelfHosted();
 
-
 	// Use global atom for dialog open state so it can be controlled from anywhere
 	const [isOpen, setIsOpen] = useAtom(connectorDialogOpenAtom);
 	const [activeTab, setActiveTab] = useState("all");
@@ -463,10 +462,16 @@ export const useConnectorDialog = () => {
 
 			setConnectingConnectorType(connectorType);
 		},
-		[searchSpaceId, selfHosted, isDesktop, setIsOpen, setFolderWatchOpen, setFolderWatchInitialFolder]
+		[
+			searchSpaceId,
+			selfHosted,
+			isDesktop,
+			setIsOpen,
+			setFolderWatchOpen,
+			setFolderWatchInitialFolder,
+		]
 	);
 
-
 	// Handle submitting connect form
 	const handleSubmitConnectForm = useCallback(
 		async (
@@ -787,8 +792,8 @@ export const useConnectorDialog = () => {
 				const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined;
 
 				// Update connector with summary, periodic sync settings, and config changes
-			if (enableSummary || enableVisionLlm || periodicEnabled || indexingConnectorConfig) {
-				const frequency = periodicEnabled ? parseInt(frequencyMinutes, 10) : undefined;
+				if (enableSummary || enableVisionLlm || periodicEnabled || indexingConnectorConfig) {
+					const frequency = periodicEnabled ? parseInt(frequencyMinutes, 10) : undefined;
 					await updateConnector({
 						id: indexingConfig.connectorId,
 						data: {

surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx

@@ -2,12 +2,12 @@
 
 import { Search, Unplug } from "lucide-react";
 import type { FC } from "react";
-import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels";
 import { Button } from "@/components/ui/button";
 import { Spinner } from "@/components/ui/spinner";
 import { TabsContent } from "@/components/ui/tabs";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
 import type { SearchSourceConnector } from "@/contracts/types/connector.types";
+import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels";
 import { cn } from "@/lib/utils";
 import { COMPOSIO_CONNECTORS, OAUTH_CONNECTORS } from "../constants/connector-constants";
 import { getDocumentCountForConnector } from "../utils/connector-document-mapping";

surfsense_web/components/assistant-ui/thread.tsx

@@ -60,7 +60,6 @@ import {
 } from "@/components/assistant-ui/inline-mention-editor";
 import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button";
 import { UserMessage } from "@/components/assistant-ui/user-message";
-import { SLIDEOUT_PANEL_OPENED_EVENT } from "@/lib/layout-events";
 import {
 	DocumentMentionPicker,
 	type DocumentMentionPickerRef,
@@ -90,6 +89,7 @@ import { useBatchCommentsPreload } from "@/hooks/use-comments";
 import { useCommentsSync } from "@/hooks/use-comments-sync";
 import { useMediaQuery } from "@/hooks/use-media-query";
 import { useElectronAPI } from "@/hooks/use-platform";
+import { SLIDEOUT_PANEL_OPENED_EVENT } from "@/lib/layout-events";
 import { cn } from "@/lib/utils";
 
 const COMPOSER_PLACEHOLDER = "Ask anything, type / for prompts, type @ to mention docs";
@@ -132,9 +132,9 @@ const ThreadContent: FC = () => {
 					style={{ paddingBottom: "max(1rem, env(safe-area-inset-bottom))" }}
 				>
 					<ThreadScrollToBottom />
-				<AuiIf condition={({ thread }) => !thread.isEmpty}>
-					<Composer />
-				</AuiIf>
+					<AuiIf condition={({ thread }) => !thread.isEmpty}>
+						<Composer />
+					</AuiIf>
 				</ThreadPrimitive.ViewportFooter>
 			</ThreadPrimitive.Viewport>
 		</ThreadPrimitive.Root>

surfsense_web/components/assistant-ui/token-usage-context.tsx

@@ -1,13 +1,26 @@
 "use client";
 
-import { createContext, useContext, useCallback, useSyncExternalStore, type FC, type ReactNode } from "react";
+import {
+	createContext,
+	type FC,
+	type ReactNode,
+	useCallback,
+	useContext,
+	useSyncExternalStore,
+} from "react";
 
 export interface TokenUsageData {
 	prompt_tokens: number;
 	completion_tokens: number;
 	total_tokens: number;
-	usage?: Record<string, { prompt_tokens: number; completion_tokens: number; total_tokens: number }>;
-	model_breakdown?: Record<string, { prompt_tokens: number; completion_tokens: number; total_tokens: number }>;
+	usage?: Record<
+		string,
+		{ prompt_tokens: number; completion_tokens: number; total_tokens: number }
+	>;
+	model_breakdown?: Record<
+		string,
+		{ prompt_tokens: number; completion_tokens: number; total_tokens: number }
+	>;
 }
 
 type Listener = () => void;
@@ -51,9 +64,10 @@ class TokenUsageStore {
 
 const TokenUsageContext = createContext<TokenUsageStore | null>(null);
 
-export const TokenUsageProvider: FC<{ store: TokenUsageStore; children: ReactNode }> = ({ store, children }) => (
-	<TokenUsageContext.Provider value={store}>{children}</TokenUsageContext.Provider>
-);
+export const TokenUsageProvider: FC<{ store: TokenUsageStore; children: ReactNode }> = ({
+	store,
+	children,
+}) => <TokenUsageContext.Provider value={store}>{children}</TokenUsageContext.Provider>;
 
 export function useTokenUsageStore(): TokenUsageStore {
 	const store = useContext(TokenUsageContext);
@@ -65,11 +79,11 @@ export function useTokenUsage(messageId: string | undefined): TokenUsageData | u
 	const store = useContext(TokenUsageContext);
 	const getSnapshot = useCallback(
 		() => (store && messageId ? store.get(messageId) : undefined),
-		[store, messageId],
+		[store, messageId]
 	);
 	const subscribe = useCallback(
 		(onStoreChange: () => void) => (store ? store.subscribe(onStoreChange) : () => {}),
-		[store],
+		[store]
 	);
 	return useSyncExternalStore(subscribe, getSnapshot, getSnapshot);
 }