feat: add processing mode support for document uploads and ETL pipeline, improded error handling ux

- Introduced a `ProcessingMode` enum to differentiate between basic and premium processing modes.
- Updated `EtlRequest` to include a `processing_mode` field, defaulting to basic.
- Enhanced ETL pipeline services to utilize the selected processing mode for Azure Document Intelligence and LlamaCloud parsing.
- Modified various routes and services to handle processing mode, affecting document upload and indexing tasks.
- Improved error handling and logging to include processing mode details.
- Added tests to validate processing mode functionality and its impact on ETL operations.

surfsense_web/lib/apis/base-api.service.ts

@@ -1,6 +1,13 @@
 import type { ZodType } from "zod";
 import { getBearerToken, handleUnauthorized, refreshAccessToken } from "../auth-utils";
-import { AppError, AuthenticationError, AuthorizationError, NotFoundError } from "../error";
+import {
+	AbortedError,
+	AppError,
+	AuthenticationError,
+	AuthorizationError,
+	NetworkError,
+	NotFoundError,
+} from "../error";
 
 enum ResponseType {
 	JSON = "json",
@@ -137,12 +144,21 @@ class BaseApiService {
 					throw new AppError("Failed to parse response", response.status, response.statusText);
 				}
 
+				// Extract structured fields from new envelope or legacy shape
+				const envelope = typeof data === "object" && data?.error;
+				const errorMessage: string =
+					envelope?.message ??
+					(typeof data === "object" && typeof data?.detail === "string" ? data.detail : "");
+				const errorCode: string | undefined = envelope?.code;
+				const requestId: string | undefined =
+					envelope?.request_id ?? response.headers.get("X-Request-ID") ?? undefined;
+				const reportUrl: string | undefined = envelope?.report_url;
+
 				// Handle 401 - try to refresh token first (only once)
 				if (response.status === 401) {
 					if (!options?._isRetry) {
 						const newToken = await refreshAccessToken();
 						if (newToken) {
-							// Retry the request with the new token
 							return this.request(url, responseSchema, {
 								...mergedOptions,
 								headers: {
@@ -153,34 +169,37 @@ class BaseApiService {
 							} as RequestOptions & { responseType?: R });
 						}
 					}
-					// Refresh failed or retry failed, redirect to login
 					handleUnauthorized();
 					throw new AuthenticationError(
-						typeof data === "object" && "detail" in data
-							? data.detail
-							: "You are not authenticated. Please login again.",
+						errorMessage || "You are not authenticated. Please login again.",
 						response.status,
 						response.statusText
 					);
 				}
 
-				// For fastapi errors response
-				if (typeof data === "object" && "detail" in data) {
-					throw new AppError(data.detail, response.status, response.statusText);
-				}
-
+				// Map status to typed error
 				switch (response.status) {
 					case 403:
 						throw new AuthorizationError(
-							"You don't have permission to access this resource.",
+							errorMessage || "You don't have permission to access this resource.",
 							response.status,
 							response.statusText
 						);
 					case 404:
-						throw new NotFoundError("Resource not found", response.status, response.statusText);
-					//  Add more cases as needed
+						throw new NotFoundError(
+							errorMessage || "Resource not found",
+							response.status,
+							response.statusText
+						);
 					default:
-						throw new AppError("Something went wrong", response.status, response.statusText);
+						throw new AppError(
+							errorMessage || "Something went wrong",
+							response.status,
+							response.statusText,
+							errorCode,
+							requestId,
+							reportUrl
+						);
 				}
 			}
 
@@ -231,6 +250,16 @@ class BaseApiService {
 
 			return data;
 		} catch (error) {
+			// Normalize browser-level fetch failures before anything else
+			if (error instanceof DOMException && error.name === "AbortError") {
+				throw new AbortedError();
+			}
+			if (error instanceof TypeError && !(error instanceof AppError)) {
+				throw new NetworkError(
+					"Unable to connect to the server. Check your internet connection and try again."
+				);
+			}
+
 			console.error("Request failed:", JSON.stringify(error));
 			if (!(error instanceof AuthenticationError)) {
 				import("posthog-js")
@@ -241,11 +270,12 @@ class BaseApiService {
 							...(error instanceof AppError && {
 								status_code: error.status,
 								status_text: error.statusText,
+								error_code: error.code,
+								request_id: error.requestId,
 							}),
 						});
 					})
 					.catch(() => {
-						// PostHog is not available in the current environment
 						console.error("Failed to capture exception in PostHog");
 					});
 			}

surfsense_web/lib/apis/documents-api.service.ts

@@ -127,7 +127,8 @@ class DocumentsApiService {
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}
 
-		const { files, search_space_id, should_summarize, use_vision_llm } = parsedRequest.data;
+		const { files, search_space_id, should_summarize, use_vision_llm, processing_mode } =
+			parsedRequest.data;
 		const UPLOAD_BATCH_SIZE = 5;
 
 		const batches: File[][] = [];
@@ -147,6 +148,7 @@ class DocumentsApiService {
 			formData.append("search_space_id", String(search_space_id));
 			formData.append("should_summarize", String(should_summarize));
 			formData.append("use_vision_llm", String(use_vision_llm));
+			formData.append("processing_mode", processing_mode);
 
 			const controller = new AbortController();
 			const timeoutId = setTimeout(() => controller.abort(), 120_000);
@@ -444,6 +446,7 @@ class DocumentsApiService {
 			root_folder_id?: number | null;
 			enable_summary?: boolean;
 			use_vision_llm?: boolean;
+			processing_mode?: "basic" | "premium";
 		},
 		signal?: AbortSignal
 	): Promise<{ message: string; status: string; root_folder_id: number; file_count: number }> => {
@@ -459,6 +462,7 @@ class DocumentsApiService {
 		}
 		formData.append("enable_summary", String(metadata.enable_summary ?? false));
 		formData.append("use_vision_llm", String(metadata.use_vision_llm ?? false));
+		formData.append("processing_mode", metadata.processing_mode ?? "basic");
 
 		const totalSize = files.reduce((acc, f) => acc + f.size, 0);
 		const timeoutMs = Math.min(Math.max((totalSize / (1024 * 1024)) * 5000, 30_000), 600_000);

surfsense_web/lib/chat/message-utils.ts

@@ -39,19 +39,20 @@ export function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike {
 		content = [{ type: "text", text: String(msg.content) }];
 	}
 
-	const metadata = (msg.author_id || msg.token_usage)
-		? {
-				custom: {
-					...(msg.author_id && {
-						author: {
-							displayName: msg.author_display_name ?? null,
-							avatarUrl: msg.author_avatar_url ?? null,
-						},
-					}),
-					...(msg.token_usage && { usage: msg.token_usage }),
-				},
-			}
-		: undefined;
+	const metadata =
+		msg.author_id || msg.token_usage
+			? {
+					custom: {
+						...(msg.author_id && {
+							author: {
+								displayName: msg.author_display_name ?? null,
+								avatarUrl: msg.author_avatar_url ?? null,
+							},
+						}),
+						...(msg.token_usage && { usage: msg.token_usage }),
+					},
+				}
+			: undefined;
 
 	return {
 		id: `msg-${msg.id}`,

surfsense_web/lib/chat/streaming-state.ts

@@ -149,7 +149,7 @@ export function addToolCall(
 	toolCallId: string,
 	toolName: string,
 	args: Record<string, unknown>,
-	force = false,
+	force = false
 ): void {
 	if (force || toolsWithUI.has(toolName)) {
 		state.contentParts.push({
@@ -241,11 +241,19 @@ export type SSEEvent =
 	| {
 			type: "data-token-usage";
 			data: {
-				usage: Record<string, { prompt_tokens: number; completion_tokens: number; total_tokens: number }>;
+				usage: Record<
+					string,
+					{ prompt_tokens: number; completion_tokens: number; total_tokens: number }
+				>;
 				prompt_tokens: number;
 				completion_tokens: number;
 				total_tokens: number;
-				call_details: Array<{ model: string; prompt_tokens: number; completion_tokens: number; total_tokens: number }>;
+				call_details: Array<{
+					model: string;
+					prompt_tokens: number;
+					completion_tokens: number;
+					total_tokens: number;
+				}>;
 			};
 	  }
 	| { type: "error"; errorText: string };

surfsense_web/lib/chat/thread-persistence.ts

@@ -30,7 +30,10 @@ export interface TokenUsageSummary {
 	prompt_tokens: number;
 	completion_tokens: number;
 	total_tokens: number;
-	model_breakdown?: Record<string, { prompt_tokens: number; completion_tokens: number; total_tokens: number }> | null;
+	model_breakdown?: Record<
+		string,
+		{ prompt_tokens: number; completion_tokens: number; total_tokens: number }
+	> | null;
 }
 
 export interface MessageRecord {

surfsense_web/lib/error-toast.ts

@@ -0,0 +1,72 @@
+import { toast } from "sonner";
+import { AbortedError, AppError, AuthenticationError, SURFSENSE_ISSUES_URL } from "./error";
+
+/**
+ * Build a GitHub issue URL pre-filled with diagnostic context.
+ * Avoids including PII — only structural metadata.
+ */
+export function buildIssueUrl(error: unknown): string {
+	const params = new URLSearchParams();
+
+	const lines: string[] = ["## Bug Report", "", "**Describe what happened:**", "", ""];
+
+	if (error instanceof AppError) {
+		lines.push("## Diagnostics (auto-filled)", "");
+		if (error.code) lines.push(`- **Error code:** \`${error.code}\``);
+		if (error.requestId) lines.push(`- **Request ID:** \`${error.requestId}\``);
+		if (error.status) lines.push(`- **HTTP status:** ${error.status}`);
+		lines.push(`- **Message:** ${error.message}`);
+	} else if (error instanceof Error) {
+		lines.push("## Diagnostics (auto-filled)", "");
+		lines.push(`- **Error:** ${error.message}`);
+	}
+
+	lines.push(`- **Timestamp:** ${new Date().toISOString()}`);
+	lines.push(
+		`- **Page:** \`${typeof window !== "undefined" ? window.location.pathname : "unknown"}\``
+	);
+	lines.push(
+		`- **User Agent:** \`${typeof navigator !== "undefined" ? navigator.userAgent : "unknown"}\``
+	);
+
+	params.set("body", lines.join("\n"));
+	params.set("labels", "bug");
+
+	return `${SURFSENSE_ISSUES_URL}/new?${params.toString()}`;
+}
+
+/**
+ * Show a standardized error toast with a "Report Issue" action.
+ *
+ * Suppressed for:
+ * - AbortedError (user-initiated cancellation)
+ * - AuthenticationError (handled by redirect)
+ */
+export function showErrorToast(error: unknown, fallbackMessage?: string) {
+	if (error instanceof AbortedError || error instanceof AuthenticationError) {
+		return;
+	}
+
+	const message =
+		error instanceof AppError
+			? error.message
+			: error instanceof Error
+				? error.message
+				: (fallbackMessage ?? "An unexpected error occurred.");
+
+	const code = error instanceof AppError ? error.code : undefined;
+	const requestId = error instanceof AppError ? error.requestId : undefined;
+
+	const descParts: string[] = [];
+	if (code) descParts.push(`Error: ${code}`);
+	if (requestId) descParts.push(`ID: ${requestId}`);
+
+	toast.error(message, {
+		description: descParts.length > 0 ? descParts.join(" | ") : undefined,
+		duration: 8000,
+		action: {
+			label: "Report Issue",
+			onClick: () => window.open(buildIssueUrl(error), "_blank"),
+		},
+	});
+}

surfsense_web/lib/error.ts

@@ -1,40 +1,61 @@
+export const SURFSENSE_ISSUES_URL = "https://github.com/MODSetter/SurfSense/issues";
+
 export class AppError extends Error {
 	status?: number;
 	statusText?: string;
-	constructor(message: string, status?: number, statusText?: string) {
+	code?: string;
+	requestId?: string;
+	reportUrl?: string;
+	constructor(
+		message: string,
+		status?: number,
+		statusText?: string,
+		code?: string,
+		requestId?: string,
+		reportUrl?: string
+	) {
 		super(message);
-		this.name = this.constructor.name; // User friendly
+		this.name = this.constructor.name;
 		this.status = status;
-		this.statusText = statusText; // Dev friendly
+		this.statusText = statusText;
+		this.code = code;
+		this.requestId = requestId;
+		this.reportUrl = reportUrl ?? SURFSENSE_ISSUES_URL;
 	}
 }
 
 export class NetworkError extends AppError {
 	constructor(message: string, status?: number, statusText?: string) {
-		super(message, status, statusText);
+		super(message, status, statusText, "NETWORK_ERROR");
+	}
+}
+
+export class AbortedError extends AppError {
+	constructor(message = "Request was cancelled.") {
+		super(message, undefined, undefined, "REQUEST_ABORTED");
 	}
 }
 
 export class ValidationError extends AppError {
 	constructor(message: string, status?: number, statusText?: string) {
-		super(message, status, statusText);
+		super(message, status, statusText, "VALIDATION_ERROR");
 	}
 }
 
 export class AuthenticationError extends AppError {
 	constructor(message: string, status?: number, statusText?: string) {
-		super(message, status, statusText);
+		super(message, status, statusText, "UNAUTHORIZED");
 	}
 }
 
 export class AuthorizationError extends AppError {
 	constructor(message: string, status?: number, statusText?: string) {
-		super(message, status, statusText);
+		super(message, status, statusText, "FORBIDDEN");
 	}
 }
 
 export class NotFoundError extends AppError {
 	constructor(message: string, status?: number, statusText?: string) {
-		super(message, status, statusText);
+		super(message, status, statusText, "NOT_FOUND");
 	}
 }

surfsense_web/lib/folder-sync-upload.ts

@@ -17,6 +17,7 @@ export interface FolderSyncParams {
 	excludePatterns: string[];
 	fileExtensions: string[];
 	enableSummary: boolean;
+	processingMode?: "basic" | "premium";
 	rootFolderId?: number | null;
 	onProgress?: (progress: FolderSyncProgress) => void;
 	signal?: AbortSignal;
@@ -62,6 +63,7 @@ async function uploadBatchesWithConcurrency(
 		searchSpaceId: number;
 		rootFolderId: number | null;
 		enableSummary: boolean;
+		processingMode?: "basic" | "premium";
 		signal?: AbortSignal;
 		onBatchComplete?: (filesInBatch: number) => void;
 	}
@@ -99,6 +101,7 @@ async function uploadBatchesWithConcurrency(
 						relative_paths: batch.map((e) => e.relativePath),
 						root_folder_id: resolvedRootFolderId,
 						enable_summary: params.enableSummary,
+						processing_mode: params.processingMode,
 					},
 					params.signal
 				);
@@ -145,6 +148,7 @@ export async function uploadFolderScan(params: FolderSyncParams): Promise<number
 		excludePatterns,
 		fileExtensions,
 		enableSummary,
+		processingMode,
 		signal,
 	} = params;
 	let rootFolderId = params.rootFolderId ?? null;
@@ -190,6 +194,7 @@ export async function uploadFolderScan(params: FolderSyncParams): Promise<number
 			searchSpaceId,
 			rootFolderId: rootFolderId ?? null,
 			enableSummary,
+			processingMode,
 			signal,
 			onBatchComplete: (count) => {
 				uploaded += count;

surfsense_web/lib/hitl/index.ts

@@ -1,8 +1,8 @@
-export { isInterruptResult } from "./types";
 export type {
 	HitlDecision,
 	InterruptActionRequest,
 	InterruptResult,
 	InterruptReviewConfig,
 } from "./types";
+export { isInterruptResult } from "./types";
 export { useHitlDecision } from "./use-hitl-decision";

surfsense_web/lib/hitl/use-hitl-decision.ts

@@ -10,9 +10,7 @@ import type { HitlDecision } from "./types";
 
 export function useHitlDecision() {
 	const dispatch = useCallback((decisions: HitlDecision[]) => {
-		window.dispatchEvent(
-			new CustomEvent("hitl-decision", { detail: { decisions } }),
-		);
+		window.dispatchEvent(new CustomEvent("hitl-decision", { detail: { decisions } }));
 	}, []);
 
 	return { dispatch };

surfsense_web/lib/query-client/client.ts

@@ -1,4 +1,5 @@
-import { QueryClient } from "@tanstack/react-query";
+import { MutationCache, QueryCache, QueryClient } from "@tanstack/react-query";
+import { showErrorToast } from "../error-toast";
 
 export const queryClient = new QueryClient({
 	defaultOptions: {
@@ -7,4 +8,16 @@ export const queryClient = new QueryClient({
 			refetchOnWindowFocus: false,
 		},
 	},
+	queryCache: new QueryCache({
+		onError: (error, query) => {
+			if (query.meta?.suppressGlobalErrorToast) return;
+			showErrorToast(error);
+		},
+	}),
+	mutationCache: new MutationCache({
+		onError: (error, _variables, _context, mutation) => {
+			if (mutation.meta?.suppressGlobalErrorToast) return;
+			showErrorToast(error);
+		},
+	}),
 });