feat: add processing mode support for document uploads and ETL pipeline, improded error handling ux

- Introduced a `ProcessingMode` enum to differentiate between basic and premium processing modes.
- Updated `EtlRequest` to include a `processing_mode` field, defaulting to basic.
- Enhanced ETL pipeline services to utilize the selected processing mode for Azure Document Intelligence and LlamaCloud parsing.
- Modified various routes and services to handle processing mode, affecting document upload and indexing tasks.
- Improved error handling and logging to include processing mode details.
- Added tests to validate processing mode functionality and its impact on ETL operations.

surfsense_web/lib/chat/message-utils.ts

@@ -39,19 +39,20 @@ export function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike {
 		content = [{ type: "text", text: String(msg.content) }];
 	}
 
-	const metadata = (msg.author_id || msg.token_usage)
-		? {
-				custom: {
-					...(msg.author_id && {
-						author: {
-							displayName: msg.author_display_name ?? null,
-							avatarUrl: msg.author_avatar_url ?? null,
-						},
-					}),
-					...(msg.token_usage && { usage: msg.token_usage }),
-				},
-			}
-		: undefined;
+	const metadata =
+		msg.author_id || msg.token_usage
+			? {
+					custom: {
+						...(msg.author_id && {
+							author: {
+								displayName: msg.author_display_name ?? null,
+								avatarUrl: msg.author_avatar_url ?? null,
+							},
+						}),
+						...(msg.token_usage && { usage: msg.token_usage }),
+					},
+				}
+			: undefined;
 
 	return {
 		id: `msg-${msg.id}`,

surfsense_web/lib/chat/streaming-state.ts

@@ -149,7 +149,7 @@ export function addToolCall(
 	toolCallId: string,
 	toolName: string,
 	args: Record<string, unknown>,
-	force = false,
+	force = false
 ): void {
 	if (force || toolsWithUI.has(toolName)) {
 		state.contentParts.push({
@@ -241,11 +241,19 @@ export type SSEEvent =
 	| {
 			type: "data-token-usage";
 			data: {
-				usage: Record<string, { prompt_tokens: number; completion_tokens: number; total_tokens: number }>;
+				usage: Record<
+					string,
+					{ prompt_tokens: number; completion_tokens: number; total_tokens: number }
+				>;
 				prompt_tokens: number;
 				completion_tokens: number;
 				total_tokens: number;
-				call_details: Array<{ model: string; prompt_tokens: number; completion_tokens: number; total_tokens: number }>;
+				call_details: Array<{
+					model: string;
+					prompt_tokens: number;
+					completion_tokens: number;
+					total_tokens: number;
+				}>;
 			};
 	  }
 	| { type: "error"; errorText: string };

surfsense_web/lib/chat/thread-persistence.ts

@@ -30,7 +30,10 @@ export interface TokenUsageSummary {
 	prompt_tokens: number;
 	completion_tokens: number;
 	total_tokens: number;
-	model_breakdown?: Record<string, { prompt_tokens: number; completion_tokens: number; total_tokens: number }> | null;
+	model_breakdown?: Record<
+		string,
+		{ prompt_tokens: number; completion_tokens: number; total_tokens: number }
+	> | null;
 }
 
 export interface MessageRecord {