feat: add processing mode support for document uploads and ETL pipeline, improded error handling ux
Some checks are pending
Build and Push Docker Images / tag_release (push) Waiting to run
Build and Push Docker Images / build (./surfsense_backend, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-24.04-arm, linux/arm64, arm64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_backend, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-latest, linux/amd64, amd64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_web, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-24.04-arm, linux/arm64, arm64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_web, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-latest, linux/amd64, amd64) (push) Blocked by required conditions
Build and Push Docker Images / create_manifest (backend, surfsense-backend) (push) Blocked by required conditions
Build and Push Docker Images / create_manifest (web, surfsense-web) (push) Blocked by required conditions

- Introduced a `ProcessingMode` enum to differentiate between basic and premium processing modes.
- Updated `EtlRequest` to include a `processing_mode` field, defaulting to basic.
- Enhanced ETL pipeline services to utilize the selected processing mode for Azure Document Intelligence and LlamaCloud parsing.
- Modified various routes and services to handle processing mode, affecting document upload and indexing tasks.
- Improved error handling and logging to include processing mode details.
- Added tests to validate processing mode functionality and its impact on ETL operations.
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-04-14 21:26:00 -07:00
parent b659f41bab
commit 656e061f84
104 changed files with 1900 additions and 909 deletions

View file

@ -15,9 +15,9 @@ import {
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
interface ConfluenceAccount {
id: number;
@ -36,7 +36,7 @@ type CreateConfluencePageInterruptContext = {
accounts?: ConfluenceAccount[];
spaces?: ConfluenceSpace[];
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -6,9 +6,9 @@ import { useCallback, useEffect, useState } from "react";
import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
type DeleteConfluencePageInterruptContext = {
account?: {
@ -26,7 +26,7 @@ type DeleteConfluencePageInterruptContext = {
indexed_at?: string;
};
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -8,9 +8,9 @@ import { openHitlEditPanelAtom } from "@/atoms/chat/hitl-edit-panel.atom";
import { PlateEditor } from "@/components/editor/plate-editor";
import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
type UpdateConfluencePageInterruptContext = {
account?: {
@ -29,7 +29,7 @@ type UpdateConfluencePageInterruptContext = {
indexed_at?: string;
};
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -16,8 +16,8 @@ import {
SelectValue,
} from "@/components/ui/select";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
interface DropboxAccount {
id: number;
@ -36,7 +36,7 @@ type DropboxCreateFileContext = {
parent_folders?: Record<number, Array<{ folder_path: string; name: string }>>;
supported_types?: SupportedType[];
error?: string;
}
};
interface SuccessResult {
status: "success";
@ -57,7 +57,11 @@ interface AuthErrorResult {
connector_type?: string;
}
type CreateDropboxFileResult = InterruptResult<DropboxCreateFileContext> | SuccessResult | ErrorResult | AuthErrorResult;
type CreateDropboxFileResult =
| InterruptResult<DropboxCreateFileContext>
| SuccessResult
| ErrorResult
| AuthErrorResult;
function isErrorResult(result: unknown): result is ErrorResult {
return (

View file

@ -7,8 +7,8 @@ import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
interface DropboxAccount {
id: number;
@ -28,7 +28,7 @@ type DropboxTrashFileContext = {
account?: DropboxAccount;
file?: DropboxFile;
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -9,8 +9,8 @@ import { Input } from "@/components/ui/input";
import { Textarea } from "@/components/ui/textarea";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import { connectorsApiService } from "@/lib/apis/connectors-api.service";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
function ParamEditor({
params,
@ -216,11 +216,7 @@ function GenericApprovalCard({
</Button>
)}
{isMCPTool && (
<Button
size="sm"
className="rounded-lg"
onClick={handleAlwaysAllow}
>
<Button size="sm" className="rounded-lg" onClick={handleAlwaysAllow}>
Always Allow
</Button>
)}

View file

@ -16,9 +16,9 @@ import {
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
interface GmailAccount {
id: number;
@ -30,7 +30,7 @@ interface GmailAccount {
type GmailCreateDraftContext = {
accounts?: GmailAccount[];
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -16,9 +16,9 @@ import {
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
interface GmailAccount {
id: number;
@ -30,7 +30,7 @@ interface GmailAccount {
type GmailSendEmailContext = {
accounts?: GmailAccount[];
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -6,9 +6,9 @@ import { useCallback, useEffect, useState } from "react";
import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
interface GmailAccount {
id: number;
@ -31,7 +31,7 @@ type GmailTrashEmailContext = {
account?: GmailAccount;
email?: GmailMessage;
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -9,9 +9,9 @@ import { openHitlEditPanelAtom } from "@/atoms/chat/hitl-edit-panel.atom";
import { PlateEditor } from "@/components/editor/plate-editor";
import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
interface GmailAccount {
id: number;
@ -36,7 +36,7 @@ type GmailUpdateDraftContext = {
draft_id?: string;
existing_body?: string;
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -16,9 +16,9 @@ import {
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
interface GoogleCalendarAccount {
id: number;
@ -37,7 +37,7 @@ type CalendarCreateEventContext = {
calendars?: CalendarEntry[];
timezone?: string;
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -6,9 +6,9 @@ import { useCallback, useEffect, useState } from "react";
import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
interface GoogleCalendarAccount {
id: number;
@ -33,7 +33,7 @@ type CalendarDeleteEventContext = {
account?: GoogleCalendarAccount;
event?: CalendarEvent;
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -16,9 +16,9 @@ import { openHitlEditPanelAtom } from "@/atoms/chat/hitl-edit-panel.atom";
import { PlateEditor } from "@/components/editor/plate-editor";
import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
interface GoogleCalendarAccount {
id: number;
@ -43,7 +43,7 @@ type CalendarUpdateEventContext = {
account?: GoogleCalendarAccount;
event?: CalendarEvent;
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -16,8 +16,8 @@ import {
SelectValue,
} from "@/components/ui/select";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
interface GoogleDriveAccount {
id: number;
@ -30,7 +30,7 @@ type DriveCreateFileContext = {
supported_types?: string[];
parent_folders?: Record<number, Array<{ folder_id: string; name: string }>>;
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -7,8 +7,8 @@ import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
interface GoogleDriveAccount {
id: number;
@ -27,7 +27,7 @@ type DriveTrashFileContext = {
account?: GoogleDriveAccount;
file?: GoogleDriveFile;
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -15,9 +15,9 @@ import {
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
interface JiraAccount {
id: number;
@ -48,7 +48,7 @@ type CreateJiraIssueInterruptContext = {
issue_types?: JiraIssueType[];
priorities?: JiraPriority[];
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -6,9 +6,9 @@ import { useCallback, useEffect, useState } from "react";
import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
interface JiraAccount {
id: number;
@ -29,7 +29,7 @@ type DeleteJiraIssueInterruptContext = {
account?: JiraAccount;
issue?: JiraIssue;
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -16,9 +16,9 @@ import {
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
interface JiraIssue {
issue_id: string;
@ -50,7 +50,7 @@ type UpdateJiraIssueInterruptContext = {
issue?: JiraIssue;
priorities?: JiraPriority[];
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -18,8 +18,8 @@ import {
} from "@/components/ui/select";
import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
interface LinearLabel {
id: string;
@ -69,7 +69,7 @@ interface LinearWorkspace {
type LinearCreateIssueContext = {
workspaces?: LinearWorkspace[];
error?: string;
}
};
interface SuccessResult {
status: "success";
@ -91,7 +91,11 @@ interface AuthErrorResult {
connector_type: string;
}
type CreateLinearIssueResult = InterruptResult<LinearCreateIssueContext> | SuccessResult | ErrorResult | AuthErrorResult;
type CreateLinearIssueResult =
| InterruptResult<LinearCreateIssueContext>
| SuccessResult
| ErrorResult
| AuthErrorResult;
function isErrorResult(result: unknown): result is ErrorResult {
return (

View file

@ -7,8 +7,8 @@ import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
type LinearDeleteIssueContext = {
workspace?: { id: number; organization_name: string };
@ -21,7 +21,7 @@ type LinearDeleteIssueContext = {
indexed_at?: string;
};
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -18,8 +18,8 @@ import {
} from "@/components/ui/select";
import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
interface LinearLabel {
id: string;
@ -72,7 +72,7 @@ type LinearUpdateIssueContext = {
labels: LinearLabel[];
};
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -16,8 +16,8 @@ import {
SelectValue,
} from "@/components/ui/select";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
type NotionCreatePageContext = {
accounts?: Array<{
@ -37,7 +37,7 @@ type NotionCreatePageContext = {
}>
>;
error?: string;
}
};
interface SuccessResult {
status: "success";
@ -61,7 +61,11 @@ interface AuthErrorResult {
connector_type: string;
}
type CreateNotionPageResult = InterruptResult<NotionCreatePageContext> | SuccessResult | ErrorResult | AuthErrorResult;
type CreateNotionPageResult =
| InterruptResult<NotionCreatePageContext>
| SuccessResult
| ErrorResult
| AuthErrorResult;
function isAuthErrorResult(result: unknown): result is AuthErrorResult {
return (

View file

@ -7,8 +7,8 @@ import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
type NotionDeletePageContext = {
account?: {
@ -23,7 +23,7 @@ type NotionDeletePageContext = {
document_id?: number;
indexed_at?: string;
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -9,8 +9,8 @@ import { PlateEditor } from "@/components/editor/plate-editor";
import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
type NotionUpdatePageContext = {
account?: {
@ -25,7 +25,7 @@ type NotionUpdatePageContext = {
document_id?: number;
indexed_at?: string;
error?: string;
}
};
interface SuccessResult {
status: "success";

View file

@ -16,8 +16,8 @@ import {
SelectValue,
} from "@/components/ui/select";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
interface OneDriveAccount {
id: number;
@ -30,7 +30,7 @@ type OneDriveCreateFileContext = {
accounts?: OneDriveAccount[];
parent_folders?: Record<number, Array<{ folder_id: string; name: string }>>;
error?: string;
}
};
interface SuccessResult {
status: "success";
@ -51,7 +51,11 @@ interface AuthErrorResult {
connector_type?: string;
}
type CreateOneDriveFileResult = InterruptResult<OneDriveCreateFileContext> | SuccessResult | ErrorResult | AuthErrorResult;
type CreateOneDriveFileResult =
| InterruptResult<OneDriveCreateFileContext>
| SuccessResult
| ErrorResult
| AuthErrorResult;
function isErrorResult(result: unknown): result is ErrorResult {
return (

View file

@ -7,8 +7,8 @@ import { TextShimmerLoader } from "@/components/prompt-kit/loader";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { useHitlPhase } from "@/hooks/use-hitl-phase";
import type { HitlDecision, InterruptResult } from "@/lib/hitl";
import { isInterruptResult, useHitlDecision } from "@/lib/hitl";
import type { InterruptResult, HitlDecision } from "@/lib/hitl";
interface OneDriveAccount {
id: number;
@ -28,7 +28,7 @@ type OneDriveTrashFileContext = {
account?: OneDriveAccount;
file?: OneDriveFile;
error?: string;
}
};
interface SuccessResult {
status: "success";