mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-04-27 09:46:25 +02:00
feat: add processing mode support for document uploads and ETL pipeline, improded error handling ux
Some checks are pending
Build and Push Docker Images / tag_release (push) Waiting to run
Build and Push Docker Images / build (./surfsense_backend, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-24.04-arm, linux/arm64, arm64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_backend, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-latest, linux/amd64, amd64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_web, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-24.04-arm, linux/arm64, arm64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_web, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-latest, linux/amd64, amd64) (push) Blocked by required conditions
Build and Push Docker Images / create_manifest (backend, surfsense-backend) (push) Blocked by required conditions
Build and Push Docker Images / create_manifest (web, surfsense-web) (push) Blocked by required conditions
Some checks are pending
Build and Push Docker Images / tag_release (push) Waiting to run
Build and Push Docker Images / build (./surfsense_backend, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-24.04-arm, linux/arm64, arm64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_backend, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-latest, linux/amd64, amd64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_web, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-24.04-arm, linux/arm64, arm64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_web, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-latest, linux/amd64, amd64) (push) Blocked by required conditions
Build and Push Docker Images / create_manifest (backend, surfsense-backend) (push) Blocked by required conditions
Build and Push Docker Images / create_manifest (web, surfsense-web) (push) Blocked by required conditions
- Introduced a `ProcessingMode` enum to differentiate between basic and premium processing modes. - Updated `EtlRequest` to include a `processing_mode` field, defaulting to basic. - Enhanced ETL pipeline services to utilize the selected processing mode for Azure Document Intelligence and LlamaCloud parsing. - Modified various routes and services to handle processing mode, affecting document upload and indexing tasks. - Improved error handling and logging to include processing mode details. - Added tests to validate processing mode functionality and its impact on ETL operations.
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
b659f41bab
commit
656e061f84
104 changed files with 1900 additions and 909 deletions
|
|
@ -86,8 +86,7 @@ export const DocumentNode = React.memo(function DocumentNode({
|
|||
const isProcessing = statusState === "pending" || statusState === "processing";
|
||||
const isUnavailable = isProcessing || isFailed;
|
||||
const isSelectable = !isUnavailable;
|
||||
const isEditable =
|
||||
EDITABLE_DOCUMENT_TYPES.has(doc.document_type) && !isUnavailable;
|
||||
const isEditable = EDITABLE_DOCUMENT_TYPES.has(doc.document_type) && !isUnavailable;
|
||||
|
||||
const handleCheckChange = useCallback(() => {
|
||||
if (isSelectable) {
|
||||
|
|
@ -261,38 +260,38 @@ export const DocumentNode = React.memo(function DocumentNode({
|
|||
className="w-40"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
<DropdownMenuItem onClick={() => onPreview(doc)} disabled={isUnavailable}>
|
||||
<Eye className="mr-2 h-4 w-4" />
|
||||
Open
|
||||
</DropdownMenuItem>
|
||||
{isEditable && (
|
||||
<DropdownMenuItem onClick={() => onEdit(doc)}>
|
||||
<PenLine className="mr-2 h-4 w-4" />
|
||||
Edit
|
||||
<DropdownMenuItem onClick={() => onPreview(doc)} disabled={isUnavailable}>
|
||||
<Eye className="mr-2 h-4 w-4" />
|
||||
Open
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
<DropdownMenuItem onClick={() => onMove(doc)}>
|
||||
<Move className="mr-2 h-4 w-4" />
|
||||
Move to...
|
||||
</DropdownMenuItem>
|
||||
{onExport && (
|
||||
<DropdownMenuSub>
|
||||
<DropdownMenuSubTrigger disabled={isUnavailable}>
|
||||
<Download className="mr-2 h-4 w-4" />
|
||||
Export
|
||||
</DropdownMenuSubTrigger>
|
||||
<DropdownMenuSubContent className="min-w-[180px]">
|
||||
<ExportDropdownItems onExport={handleExport} exporting={exporting} />
|
||||
</DropdownMenuSubContent>
|
||||
</DropdownMenuSub>
|
||||
)}
|
||||
{onVersionHistory && isVersionableType(doc.document_type) && (
|
||||
<DropdownMenuItem disabled={isUnavailable} onClick={() => onVersionHistory(doc)}>
|
||||
<History className="mr-2 h-4 w-4" />
|
||||
Versions
|
||||
{isEditable && (
|
||||
<DropdownMenuItem onClick={() => onEdit(doc)}>
|
||||
<PenLine className="mr-2 h-4 w-4" />
|
||||
Edit
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
<DropdownMenuItem onClick={() => onMove(doc)}>
|
||||
<Move className="mr-2 h-4 w-4" />
|
||||
Move to...
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
<DropdownMenuItem disabled={isProcessing} onClick={() => onDelete(doc)}>
|
||||
{onExport && (
|
||||
<DropdownMenuSub>
|
||||
<DropdownMenuSubTrigger disabled={isUnavailable}>
|
||||
<Download className="mr-2 h-4 w-4" />
|
||||
Export
|
||||
</DropdownMenuSubTrigger>
|
||||
<DropdownMenuSubContent className="min-w-[180px]">
|
||||
<ExportDropdownItems onExport={handleExport} exporting={exporting} />
|
||||
</DropdownMenuSubContent>
|
||||
</DropdownMenuSub>
|
||||
)}
|
||||
{onVersionHistory && isVersionableType(doc.document_type) && (
|
||||
<DropdownMenuItem disabled={isUnavailable} onClick={() => onVersionHistory(doc)}>
|
||||
<History className="mr-2 h-4 w-4" />
|
||||
Versions
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
<DropdownMenuItem disabled={isProcessing} onClick={() => onDelete(doc)}>
|
||||
<Trash2 className="mr-2 h-4 w-4" />
|
||||
Delete
|
||||
</DropdownMenuItem>
|
||||
|
|
@ -304,38 +303,38 @@ export const DocumentNode = React.memo(function DocumentNode({
|
|||
|
||||
{contextMenuOpen && (
|
||||
<ContextMenuContent className="w-40" onClick={(e) => e.stopPropagation()}>
|
||||
<ContextMenuItem onClick={() => onPreview(doc)} disabled={isUnavailable}>
|
||||
<Eye className="mr-2 h-4 w-4" />
|
||||
Open
|
||||
</ContextMenuItem>
|
||||
{isEditable && (
|
||||
<ContextMenuItem onClick={() => onEdit(doc)}>
|
||||
<PenLine className="mr-2 h-4 w-4" />
|
||||
Edit
|
||||
<ContextMenuItem onClick={() => onPreview(doc)} disabled={isUnavailable}>
|
||||
<Eye className="mr-2 h-4 w-4" />
|
||||
Open
|
||||
</ContextMenuItem>
|
||||
)}
|
||||
<ContextMenuItem onClick={() => onMove(doc)}>
|
||||
<Move className="mr-2 h-4 w-4" />
|
||||
Move to...
|
||||
</ContextMenuItem>
|
||||
{onExport && (
|
||||
<ContextMenuSub>
|
||||
<ContextMenuSubTrigger disabled={isUnavailable}>
|
||||
<Download className="mr-2 h-4 w-4" />
|
||||
Export
|
||||
</ContextMenuSubTrigger>
|
||||
<ContextMenuSubContent className="min-w-[180px]">
|
||||
<ExportContextItems onExport={handleExport} exporting={exporting} />
|
||||
</ContextMenuSubContent>
|
||||
</ContextMenuSub>
|
||||
)}
|
||||
{onVersionHistory && isVersionableType(doc.document_type) && (
|
||||
<ContextMenuItem disabled={isUnavailable} onClick={() => onVersionHistory(doc)}>
|
||||
<History className="mr-2 h-4 w-4" />
|
||||
Versions
|
||||
{isEditable && (
|
||||
<ContextMenuItem onClick={() => onEdit(doc)}>
|
||||
<PenLine className="mr-2 h-4 w-4" />
|
||||
Edit
|
||||
</ContextMenuItem>
|
||||
)}
|
||||
<ContextMenuItem onClick={() => onMove(doc)}>
|
||||
<Move className="mr-2 h-4 w-4" />
|
||||
Move to...
|
||||
</ContextMenuItem>
|
||||
)}
|
||||
<ContextMenuItem disabled={isProcessing} onClick={() => onDelete(doc)}>
|
||||
{onExport && (
|
||||
<ContextMenuSub>
|
||||
<ContextMenuSubTrigger disabled={isUnavailable}>
|
||||
<Download className="mr-2 h-4 w-4" />
|
||||
Export
|
||||
</ContextMenuSubTrigger>
|
||||
<ContextMenuSubContent className="min-w-[180px]">
|
||||
<ExportContextItems onExport={handleExport} exporting={exporting} />
|
||||
</ContextMenuSubContent>
|
||||
</ContextMenuSub>
|
||||
)}
|
||||
{onVersionHistory && isVersionableType(doc.document_type) && (
|
||||
<ContextMenuItem disabled={isUnavailable} onClick={() => onVersionHistory(doc)}>
|
||||
<History className="mr-2 h-4 w-4" />
|
||||
Versions
|
||||
</ContextMenuItem>
|
||||
)}
|
||||
<ContextMenuItem disabled={isProcessing} onClick={() => onDelete(doc)}>
|
||||
<Trash2 className="mr-2 h-4 w-4" />
|
||||
Delete
|
||||
</ContextMenuItem>
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ export function getDocumentTypeIcon(type: string, className?: string): React.Rea
|
|||
return getConnectorIcon(type, className);
|
||||
}
|
||||
|
||||
|
||||
export function DocumentTypeChip({ type, className }: { type: string; className?: string }) {
|
||||
const icon = getDocumentTypeIcon(type, "h-4 w-4");
|
||||
const fullLabel = getDocumentTypeLabel(type);
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover
|
|||
import { Spinner } from "@/components/ui/spinner";
|
||||
import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { cn } from "@/lib/utils";
|
||||
import type { DocumentTypeEnum } from "@/contracts/types/document.types";
|
||||
import { getDocumentTypeIcon } from "./DocumentTypeIcon";
|
||||
import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { getDocumentTypeIcon } from "./DocumentTypeIcon";
|
||||
|
||||
export function DocumentsFilters({
|
||||
typeCounts: typeCountsRecord,
|
||||
|
|
@ -81,18 +81,18 @@ export function DocumentsFilters({
|
|||
<TooltipTrigger asChild>
|
||||
<ToggleGroupItem
|
||||
value="folder"
|
||||
className="h-9 w-9 shrink-0 border-sidebar-border text-muted-foreground hover:text-foreground hover:border-sidebar-border bg-sidebar"
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
onCreateFolder();
|
||||
className="h-9 w-9 shrink-0 border-sidebar-border text-muted-foreground hover:text-foreground hover:border-sidebar-border bg-sidebar"
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
onCreateFolder();
|
||||
}}
|
||||
>
|
||||
<FolderPlus size={14} />
|
||||
</ToggleGroupItem>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>New folder</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
</Tooltip>
|
||||
)}
|
||||
|
||||
{onToggleAiSort && (
|
||||
<Tooltip>
|
||||
|
|
@ -114,9 +114,9 @@ export function DocumentsFilters({
|
|||
aria-label={aiSortEnabled ? "Disable AI sort" : "Enable AI sort"}
|
||||
aria-pressed={aiSortEnabled}
|
||||
>
|
||||
{aiSortBusy ? (
|
||||
<Spinner size="xs" />
|
||||
) : aiSortEnabled ? (
|
||||
{aiSortBusy ? (
|
||||
<Spinner size="xs" />
|
||||
) : aiSortEnabled ? (
|
||||
<IconBinaryTreeFilled size={16} />
|
||||
) : (
|
||||
<IconBinaryTree size={16} />
|
||||
|
|
|
|||
|
|
@ -247,10 +247,8 @@ export function FolderTreeView({
|
|||
function renderLevel(parentId: number | null, depth: number): React.ReactNode[] {
|
||||
const key = parentId ?? "root";
|
||||
const childFolders = (foldersByParent[key] ?? []).slice().sort((a, b) => {
|
||||
const aIsDate =
|
||||
a.metadata?.ai_sort === true && a.metadata?.ai_sort_level === 2;
|
||||
const bIsDate =
|
||||
b.metadata?.ai_sort === true && b.metadata?.ai_sort_level === 2;
|
||||
const aIsDate = a.metadata?.ai_sort === true && a.metadata?.ai_sort_level === 2;
|
||||
const bIsDate = b.metadata?.ai_sort === true && b.metadata?.ai_sort_level === 2;
|
||||
if (aIsDate && bIsDate) {
|
||||
return b.name.localeCompare(a.name);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue