feat: added ai file sorting

surfsense_web/components/documents/DocumentsFilters.tsx

@@ -1,6 +1,8 @@
 "use client";
 
+import { IconBinaryTree, IconBinaryTreeFilled } from "@tabler/icons-react";
 import { FolderPlus, ListFilter, Search, Upload, X } from "lucide-react";
+import { AnimatePresence, motion } from "motion/react";
 import { useTranslations } from "next-intl";
 import React, { useCallback, useMemo, useRef, useState } from "react";
 import { useDocumentUploadDialog } from "@/components/assistant-ui/document-upload-popup";
@@ -10,6 +12,7 @@ import { Input } from "@/components/ui/input";
 import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
 import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
+import { cn } from "@/lib/utils";
 import type { DocumentTypeEnum } from "@/contracts/types/document.types";
 import { getDocumentTypeIcon, getDocumentTypeLabel } from "./DocumentTypeIcon";
 
@@ -20,6 +23,9 @@ export function DocumentsFilters({
 	onToggleType,
 	activeTypes,
 	onCreateFolder,
+	aiSortEnabled = false,
+	aiSortBusy = false,
+	onToggleAiSort,
 }: {
 	typeCounts: Partial<Record<DocumentTypeEnum, number>>;
 	onSearch: (v: string) => void;
@@ -27,6 +33,9 @@ export function DocumentsFilters({
 	onToggleType: (type: DocumentTypeEnum, checked: boolean) => void;
 	activeTypes: DocumentTypeEnum[];
 	onCreateFolder?: () => void;
+	aiSortEnabled?: boolean;
+	aiSortBusy?: boolean;
+	onToggleAiSort?: () => void;
 }) {
 	const t = useTranslations("documents");
 	const id = React.useId();
@@ -64,7 +73,7 @@ export function DocumentsFilters({
 	return (
 		<div className="flex select-none">
 			<div className="flex items-center gap-2 w-full">
-				{/* Filter + New Folder Toggle Group */}
+				{/* New Folder + Filter Toggle Group */}
 				<ToggleGroup type="multiple" variant="outline" value={[]} className="overflow-visible">
 					{onCreateFolder && (
 						<Tooltip>
@@ -172,6 +181,70 @@ export function DocumentsFilters({
 					</Popover>
 				</ToggleGroup>
 
+				{/* AI Sort Toggle */}
+				{onToggleAiSort && (
+					<Tooltip>
+						<TooltipTrigger asChild>
+							<button
+								type="button"
+								disabled={aiSortBusy}
+								onClick={onToggleAiSort}
+								className={cn(
+									"relative h-9 w-9 shrink-0 rounded-md border inline-flex items-center justify-center transition-all duration-300 ease-out",
+									"focus-visible:border-ring focus-visible:ring-[3px] focus-visible:ring-ring/50 outline-none",
+									"disabled:pointer-events-none disabled:opacity-50",
+									aiSortEnabled
+										? "border-violet-400/60 bg-violet-50 text-violet-600 shadow-[0_0_8px_-1px_rgba(139,92,246,0.3)] hover:bg-violet-100 dark:border-violet-500/40 dark:bg-violet-500/15 dark:text-violet-400 dark:shadow-[0_0_8px_-1px_rgba(139,92,246,0.2)] dark:hover:bg-violet-500/25"
+										: "border-sidebar-border bg-sidebar text-sidebar-foreground/60 hover:text-sidebar-foreground hover:border-sidebar-border hover:bg-accent"
+								)}
+								aria-label={aiSortEnabled ? "Disable AI sort" : "Enable AI sort"}
+								aria-pressed={aiSortEnabled}
+							>
+								<AnimatePresence mode="wait" initial={false}>
+									{aiSortBusy ? (
+										<motion.div
+											key="busy"
+											initial={{ opacity: 0, scale: 0.6, rotate: -90 }}
+											animate={{ opacity: 1, scale: 1, rotate: 0 }}
+											exit={{ opacity: 0, scale: 0.6, rotate: 90 }}
+											transition={{ duration: 0.2, ease: "easeInOut" }}
+										>
+											<IconBinaryTree size={16} className="animate-pulse" />
+										</motion.div>
+									) : aiSortEnabled ? (
+										<motion.div
+											key="on"
+											initial={{ opacity: 0, scale: 0.6, rotate: -90 }}
+											animate={{ opacity: 1, scale: 1, rotate: 0 }}
+											exit={{ opacity: 0, scale: 0.6, rotate: 90 }}
+											transition={{ duration: 0.25, ease: "easeInOut" }}
+										>
+											<IconBinaryTreeFilled size={16} />
+										</motion.div>
+									) : (
+										<motion.div
+											key="off"
+											initial={{ opacity: 0, scale: 0.6, rotate: 90 }}
+											animate={{ opacity: 1, scale: 1, rotate: 0 }}
+											exit={{ opacity: 0, scale: 0.6, rotate: -90 }}
+											transition={{ duration: 0.25, ease: "easeInOut" }}
+										>
+											<IconBinaryTree size={16} />
+										</motion.div>
+									)}
+								</AnimatePresence>
+							</button>
+						</TooltipTrigger>
+						<TooltipContent>
+							{aiSortBusy
+								? "AI sort in progress..."
+								: aiSortEnabled
+									? "AI sort active — click to disable"
+									: "Enable AI sort"}
+						</TooltipContent>
+					</Tooltip>
+				)}
+
 				{/* Search Input */}
 				<div className="relative flex-1 min-w-0">
 					<div className="pointer-events-none absolute inset-y-0 left-0 flex items-center pl-3 text-muted-foreground">

surfsense_web/components/documents/FolderTreeView.tsx

@@ -90,6 +90,8 @@ export function FolderTreeView({
 
 	const [openContextMenuId, setOpenContextMenuId] = useState<string | null>(null);
 
+	const [manuallyCollapsedAiIds, setManuallyCollapsedAiIds] = useState<Set<number>>(new Set());
+
 	// Single subscription for rename state — derived boolean passed to each FolderNode
 	const [renamingFolderId, setRenamingFolderId] = useAtom(renamingFolderIdAtom);
 	const handleStartRename = useCallback(
@@ -98,6 +100,38 @@ export function FolderTreeView({
 	);
 	const handleCancelRename = useCallback(() => setRenamingFolderId(null), [setRenamingFolderId]);
 
+	const aiSortFolderLevels = useMemo(() => {
+		const map = new Map<number, number>();
+		for (const f of folders) {
+			if (f.metadata?.ai_sort === true && typeof f.metadata?.ai_sort_level === "number") {
+				map.set(f.id, f.metadata.ai_sort_level as number);
+			}
+		}
+		return map;
+	}, [folders]);
+
+	const handleToggleExpand = useCallback(
+		(folderId: number) => {
+			const aiLevel = aiSortFolderLevels.get(folderId);
+			if (aiLevel !== undefined && aiLevel < 4) {
+				// AI-auto-expanded folder: only toggle the manual-collapse set.
+				// Calling onToggleExpand would add it to expandedIds and fight auto-expand.
+				setManuallyCollapsedAiIds((prev) => {
+					const next = new Set(prev);
+					if (next.has(folderId)) {
+						next.delete(folderId);
+					} else {
+						next.add(folderId);
+					}
+					return next;
+				});
+				return;
+			}
+			onToggleExpand(folderId);
+		},
+		[onToggleExpand, aiSortFolderLevels]
+	);
+
 	const effectiveActiveTypes = useMemo(() => {
 		if (
 			activeTypes.includes("FILE" as DocumentTypeEnum) &&
@@ -212,9 +246,16 @@ export function FolderTreeView({
 
 	function renderLevel(parentId: number | null, depth: number): React.ReactNode[] {
 		const key = parentId ?? "root";
-		const childFolders = (foldersByParent[key] ?? [])
-			.slice()
-			.sort((a, b) => a.position.localeCompare(b.position));
+		const childFolders = (foldersByParent[key] ?? []).slice().sort((a, b) => {
+			const aIsDate =
+				a.metadata?.ai_sort === true && a.metadata?.ai_sort_level === 2;
+			const bIsDate =
+				b.metadata?.ai_sort === true && b.metadata?.ai_sort_level === 2;
+			if (aIsDate && bIsDate) {
+				return b.name.localeCompare(a.name);
+			}
+			return a.position.localeCompare(b.position);
+		});
 		const visibleFolders = hasDescendantMatch
 			? childFolders.filter((f) => hasDescendantMatch[f.id])
 			: childFolders;
@@ -226,6 +267,32 @@ export function FolderTreeView({
 
 		const nodes: React.ReactNode[] = [];
 
+		if (parentId === null) {
+			const processingDocs = childDocs.filter((d) => {
+				const state = d.status?.state;
+				return state === "pending" || state === "processing";
+			});
+			for (const d of processingDocs) {
+				nodes.push(
+					<DocumentNode
+						key={`doc-${d.id}`}
+						doc={d}
+						depth={depth}
+						isMentioned={mentionedDocIds.has(d.id)}
+						onToggleChatMention={onToggleChatMention}
+						onPreview={onPreviewDocument}
+						onEdit={onEditDocument}
+						onDelete={onDeleteDocument}
+						onMove={onMoveDocument}
+						onExport={onExportDocument}
+						onVersionHistory={onVersionHistory}
+						contextMenuOpen={openContextMenuId === `doc-${d.id}`}
+						onContextMenuOpenChange={(open) => setOpenContextMenuId(open ? `doc-${d.id}` : null)}
+					/>
+				);
+			}
+		}
+
 		for (let i = 0; i < visibleFolders.length; i++) {
 			const f = visibleFolders[i];
 			const siblingPositions = {
@@ -233,8 +300,15 @@ export function FolderTreeView({
 				after: i < visibleFolders.length - 1 ? visibleFolders[i + 1].position : null,
 			};
 
-			const isAutoExpanded = !!searchQuery && !!hasDescendantMatch?.[f.id];
-			const isExpanded = expandedIds.has(f.id) || isAutoExpanded;
+			const isSearchAutoExpanded = !!searchQuery && !!hasDescendantMatch?.[f.id];
+			const isAiAutoExpandCandidate =
+				f.metadata?.ai_sort === true &&
+				typeof f.metadata?.ai_sort_level === "number" &&
+				(f.metadata.ai_sort_level as number) < 4;
+			const isManuallyCollapsed = manuallyCollapsedAiIds.has(f.id);
+			const isExpanded = isManuallyCollapsed
+				? isSearchAutoExpanded
+				: expandedIds.has(f.id) || isSearchAutoExpanded || isAiAutoExpandCandidate;
 
 			nodes.push(
 				<FolderNode
@@ -246,7 +320,7 @@ export function FolderTreeView({
 					selectionState={folderSelectionStates[f.id] ?? "none"}
 					processingState={folderProcessingStates[f.id] ?? "idle"}
 					onToggleSelect={onToggleFolderSelect}
-					onToggleExpand={onToggleExpand}
+					onToggleExpand={handleToggleExpand}
 					onRename={onRenameFolder}
 					onStartRename={handleStartRename}
 					onCancelRename={handleCancelRename}
@@ -270,7 +344,15 @@ export function FolderTreeView({
 			}
 		}
 
-		for (const d of childDocs) {
+		const remainingDocs =
+			parentId === null
+				? childDocs.filter((d) => {
+						const state = d.status?.state;
+						return state !== "pending" && state !== "processing";
+					})
+				: childDocs;
+
+		for (const d of remainingDocs) {
 			nodes.push(
 				<DocumentNode
 					key={`doc-${d.id}`}

surfsense_web/components/editor/utils/escape-mdx.ts

@@ -1,25 +1,60 @@
 // ---------------------------------------------------------------------------
-// MDX curly-brace escaping helper
+// MDX pre-processing helpers
 // ---------------------------------------------------------------------------
-// remarkMdx treats { } as JSX expression delimiters. Arbitrary markdown
-// (e.g. AI-generated reports) can contain curly braces that are NOT valid JS
-// expressions, which makes acorn throw "Could not parse expression".
-// We escape unescaped { and } *outside* of fenced code blocks and inline code
-// so remarkMdx treats them as literal characters while still parsing
-// <mark>, <u>, <kbd>, etc. tags correctly.
+// remarkMdx treats { } as JSX expression delimiters and does NOT support
+// HTML comments (<!-- -->). Arbitrary markdown from document conversions
+// (e.g. PDF-to-markdown via Azure/DocIntel) can contain constructs that
+// break the MDX parser. This module sanitises them before deserialization.
 // ---------------------------------------------------------------------------
 
 const FENCED_OR_INLINE_CODE = /(```[\s\S]*?```|`[^`\n]+`)/g;
 
-export function escapeMdxExpressions(md: string): string {
+// Strip HTML comments that MDX cannot parse.
+// PDF converters emit <!-- PageHeader="..." -->, <!-- PageBreak -->, etc.
+// MDX uses JSX-style comments and chokes on HTML comments, causing the
+// parser to stop at the first occurrence.
+// - <!-- PageBreak --> becomes a thematic break (---)
+// - All other HTML comments are removed
+function stripHtmlComments(md: string): string {
+	return md
+		.replace(/<!--\s*PageBreak\s*-->/gi, "\n---\n")
+		.replace(/<!--[\s\S]*?-->/g, "");
+}
+
+// Convert <figure>...</figure> blocks to plain text blockquotes.
+// <figure> with arbitrary text content is not valid JSX, causing the MDX
+// parser to fail.
+function convertFigureBlocks(md: string): string {
+	return md.replace(/<figure[^>]*>([\s\S]*?)<\/figure>/gi, (_match, inner: string) => {
+		const trimmed = (inner as string).trim();
+		if (!trimmed) return "";
+		const quoted = trimmed
+			.split("\n")
+			.map((line) => `> ${line}`)
+			.join("\n");
+		return `\n${quoted}\n`;
+	});
+}
+
+// Escape unescaped { and } outside of fenced/inline code so remarkMdx
+// treats them as literal characters rather than JSX expression delimiters.
+function escapeCurlyBraces(md: string): string {
 	const parts = md.split(FENCED_OR_INLINE_CODE);
 
 	return parts
 		.map((part, i) => {
-			// Odd indices are code blocks / inline code – leave untouched
 			if (i % 2 === 1) return part;
-			// Escape { and } that are NOT already escaped (no preceding \)
 			return part.replace(/(?<!\\)\{/g, "\\{").replace(/(?<!\\)\}/g, "\\}");
 		})
 		.join("");
 }
+
+// Pre-process raw markdown so it can be safely parsed by the MDX-enabled
+// Plate editor. Applies all sanitisation steps in order.
+export function escapeMdxExpressions(md: string): string {
+	let result = md;
+	result = stripHtmlComments(result);
+	result = convertFigureBlocks(result);
+	result = escapeCurlyBraces(result);
+	return result;
+}

surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx

@@ -15,6 +15,7 @@ import { expandedFolderIdsAtom } from "@/atoms/documents/folder.atoms";
 import { agentCreatedDocumentsAtom } from "@/atoms/documents/ui.atoms";
 import { openEditorPanelAtom } from "@/atoms/editor/editor-panel.atom";
 import { rightPanelCollapsedAtom } from "@/atoms/layout/right-panel.atom";
+import { searchSpacesAtom } from "@/atoms/search-spaces/search-space-query.atoms";
 import { CreateFolderDialog } from "@/components/documents/CreateFolderDialog";
 import type { DocumentNodeDoc } from "@/components/documents/DocumentNode";
 import { DocumentsFilters } from "@/components/documents/DocumentsFilters";
@@ -49,6 +50,7 @@ import { useMediaQuery } from "@/hooks/use-media-query";
 import { useElectronAPI } from "@/hooks/use-platform";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { foldersApiService } from "@/lib/apis/folders-api.service";
+import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service";
 import { authenticatedFetch } from "@/lib/auth-utils";
 import { uploadFolderScan } from "@/lib/folder-sync-upload";
 import { getSupportedExtensionsSet } from "@/lib/supported-extensions";
@@ -108,6 +110,47 @@ export function DocumentsSidebar({
 	const [watchInitialFolder, setWatchInitialFolder] = useState<SelectedFolder | null>(null);
 	const isElectron = typeof window !== "undefined" && !!window.electronAPI;
 
+	// AI File Sort state
+	const { data: searchSpaces, refetch: refetchSearchSpaces } = useAtomValue(searchSpacesAtom);
+	const activeSearchSpace = useMemo(
+		() => searchSpaces?.find((s) => s.id === searchSpaceId),
+		[searchSpaces, searchSpaceId]
+	);
+	const aiSortEnabled = activeSearchSpace?.ai_file_sort_enabled ?? false;
+	const [aiSortBusy, setAiSortBusy] = useState(false);
+	const [aiSortConfirmOpen, setAiSortConfirmOpen] = useState(false);
+
+	const handleToggleAiSort = useCallback(() => {
+		if (aiSortEnabled) {
+			// Disable: just update the setting, no confirmation needed
+			setAiSortBusy(true);
+			searchSpacesApiService
+				.updateSearchSpace({ id: searchSpaceId, data: { ai_file_sort_enabled: false } })
+				.then(() => {
+					refetchSearchSpaces();
+					toast.success("AI file sorting disabled");
+				})
+				.catch(() => toast.error("Failed to disable AI file sorting"))
+				.finally(() => setAiSortBusy(false));
+		} else {
+			setAiSortConfirmOpen(true);
+		}
+	}, [aiSortEnabled, searchSpaceId, refetchSearchSpaces]);
+
+	const handleConfirmEnableAiSort = useCallback(() => {
+		setAiSortConfirmOpen(false);
+		setAiSortBusy(true);
+		searchSpacesApiService
+			.updateSearchSpace({ id: searchSpaceId, data: { ai_file_sort_enabled: true } })
+			.then(() => searchSpacesApiService.triggerAiSort(searchSpaceId))
+			.then(() => {
+				refetchSearchSpaces();
+				toast.success("AI file sorting enabled — organizing your documents in the background");
+			})
+			.catch(() => toast.error("Failed to enable AI file sorting"))
+			.finally(() => setAiSortBusy(false));
+	}, [searchSpaceId, refetchSearchSpaces]);
+
 	const handleWatchLocalFolder = useCallback(async () => {
 		const api = window.electronAPI;
 		if (!api?.selectFolder) return;
@@ -905,6 +948,9 @@ export function DocumentsSidebar({
 						onToggleType={onToggleType}
 						activeTypes={activeTypes}
 						onCreateFolder={() => handleCreateFolder(null)}
+						aiSortEnabled={aiSortEnabled}
+						aiSortBusy={aiSortBusy}
+						onToggleAiSort={handleToggleAiSort}
 					/>
 				</div>
 
@@ -1066,6 +1112,25 @@ export function DocumentsSidebar({
 					</AlertDialogFooter>
 				</AlertDialogContent>
 			</AlertDialog>
+
+			<AlertDialog open={aiSortConfirmOpen} onOpenChange={setAiSortConfirmOpen}>
+				<AlertDialogContent>
+					<AlertDialogHeader>
+						<AlertDialogTitle>Enable AI File Sorting?</AlertDialogTitle>
+						<AlertDialogDescription>
+							All documents in this search space will be organized into folders by
+							connector type, date, and AI-generated categories. New documents will
+							also be sorted automatically. You can disable this at any time.
+						</AlertDialogDescription>
+					</AlertDialogHeader>
+					<AlertDialogFooter>
+						<AlertDialogCancel>Cancel</AlertDialogCancel>
+						<AlertDialogAction onClick={handleConfirmEnableAiSort}>
+							Enable
+						</AlertDialogAction>
+					</AlertDialogFooter>
+				</AlertDialogContent>
+			</AlertDialog>
 		</>
 	);
 

surfsense_web/contracts/types/search-space.types.ts

@@ -10,6 +10,7 @@ export const searchSpace = z.object({
 	citations_enabled: z.boolean(),
 	qna_custom_instructions: z.string().nullable(),
 	shared_memory_md: z.string().nullable().optional(),
+	ai_file_sort_enabled: z.boolean().optional().default(false),
 	member_count: z.number(),
 	is_owner: z.boolean(),
 });
@@ -56,6 +57,7 @@ export const updateSearchSpaceRequest = z.object({
 			citations_enabled: true,
 			qna_custom_instructions: true,
 			shared_memory_md: true,
+			ai_file_sort_enabled: true,
 		})
 		.partial(),
 });

surfsense_web/lib/apis/search-spaces-api.service.ts

@@ -1,3 +1,4 @@
+import { z } from "zod";
 import {
 	type CreateSearchSpaceRequest,
 	createSearchSpaceRequest,
@@ -117,6 +118,17 @@ class SearchSpacesApiService {
 		return baseApiService.delete(`/api/v1/searchspaces/${request.id}`, deleteSearchSpaceResponse);
 	};
 
+	/**
+	 * Trigger AI file sorting for all documents in a search space
+	 */
+	triggerAiSort = async (searchSpaceId: number) => {
+		return baseApiService.post(
+			`/api/v1/searchspaces/${searchSpaceId}/ai-sort`,
+			z.object({ message: z.string() }),
+			{}
+		);
+	};
+
 	/**
 	 * Leave a search space (remove own membership)
 	 * This is used by non-owners to leave a shared search space