Merge remote-tracking branch 'upstream/dev' into feat/token-calculation

surfsense_web/components/assistant-ui/assistant-message.tsx

@@ -53,6 +53,7 @@ import { useElectronAPI } from "@/hooks/use-platform";
 import { useTokenUsage } from "@/components/assistant-ui/token-usage-context";
 import { getProviderIcon } from "@/lib/provider-icons";
 import { cn } from "@/lib/utils";
+import { openSafeNavigationHref, resolveSafeNavigationHref } from "@/components/tool-ui/shared/media";
 
 // Captured once at module load — survives client-side navigations that strip the query param.
 const IS_QUICK_ASSIST_WINDOW =
@@ -482,6 +483,7 @@ const AssistantMessageInner: FC = () => {
 								generate_image: GenerateImageToolUI,
 								update_memory: UpdateMemoryToolUI,
 								execute: SandboxExecuteToolUI,
+								execute_code: SandboxExecuteToolUI,
 								create_notion_page: CreateNotionPageToolUI,
 								update_notion_page: UpdateNotionPageToolUI,
 								delete_notion_page: DeleteNotionPageToolUI,

surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx

@@ -2,7 +2,7 @@
 
 import { Search, Unplug } from "lucide-react";
 import type { FC } from "react";
-import { getDocumentTypeLabel } from "@/components/documents/DocumentTypeIcon";
+import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels";
 import { Button } from "@/components/ui/button";
 import { Spinner } from "@/components/ui/spinner";
 import { TabsContent } from "@/components/ui/tabs";

surfsense_web/components/assistant-ui/thread.tsx

@@ -60,7 +60,7 @@ import {
 } from "@/components/assistant-ui/inline-mention-editor";
 import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button";
 import { UserMessage } from "@/components/assistant-ui/user-message";
-import { SLIDEOUT_PANEL_OPENED_EVENT } from "@/components/layout/ui/sidebar/SidebarSlideOutPanel";
+import { SLIDEOUT_PANEL_OPENED_EVENT } from "@/lib/layout-events";
 import {
 	DocumentMentionPicker,
 	type DocumentMentionPickerRef,

surfsense_web/components/chat-comments/comment-item/comment-item.tsx

@@ -9,6 +9,7 @@ import { Button } from "@/components/ui/button";
 import { cn } from "@/lib/utils";
 import { CommentComposer } from "../comment-composer/comment-composer";
 import { CommentActions } from "./comment-actions";
+import { convertRenderedToDisplay } from "@/lib/comments/utils";
 import type { CommentItemProps } from "./types";
 
 function getInitials(name: string | null, email: string): string {
@@ -69,10 +70,6 @@ function formatTimestamp(dateString: string): string {
 	);
 }
 
-export function convertRenderedToDisplay(contentRendered: string): string {
-	// Convert @{DisplayName} format to @DisplayName for editing
-	return contentRendered.replace(/@\{([^}]+)\}/g, "@$1");
-}
 
 function renderMentions(content: string): React.ReactNode {
 	// Match @{DisplayName} format from backend

surfsense_web/components/documents/DocumentTypeIcon.tsx

@@ -4,52 +4,12 @@ import type React from "react";
 import { useEffect, useRef, useState } from "react";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
+import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels";
 
 export function getDocumentTypeIcon(type: string, className?: string): React.ReactNode {
 	return getConnectorIcon(type, className);
 }
 
-export function getDocumentTypeLabel(type: string): string {
-	const labelMap: Record<string, string> = {
-		EXTENSION: "Extension",
-		CRAWLED_URL: "Web Page",
-		FILE: "File",
-		SLACK_CONNECTOR: "Slack",
-		TEAMS_CONNECTOR: "Microsoft Teams",
-		ONEDRIVE_FILE: "OneDrive",
-		DROPBOX_FILE: "Dropbox",
-		NOTION_CONNECTOR: "Notion",
-		YOUTUBE_VIDEO: "YouTube Video",
-		GITHUB_CONNECTOR: "GitHub",
-		LINEAR_CONNECTOR: "Linear",
-		DISCORD_CONNECTOR: "Discord",
-		JIRA_CONNECTOR: "Jira",
-		CONFLUENCE_CONNECTOR: "Confluence",
-		CLICKUP_CONNECTOR: "ClickUp",
-		GOOGLE_CALENDAR_CONNECTOR: "Google Calendar",
-		GOOGLE_GMAIL_CONNECTOR: "Gmail",
-		GOOGLE_DRIVE_FILE: "Google Drive",
-		AIRTABLE_CONNECTOR: "Airtable",
-		LUMA_CONNECTOR: "Luma",
-		ELASTICSEARCH_CONNECTOR: "Elasticsearch",
-		BOOKSTACK_CONNECTOR: "BookStack",
-		CIRCLEBACK: "Circleback",
-		OBSIDIAN_CONNECTOR: "Obsidian",
-		LOCAL_FOLDER_FILE: "Local Folder",
-		SURFSENSE_DOCS: "SurfSense Docs",
-		NOTE: "Note",
-		COMPOSIO_GOOGLE_DRIVE_CONNECTOR: "Composio Google Drive",
-		COMPOSIO_GMAIL_CONNECTOR: "Composio Gmail",
-		COMPOSIO_GOOGLE_CALENDAR_CONNECTOR: "Composio Google Calendar",
-	};
-	return (
-		labelMap[type] ||
-		type
-			.split("_")
-			.map((word) => word.charAt(0) + word.slice(1).toLowerCase())
-			.join(" ")
-	);
-}
 
 export function DocumentTypeChip({ type, className }: { type: string; className?: string }) {
 	const icon = getDocumentTypeIcon(type, "h-4 w-4");

surfsense_web/components/documents/DocumentsFilters.tsx

@@ -1,6 +1,8 @@
 "use client";
 
+import { IconBinaryTree, IconBinaryTreeFilled } from "@tabler/icons-react";
 import { FolderPlus, ListFilter, Search, Upload, X } from "lucide-react";
+import { AnimatePresence, motion } from "motion/react";
 import { useTranslations } from "next-intl";
 import React, { useCallback, useMemo, useRef, useState } from "react";
 import { useDocumentUploadDialog } from "@/components/assistant-ui/document-upload-popup";
@@ -10,8 +12,10 @@ import { Input } from "@/components/ui/input";
 import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
 import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
+import { cn } from "@/lib/utils";
 import type { DocumentTypeEnum } from "@/contracts/types/document.types";
-import { getDocumentTypeIcon, getDocumentTypeLabel } from "./DocumentTypeIcon";
+import { getDocumentTypeIcon } from "./DocumentTypeIcon";
+import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels";
 
 export function DocumentsFilters({
 	typeCounts: typeCountsRecord,
@@ -20,6 +24,9 @@ export function DocumentsFilters({
 	onToggleType,
 	activeTypes,
 	onCreateFolder,
+	aiSortEnabled = false,
+	aiSortBusy = false,
+	onToggleAiSort,
 }: {
 	typeCounts: Partial<Record<DocumentTypeEnum, number>>;
 	onSearch: (v: string) => void;
@@ -27,6 +34,9 @@ export function DocumentsFilters({
 	onToggleType: (type: DocumentTypeEnum, checked: boolean) => void;
 	activeTypes: DocumentTypeEnum[];
 	onCreateFolder?: () => void;
+	aiSortEnabled?: boolean;
+	aiSortBusy?: boolean;
+	onToggleAiSort?: () => void;
 }) {
 	const t = useTranslations("documents");
 	const id = React.useId();
@@ -64,7 +74,7 @@ export function DocumentsFilters({
 	return (
 		<div className="flex select-none">
 			<div className="flex items-center gap-2 w-full">
-				{/* Filter + New Folder Toggle Group */}
+				{/* New Folder + Filter Toggle Group */}
 				<ToggleGroup type="multiple" variant="outline" value={[]} className="overflow-visible">
 					{onCreateFolder && (
 						<Tooltip>
@@ -172,6 +182,70 @@ export function DocumentsFilters({
 					</Popover>
 				</ToggleGroup>
 
+				{/* AI Sort Toggle */}
+				{onToggleAiSort && (
+					<Tooltip>
+						<TooltipTrigger asChild>
+							<button
+								type="button"
+								disabled={aiSortBusy}
+								onClick={onToggleAiSort}
+								className={cn(
+									"relative h-9 w-9 shrink-0 rounded-md border inline-flex items-center justify-center transition-all duration-300 ease-out",
+									"focus-visible:border-ring focus-visible:ring-[3px] focus-visible:ring-ring/50 outline-none",
+									"disabled:pointer-events-none disabled:opacity-50",
+									aiSortEnabled
+										? "border-violet-400/60 bg-violet-50 text-violet-600 shadow-[0_0_8px_-1px_rgba(139,92,246,0.3)] hover:bg-violet-100 dark:border-violet-500/40 dark:bg-violet-500/15 dark:text-violet-400 dark:shadow-[0_0_8px_-1px_rgba(139,92,246,0.2)] dark:hover:bg-violet-500/25"
+										: "border-sidebar-border bg-sidebar text-sidebar-foreground/60 hover:text-sidebar-foreground hover:border-sidebar-border hover:bg-accent"
+								)}
+								aria-label={aiSortEnabled ? "Disable AI sort" : "Enable AI sort"}
+								aria-pressed={aiSortEnabled}
+							>
+								<AnimatePresence mode="wait" initial={false}>
+									{aiSortBusy ? (
+										<motion.div
+											key="busy"
+											initial={{ opacity: 0, scale: 0.6, rotate: -90 }}
+											animate={{ opacity: 1, scale: 1, rotate: 0 }}
+											exit={{ opacity: 0, scale: 0.6, rotate: 90 }}
+											transition={{ duration: 0.2, ease: "easeInOut" }}
+										>
+											<IconBinaryTree size={16} className="animate-pulse" />
+										</motion.div>
+									) : aiSortEnabled ? (
+										<motion.div
+											key="on"
+											initial={{ opacity: 0, scale: 0.6, rotate: -90 }}
+											animate={{ opacity: 1, scale: 1, rotate: 0 }}
+											exit={{ opacity: 0, scale: 0.6, rotate: 90 }}
+											transition={{ duration: 0.25, ease: "easeInOut" }}
+										>
+											<IconBinaryTreeFilled size={16} />
+										</motion.div>
+									) : (
+										<motion.div
+											key="off"
+											initial={{ opacity: 0, scale: 0.6, rotate: 90 }}
+											animate={{ opacity: 1, scale: 1, rotate: 0 }}
+											exit={{ opacity: 0, scale: 0.6, rotate: -90 }}
+											transition={{ duration: 0.25, ease: "easeInOut" }}
+										>
+											<IconBinaryTree size={16} />
+										</motion.div>
+									)}
+								</AnimatePresence>
+							</button>
+						</TooltipTrigger>
+						<TooltipContent>
+							{aiSortBusy
+								? "AI sort in progress..."
+								: aiSortEnabled
+									? "AI sort active — click to disable"
+									: "Enable AI sort"}
+						</TooltipContent>
+					</Tooltip>
+				)}
+
 				{/* Search Input */}
 				<div className="relative flex-1 min-w-0">
 					<div className="pointer-events-none absolute inset-y-0 left-0 flex items-center pl-3 text-muted-foreground">

surfsense_web/components/documents/FolderTreeView.tsx

@@ -90,6 +90,8 @@ export function FolderTreeView({
 
 	const [openContextMenuId, setOpenContextMenuId] = useState<string | null>(null);
 
+	const [manuallyCollapsedAiIds, setManuallyCollapsedAiIds] = useState<Set<number>>(new Set());
+
 	// Single subscription for rename state — derived boolean passed to each FolderNode
 	const [renamingFolderId, setRenamingFolderId] = useAtom(renamingFolderIdAtom);
 	const handleStartRename = useCallback(
@@ -98,6 +100,38 @@ export function FolderTreeView({
 	);
 	const handleCancelRename = useCallback(() => setRenamingFolderId(null), [setRenamingFolderId]);
 
+	const aiSortFolderLevels = useMemo(() => {
+		const map = new Map<number, number>();
+		for (const f of folders) {
+			if (f.metadata?.ai_sort === true && typeof f.metadata?.ai_sort_level === "number") {
+				map.set(f.id, f.metadata.ai_sort_level as number);
+			}
+		}
+		return map;
+	}, [folders]);
+
+	const handleToggleExpand = useCallback(
+		(folderId: number) => {
+			const aiLevel = aiSortFolderLevels.get(folderId);
+			if (aiLevel !== undefined && aiLevel < 4) {
+				// AI-auto-expanded folder: only toggle the manual-collapse set.
+				// Calling onToggleExpand would add it to expandedIds and fight auto-expand.
+				setManuallyCollapsedAiIds((prev) => {
+					const next = new Set(prev);
+					if (next.has(folderId)) {
+						next.delete(folderId);
+					} else {
+						next.add(folderId);
+					}
+					return next;
+				});
+				return;
+			}
+			onToggleExpand(folderId);
+		},
+		[onToggleExpand, aiSortFolderLevels]
+	);
+
 	const effectiveActiveTypes = useMemo(() => {
 		if (
 			activeTypes.includes("FILE" as DocumentTypeEnum) &&
@@ -212,9 +246,16 @@ export function FolderTreeView({
 
 	function renderLevel(parentId: number | null, depth: number): React.ReactNode[] {
 		const key = parentId ?? "root";
-		const childFolders = (foldersByParent[key] ?? [])
-			.slice()
-			.sort((a, b) => a.position.localeCompare(b.position));
+		const childFolders = (foldersByParent[key] ?? []).slice().sort((a, b) => {
+			const aIsDate =
+				a.metadata?.ai_sort === true && a.metadata?.ai_sort_level === 2;
+			const bIsDate =
+				b.metadata?.ai_sort === true && b.metadata?.ai_sort_level === 2;
+			if (aIsDate && bIsDate) {
+				return b.name.localeCompare(a.name);
+			}
+			return a.position.localeCompare(b.position);
+		});
 		const visibleFolders = hasDescendantMatch
 			? childFolders.filter((f) => hasDescendantMatch[f.id])
 			: childFolders;
@@ -226,6 +267,32 @@ export function FolderTreeView({
 
 		const nodes: React.ReactNode[] = [];
 
+		if (parentId === null) {
+			const processingDocs = childDocs.filter((d) => {
+				const state = d.status?.state;
+				return state === "pending" || state === "processing";
+			});
+			for (const d of processingDocs) {
+				nodes.push(
+					<DocumentNode
+						key={`doc-${d.id}`}
+						doc={d}
+						depth={depth}
+						isMentioned={mentionedDocIds.has(d.id)}
+						onToggleChatMention={onToggleChatMention}
+						onPreview={onPreviewDocument}
+						onEdit={onEditDocument}
+						onDelete={onDeleteDocument}
+						onMove={onMoveDocument}
+						onExport={onExportDocument}
+						onVersionHistory={onVersionHistory}
+						contextMenuOpen={openContextMenuId === `doc-${d.id}`}
+						onContextMenuOpenChange={(open) => setOpenContextMenuId(open ? `doc-${d.id}` : null)}
+					/>
+				);
+			}
+		}
+
 		for (let i = 0; i < visibleFolders.length; i++) {
 			const f = visibleFolders[i];
 			const siblingPositions = {
@@ -233,8 +300,15 @@ export function FolderTreeView({
 				after: i < visibleFolders.length - 1 ? visibleFolders[i + 1].position : null,
 			};
 
-			const isAutoExpanded = !!searchQuery && !!hasDescendantMatch?.[f.id];
-			const isExpanded = expandedIds.has(f.id) || isAutoExpanded;
+			const isSearchAutoExpanded = !!searchQuery && !!hasDescendantMatch?.[f.id];
+			const isAiAutoExpandCandidate =
+				f.metadata?.ai_sort === true &&
+				typeof f.metadata?.ai_sort_level === "number" &&
+				(f.metadata.ai_sort_level as number) < 4;
+			const isManuallyCollapsed = manuallyCollapsedAiIds.has(f.id);
+			const isExpanded = isManuallyCollapsed
+				? isSearchAutoExpanded
+				: expandedIds.has(f.id) || isSearchAutoExpanded || isAiAutoExpandCandidate;
 
 			nodes.push(
 				<FolderNode
@@ -246,7 +320,7 @@ export function FolderTreeView({
 					selectionState={folderSelectionStates[f.id] ?? "none"}
 					processingState={folderProcessingStates[f.id] ?? "idle"}
 					onToggleSelect={onToggleFolderSelect}
-					onToggleExpand={onToggleExpand}
+					onToggleExpand={handleToggleExpand}
 					onRename={onRenameFolder}
 					onStartRename={handleStartRename}
 					onCancelRename={handleCancelRename}
@@ -270,7 +344,15 @@ export function FolderTreeView({
 			}
 		}
 
-		for (const d of childDocs) {
+		const remainingDocs =
+			parentId === null
+				? childDocs.filter((d) => {
+						const state = d.status?.state;
+						return state !== "pending" && state !== "processing";
+					})
+				: childDocs;
+
+		for (const d of remainingDocs) {
 			nodes.push(
 				<DocumentNode
 					key={`doc-${d.id}`}

surfsense_web/components/editor/utils/escape-mdx.ts

@@ -1,25 +1,60 @@
 // ---------------------------------------------------------------------------
-// MDX curly-brace escaping helper
+// MDX pre-processing helpers
 // ---------------------------------------------------------------------------
-// remarkMdx treats { } as JSX expression delimiters. Arbitrary markdown
-// (e.g. AI-generated reports) can contain curly braces that are NOT valid JS
-// expressions, which makes acorn throw "Could not parse expression".
-// We escape unescaped { and } *outside* of fenced code blocks and inline code
-// so remarkMdx treats them as literal characters while still parsing
-// <mark>, <u>, <kbd>, etc. tags correctly.
+// remarkMdx treats { } as JSX expression delimiters and does NOT support
+// HTML comments (<!-- -->). Arbitrary markdown from document conversions
+// (e.g. PDF-to-markdown via Azure/DocIntel) can contain constructs that
+// break the MDX parser. This module sanitises them before deserialization.
 // ---------------------------------------------------------------------------
 
 const FENCED_OR_INLINE_CODE = /(```[\s\S]*?```|`[^`\n]+`)/g;
 
-export function escapeMdxExpressions(md: string): string {
+// Strip HTML comments that MDX cannot parse.
+// PDF converters emit <!-- PageHeader="..." -->, <!-- PageBreak -->, etc.
+// MDX uses JSX-style comments and chokes on HTML comments, causing the
+// parser to stop at the first occurrence.
+// - <!-- PageBreak --> becomes a thematic break (---)
+// - All other HTML comments are removed
+function stripHtmlComments(md: string): string {
+	return md
+		.replace(/<!--\s*PageBreak\s*-->/gi, "\n---\n")
+		.replace(/<!--[\s\S]*?-->/g, "");
+}
+
+// Convert <figure>...</figure> blocks to plain text blockquotes.
+// <figure> with arbitrary text content is not valid JSX, causing the MDX
+// parser to fail.
+function convertFigureBlocks(md: string): string {
+	return md.replace(/<figure[^>]*>([\s\S]*?)<\/figure>/gi, (_match, inner: string) => {
+		const trimmed = (inner as string).trim();
+		if (!trimmed) return "";
+		const quoted = trimmed
+			.split("\n")
+			.map((line) => `> ${line}`)
+			.join("\n");
+		return `\n${quoted}\n`;
+	});
+}
+
+// Escape unescaped { and } outside of fenced/inline code so remarkMdx
+// treats them as literal characters rather than JSX expression delimiters.
+function escapeCurlyBraces(md: string): string {
 	const parts = md.split(FENCED_OR_INLINE_CODE);
 
 	return parts
 		.map((part, i) => {
-			// Odd indices are code blocks / inline code – leave untouched
 			if (i % 2 === 1) return part;
-			// Escape { and } that are NOT already escaped (no preceding \)
 			return part.replace(/(?<!\\)\{/g, "\\{").replace(/(?<!\\)\}/g, "\\}");
 		})
 		.join("");
 }
+
+// Pre-process raw markdown so it can be safely parsed by the MDX-enabled
+// Plate editor. Applies all sanitisation steps in order.
+export function escapeMdxExpressions(md: string): string {
+	let result = md;
+	result = stripHtmlComments(result);
+	result = convertFigureBlocks(result);
+	result = escapeCurlyBraces(result);
+	return result;
+}

surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx

@@ -15,6 +15,7 @@ import { expandedFolderIdsAtom } from "@/atoms/documents/folder.atoms";
 import { agentCreatedDocumentsAtom } from "@/atoms/documents/ui.atoms";
 import { openEditorPanelAtom } from "@/atoms/editor/editor-panel.atom";
 import { rightPanelCollapsedAtom } from "@/atoms/layout/right-panel.atom";
+import { searchSpacesAtom } from "@/atoms/search-spaces/search-space-query.atoms";
 import { CreateFolderDialog } from "@/components/documents/CreateFolderDialog";
 import type { DocumentNodeDoc } from "@/components/documents/DocumentNode";
 import { DocumentsFilters } from "@/components/documents/DocumentsFilters";
@@ -49,6 +50,7 @@ import { useMediaQuery } from "@/hooks/use-media-query";
 import { useElectronAPI } from "@/hooks/use-platform";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { foldersApiService } from "@/lib/apis/folders-api.service";
+import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service";
 import { authenticatedFetch } from "@/lib/auth-utils";
 import { uploadFolderScan } from "@/lib/folder-sync-upload";
 import { getSupportedExtensionsSet } from "@/lib/supported-extensions";
@@ -108,6 +110,47 @@ export function DocumentsSidebar({
 	const [watchInitialFolder, setWatchInitialFolder] = useState<SelectedFolder | null>(null);
 	const isElectron = typeof window !== "undefined" && !!window.electronAPI;
 
+	// AI File Sort state
+	const { data: searchSpaces, refetch: refetchSearchSpaces } = useAtomValue(searchSpacesAtom);
+	const activeSearchSpace = useMemo(
+		() => searchSpaces?.find((s) => s.id === searchSpaceId),
+		[searchSpaces, searchSpaceId]
+	);
+	const aiSortEnabled = activeSearchSpace?.ai_file_sort_enabled ?? false;
+	const [aiSortBusy, setAiSortBusy] = useState(false);
+	const [aiSortConfirmOpen, setAiSortConfirmOpen] = useState(false);
+
+	const handleToggleAiSort = useCallback(() => {
+		if (aiSortEnabled) {
+			// Disable: just update the setting, no confirmation needed
+			setAiSortBusy(true);
+			searchSpacesApiService
+				.updateSearchSpace({ id: searchSpaceId, data: { ai_file_sort_enabled: false } })
+				.then(() => {
+					refetchSearchSpaces();
+					toast.success("AI file sorting disabled");
+				})
+				.catch(() => toast.error("Failed to disable AI file sorting"))
+				.finally(() => setAiSortBusy(false));
+		} else {
+			setAiSortConfirmOpen(true);
+		}
+	}, [aiSortEnabled, searchSpaceId, refetchSearchSpaces]);
+
+	const handleConfirmEnableAiSort = useCallback(() => {
+		setAiSortConfirmOpen(false);
+		setAiSortBusy(true);
+		searchSpacesApiService
+			.updateSearchSpace({ id: searchSpaceId, data: { ai_file_sort_enabled: true } })
+			.then(() => searchSpacesApiService.triggerAiSort(searchSpaceId))
+			.then(() => {
+				refetchSearchSpaces();
+				toast.success("AI file sorting enabled — organizing your documents in the background");
+			})
+			.catch(() => toast.error("Failed to enable AI file sorting"))
+			.finally(() => setAiSortBusy(false));
+	}, [searchSpaceId, refetchSearchSpaces]);
+
 	const handleWatchLocalFolder = useCallback(async () => {
 		const api = window.electronAPI;
 		if (!api?.selectFolder) return;
@@ -905,6 +948,9 @@ export function DocumentsSidebar({
 						onToggleType={onToggleType}
 						activeTypes={activeTypes}
 						onCreateFolder={() => handleCreateFolder(null)}
+						aiSortEnabled={aiSortEnabled}
+						aiSortBusy={aiSortBusy}
+						onToggleAiSort={handleToggleAiSort}
 					/>
 				</div>
 
@@ -1066,6 +1112,25 @@ export function DocumentsSidebar({
 					</AlertDialogFooter>
 				</AlertDialogContent>
 			</AlertDialog>
+
+			<AlertDialog open={aiSortConfirmOpen} onOpenChange={setAiSortConfirmOpen}>
+				<AlertDialogContent>
+					<AlertDialogHeader>
+						<AlertDialogTitle>Enable AI File Sorting?</AlertDialogTitle>
+						<AlertDialogDescription>
+							All documents in this search space will be organized into folders by
+							connector type, date, and AI-generated categories. New documents will
+							also be sorted automatically. You can disable this at any time.
+						</AlertDialogDescription>
+					</AlertDialogHeader>
+					<AlertDialogFooter>
+						<AlertDialogCancel>Cancel</AlertDialogCancel>
+						<AlertDialogAction onClick={handleConfirmEnableAiSort}>
+							Enable
+						</AlertDialogAction>
+					</AlertDialogFooter>
+				</AlertDialogContent>
+			</AlertDialog>
 		</>
 	);
 

surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx

@@ -22,8 +22,8 @@ import { useParams, useRouter } from "next/navigation";
 import { useTranslations } from "next-intl";
 import { useCallback, useDeferredValue, useEffect, useMemo, useRef, useState } from "react";
 import { setTargetCommentIdAtom } from "@/atoms/chat/current-thread.atom";
-import { convertRenderedToDisplay } from "@/components/chat-comments/comment-item/comment-item";
-import { getDocumentTypeLabel } from "@/components/documents/DocumentTypeIcon";
+import { convertRenderedToDisplay } from "@/lib/comments/utils";
+import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels";
 import { Tabs, TabsList, TabsTrigger } from "@/components/ui/animated-tabs";
 import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar";
 import { Button } from "@/components/ui/button";

surfsense_web/components/layout/ui/sidebar/SidebarSlideOutPanel.tsx

@@ -3,8 +3,8 @@
 import { AnimatePresence, motion } from "motion/react";
 import { useCallback, useEffect } from "react";
 import { useMediaQuery } from "@/hooks/use-media-query";
+import { SLIDEOUT_PANEL_OPENED_EVENT } from "@/lib/layout-events";
 
-export const SLIDEOUT_PANEL_OPENED_EVENT = "slideout-panel-opened";
 
 interface SidebarSlideOutPanelProps {
 	open: boolean;

surfsense_web/components/new-chat/chat-header.tsx

@@ -44,21 +44,28 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
 	const [isVisionGlobal, setIsVisionGlobal] = useState(false);
 	const [visionDialogMode, setVisionDialogMode] = useState<"create" | "edit" | "view">("view");
 
+	// Default provider for create dialogs
+	const [defaultLLMProvider, setDefaultLLMProvider] = useState<string | undefined>();
+	const [defaultImageProvider, setDefaultImageProvider] = useState<string | undefined>();
+	const [defaultVisionProvider, setDefaultVisionProvider] = useState<string | undefined>();
+
 	// LLM handlers
 	const handleEditLLMConfig = useCallback(
 		(config: NewLLMConfigPublic | GlobalNewLLMConfig, global: boolean) => {
 			setSelectedConfig(config);
 			setIsGlobal(global);
 			setDialogMode(global ? "view" : "edit");
+			setDefaultLLMProvider(undefined);
 			setDialogOpen(true);
 		},
 		[]
 	);
 
-	const handleAddNewLLM = useCallback(() => {
+	const handleAddNewLLM = useCallback((provider?: string) => {
 		setSelectedConfig(null);
 		setIsGlobal(false);
 		setDialogMode("create");
+		setDefaultLLMProvider(provider);
 		setDialogOpen(true);
 	}, []);
 
@@ -68,10 +75,11 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
 	}, []);
 
 	// Image model handlers
-	const handleAddImageModel = useCallback(() => {
+	const handleAddImageModel = useCallback((provider?: string) => {
 		setSelectedImageConfig(null);
 		setIsImageGlobal(false);
 		setImageDialogMode("create");
+		setDefaultImageProvider(provider);
 		setImageDialogOpen(true);
 	}, []);
 
@@ -80,6 +88,7 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
 			setSelectedImageConfig(config);
 			setIsImageGlobal(global);
 			setImageDialogMode(global ? "view" : "edit");
+			setDefaultImageProvider(undefined);
 			setImageDialogOpen(true);
 		},
 		[]
@@ -91,10 +100,11 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
 	}, []);
 
 	// Vision model handlers
-	const handleAddVisionModel = useCallback(() => {
+	const handleAddVisionModel = useCallback((provider?: string) => {
 		setSelectedVisionConfig(null);
 		setIsVisionGlobal(false);
 		setVisionDialogMode("create");
+		setDefaultVisionProvider(provider);
 		setVisionDialogOpen(true);
 	}, []);
 
@@ -103,6 +113,7 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
 			setSelectedVisionConfig(config);
 			setIsVisionGlobal(global);
 			setVisionDialogMode(global ? "view" : "edit");
+			setDefaultVisionProvider(undefined);
 			setVisionDialogOpen(true);
 		},
 		[]
@@ -131,6 +142,7 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
 				isGlobal={isGlobal}
 				searchSpaceId={searchSpaceId}
 				mode={dialogMode}
+				defaultProvider={defaultLLMProvider}
 			/>
 			<ImageConfigDialog
 				open={imageDialogOpen}
@@ -139,6 +151,7 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
 				isGlobal={isImageGlobal}
 				searchSpaceId={searchSpaceId}
 				mode={imageDialogMode}
+				defaultProvider={defaultImageProvider}
 			/>
 			<VisionConfigDialog
 				open={visionDialogOpen}
@@ -147,6 +160,7 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
 				isGlobal={isVisionGlobal}
 				searchSpaceId={searchSpaceId}
 				mode={visionDialogMode}
+				defaultProvider={defaultVisionProvider}
 			/>
 		</div>
 	);

surfsense_web/components/pricing/pricing-section.tsx

@@ -1,4 +1,9 @@
+"use client";
+import React, { useRef, useEffect, useState } from "react";
+import { AnimatePresence, motion } from "motion/react";
+import { IconPlus } from "@tabler/icons-react";
 import { Pricing } from "@/components/pricing";
+import { cn } from "@/lib/utils";
 
 const demoPlans = [
 	{
@@ -59,13 +64,280 @@ const demoPlans = [
 	},
 ];
 
+interface FAQItem {
+	question: string;
+	answer: string;
+}
+
+interface FAQSection {
+	title: string;
+	items: FAQItem[];
+}
+
+const faqData: FAQSection[] = [
+	{
+		title: "Pages & Billing",
+		items: [
+			{
+				question: "What exactly is a \"page\" in SurfSense?",
+				answer:
+					"A page is a simple billing unit that measures how much content you add to your knowledge base. For PDFs, one page equals one real PDF page. For other document types like Word, PowerPoint, and Excel files, pages are automatically estimated based on the file. Every file uses at least 1 page.",
+			},
+			{
+				question: "How does the Pay As You Go plan work?",
+				answer:
+					"There's no monthly subscription. When you need more pages, simply purchase 1,000-page packs at $1 each. Purchased pages are added to your account immediately so you can keep indexing right away. You only pay when you actually need more.",
+			},
+			{
+				question: "What happens if I run out of pages?",
+				answer:
+					"SurfSense checks your remaining pages before processing each file. If you don't have enough, the upload is paused and you'll be notified. You can purchase additional page packs at any time to continue. For cloud connector syncs, a small overage may be allowed so your sync doesn't partially fail.",
+			},
+			{
+				question: "If I delete a document, do I get my pages back?",
+				answer:
+					"No. Deleting a document removes it from your knowledge base, but the pages it used are not refunded. Pages track your total usage over time, not how much is currently stored. So be mindful of what you index. Once pages are spent, they're spent even if you later remove the document.",
+			},
+		],
+	},
+	{
+		title: "File Types & Connectors",
+		items: [
+			{
+				question: "Which file types count toward my page limit?",
+				answer:
+					"Page limits only apply to document files that need processing, including PDFs, Word documents (DOC, DOCX, ODT, RTF), presentations (PPT, PPTX, ODP), spreadsheets (XLS, XLSX, ODS), ebooks (EPUB), and images (JPG, PNG, TIFF, WebP, BMP). Plain text files, code files, Markdown, CSV, TSV, HTML, audio, and video files do not consume any pages.",
+			},
+			{
+				question: "How are pages consumed?",
+				answer:
+					"Pages are deducted whenever a document file is successfully indexed into your knowledge base, whether through direct uploads or file-based connector syncs (Google Drive, OneDrive, Dropbox, Local Folder). SurfSense checks your remaining pages before processing and only charges you after the file is indexed. Duplicate documents are automatically detected and won't cost you extra pages.",
+			},
+			{
+				question: "Do connectors like Slack, Notion, or Gmail use pages?",
+				answer:
+					"No. Connectors that work with structured text data like Slack, Discord, Notion, Confluence, Jira, Linear, ClickUp, GitHub, Gmail, Google Calendar, Microsoft Teams, Airtable, Elasticsearch, Web Crawler, BookStack, Obsidian, and Luma do not use pages at all. Page limits only apply to file-based connectors that need document processing, such as Google Drive, OneDrive, Dropbox, and Local Folder syncs.",
+			},
+		],
+	},
+	{
+		title: "Self-Hosting",
+		items: [
+			{
+				question: "Can I self-host SurfSense with unlimited pages?",
+				answer:
+					"Yes! When self-hosting, you have full control over your page limits. The default self-hosted setup gives you effectively unlimited pages, so you can index as much data as your infrastructure supports.",
+			},
+		],
+	},
+];
+
+const GridLineHorizontal = ({
+	className,
+	offset,
+}: {
+	className?: string;
+	offset?: string;
+}) => {
+	return (
+		<div
+			style={
+				{
+					"--background": "#ffffff",
+					"--color": "rgba(0, 0, 0, 0.2)",
+					"--height": "1px",
+					"--width": "5px",
+					"--fade-stop": "90%",
+					"--offset": offset || "200px",
+					"--color-dark": "rgba(255, 255, 255, 0.2)",
+					maskComposite: "exclude",
+				} as React.CSSProperties
+			}
+			className={cn(
+				"[--background:var(--color-neutral-200)] [--color:var(--color-neutral-400)] dark:[--background:var(--color-neutral-800)] dark:[--color:var(--color-neutral-600)]",
+				"absolute left-[calc(var(--offset)/2*-1)] h-(--height) w-[calc(100%+var(--offset))]",
+				"bg-[linear-gradient(to_right,var(--color),var(--color)_50%,transparent_0,transparent)]",
+				"bg-size-[var(--width)_var(--height)]",
+				"[mask:linear-gradient(to_left,var(--background)_var(--fade-stop),transparent),linear-gradient(to_right,var(--background)_var(--fade-stop),transparent),linear-gradient(black,black)]",
+				"mask-exclude",
+				"z-30",
+				"dark:bg-[linear-gradient(to_right,var(--color-dark),var(--color-dark)_50%,transparent_0,transparent)]",
+				className,
+			)}
+		/>
+	);
+};
+
+const GridLineVertical = ({
+	className,
+	offset,
+}: {
+	className?: string;
+	offset?: string;
+}) => {
+	return (
+		<div
+			style={
+				{
+					"--background": "#ffffff",
+					"--color": "rgba(0, 0, 0, 0.2)",
+					"--height": "5px",
+					"--width": "1px",
+					"--fade-stop": "90%",
+					"--offset": offset || "150px",
+					"--color-dark": "rgba(255, 255, 255, 0.2)",
+					maskComposite: "exclude",
+				} as React.CSSProperties
+			}
+			className={cn(
+				"absolute top-[calc(var(--offset)/2*-1)] h-[calc(100%+var(--offset))] w-(--width)",
+				"bg-[linear-gradient(to_bottom,var(--color),var(--color)_50%,transparent_0,transparent)]",
+				"bg-size-[var(--width)_var(--height)]",
+				"[mask:linear-gradient(to_top,var(--background)_var(--fade-stop),transparent),linear-gradient(to_bottom,var(--background)_var(--fade-stop),transparent),linear-gradient(black,black)]",
+				"mask-exclude",
+				"z-30",
+				"dark:bg-[linear-gradient(to_bottom,var(--color-dark),var(--color-dark)_50%,transparent_0,transparent)]",
+				className,
+			)}
+		/>
+	);
+};
+
+function PricingFAQ() {
+	const [activeId, setActiveId] = useState<string | null>(null);
+	const containerRef = useRef<HTMLDivElement>(null);
+
+	useEffect(() => {
+		function handleClickOutside(event: MouseEvent) {
+			if (
+				containerRef.current &&
+				!containerRef.current.contains(event.target as Node)
+			) {
+				setActiveId(null);
+			}
+		}
+
+		document.addEventListener("mousedown", handleClickOutside);
+		return () => document.removeEventListener("mousedown", handleClickOutside);
+	}, []);
+
+	const toggleQuestion = (id: string) => {
+		setActiveId(activeId === id ? null : id);
+	};
+
+	return (
+		<div className="mx-auto w-full max-w-4xl overflow-hidden px-4 py-20 md:px-8 md:py-32">
+			<div className="text-center">
+				<h2 className="text-4xl font-bold tracking-tight sm:text-5xl">
+					Frequently Asked Questions
+				</h2>
+				<p className="mx-auto mt-4 max-w-2xl text-lg text-muted-foreground">
+					Everything you need to know about SurfSense pages and billing.
+					Can&apos;t find what you need? Reach out at{" "}
+					<a
+						href="mailto:rohan@surfsense.com"
+						className="text-blue-500 underline"
+					>
+						rohan@surfsense.com
+					</a>
+				</p>
+			</div>
+
+			<div
+				ref={containerRef}
+				className="relative mt-16 flex w-full flex-col gap-12 px-4 md:px-8"
+			>
+				{faqData.map((section) => (
+					<div key={section.title + "faq"}>
+						<h3 className="mb-6 text-lg font-medium text-neutral-800 dark:text-neutral-200">
+							{section.title}
+						</h3>
+						<div className="flex flex-col gap-3">
+							{section.items.map((item, index) => {
+								const id = `${section.title}-${index}`;
+								const isActive = activeId === id;
+
+								return (
+									<div
+										key={id + "faq-item"}
+										className={cn(
+											"relative rounded-lg transition-all duration-200",
+											isActive
+												? "bg-white shadow-sm ring-1 shadow-black/10 ring-black/10 dark:bg-neutral-900 dark:shadow-white/5 dark:ring-white/10"
+												: "hover:bg-neutral-50 dark:hover:bg-neutral-900",
+										)}
+									>
+										{isActive && (
+											<div className="absolute inset-0">
+												<GridLineHorizontal
+													className="-top-[2px]"
+													offset="100px"
+												/>
+												<GridLineHorizontal
+													className="-bottom-[2px]"
+													offset="100px"
+												/>
+												<GridLineVertical
+													className="-left-[2px]"
+													offset="100px"
+												/>
+												<GridLineVertical
+													className="-right-[2px] left-auto"
+													offset="100px"
+												/>
+											</div>
+										)}
+										<button
+											onClick={() => toggleQuestion(id)}
+											className="flex w-full items-center justify-between px-4 py-4 text-left"
+										>
+											<span className="text-sm font-medium text-neutral-700 md:text-base dark:text-neutral-300">
+												{item.question}
+											</span>
+											<motion.div
+												animate={{ rotate: isActive ? 45 : 0 }}
+												transition={{ duration: 0.2 }}
+												className="ml-4 shrink-0"
+											>
+												<IconPlus className="size-5 text-neutral-500 dark:text-neutral-400" />
+											</motion.div>
+										</button>
+										<AnimatePresence initial={false}>
+											{isActive && (
+												<motion.div
+													initial={{ height: 0, opacity: 0 }}
+													animate={{ height: "auto", opacity: 1 }}
+													exit={{ height: 0, opacity: 0 }}
+													transition={{ duration: 0.15, ease: "easeInOut" }}
+													className="relative"
+												>
+													<p className="max-w-[90%] px-4 pb-4 text-sm text-neutral-600 dark:text-neutral-400">
+														{item.answer}
+													</p>
+												</motion.div>
+											)}
+										</AnimatePresence>
+									</div>
+								);
+							})}
+						</div>
+					</div>
+				))}
+			</div>
+		</div>
+	);
+}
+
 function PricingBasic() {
 	return (
-		<Pricing
-			plans={demoPlans}
-			title="SurfSense Pricing"
-			description="Start free with 500 pages and pay as you go."
-		/>
+		<>
+			<Pricing
+				plans={demoPlans}
+				title="SurfSense Pricing"
+				description="Start free with 500 pages and pay as you go."
+			/>
+			<PricingFAQ />
+		</>
 	);
 }
 

surfsense_web/components/shared/image-config-dialog.tsx

@@ -48,6 +48,7 @@ interface ImageConfigDialogProps {
 	isGlobal: boolean;
 	searchSpaceId: number;
 	mode: "create" | "edit" | "view";
+	defaultProvider?: string;
 }
 
 const INITIAL_FORM = {
@@ -67,6 +68,7 @@ export function ImageConfigDialog({
 	isGlobal,
 	searchSpaceId,
 	mode,
+	defaultProvider,
 }: ImageConfigDialogProps) {
 	const [isSubmitting, setIsSubmitting] = useState(false);
 	const [formData, setFormData] = useState(INITIAL_FORM);
@@ -87,11 +89,11 @@ export function ImageConfigDialog({
 					api_version: config.api_version || "",
 				});
 			} else if (mode === "create") {
-				setFormData(INITIAL_FORM);
+				setFormData({ ...INITIAL_FORM, provider: defaultProvider ?? "" });
 			}
 			setScrollPos("top");
 		}
-	}, [open, mode, config, isGlobal]);
+	}, [open, mode, config, isGlobal, defaultProvider]);
 
 	const { mutateAsync: createConfig } = useAtomValue(createImageGenConfigMutationAtom);
 	const { mutateAsync: updateConfig } = useAtomValue(updateImageGenConfigMutationAtom);

surfsense_web/components/shared/model-config-dialog.tsx

@@ -28,6 +28,7 @@ interface ModelConfigDialogProps {
 	isGlobal: boolean;
 	searchSpaceId: number;
 	mode: "create" | "edit" | "view";
+	defaultProvider?: string;
 }
 
 export function ModelConfigDialog({
@@ -37,6 +38,7 @@ export function ModelConfigDialog({
 	isGlobal,
 	searchSpaceId,
 	mode,
+	defaultProvider,
 }: ModelConfigDialogProps) {
 	const [isSubmitting, setIsSubmitting] = useState(false);
 	const [scrollPos, setScrollPos] = useState<"top" | "middle" | "bottom">("top");
@@ -194,10 +196,12 @@ export function ModelConfigDialog({
 
 					{mode === "create" ? (
 						<LLMConfigForm
+							key={defaultProvider ?? "no-provider"}
 							searchSpaceId={searchSpaceId}
 							onSubmit={handleSubmit}
 							mode="create"
 							formId="model-config-form"
+							initialData={defaultProvider ? { provider: defaultProvider as LiteLLMProvider } : undefined}
 						/>
 					) : isGlobal && config ? (
 						<div className="space-y-6">

surfsense_web/components/shared/vision-config-dialog.tsx

@@ -49,6 +49,7 @@ interface VisionConfigDialogProps {
 	isGlobal: boolean;
 	searchSpaceId: number;
 	mode: "create" | "edit" | "view";
+	defaultProvider?: string;
 }
 
 const INITIAL_FORM = {
@@ -68,6 +69,7 @@ export function VisionConfigDialog({
 	isGlobal,
 	searchSpaceId,
 	mode,
+	defaultProvider,
 }: VisionConfigDialogProps) {
 	const [isSubmitting, setIsSubmitting] = useState(false);
 	const [formData, setFormData] = useState(INITIAL_FORM);
@@ -87,11 +89,11 @@ export function VisionConfigDialog({
 					api_version: (config as VisionLLMConfig).api_version || "",
 				});
 			} else if (mode === "create") {
-				setFormData(INITIAL_FORM);
+				setFormData({ ...INITIAL_FORM, provider: defaultProvider ?? "" });
 			}
 			setScrollPos("top");
 		}
-	}, [open, mode, config, isGlobal]);
+	}, [open, mode, config, isGlobal, defaultProvider]);
 
 	const { mutateAsync: createConfig } = useAtomValue(createVisionLLMConfigMutationAtom);
 	const { mutateAsync: updateConfig } = useAtomValue(updateVisionLLMConfigMutationAtom);

surfsense_web/components/tool-ui/image/index.tsx

@@ -288,7 +288,7 @@ export function Image({
 							alt={alt}
 							width={0}
 							height={0}
-							sizes="100vw"
+							sizes={`(max-width: ${maxWidth}) 100vw, ${maxWidth}`}
 							loading="eager"
 							className={cn(
 								"w-full h-auto transition-transform duration-300",
@@ -307,7 +307,7 @@ export function Image({
 							src={src}
 							alt={alt}
 							fill
-							sizes="(max-width: 512px) 100vw, 512px"
+							sizes={`(max-width: ${maxWidth}) 100vw, ${maxWidth}`}
 							className={cn(
 								"transition-transform duration-300",
 								fit === "cover" ? "object-cover" : "object-contain",