Merge branch 'dev' into fix/fetch-abort-controller

surfsense_web/app/(home)/changelog/page.tsx

@@ -29,7 +29,7 @@ interface ChangelogPageItem {
 
 export default async function ChangelogPage() {
 	const allPages = source.getPages() as ChangelogPageItem[];
-	const sortedChangelogs = allPages.sort((a, b) => {
+	const sortedChangelogs = allPages.toSorted((a, b) => {
 		const dateA = new Date(a.data.date).getTime();
 		const dateB = new Date(b.data.date).getTime();
 		return dateB - dateA;

surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx

@@ -329,14 +329,15 @@ export function DocumentsTableShell({
 
 	const handleViewDocument = useCallback(async (doc: Document) => {
 		setViewingDoc(doc);
-		if (doc.content) {
-			setViewingContent(doc.content);
+		const preview = doc.content_preview || doc.content;
+		if (preview) {
+			setViewingContent(preview);
 			return;
 		}
 		setViewingLoading(true);
 		try {
 			const fullDoc = await documentsApiService.getDocument({ id: doc.id });
-			setViewingContent(fullDoc.content);
+			setViewingContent(fullDoc.content_preview || fullDoc.content);
 		} catch (err) {
 			console.error("[DocumentsTableShell] Failed to fetch document content:", err);
 			setViewingContent("Failed to load document content.");
@@ -951,7 +952,30 @@ export function DocumentsTableShell({
 								<Spinner size="lg" className="text-muted-foreground" />
 							</div>
 						) : (
-							<MarkdownViewer content={viewingContent} />
+							<>
+								<MarkdownViewer content={viewingContent} maxLength={50_000} />
+								{viewingDoc && (
+									<div className="mt-4 flex justify-center">
+										<Button
+											variant="outline"
+											size="sm"
+											onClick={() => {
+												if (viewingDoc) {
+													openEditor({
+														documentId: viewingDoc.id,
+														searchSpaceId: Number(searchSpaceId),
+														title: viewingDoc.title,
+													});
+													handleCloseViewer();
+												}
+											}}
+										>
+											<Eye className="h-3.5 w-3.5 mr-1.5" />
+											View full document
+										</Button>
+									</div>
+								)}
+							</>
 						)}
 					</div>
 				</DrawerContent>

surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/types.ts

@@ -9,9 +9,9 @@ export type Document = {
 	id: number;
 	title: string;
 	document_type: DocumentType;
-	// Optional: Only needed when viewing document details (lazy loaded)
 	document_metadata?: any;
 	content?: string;
+	content_preview?: string;
 	created_at: string;
 	search_space_id: number;
 	created_by_id?: string | null;

surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx

@@ -228,13 +228,14 @@ export default function NewChatPage() {
 					return prev;
 				}
 
+				const memberById = new Map(membersData?.map((m) => [m.user_id, m]) ?? []);
+				const prevById = new Map(prev.map((m) => [m.id, m]));
+
 				return syncedMessages.map((msg) => {
-					const member = msg.author_id
-						? membersData?.find((m) => m.user_id === msg.author_id)
-						: null;
+					const member = msg.author_id ? (memberById.get(msg.author_id) ?? null) : null;
 
 					// Preserve existing author info if member lookup fails (e.g., cloned chats)
-					const existingMsg = prev.find((m) => m.id === `msg-${msg.id}`);
+					const existingMsg = prevById.get(`msg-${msg.id}`);
 					const existingAuthor = existingMsg?.metadata?.custom?.author as
 						| { displayName?: string | null; avatarUrl?: string | null }
 						| undefined;

surfsense_web/app/docs/[[...slug]]/page.tsx

@@ -1,8 +1,8 @@
 import { DocsBody, DocsDescription, DocsPage, DocsTitle } from "fumadocs-ui/page";
 import { notFound } from "next/navigation";
+import { cache } from "react";
 import { source } from "@/lib/source";
 import { getMDXComponents } from "@/mdx-components";
-import { cache } from "react";
 
 const getDocPage = cache((slug?: string[]) => {
 	return source.getPage(slug);

surfsense_web/app/error.tsx

@@ -1,6 +1,5 @@
 "use client";
 
-import posthog from "posthog-js";
 import { useEffect } from "react";
 
 export default function ErrorPage({
@@ -11,7 +10,11 @@ export default function ErrorPage({
 	reset: () => void;
 }) {
 	useEffect(() => {
-		posthog.captureException(error);
+		import("posthog-js")
+			.then(({ default: posthog }) => {
+				posthog.captureException(error);
+			})
+			.catch(() => {});
 	}, [error]);
 
 	return (

surfsense_web/components/assistant-ui/thread.tsx

@@ -834,6 +834,7 @@ const ComposerAction: FC<ComposerActionProps> = ({ isBlockedByOtherUser = false
 
 	const { data: agentTools } = useAtomValue(agentToolsAtom);
 	const disabledTools = useAtomValue(disabledToolsAtom);
+	const disabledToolsSet = useMemo(() => new Set(disabledTools), [disabledTools]);
 	const toggleTool = useSetAtom(toggleToolAtom);
 	const setDisabledTools = useSetAtom(disabledToolsAtom);
 	const hydrateDisabled = useSetAtom(hydrateDisabledToolsAtom);
@@ -846,18 +847,18 @@ const ComposerAction: FC<ComposerActionProps> = ({ isBlockedByOtherUser = false
 
 	const toggleToolGroup = useCallback(
 		(toolNames: string[]) => {
-			const allDisabled = toolNames.every((name) => disabledTools.includes(name));
+			const allDisabled = toolNames.every((name) => disabledToolsSet.has(name));
 			if (allDisabled) {
 				setDisabledTools((prev) => prev.filter((t) => !toolNames.includes(t)));
 			} else {
 				setDisabledTools((prev) => [...new Set([...prev, ...toolNames])]);
 			}
 		},
-		[disabledTools, setDisabledTools]
+		[disabledToolsSet, setDisabledTools]
 	);
 
 	const hasWebSearchTool = agentTools?.some((t) => t.name === "web_search") ?? false;
-	const isWebSearchEnabled = hasWebSearchTool && !disabledTools.includes("web_search");
+	const isWebSearchEnabled = hasWebSearchTool && !disabledToolsSet.has("web_search");
 	const filteredTools = useMemo(
 		() => agentTools?.filter((t) => t.name !== "web_search"),
 		[agentTools]
@@ -957,7 +958,7 @@ const ComposerAction: FC<ComposerActionProps> = ({ isBlockedByOtherUser = false
 													{group.label}
 												</div>
 												{group.tools.map((tool) => {
-													const isDisabled = disabledTools.includes(tool.name);
+													const isDisabled = disabledToolsSet.has(tool.name);
 													const ToolIcon = getToolIcon(tool.name);
 													return (
 														<div
@@ -989,7 +990,7 @@ const ComposerAction: FC<ComposerActionProps> = ({ isBlockedByOtherUser = false
 													const iconKey = group.connectorIcon ?? "";
 													const iconInfo = CONNECTOR_TOOL_ICON_PATHS[iconKey];
 													const toolNames = group.tools.map((t) => t.name);
-													const allDisabled = toolNames.every((n) => disabledTools.includes(n));
+													const allDisabled = toolNames.every((n) => disabledToolsSet.has(n));
 													return (
 														<div
 															key={group.label}
@@ -1078,7 +1079,7 @@ const ComposerAction: FC<ComposerActionProps> = ({ isBlockedByOtherUser = false
 												{group.label}
 											</div>
 											{group.tools.map((tool) => {
-												const isDisabled = disabledTools.includes(tool.name);
+												const isDisabled = disabledToolsSet.has(tool.name);
 												const ToolIcon = getToolIcon(tool.name);
 												const row = (
 													<div className="flex w-full items-center gap-2 sm:gap-3 px-2.5 sm:px-3 py-1 sm:py-1.5 hover:bg-muted-foreground/10 transition-colors">
@@ -1115,7 +1116,7 @@ const ComposerAction: FC<ComposerActionProps> = ({ isBlockedByOtherUser = false
 												const iconKey = group.connectorIcon ?? "";
 												const iconInfo = CONNECTOR_TOOL_ICON_PATHS[iconKey];
 												const toolNames = group.tools.map((t) => t.name);
-												const allDisabled = toolNames.every((n) => disabledTools.includes(n));
+												const allDisabled = toolNames.every((n) => disabledToolsSet.has(n));
 												const groupDef = TOOL_GROUPS.find((g) => g.label === group.label);
 												const row = (
 													<div className="flex w-full items-center gap-2 sm:gap-3 px-2.5 sm:px-3 py-1 sm:py-1.5 hover:bg-muted-foreground/10 transition-colors">

surfsense_web/components/chat-comments/comment-composer/comment-composer.tsx

@@ -16,10 +16,7 @@ function convertDisplayToData(displayContent: string, mentions: InsertedMention[
 	const sortedMentions = [...mentions].sort((a, b) => b.displayName.length - a.displayName.length);
 
 	const mentionPatterns = sortedMentions.map((mention) => ({
-		pattern: new RegExp(
-			`@${escapeRegExp(mention.displayName)}(?=\\s|$|[.,!?;:])`,
-			"g"
-		),
+		pattern: new RegExp(`@${escapeRegExp(mention.displayName)}(?=\\s|$|[.,!?;:])`, "g"),
 		dataFormat: `@[${mention.id}]`,
 	}));
 

surfsense_web/components/editor-panel/editor-panel.tsx

@@ -1,12 +1,13 @@
 "use client";
 
 import { useAtomValue, useSetAtom } from "jotai";
-import { AlertCircle, XIcon } from "lucide-react";
+import { AlertCircle, Download, FileText, Loader2, XIcon } from "lucide-react";
 import dynamic from "next/dynamic";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { toast } from "sonner";
 import { closeEditorPanelAtom, editorPanelAtom } from "@/atoms/editor/editor-panel.atom";
 import { MarkdownViewer } from "@/components/markdown-viewer";
+import { Alert, AlertDescription } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
 import { Drawer, DrawerContent, DrawerHandle, DrawerTitle } from "@/components/ui/drawer";
 import { Skeleton } from "@/components/ui/skeleton";
@@ -18,11 +19,16 @@ const PlateEditor = dynamic(
 	{ ssr: false, loading: () => <Skeleton className="h-64 w-full" /> }
 );
 
+const LARGE_DOCUMENT_THRESHOLD = 2 * 1024 * 1024; // 2MB
+
 interface EditorContent {
 	document_id: number;
 	title: string;
 	document_type?: string;
 	source_markdown: string;
+	content_size_bytes?: number;
+	chunk_count?: number;
+	truncated?: boolean;
 }
 
 const EDITABLE_DOCUMENT_TYPES = new Set(["FILE", "NOTE"]);
@@ -62,6 +68,7 @@ export function EditorPanelContent({
 	const [isLoading, setIsLoading] = useState(true);
 	const [error, setError] = useState<string | null>(null);
 	const [saving, setSaving] = useState(false);
+	const [downloading, setDownloading] = useState(false);
 
 	const [editedMarkdown, setEditedMarkdown] = useState<string | null>(null);
 	const markdownRef = useRef<string>("");
@@ -69,6 +76,8 @@ export function EditorPanelContent({
 	const changeCountRef = useRef(0);
 	const [displayTitle, setDisplayTitle] = useState(title || "Untitled");
 
+	const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD;
+
 	useEffect(() => {
 		const controller = new AbortController();
 		setIsLoading(true);
@@ -89,7 +98,12 @@ export function EditorPanelContent({
 				const response = await authenticatedFetch(
 					`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`,
 					{ method: "GET", signal: controller.signal }
+				const url = new URL(
+					`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`
 				);
+				url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD));
+
+				const response = await authenticatedFetch(url.toString(), { method: "GET" });
 
 				if (controller.signal.aborted) return;
 
@@ -173,7 +187,7 @@ export function EditorPanelContent({
 	}, [documentId, searchSpaceId]);
 
 	const isEditableType = editorDoc
-		? EDITABLE_DOCUMENT_TYPES.has(editorDoc.document_type ?? "")
+		? EDITABLE_DOCUMENT_TYPES.has(editorDoc.document_type ?? "") && !isLargeDocument
 		: false;
 
 	return (
@@ -204,6 +218,59 @@ export function EditorPanelContent({
 							<p className="text-sm text-red-500 mt-1">{error || "An unknown error occurred"}</p>
 						</div>
 					</div>
+				) : isLargeDocument ? (
+					<div className="h-full overflow-y-auto px-5 py-4">
+						<Alert className="mb-4">
+							<FileText className="size-4" />
+							<AlertDescription className="flex items-center justify-between gap-4">
+								<span>
+									This document is too large for the editor (
+									{Math.round((editorDoc.content_size_bytes ?? 0) / 1024 / 1024)}MB,{" "}
+									{editorDoc.chunk_count ?? 0} chunks). Showing a preview below.
+								</span>
+								<Button
+									variant="outline"
+									size="sm"
+									className="shrink-0 gap-1.5"
+									disabled={downloading}
+									onClick={async () => {
+										setDownloading(true);
+										try {
+											const response = await authenticatedFetch(
+												`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/download-markdown`,
+												{ method: "GET" }
+											);
+											if (!response.ok) throw new Error("Download failed");
+											const blob = await response.blob();
+											const url = URL.createObjectURL(blob);
+											const a = document.createElement("a");
+											a.href = url;
+											const disposition = response.headers.get("content-disposition");
+											const match = disposition?.match(/filename="(.+)"/);
+											a.download = match?.[1] ?? `${editorDoc.title || "document"}.md`;
+											document.body.appendChild(a);
+											a.click();
+											a.remove();
+											URL.revokeObjectURL(url);
+											toast.success("Download started");
+										} catch {
+											toast.error("Failed to download document");
+										} finally {
+											setDownloading(false);
+										}
+									}}
+								>
+									{downloading ? (
+										<Loader2 className="size-3.5 animate-spin" />
+									) : (
+										<Download className="size-3.5" />
+									)}
+									{downloading ? "Preparing..." : "Download .md"}
+								</Button>
+							</AlertDescription>
+						</Alert>
+						<MarkdownViewer content={editorDoc.source_markdown} />
+					</div>
 				) : isEditableType ? (
 					<PlateEditor
 						key={documentId}

surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx

@@ -1,18 +1,24 @@
 "use client";
 
-import { AlertCircle, Pencil } from "lucide-react";
+import { AlertCircle, Download, FileText, Loader2, Pencil } from "lucide-react";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { toast } from "sonner";
 import { PlateEditor } from "@/components/editor/plate-editor";
 import { MarkdownViewer } from "@/components/markdown-viewer";
+import { Alert, AlertDescription } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
 import { authenticatedFetch, getBearerToken, redirectToLogin } from "@/lib/auth-utils";
 
+const LARGE_DOCUMENT_THRESHOLD = 2 * 1024 * 1024; // 2MB
+
 interface DocumentContent {
 	document_id: number;
 	title: string;
 	document_type?: string;
 	source_markdown: string;
+	content_size_bytes?: number;
+	chunk_count?: number;
+	truncated?: boolean;
 }
 
 function DocumentSkeleton() {
@@ -49,11 +55,14 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 	const [error, setError] = useState<string | null>(null);
 	const [isEditing, setIsEditing] = useState(false);
 	const [saving, setSaving] = useState(false);
+	const [downloading, setDownloading] = useState(false);
 	const [editedMarkdown, setEditedMarkdown] = useState<string | null>(null);
 	const markdownRef = useRef<string>("");
 	const initialLoadDone = useRef(false);
 	const changeCountRef = useRef(0);
 
+	const isLargeDocument = (doc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD;
+
 	useEffect(() => {
 		const controller = new AbortController();
 		setIsLoading(true);
@@ -75,7 +84,12 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 				const response = await authenticatedFetch(
 					`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`,
 					{ method: "GET", signal: controller.signal }
+				const url = new URL(
+					`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`
 				);
+				url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD));
+
+				const response = await authenticatedFetch(url.toString(), { method: "GET" });
 
 				if (controller.signal.aborted) return;
 
@@ -171,9 +185,9 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 		);
 	}
 
-	const isEditable = EDITABLE_DOCUMENT_TYPES.has(doc.document_type ?? "");
+	const isEditable = EDITABLE_DOCUMENT_TYPES.has(doc.document_type ?? "") && !isLargeDocument;
 
-	if (isEditing) {
+	if (isEditing && !isLargeDocument) {
 		return (
 			<div className="flex flex-col h-full overflow-hidden">
 				<div className="flex items-center justify-between px-6 py-3 border-b shrink-0">
@@ -234,7 +248,62 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 			</div>
 			<div className="flex-1 overflow-auto">
 				<div className="max-w-4xl mx-auto px-6 py-6">
-					<MarkdownViewer content={doc.source_markdown} />
+					{isLargeDocument ? (
+						<>
+							<Alert className="mb-4">
+								<FileText className="size-4" />
+								<AlertDescription className="flex items-center justify-between gap-4">
+									<span>
+										This document is too large for the editor (
+										{Math.round((doc.content_size_bytes ?? 0) / 1024 / 1024)}MB,{" "}
+										{doc.chunk_count ?? 0} chunks). Showing a preview below.
+									</span>
+									<Button
+										variant="outline"
+										size="sm"
+										className="shrink-0 gap-1.5"
+										disabled={downloading}
+										onClick={async () => {
+											setDownloading(true);
+											try {
+												const response = await authenticatedFetch(
+													`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/download-markdown`,
+													{ method: "GET" }
+												);
+												if (!response.ok) throw new Error("Download failed");
+												const blob = await response.blob();
+												const url = URL.createObjectURL(blob);
+												const a = document.createElement("a");
+												a.href = url;
+												const disposition = response.headers.get("content-disposition");
+												const match = disposition?.match(/filename="(.+)"/);
+												a.download = match?.[1] ?? `${doc.title || "document"}.md`;
+												document.body.appendChild(a);
+												a.click();
+												a.remove();
+												URL.revokeObjectURL(url);
+												toast.success("Download started");
+											} catch {
+												toast.error("Failed to download document");
+											} finally {
+												setDownloading(false);
+											}
+										}}
+									>
+										{downloading ? (
+											<Loader2 className="size-3.5 animate-spin" />
+										) : (
+											<Download className="size-3.5" />
+										)}
+										{downloading ? "Preparing..." : "Download .md"}
+									</Button>
+								</AlertDescription>
+							</Alert>
+							<MarkdownViewer content={doc.source_markdown} />
+						</>
+					) : (
+						<MarkdownViewer content={doc.source_markdown} />
+					)}
 				</div>
 			</div>
 		</div>

surfsense_web/components/markdown-viewer.tsx

@@ -15,6 +15,7 @@ const math = createMathPlugin({
 interface MarkdownViewerProps {
 	content: string;
 	className?: string;
+	maxLength?: number;
 }
 
 /**
@@ -79,8 +80,10 @@ function convertLatexDelimiters(content: string): string {
 	return content;
 }
 
-export function MarkdownViewer({ content, className }: MarkdownViewerProps) {
-	const processedContent = convertLatexDelimiters(stripOuterMarkdownFence(content));
+export function MarkdownViewer({ content, className, maxLength }: MarkdownViewerProps) {
+	const isTruncated = maxLength != null && content.length > maxLength;
+	const displayContent = isTruncated ? content.slice(0, maxLength) : content;
+	const processedContent = convertLatexDelimiters(stripOuterMarkdownFence(displayContent));
 	const components: StreamdownProps["components"] = {
 		p: ({ children, ...props }) => (
 			<p className="my-2" {...props}>
@@ -171,6 +174,12 @@ export function MarkdownViewer({ content, className }: MarkdownViewerProps) {
 			>
 				{processedContent}
 			</Streamdown>
+			{isTruncated && (
+				<p className="mt-4 text-sm text-muted-foreground italic">
+					Content truncated ({Math.round(content.length / 1024)}KB total). Showing first{" "}
+					{Math.round(maxLength / 1024)}KB.
+				</p>
+			)}
 		</div>
 	);
 }

surfsense_web/components/new-chat/source-detail-panel.tsx

@@ -1,7 +1,17 @@
 "use client";
 
 import { useQuery } from "@tanstack/react-query";
-import { BookOpen, ChevronDown, ExternalLink, FileText, Hash, Sparkles, X } from "lucide-react";
+import {
+	BookOpen,
+	ChevronDown,
+	ChevronUp,
+	ExternalLink,
+	FileText,
+	Hash,
+	Loader2,
+	Sparkles,
+	X,
+} from "lucide-react";
 import { AnimatePresence, motion, useReducedMotion } from "motion/react";
 import { useTranslations } from "next-intl";
 import type React from "react";
@@ -10,7 +20,6 @@ import { createPortal } from "react-dom";
 import { MarkdownViewer } from "@/components/markdown-viewer";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
-import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
 import { ScrollArea } from "@/components/ui/scroll-area";
 import { Spinner } from "@/components/ui/spinner";
 import type {
@@ -48,7 +57,8 @@ const formatDocumentType = (type: string) => {
 // which break auto-scroll functionality
 interface ChunkCardProps {
 	chunk: { id: number; content: string };
-	index: number;
+	localIndex: number;
+	chunkNumber: number;
 	totalChunks: number;
 	isCited: boolean;
 	isActive: boolean;
@@ -56,51 +66,52 @@ interface ChunkCardProps {
 }
 
 const ChunkCard = memo(
-	forwardRef<HTMLDivElement, ChunkCardProps>(({ chunk, index, totalChunks, isCited }, ref) => {
-		return (
-			<div
-				ref={ref}
-				data-chunk-index={index}
-				className={cn(
-					"group relative rounded-2xl border-2 transition-all duration-300",
-					isCited
-						? "bg-linear-to-br from-primary/5 via-primary/10 to-primary/5 border-primary shadow-lg shadow-primary/10"
-						: "bg-card border-border/50 hover:border-border hover:shadow-md"
-				)}
-			>
-				{/* Cited indicator glow effect */}
-				{isCited && <div className="absolute inset-0 rounded-2xl bg-primary/5 blur-xl -z-10" />}
-
-				{/* Header */}
-				<div className="flex items-center justify-between px-5 py-4 border-b border-border/50">
-					<div className="flex items-center gap-3">
-						<div
-							className={cn(
-								"flex items-center justify-center w-8 h-8 rounded-full text-sm font-semibold transition-colors",
-								isCited
-									? "bg-primary text-primary-foreground"
-									: "bg-muted text-muted-foreground group-hover:bg-muted/80"
-							)}
-						>
-							{index + 1}
-						</div>
-						<span className="text-sm text-muted-foreground">of {totalChunks} chunks</span>
-					</div>
-					{isCited && (
-						<Badge variant="default" className="gap-1.5 px-3 py-1">
-							<Sparkles className="h-3 w-3" />
-							Cited Source
-						</Badge>
+	forwardRef<HTMLDivElement, ChunkCardProps>(
+		({ chunk, localIndex, chunkNumber, totalChunks, isCited }, ref) => {
+			return (
+				<div
+					ref={ref}
+					data-chunk-index={localIndex}
+					className={cn(
+						"group relative rounded-2xl border-2 transition-all duration-300",
+						isCited
+							? "bg-linear-to-br from-primary/5 via-primary/10 to-primary/5 border-primary shadow-lg shadow-primary/10"
+							: "bg-card border-border/50 hover:border-border hover:shadow-md"
 					)}
-				</div>
+				>
+					{isCited && <div className="absolute inset-0 rounded-2xl bg-primary/5 blur-xl -z-10" />}
 
-				{/* Content */}
-				<div className="p-5 overflow-hidden">
-					<MarkdownViewer content={chunk.content} />
+					<div className="flex items-center justify-between px-5 py-4 border-b border-border/50">
+						<div className="flex items-center gap-3">
+							<div
+								className={cn(
+									"flex items-center justify-center w-8 h-8 rounded-full text-sm font-semibold transition-colors",
+									isCited
+										? "bg-primary text-primary-foreground"
+										: "bg-muted text-muted-foreground group-hover:bg-muted/80"
+								)}
+							>
+								{chunkNumber}
+							</div>
+							<span className="text-sm text-muted-foreground">
+								Chunk {chunkNumber} of {totalChunks}
+							</span>
+						</div>
+						{isCited && (
+							<Badge variant="default" className="gap-1.5 px-3 py-1">
+								<Sparkles className="h-3 w-3" />
+								Cited Source
+							</Badge>
+						)}
+					</div>
+
+					<div className="p-5 overflow-hidden">
+						<MarkdownViewer content={chunk.content} maxLength={100_000} />
+					</div>
 				</div>
-			</div>
-		);
-	})
+			);
+		}
+	)
 );
 ChunkCard.displayName = "ChunkCard";
 
@@ -118,7 +129,6 @@ export function SourceDetailPanel({
 	const t = useTranslations("dashboard");
 	const scrollAreaRef = useRef<HTMLDivElement>(null);
 	const hasScrolledRef = useRef(false); // Use ref to avoid stale closures
-	const [summaryOpen, setSummaryOpen] = useState(false);
 	const [activeChunkIndex, setActiveChunkIndex] = useState<number | null>(null);
 	const [mounted, setMounted] = useState(false);
 	const [_hasScrolledToCited, setHasScrolledToCited] = useState(false);
@@ -140,20 +150,93 @@ export function SourceDetailPanel({
 			if (isDocsChunk) {
 				return documentsApiService.getSurfsenseDocByChunk(chunkId);
 			}
-			return documentsApiService.getDocumentByChunk({ chunk_id: chunkId });
+			return documentsApiService.getDocumentByChunk({ chunk_id: chunkId, chunk_window: 5 });
 		},
 		enabled: !!chunkId && open,
 		staleTime: 5 * 60 * 1000,
 	});
 
+	const totalChunks =
+		documentData && "total_chunks" in documentData
+			? (documentData.total_chunks ?? documentData.chunks.length)
+			: (documentData?.chunks?.length ?? 0);
+	const [beforeChunks, setBeforeChunks] = useState<
+		Array<{ id: number; content: string; created_at: string }>
+	>([]);
+	const [afterChunks, setAfterChunks] = useState<
+		Array<{ id: number; content: string; created_at: string }>
+	>([]);
+	const [loadingBefore, setLoadingBefore] = useState(false);
+	const [loadingAfter, setLoadingAfter] = useState(false);
+
+	useEffect(() => {
+		setBeforeChunks([]);
+		setAfterChunks([]);
+	}, [chunkId, open]);
+
+	const chunkStartIndex =
+		documentData && "chunk_start_index" in documentData ? (documentData.chunk_start_index ?? 0) : 0;
+	const initialChunks = documentData?.chunks ?? [];
+	const allChunks = [...beforeChunks, ...initialChunks, ...afterChunks];
+	const absoluteStart = chunkStartIndex - beforeChunks.length;
+	const absoluteEnd = chunkStartIndex + initialChunks.length + afterChunks.length;
+	const canLoadBefore = absoluteStart > 0;
+	const canLoadAfter = absoluteEnd < totalChunks;
+
+	const EXPAND_SIZE = 10;
+
+	const loadBefore = useCallback(async () => {
+		if (!documentData || !("search_space_id" in documentData) || !canLoadBefore) return;
+		setLoadingBefore(true);
+		try {
+			const count = Math.min(EXPAND_SIZE, absoluteStart);
+			const result = await documentsApiService.getDocumentChunks({
+				document_id: documentData.id,
+				page: 0,
+				page_size: count,
+				start_offset: absoluteStart - count,
+			});
+			const existingIds = new Set(allChunks.map((c) => c.id));
+			const newChunks = result.items
+				.filter((c) => !existingIds.has(c.id))
+				.map((c) => ({ id: c.id, content: c.content, created_at: c.created_at }));
+			setBeforeChunks((prev) => [...newChunks, ...prev]);
+		} catch (err) {
+			console.error("Failed to load earlier chunks:", err);
+		} finally {
+			setLoadingBefore(false);
+		}
+	}, [documentData, absoluteStart, canLoadBefore, allChunks]);
+
+	const loadAfter = useCallback(async () => {
+		if (!documentData || !("search_space_id" in documentData) || !canLoadAfter) return;
+		setLoadingAfter(true);
+		try {
+			const result = await documentsApiService.getDocumentChunks({
+				document_id: documentData.id,
+				page: 0,
+				page_size: EXPAND_SIZE,
+				start_offset: absoluteEnd,
+			});
+			const existingIds = new Set(allChunks.map((c) => c.id));
+			const newChunks = result.items
+				.filter((c) => !existingIds.has(c.id))
+				.map((c) => ({ id: c.id, content: c.content, created_at: c.created_at }));
+			setAfterChunks((prev) => [...prev, ...newChunks]);
+		} catch (err) {
+			console.error("Failed to load later chunks:", err);
+		} finally {
+			setLoadingAfter(false);
+		}
+	}, [documentData, absoluteEnd, canLoadAfter, allChunks]);
+
 	const isDirectRenderSource =
 		sourceType === "TAVILY_API" ||
 		sourceType === "LINKUP_API" ||
 		sourceType === "SEARXNG_API" ||
 		sourceType === "BAIDU_SEARCH_API";
 
-	// Find cited chunk index
-	const citedChunkIndex = documentData?.chunks?.findIndex((chunk) => chunk.id === chunkId) ?? -1;
+	const citedChunkIndex = allChunks.findIndex((chunk) => chunk.id === chunkId);
 
 	// Simple scroll function that scrolls to a chunk by index
 	const scrollToChunkByIndex = useCallback(
@@ -336,10 +419,10 @@ export function SourceDetailPanel({
 									{documentData && "document_type" in documentData
 										? formatDocumentType(documentData.document_type)
 										: sourceType && formatDocumentType(sourceType)}
-									{documentData?.chunks && (
+									{totalChunks > 0 && (
 										<span className="ml-2">
-											• {documentData.chunks.length} chunk
-											{documentData.chunks.length !== 1 ? "s" : ""}
+											• {totalChunks} chunk{totalChunks !== 1 ? "s" : ""}
+											{allChunks.length < totalChunks && ` (showing ${allChunks.length})`}
 										</span>
 									)}
 								</p>
@@ -450,7 +533,7 @@ export function SourceDetailPanel({
 						{!isDirectRenderSource && documentData && (
 							<div className="flex-1 flex overflow-hidden">
 								{/* Chunk Navigation Sidebar */}
-								{documentData.chunks.length > 1 && (
+								{allChunks.length > 1 && (
 									<motion.div
 										initial={{ opacity: 0, x: -20 }}
 										animate={{ opacity: 1, x: 0 }}
@@ -459,7 +542,8 @@ export function SourceDetailPanel({
 									>
 										<ScrollArea className="flex-1 h-full">
 											<div className="p-2 pt-3 flex flex-col gap-1.5">
-												{documentData.chunks.map((chunk, idx) => {
+												{allChunks.map((chunk, idx) => {
+													const absNum = absoluteStart + idx + 1;
 													const isCited = chunk.id === chunkId;
 													const isActive = activeChunkIndex === idx;
 													return (
@@ -478,9 +562,9 @@ export function SourceDetailPanel({
 																		? "bg-muted text-foreground"
 																		: "bg-muted/50 text-muted-foreground hover:bg-muted hover:text-foreground"
 															)}
-															title={isCited ? `Chunk ${idx + 1} (Cited)` : `Chunk ${idx + 1}`}
+															title={isCited ? `Chunk ${absNum} (Cited)` : `Chunk ${absNum}`}
 														>
-															{idx + 1}
+															{absNum}
 															{isCited && (
 																<span className="absolute -top-1.5 -right-1.5 flex items-center justify-center w-4 h-4 bg-primary rounded-full border-2 border-background shadow-sm">
 																	<Sparkles className="h-2.5 w-2.5 text-primary-foreground" />
@@ -524,44 +608,11 @@ export function SourceDetailPanel({
 												</motion.div>
 											)}
 
-										{/* Summary Collapsible */}
-										{documentData.content && (
-											<motion.div
-												initial={{ opacity: 0, y: 10 }}
-												animate={{ opacity: 1, y: 0 }}
-												transition={{ delay: 0.15 }}
-											>
-												<Collapsible open={summaryOpen} onOpenChange={setSummaryOpen}>
-													<CollapsibleTrigger className="w-full flex items-center justify-between p-5 rounded-2xl bg-linear-to-r from-muted/50 to-muted/30 border hover:from-muted/70 hover:to-muted/50 transition-all duration-200">
-														<span className="font-semibold flex items-center gap-2">
-															<BookOpen className="h-4 w-4" />
-															Document Summary
-														</span>
-														<motion.div
-															animate={{ rotate: summaryOpen ? 180 : 0 }}
-															transition={{ duration: 0.2 }}
-														>
-															<ChevronDown className="h-5 w-5 text-muted-foreground" />
-														</motion.div>
-													</CollapsibleTrigger>
-													<CollapsibleContent>
-														<motion.div
-															initial={{ opacity: 0 }}
-															animate={{ opacity: 1 }}
-															className="mt-3 p-5 bg-muted/20 rounded-2xl border"
-														>
-															<MarkdownViewer content={documentData.content} />
-														</motion.div>
-													</CollapsibleContent>
-												</Collapsible>
-											</motion.div>
-										)}
-
 										{/* Chunks Header */}
-										<div className="flex items-center justify-between pt-4">
+										<div className="flex items-center justify-between pt-2">
 											<h3 className="text-sm font-semibold text-muted-foreground uppercase tracking-wider flex items-center gap-2">
 												<Hash className="h-4 w-4" />
-												Content Chunks
+												Chunks {absoluteStart + 1}–{absoluteEnd} of {totalChunks}
 											</h3>
 											{citedChunkIndex !== -1 && (
 												<Button
@@ -576,24 +627,70 @@ export function SourceDetailPanel({
 											)}
 										</div>
 
+										{/* Load Earlier */}
+										{canLoadBefore && (
+											<div className="flex items-center justify-center">
+												<Button
+													variant="outline"
+													size="sm"
+													onClick={loadBefore}
+													disabled={loadingBefore}
+													className="gap-2"
+												>
+													{loadingBefore ? (
+														<Loader2 className="h-3.5 w-3.5 animate-spin" />
+													) : (
+														<ChevronUp className="h-3.5 w-3.5" />
+													)}
+													{loadingBefore
+														? "Loading..."
+														: `Load ${Math.min(EXPAND_SIZE, absoluteStart)} earlier chunks`}
+												</Button>
+											</div>
+										)}
+
 										{/* Chunks */}
 										<div className="space-y-4">
-											{documentData.chunks.map((chunk, idx) => {
+											{allChunks.map((chunk, idx) => {
 												const isCited = chunk.id === chunkId;
+												const chunkNumber = absoluteStart + idx + 1;
 												return (
 													<ChunkCard
 														key={chunk.id}
 														ref={isCited ? citedChunkRefCallback : undefined}
 														chunk={chunk}
-														index={idx}
-														totalChunks={documentData.chunks.length}
+														localIndex={idx}
+														chunkNumber={chunkNumber}
+														totalChunks={totalChunks}
 														isCited={isCited}
 														isActive={activeChunkIndex === idx}
-														disableLayoutAnimation={documentData.chunks.length > 30}
+														disableLayoutAnimation={allChunks.length > 30}
 													/>
 												);
 											})}
 										</div>
+
+										{/* Load Later */}
+										{canLoadAfter && (
+											<div className="flex items-center justify-center py-3">
+												<Button
+													variant="outline"
+													size="sm"
+													onClick={loadAfter}
+													disabled={loadingAfter}
+													className="gap-2"
+												>
+													{loadingAfter ? (
+														<Loader2 className="h-3.5 w-3.5 animate-spin" />
+													) : (
+														<ChevronDown className="h-3.5 w-3.5" />
+													)}
+													{loadingAfter
+														? "Loading..."
+														: `Load ${Math.min(EXPAND_SIZE, totalChunks - absoluteEnd)} later chunks`}
+												</Button>
+											</div>
+										)}
 									</div>
 								</ScrollArea>
 							</div>

surfsense_web/components/sources/DocumentUploadTab.tsx

@@ -1,10 +1,10 @@
 "use client";
 
 import { useAtom } from "jotai";
-import { CheckCircle2, FileType, Info, Upload, X } from "lucide-react";
+import { CheckCircle2, FileType, FolderOpen, Info, Upload, X } from "lucide-react";
 
 import { useTranslations } from "next-intl";
-import { useCallback, useMemo, useRef, useState } from "react";
+import { type ChangeEvent, useCallback, useMemo, useRef, useState } from "react";
 import { useDropzone } from "react-dropzone";
 import { toast } from "sonner";
 import { uploadDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
@@ -51,6 +51,7 @@ const commonTypes = {
 	"application/vnd.openxmlformats-officedocument.presentationml.presentation": [".pptx"],
 	"text/html": [".html", ".htm"],
 	"text/csv": [".csv"],
+	"text/tab-separated-values": [".tsv"],
 	"image/jpeg": [".jpg", ".jpeg"],
 	"image/png": [".png"],
 	"image/bmp": [".bmp"],
@@ -76,7 +77,6 @@ const FILE_TYPE_CONFIG: Record<string, Record<string, string[]>> = {
 		"application/rtf": [".rtf"],
 		"application/xml": [".xml"],
 		"application/epub+zip": [".epub"],
-		"text/tab-separated-values": [".tsv"],
 		"text/html": [".html", ".htm", ".web"],
 		"image/gif": [".gif"],
 		"image/svg+xml": [".svg"],
@@ -102,7 +102,6 @@ const FILE_TYPE_CONFIG: Record<string, Record<string, string[]>> = {
 		"application/vnd.ms-powerpoint": [".ppt"],
 		"text/x-rst": [".rst"],
 		"application/rtf": [".rtf"],
-		"text/tab-separated-values": [".tsv"],
 		"application/vnd.ms-excel": [".xls"],
 		"application/xml": [".xml"],
 		...audioFileTypes,
@@ -116,10 +115,8 @@ interface FileWithId {
 
 const cardClass = "border border-border bg-slate-400/5 dark:bg-white/5";
 
-// Upload limits — files are sent in batches of 5 to avoid proxy timeouts
-const MAX_FILES = 50;
-const MAX_TOTAL_SIZE_MB = 200;
-const MAX_TOTAL_SIZE_BYTES = MAX_TOTAL_SIZE_MB * 1024 * 1024;
+const MAX_FILE_SIZE_MB = 500;
+const MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024;
 
 export function DocumentUploadTab({
 	searchSpaceId,
@@ -134,6 +131,7 @@ export function DocumentUploadTab({
 	const [uploadDocumentMutation] = useAtom(uploadDocumentMutationAtom);
 	const { mutate: uploadDocuments, isPending: isUploading } = uploadDocumentMutation;
 	const fileInputRef = useRef<HTMLInputElement>(null);
+	const folderInputRef = useRef<HTMLInputElement>(null);
 
 	const acceptedFileTypes = useMemo(() => {
 		const etlService = process.env.NEXT_PUBLIC_ETL_SERVICE;
@@ -145,49 +143,76 @@ export function DocumentUploadTab({
 		[acceptedFileTypes]
 	);
 
-	const onDrop = useCallback(
-		(acceptedFiles: File[]) => {
+	const supportedExtensionsSet = useMemo(
+		() => new Set(supportedExtensions.map((ext) => ext.toLowerCase())),
+		[supportedExtensions]
+	);
+
+	const addFiles = useCallback(
+		(incoming: File[]) => {
+			const oversized = incoming.filter((f) => f.size > MAX_FILE_SIZE_BYTES);
+			if (oversized.length > 0) {
+				toast.error(t("file_too_large"), {
+					description: t("file_too_large_desc", {
+						name: oversized[0].name,
+						maxMB: MAX_FILE_SIZE_MB,
+					}),
+				});
+			}
+			const valid = incoming.filter((f) => f.size <= MAX_FILE_SIZE_BYTES);
+			if (valid.length === 0) return;
+
 			setFiles((prev) => {
-				const newEntries = acceptedFiles.map((f) => ({
+				const newEntries = valid.map((f) => ({
 					id: crypto.randomUUID?.() ?? `file-${Date.now()}-${Math.random().toString(36)}`,
 					file: f,
 				}));
-				const newFiles = [...prev, ...newEntries];
-
-				if (newFiles.length > MAX_FILES) {
-					toast.error(t("max_files_exceeded"), {
-						description: t("max_files_exceeded_desc", { max: MAX_FILES }),
-					});
-					return prev;
-				}
-
-				const newTotalSize = newFiles.reduce((sum, entry) => sum + entry.file.size, 0);
-				if (newTotalSize > MAX_TOTAL_SIZE_BYTES) {
-					toast.error(t("max_size_exceeded"), {
-						description: t("max_size_exceeded_desc", { max: MAX_TOTAL_SIZE_MB }),
-					});
-					return prev;
-				}
-
-				return newFiles;
+				return [...prev, ...newEntries];
 			});
 		},
 		[t]
 	);
 
+	const onDrop = useCallback(
+		(acceptedFiles: File[]) => {
+			addFiles(acceptedFiles);
+		},
+		[addFiles]
+	);
+
 	const { getRootProps, getInputProps, isDragActive } = useDropzone({
 		onDrop,
 		accept: acceptedFileTypes,
-		maxSize: 50 * 1024 * 1024, // 50MB per file
+		maxSize: MAX_FILE_SIZE_BYTES,
 		noClick: false,
-		disabled: files.length >= MAX_FILES,
 	});
 
-	// Handle file input click to prevent event bubbling that might reopen dialog
 	const handleFileInputClick = useCallback((e: React.MouseEvent<HTMLInputElement>) => {
 		e.stopPropagation();
 	}, []);
 
+	const handleFolderChange = useCallback(
+		(e: ChangeEvent<HTMLInputElement>) => {
+			const fileList = e.target.files;
+			if (!fileList || fileList.length === 0) return;
+
+			const folderFiles = Array.from(fileList).filter((f) => {
+				const ext = f.name.includes(".") ? `.${f.name.split(".").pop()?.toLowerCase()}` : "";
+				return ext !== "" && supportedExtensionsSet.has(ext);
+			});
+
+			if (folderFiles.length === 0) {
+				toast.error(t("no_supported_files_in_folder"));
+				e.target.value = "";
+				return;
+			}
+
+			addFiles(folderFiles);
+			e.target.value = "";
+		},
+		[addFiles, supportedExtensionsSet, t]
+	);
+
 	const formatFileSize = (bytes: number) => {
 		if (bytes === 0) return "0 Bytes";
 		const k = 1024;
@@ -198,15 +223,6 @@ export function DocumentUploadTab({
 
 	const totalFileSize = files.reduce((total, entry) => total + entry.file.size, 0);
 
-	// Check if limits are reached
-	const isFileCountLimitReached = files.length >= MAX_FILES;
-	const isSizeLimitReached = totalFileSize >= MAX_TOTAL_SIZE_BYTES;
-	const remainingFiles = MAX_FILES - files.length;
-	const remainingSizeMB = Math.max(
-		0,
-		(MAX_TOTAL_SIZE_BYTES - totalFileSize) / (1024 * 1024)
-	).toFixed(1);
-
 	// Track accordion state changes
 	const handleAccordionChange = useCallback(
 		(value: string) => {
@@ -257,11 +273,20 @@ export function DocumentUploadTab({
 			<Alert className="border border-border bg-slate-400/5 dark:bg-white/5">
 				<Info className="h-4 w-4 shrink-0 mt-0.5" />
 				<AlertDescription className="text-xs sm:text-sm leading-relaxed pt-0.5">
-					{t("file_size_limit")}{" "}
-					{t("upload_limits", { maxFiles: MAX_FILES, maxSizeMB: MAX_TOTAL_SIZE_MB })}
+					{t("file_size_limit", { maxMB: MAX_FILE_SIZE_MB })} {t("upload_limits")}
 				</AlertDescription>
 			</Alert>
 
+			{/* Hidden folder input */}
+			<input
+				ref={folderInputRef}
+				type="file"
+				className="hidden"
+				onChange={handleFolderChange}
+				multiple
+				{...({ webkitdirectory: "", directory: "" } as React.InputHTMLAttributes<HTMLInputElement>)}
+			/>
+
 			<Card className={`relative overflow-hidden ${cardClass}`}>
 				<div className="absolute inset-0 [mask-image:radial-gradient(ellipse_at_center,white,transparent)] opacity-30">
 					<GridPattern />
@@ -269,11 +294,7 @@ export function DocumentUploadTab({
 				<CardContent className="p-4 sm:p-10 relative z-10">
 					<div
 						{...getRootProps()}
-						className={`flex flex-col items-center justify-center min-h-[200px] sm:min-h-[300px] border-2 border-dashed rounded-lg transition-colors ${
-							isFileCountLimitReached || isSizeLimitReached
-								? "border-destructive/50 bg-destructive/5 cursor-not-allowed"
-								: "border-border hover:border-primary/50 cursor-pointer"
-						}`}
+						className="flex flex-col items-center justify-center min-h-[200px] sm:min-h-[300px] border-2 border-dashed rounded-lg transition-colors border-border hover:border-primary/50 cursor-pointer"
 					>
 						<input
 							{...getInputProps()}
@@ -281,19 +302,7 @@ export function DocumentUploadTab({
 							className="hidden"
 							onClick={handleFileInputClick}
 						/>
-						{isFileCountLimitReached ? (
-							<div className="flex flex-col items-center gap-2 sm:gap-4 text-center px-4">
-								<Upload className="h-8 w-8 sm:h-12 sm:w-12 text-destructive/70" />
-								<div>
-									<p className="text-sm sm:text-lg font-medium text-destructive">
-										{t("file_limit_reached")}
-									</p>
-									<p className="text-xs sm:text-sm text-muted-foreground mt-1">
-										{t("file_limit_reached_desc", { max: MAX_FILES })}
-									</p>
-								</div>
-							</div>
-						) : isDragActive ? (
+						{isDragActive ? (
 							<div className="flex flex-col items-center gap-2 sm:gap-4">
 								<Upload className="h-8 w-8 sm:h-12 sm:w-12 text-primary" />
 								<p className="text-sm sm:text-lg font-medium text-primary">{t("drop_files")}</p>
@@ -305,29 +314,35 @@ export function DocumentUploadTab({
 									<p className="text-sm sm:text-lg font-medium">{t("drag_drop")}</p>
 									<p className="text-xs sm:text-sm text-muted-foreground mt-1">{t("or_browse")}</p>
 								</div>
-								{files.length > 0 && (
-									<p className="text-xs text-muted-foreground">
-										{t("remaining_capacity", { files: remainingFiles, sizeMB: remainingSizeMB })}
-									</p>
-								)}
-							</div>
-						)}
-						{!isFileCountLimitReached && (
-							<div className="mt-2 sm:mt-4">
-								<Button
-									variant="secondary"
-									size="sm"
-									className="text-xs sm:text-sm"
-									onClick={(e) => {
-										e.stopPropagation();
-										e.preventDefault();
-										fileInputRef.current?.click();
-									}}
-								>
-									{t("browse_files")}
-								</Button>
 							</div>
 						)}
+						<div className="mt-2 sm:mt-4 flex gap-2">
+							<Button
+								variant="secondary"
+								size="sm"
+								className="text-xs sm:text-sm"
+								onClick={(e) => {
+									e.stopPropagation();
+									e.preventDefault();
+									fileInputRef.current?.click();
+								}}
+							>
+								{t("browse_files")}
+							</Button>
+							<Button
+								variant="outline"
+								size="sm"
+								className="text-xs sm:text-sm"
+								onClick={(e) => {
+									e.stopPropagation();
+									e.preventDefault();
+									folderInputRef.current?.click();
+								}}
+							>
+								<FolderOpen className="h-4 w-4 mr-1.5" />
+								{t("browse_folder")}
+							</Button>
+						</div>
 					</div>
 				</CardContent>
 			</Card>

surfsense_web/components/ui/checkbox.tsx

@@ -1,7 +1,7 @@
 "use client";
 
-import { CheckIcon } from "lucide-react";
 import * as CheckboxPrimitive from "@radix-ui/react-checkbox";
+import { CheckIcon } from "lucide-react";
 import type * as React from "react";
 
 import { cn } from "@/lib/utils";

surfsense_web/components/ui/dropdown-menu.tsx

@@ -1,7 +1,7 @@
 "use client";
 
-import { CheckIcon, ChevronRightIcon, CircleIcon } from "lucide-react";
 import * as DropdownMenuPrimitive from "@radix-ui/react-dropdown-menu";
+import { CheckIcon, ChevronRightIcon, CircleIcon } from "lucide-react";
 import type * as React from "react";
 
 import { cn } from "@/lib/utils";

surfsense_web/components/ui/toggle-group.tsx

@@ -1,7 +1,7 @@
 "use client";
 
-import type { VariantProps } from "class-variance-authority";
 import * as ToggleGroupPrimitive from "@radix-ui/react-toggle-group";
+import type { VariantProps } from "class-variance-authority";
 import * as React from "react";
 import { toggleVariants } from "@/components/ui/toggle";
 import { cn } from "@/lib/utils";

surfsense_web/components/ui/toggle.tsx

@@ -1,7 +1,7 @@
 "use client";
 
-import { cva, type VariantProps } from "class-variance-authority";
 import * as TogglePrimitive from "@radix-ui/react-toggle";
+import { cva, type VariantProps } from "class-variance-authority";
 import type * as React from "react";
 
 import { cn } from "@/lib/utils";

surfsense_web/contracts/types/document.types.ts

@@ -39,6 +39,7 @@ export const document = z.object({
 	document_type: documentTypeEnum,
 	document_metadata: z.record(z.string(), z.any()),
 	content: z.string(),
+	content_preview: z.string().optional().default(""),
 	content_hash: z.string(),
 	unique_identifier_hash: z.string().nullable(),
 	created_at: z.string(),
@@ -69,6 +70,8 @@ export const documentWithChunks = document.extend({
 			created_at: z.string(),
 		})
 	),
+	total_chunks: z.number().optional().default(0),
+	chunk_start_index: z.number().optional().default(0),
 });
 
 /**
@@ -243,10 +246,36 @@ export const getDocumentTypeCountsResponse = z.record(z.string(), z.number());
  */
 export const getDocumentByChunkRequest = z.object({
 	chunk_id: z.number(),
+	chunk_window: z.number().optional(),
 });
 
 export const getDocumentByChunkResponse = documentWithChunks;
 
+/**
+ * Get paginated chunks for a document
+ */
+export const getDocumentChunksRequest = z.object({
+	document_id: z.number(),
+	page: z.number().optional().default(0),
+	page_size: z.number().optional().default(20),
+	start_offset: z.number().optional(),
+});
+
+export const chunkRead = z.object({
+	id: z.number(),
+	content: z.string(),
+	document_id: z.number(),
+	created_at: z.string(),
+});
+
+export const getDocumentChunksResponse = z.object({
+	items: z.array(chunkRead),
+	total: z.number(),
+	page: z.number(),
+	page_size: z.number(),
+	has_more: z.boolean(),
+});
+
 /**
  * Get Surfsense docs by chunk
  */
@@ -328,3 +357,6 @@ export type GetSurfsenseDocsByChunkRequest = z.infer<typeof getSurfsenseDocsByCh
 export type GetSurfsenseDocsByChunkResponse = z.infer<typeof getSurfsenseDocsByChunkResponse>;
 export type GetSurfsenseDocsRequest = z.infer<typeof getSurfsenseDocsRequest>;
 export type GetSurfsenseDocsResponse = z.infer<typeof getSurfsenseDocsResponse>;
+export type GetDocumentChunksRequest = z.infer<typeof getDocumentChunksRequest>;
+export type GetDocumentChunksResponse = z.infer<typeof getDocumentChunksResponse>;
+export type ChunkRead = z.infer<typeof chunkRead>;

surfsense_web/hooks/use-comments-sync.ts

@@ -118,8 +118,8 @@ function transformComments(
 
 	for (const [messageId, group] of byMessage) {
 		const comments: Comment[] = group.topLevel.map((raw) => {
-			const replies = (group.replies.get(raw.id) || [])
-				.sort((a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime())
+			const replies = (group.replies.get(raw.id) ?? [])
+				.toSorted((a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime())
 				.map((r) => transformReply(r, memberMap, currentUserId, isOwner));
 
 			return {

surfsense_web/hooks/use-documents.ts

@@ -246,9 +246,11 @@ export function useDocuments(
 					status: (doc.status as unknown as DocumentStatusType) ?? { state: "ready" },
 				}));
 
+			const liveById = new Map(validItems.map((v) => [v.id, v]));
+
 			let updated = prev.map((existing) => {
 				if (liveIds.has(existing.id)) {
-					const liveItem = validItems.find((v) => v.id === existing.id);
+					const liveItem = liveById.get(existing.id);
 					if (liveItem) {
 						return {
 							...existing,

surfsense_web/hooks/use-inbox.ts

@@ -157,8 +157,10 @@ export function useInbox(
 						}) as InboxItem
 				);
 
+			const liveById = new Map(recentItems.map((v) => [v.id, v]));
+
 			let updated = prev.map((existing) => {
-				const liveItem = recentItems.find((v) => v.id === existing.id);
+				const liveItem = liveById.get(existing.id);
 				if (liveItem) {
 					return {
 						...existing,

surfsense_web/lib/apis/base-api.service.ts

@@ -1,4 +1,3 @@
-import posthog from "posthog-js";
 import type { ZodType } from "zod";
 import { getBearerToken, handleUnauthorized, refreshAccessToken } from "../auth-utils";
 import { AppError, AuthenticationError, AuthorizationError, NotFoundError } from "../error";
@@ -234,18 +233,21 @@ class BaseApiService {
 		} catch (error) {
 			console.error("Request failed:", JSON.stringify(error));
 			if (!(error instanceof AuthenticationError)) {
-				try {
-					posthog.captureException(error, {
-						api_url: url,
-						api_method: options?.method ?? "GET",
-						...(error instanceof AppError && {
-							status_code: error.status,
-							status_text: error.statusText,
-						}),
+				import("posthog-js")
+					.then(({ default: posthog }) => {
+						posthog.captureException(error, {
+							api_url: url,
+							api_method: options?.method ?? "GET",
+							...(error instanceof AppError && {
+								status_code: error.status,
+								status_text: error.statusText,
+							}),
+						});
+					})
+					.catch(() => {
+						// PostHog is not available in the current environment
+						console.error("Failed to capture exception in PostHog");
 					});
-				} catch {
-					// PostHog capture failed — don't block the error flow
-				}
 			}
 			throw error;
 		}

surfsense_web/lib/apis/documents-api.service.ts

@@ -6,6 +6,7 @@ import {
 	deleteDocumentRequest,
 	deleteDocumentResponse,
 	type GetDocumentByChunkRequest,
+	type GetDocumentChunksRequest,
 	type GetDocumentRequest,
 	type GetDocumentsRequest,
 	type GetDocumentsStatusRequest,
@@ -13,6 +14,8 @@ import {
 	type GetSurfsenseDocsRequest,
 	getDocumentByChunkRequest,
 	getDocumentByChunkResponse,
+	getDocumentChunksRequest,
+	getDocumentChunksResponse,
 	getDocumentRequest,
 	getDocumentResponse,
 	getDocumentsRequest,
@@ -295,23 +298,52 @@ class DocumentsApiService {
 	};
 
 	/**
-	 * Get document by chunk ID (includes all chunks)
+	 * Get document by chunk ID (includes a window of chunks around the cited one)
 	 */
 	getDocumentByChunk = async (request: GetDocumentByChunkRequest) => {
-		// Validate the request
 		const parsedRequest = getDocumentByChunkRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
 			console.error("Invalid request:", parsedRequest.error);
 
-			// Format a user friendly error message
 			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}
 
+		const params = new URLSearchParams();
+		if (request.chunk_window != null) {
+			params.set("chunk_window", String(request.chunk_window));
+		}
+		const qs = params.toString();
+		const url = `/api/v1/documents/by-chunk/${request.chunk_id}${qs ? `?${qs}` : ""}`;
+
+		return baseApiService.get(url, getDocumentByChunkResponse);
+	};
+
+	/**
+	 * Get paginated chunks for a document
+	 */
+	getDocumentChunks = async (request: GetDocumentChunksRequest) => {
+		const parsedRequest = getDocumentChunksRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		const params = new URLSearchParams({
+			page: String(parsedRequest.data.page),
+			page_size: String(parsedRequest.data.page_size),
+		});
+		if (parsedRequest.data.start_offset != null) {
+			params.set("start_offset", String(parsedRequest.data.start_offset));
+		}
+
 		return baseApiService.get(
-			`/api/v1/documents/by-chunk/${request.chunk_id}`,
-			getDocumentByChunkResponse
+			`/api/v1/documents/${parsedRequest.data.document_id}/chunks?${params}`,
+			getDocumentChunksResponse
 		);
 	};
 

surfsense_web/lib/query-client/query-client.provider.tsx

@@ -1,13 +1,18 @@
 "use client";
-import { ReactQueryDevtools } from "@tanstack/react-query-devtools";
 import { QueryClientAtomProvider } from "jotai-tanstack-query/react";
+import dynamic from "next/dynamic";
 import { queryClient } from "./client";
 
+const ReactQueryDevtools = dynamic(
+	() => import("@tanstack/react-query-devtools").then((m) => ({ default: m.ReactQueryDevtools })),
+	{ ssr: false }
+);
+
 export function ReactQueryClientProvider({ children }: { children: React.ReactNode }) {
 	return (
 		<QueryClientAtomProvider client={queryClient}>
 			{children}
-			<ReactQueryDevtools initialIsOpen={false} />
+			{process.env.NODE_ENV === "development" && <ReactQueryDevtools initialIsOpen={false} />}
 		</QueryClientAtomProvider>
 	);
 }

surfsense_web/messages/en.json

@@ -376,12 +376,13 @@
 	"upload_documents": {
 		"title": "Upload Documents",
 		"subtitle": "Upload your files to make them searchable and accessible through AI-powered conversations.",
-		"file_size_limit": "Maximum file size: 50MB per file.",
-		"upload_limits": "Upload limit: {maxFiles} files, {maxSizeMB}MB total.",
-		"drop_files": "Drop files here",
-		"drag_drop": "Drag & drop files here",
-		"or_browse": "or click to browse",
+		"file_size_limit": "Maximum file size: {maxMB}MB per file.",
+		"upload_limits": "Upload files or entire folders",
+		"drop_files": "Drop files or folders here",
+		"drag_drop": "Drag & drop files or folders here",
+		"or_browse": "or click to browse files and folders",
 		"browse_files": "Browse Files",
+		"browse_folder": "Browse Folder",
 		"selected_files": "Selected Files ({count})",
 		"total_size": "Total size",
 		"clear_all": "Clear all",
@@ -394,13 +395,9 @@
 		"upload_error_desc": "Error uploading files",
 		"supported_file_types": "Supported File Types",
 		"file_types_desc": "These file types are supported based on your current ETL service configuration.",
-		"max_files_exceeded": "File Limit Exceeded",
-		"max_files_exceeded_desc": "You can upload a maximum of {max} files at a time.",
-		"max_size_exceeded": "Size Limit Exceeded",
-		"max_size_exceeded_desc": "Total file size cannot exceed {max}MB.",
-		"file_limit_reached": "Maximum Files Reached",
-		"file_limit_reached_desc": "Remove some files to add more (max {max} files).",
-		"remaining_capacity": "{files} files remaining • {sizeMB}MB available"
+		"file_too_large": "File Too Large",
+		"file_too_large_desc": "\"{name}\" exceeds the {maxMB}MB per-file limit.",
+		"no_supported_files_in_folder": "No supported file types found in the selected folder."
 	},
 	"add_webpage": {
 		"title": "Add Webpages for Crawling",

surfsense_web/messages/es.json

@@ -376,12 +376,13 @@
 	"upload_documents": {
 		"title": "Subir documentos",
 		"subtitle": "Sube tus archivos para hacerlos buscables y accesibles a través de conversaciones con IA.",
-		"file_size_limit": "Tamaño máximo de archivo: 50 MB por archivo.",
-		"upload_limits": "Límite de subida: {maxFiles} archivos, {maxSizeMB} MB en total.",
-		"drop_files": "Suelta los archivos aquí",
-		"drag_drop": "Arrastra y suelta archivos aquí",
-		"or_browse": "o haz clic para explorar",
+		"file_size_limit": "Tamaño máximo de archivo: {maxMB} MB por archivo.",
+		"upload_limits": "Sube archivos o carpetas enteras",
+		"drop_files": "Suelta archivos o carpetas aquí",
+		"drag_drop": "Arrastra y suelta archivos o carpetas aquí",
+		"or_browse": "o haz clic para explorar archivos y carpetas",
 		"browse_files": "Explorar archivos",
+		"browse_folder": "Explorar carpeta",
 		"selected_files": "Archivos seleccionados ({count})",
 		"total_size": "Tamaño total",
 		"clear_all": "Limpiar todo",
@@ -394,13 +395,9 @@
 		"upload_error_desc": "Error al subir archivos",
 		"supported_file_types": "Tipos de archivo soportados",
 		"file_types_desc": "Estos tipos de archivo son soportados según la configuración actual de tu servicio ETL.",
-		"max_files_exceeded": "Límite de archivos excedido",
-		"max_files_exceeded_desc": "Puedes subir un máximo de {max} archivos a la vez.",
-		"max_size_exceeded": "Límite de tamaño excedido",
-		"max_size_exceeded_desc": "El tamaño total de los archivos no puede exceder {max} MB.",
-		"file_limit_reached": "Máximo de archivos alcanzado",
-		"file_limit_reached_desc": "Elimina algunos archivos para agregar más (máximo {max} archivos).",
-		"remaining_capacity": "{files} archivos restantes • {sizeMB} MB disponibles"
+		"file_too_large": "Archivo demasiado grande",
+		"file_too_large_desc": "\"{name}\" excede el límite de {maxMB} MB por archivo.",
+		"no_supported_files_in_folder": "No se encontraron tipos de archivo compatibles en la carpeta seleccionada."
 	},
 	"add_webpage": {
 		"title": "Agregar páginas web para rastreo",

surfsense_web/messages/hi.json

@@ -376,12 +376,13 @@
 	"upload_documents": {
 		"title": "दस्तावेज़ अपलोड करें",
 		"subtitle": "AI-संचालित बातचीत के माध्यम से अपनी फ़ाइलों को खोजने योग्य और सुलभ बनाने के लिए अपलोड करें।",
-		"file_size_limit": "अधिकतम फ़ाइल आकार: प्रति फ़ाइल 50MB।",
-		"upload_limits": "अपलोड सीमा: {maxFiles} फ़ाइलें, कुल {maxSizeMB}MB।",
-		"drop_files": "फ़ाइलें यहां छोड़ें",
-		"drag_drop": "फ़ाइलें यहां खींचें और छोड़ें",
-		"or_browse": "या ब्राउज़ करने के लिए क्लिक करें",
+		"file_size_limit": "अधिकतम फ़ाइल आकार: प्रति फ़ाइल {maxMB}MB।",
+		"upload_limits": "फ़ाइलें या पूरे फ़ोल्डर अपलोड करें",
+		"drop_files": "फ़ाइलें या फ़ोल्डर यहां छोड़ें",
+		"drag_drop": "फ़ाइलें या फ़ोल्डर यहां खींचें और छोड़ें",
+		"or_browse": "या फ़ाइलें और फ़ोल्डर ब्राउज़ करने के लिए क्लिक करें",
 		"browse_files": "फ़ाइलें ब्राउज़ करें",
+		"browse_folder": "फ़ोल्डर ब्राउज़ करें",
 		"selected_files": "चयनित फ़ाइलें ({count})",
 		"total_size": "कुल आकार",
 		"clear_all": "सभी साफ करें",
@@ -394,13 +395,9 @@
 		"upload_error_desc": "फ़ाइलें अपलोड करने में त्रुटि",
 		"supported_file_types": "समर्थित फ़ाइल प्रकार",
 		"file_types_desc": "ये फ़ाइल प्रकार आपकी वर्तमान ETL सेवा कॉन्फ़िगरेशन के आधार पर समर्थित हैं।",
-		"max_files_exceeded": "फ़ाइल सीमा पार हो गई",
-		"max_files_exceeded_desc": "आप एक बार में अधिकतम {max} फ़ाइलें अपलोड कर सकते हैं।",
-		"max_size_exceeded": "आकार सीमा पार हो गई",
-		"max_size_exceeded_desc": "कुल फ़ाइल आकार {max}MB से अधिक नहीं हो सकता।",
-		"file_limit_reached": "अधिकतम फ़ाइलें पहुंच गई",
-		"file_limit_reached_desc": "और जोड़ने के लिए कुछ फ़ाइलें हटाएं (अधिकतम {max} फ़ाइलें)।",
-		"remaining_capacity": "{files} फ़ाइलें शेष • {sizeMB}MB उपलब्ध"
+		"file_too_large": "फ़ाइल बहुत बड़ी है",
+		"file_too_large_desc": "\"{name}\" प्रति फ़ाइल {maxMB}MB की सीमा से अधिक है।",
+		"no_supported_files_in_folder": "चयनित फ़ोल्डर में कोई समर्थित फ़ाइल प्रकार नहीं मिला।"
 	},
 	"add_webpage": {
 		"title": "क्रॉलिंग के लिए वेबपेज जोड़ें",

surfsense_web/messages/pt.json

@@ -376,12 +376,13 @@
 	"upload_documents": {
 		"title": "Enviar documentos",
 		"subtitle": "Envie seus arquivos para torná-los pesquisáveis e acessíveis através de conversas com IA.",
-		"file_size_limit": "Tamanho máximo do arquivo: 50 MB por arquivo.",
-		"upload_limits": "Limite de envio: {maxFiles} arquivos, {maxSizeMB} MB no total.",
-		"drop_files": "Solte os arquivos aqui",
-		"drag_drop": "Arraste e solte arquivos aqui",
-		"or_browse": "ou clique para navegar",
+		"file_size_limit": "Tamanho máximo do arquivo: {maxMB} MB por arquivo.",
+		"upload_limits": "Envie arquivos ou pastas inteiras",
+		"drop_files": "Solte arquivos ou pastas aqui",
+		"drag_drop": "Arraste e solte arquivos ou pastas aqui",
+		"or_browse": "ou clique para navegar arquivos e pastas",
 		"browse_files": "Navegar arquivos",
+		"browse_folder": "Navegar pasta",
 		"selected_files": "Arquivos selecionados ({count})",
 		"total_size": "Tamanho total",
 		"clear_all": "Limpar tudo",
@@ -394,13 +395,9 @@
 		"upload_error_desc": "Erro ao enviar arquivos",
 		"supported_file_types": "Tipos de arquivo suportados",
 		"file_types_desc": "Estes tipos de arquivo são suportados com base na configuração atual do seu serviço ETL.",
-		"max_files_exceeded": "Limite de arquivos excedido",
-		"max_files_exceeded_desc": "Você pode enviar no máximo {max} arquivos de uma vez.",
-		"max_size_exceeded": "Limite de tamanho excedido",
-		"max_size_exceeded_desc": "O tamanho total dos arquivos não pode exceder {max} MB.",
-		"file_limit_reached": "Máximo de arquivos atingido",
-		"file_limit_reached_desc": "Remova alguns arquivos para adicionar mais (máximo {max} arquivos).",
-		"remaining_capacity": "{files} arquivos restantes • {sizeMB} MB disponíveis"
+		"file_too_large": "Arquivo muito grande",
+		"file_too_large_desc": "\"{name}\" excede o limite de {maxMB} MB por arquivo.",
+		"no_supported_files_in_folder": "Nenhum tipo de arquivo suportado encontrado na pasta selecionada."
 	},
 	"add_webpage": {
 		"title": "Adicionar páginas web para rastreamento",

surfsense_web/messages/zh.json

@@ -360,12 +360,13 @@
 	"upload_documents": {
 		"title": "上传文档",
 		"subtitle": "上传您的文件，使其可通过 AI 对话进行搜索和访问。",
-		"file_size_limit": "最大文件大小：每个文件 50MB。",
-		"upload_limits": "上传限制：最多 {maxFiles} 个文件，总大小不超过 {maxSizeMB}MB。",
-		"drop_files": "放下文件到这里",
-		"drag_drop": "拖放文件到这里",
-		"or_browse": "或点击浏览",
+		"file_size_limit": "最大文件大小：每个文件 {maxMB}MB。",
+		"upload_limits": "上传文件或整个文件夹",
+		"drop_files": "将文件或文件夹拖放到此处",
+		"drag_drop": "将文件或文件夹拖放到此处",
+		"or_browse": "或点击浏览文件和文件夹",
 		"browse_files": "浏览文件",
+		"browse_folder": "浏览文件夹",
 		"selected_files": "已选择的文件 ({count})",
 		"total_size": "总大小",
 		"clear_all": "全部清除",
@@ -378,13 +379,9 @@
 		"upload_error_desc": "上传文件时出错",
 		"supported_file_types": "支持的文件类型",
 		"file_types_desc": "根据您当前的 ETL 服务配置支持这些文件类型。",
-		"max_files_exceeded": "超过文件数量限制",
-		"max_files_exceeded_desc": "一次最多只能上传 {max} 个文件。",
-		"max_size_exceeded": "超过文件大小限制",
-		"max_size_exceeded_desc": "文件总大小不能超过 {max}MB。",
-		"file_limit_reached": "已达到最大文件数量",
-		"file_limit_reached_desc": "移除一些文件以添加更多（最多 {max} 个文件）。",
-		"remaining_capacity": "剩余 {files} 个文件名额 • 可用 {sizeMB}MB"
+		"file_too_large": "文件过大",
+		"file_too_large_desc": "\"{name}\" 超过了每个文件 {maxMB}MB 的限制。",
+		"no_supported_files_in_folder": "所选文件夹中没有找到支持的文件类型。"
 	},
 	"add_webpage": {
 		"title": "添加网页爬取",

surfsense_web/next.config.ts

@@ -24,6 +24,16 @@ const nextConfig: NextConfig = {
 			},
 		],
 	},
+	experimental: {
+		optimizePackageImports: [
+			"lucide-react",
+			"@tabler/icons-react",
+			"date-fns",
+			"@assistant-ui/react",
+			"@assistant-ui/react-markdown",
+			"motion",
+		],
+	},
 	// Turbopack config (used during `next dev --turbopack`)
 	turbopack: {
 		rules: {