Merge pull request #1470 from AnishSarkar22/fix/platejs-pagination

feat(editor): route large documents to Monaco viewer

surfsense_backend/app/routes/editor_routes.py

@@ -38,14 +38,13 @@ logger = logging.getLogger(__name__)
 
 router = APIRouter()
 
+EDITOR_PLATE_MAX_BYTES = 5 * 1024 * 1024
+
 
 @router.get("/search-spaces/{search_space_id}/documents/{document_id}/editor-content")
 async def get_editor_content(
     search_space_id: int,
     document_id: int,
-    max_length: int | None = Query(
-        None, description="Truncate source_markdown to this many characters"
-    ),
     session: AsyncSession = Depends(get_async_session),
     user: User = Depends(current_active_user),
 ):
@@ -84,19 +83,16 @@ async def get_editor_content(
 
     def _build_response(md: str) -> dict:
         size_bytes = len(md.encode("utf-8"))
-        truncated = False
+        viewer_mode = "monaco" if size_bytes > EDITOR_PLATE_MAX_BYTES else "plate"
-        output_md = md
-        if max_length is not None and size_bytes > max_length:
-            output_md = md[:max_length]
-            truncated = True
         return {
             "document_id": document.id,
             "title": document.title,
             "document_type": document.document_type.value,
-            "source_markdown": output_md,
+            "source_markdown": md,
             "content_size_bytes": size_bytes,
             "chunk_count": chunk_count,
-            "truncated": truncated,
+            "viewer_mode": viewer_mode,
+            "editor_plate_max_bytes": EDITOR_PLATE_MAX_BYTES,
             "updated_at": document.updated_at.isoformat()
             if document.updated_at
             else None,

surfsense_web/components/editor-panel/editor-panel.tsx

@@ -12,7 +12,7 @@ import {
 	XIcon,
 } from "lucide-react";
 import dynamic from "next/dynamic";
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { toast } from "sonner";
 import { closeEditorPanelAtom, editorPanelAtom } from "@/atoms/editor/editor-panel.atom";
 import { DownloadOriginalButton } from "@/components/documents/download-original-button";
@@ -50,11 +50,13 @@ interface EditorContent {
 	source_markdown: string;
 	content_size_bytes?: number;
 	chunk_count?: number;
-	truncated?: boolean;
+	viewer_mode?: ViewerMode;
+	editor_plate_max_bytes?: number;
 }
 
 const EDITABLE_DOCUMENT_TYPES = new Set(["FILE", "NOTE"]);
 type EditorRenderMode = "rich_markdown" | "source_code";
+type ViewerMode = "plate" | "monaco";
 
 type AgentFilesystemMount = {
 	mount: string;
@@ -112,6 +114,20 @@ function EditorPanelSkeleton() {
 	);
 }
 
+function getUtf8ByteSize(value: string): number {
+	return new TextEncoder().encode(value).byteLength;
+}
+
+function formatBytes(bytes: number): string {
+	if (bytes >= 1024 * 1024) {
+		return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
+	}
+	if (bytes >= 1024) {
+		return `${Math.round(bytes / 1024)}KB`;
+	}
+	return `${bytes}B`;
+}
+
 export function EditorPanelContent({
 	kind = "document",
 	documentId,
@@ -167,7 +183,11 @@ export function EditorPanelContent({
 		[electronAPI, searchSpaceId]
 	);
 
-	const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD;
+	const plateMaxBytes = editorDoc?.editor_plate_max_bytes ?? LARGE_DOCUMENT_THRESHOLD;
+	const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > plateMaxBytes;
+	const viewerMode: ViewerMode = isMemoryMode
+		? "plate"
+		: (editorDoc?.viewer_mode ?? (isLargeDocument ? "monaco" : "plate"));
 
 	useEffect(() => {
 		const controller = new AbortController();
@@ -243,8 +263,6 @@ export function EditorPanelContent({
 				const url = new URL(
 					`${BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`
 				);
-				url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD));
-
 				const response = await authenticatedFetch(url.toString(), { method: "GET" });
 
 				if (controller.signal.aborted) return;
@@ -402,7 +420,12 @@ export function EditorPanelContent({
 				setEditorDoc((prev) => (prev ? { ...prev, source_markdown: markdownRef.current } : prev));
 				setEditedMarkdown(null);
 				if (!options?.silent) {
-					toast.success("Document saved! Reindexing in background...");
+					const savedSizeBytes = getUtf8ByteSize(markdownRef.current);
+					if (savedSizeBytes > plateMaxBytes) {
+						toast.success("Document saved. It will reopen in raw markdown mode.");
+					} else {
+						toast.success("Document saved! Reindexing in background...");
+					}
 				}
 				return true;
 			} catch (err) {
@@ -423,6 +446,7 @@ export function EditorPanelContent({
 			localFilePath,
 			memoryLimits,
 			memoryScope,
+			plateMaxBytes,
 			resolveLocalVirtualPath,
 			searchSpaceId,
 		]
@@ -432,18 +456,21 @@ export function EditorPanelContent({
 		? (isMemoryMode ||
 				editorRenderMode === "source_code" ||
 				EDITABLE_DOCUMENT_TYPES.has(editorDoc.document_type ?? "")) &&
-			!isLargeDocument
+			viewerMode === "plate"
 		: false;
-	// Render through PlateEditor for editable doc types (FILE/NOTE).
+	// Render through PlateEditor only when the backend says the rich editor is safe.
-	// Everything else (large docs, non-editable types) falls back to the
+	// Monaco mode is a raw markdown safety path for large documents.
-	// lightweight `MarkdownViewer` — Plate is heavy on multi-MB docs and
-	// non-editable types don't benefit from its editing UX.
 	const renderInPlateEditor = isEditableType;
 	const hasUnsavedChanges = editedMarkdown !== null;
 	const showDesktopHeader = !!onClose;
 	const showEditingActions = isEditableType && isEditing;
 	const localFileLanguage = inferMonacoLanguageFromPath(localFilePath);
 	const activeMarkdown = editedMarkdown ?? editorDoc?.source_markdown ?? "";
+	const activeMarkdownSizeBytes = useMemo(() => getUtf8ByteSize(activeMarkdown), [activeMarkdown]);
+	const isNearPlateLimit = activeMarkdownSizeBytes >= plateMaxBytes * 0.9;
+	const isOverPlateLimit = activeMarkdownSizeBytes > plateMaxBytes;
+	const showPlateSizeWarning =
+		showEditingActions && !isMemoryMode && !isLocalFileMode && isNearPlateLimit;
 	const memoryLimitState = isMemoryMode
 		? getMemoryLimitState(activeMarkdown.length, memoryLimits)
 		: null;
@@ -492,14 +519,14 @@ export function EditorPanelContent({
 		}
 	}, [documentId, editorDoc?.title, searchSpaceId]);
 
-	const largeDocAlert = isLargeDocument && !isLocalFileMode && editorDoc && (
+	const largeDocAlert = viewerMode === "monaco" && !isLocalFileMode && editorDoc && (
-		<Alert className="mb-4">
+		<Alert className="m-4 shrink-0">
 			<FileText className="size-4" />
 			<AlertDescription className="flex items-center justify-between gap-4">
 				<span>
 					This document is too large for the editor (
 					{Math.round((editorDoc.content_size_bytes ?? 0) / 1024 / 1024)}MB,{" "}
-					{editorDoc.chunk_count ?? 0} chunks). Showing a preview below.
+					{editorDoc.chunk_count ?? 0} chunks). Showing raw markdown below.
 				</span>
 				<Button
 					variant="outline"
@@ -753,16 +780,33 @@ export function EditorPanelContent({
 							}}
 						/>
 					</div>
-				) : isLargeDocument && !isLocalFileMode ? (
+				) : viewerMode === "monaco" && !isLocalFileMode ? (
-					// Large doc — fast Streamdown preview + download CTA.
+					// Large doc — raw markdown in Monaco. Rich renderers are intentionally skipped.
-					// Plate is heavy on multi-MB docs.
+					<div className="flex h-full min-h-0 flex-col">
-					<div className="h-full overflow-y-auto px-5 py-4">
 						{largeDocAlert}
-						<MarkdownViewer content={editorDoc.source_markdown} enableCitations />
+						<div className="min-h-0 flex-1 overflow-hidden">
+							<SourceCodeEditor
+								path={`${editorDoc.title || "document"}.md`}
+								language="markdown"
+								value={editorDoc.source_markdown}
+								readOnly
+								onChange={() => {}}
+							/>
+						</div>
 					</div>
 				) : renderInPlateEditor ? (
 					// Editable doc (FILE/NOTE) — Plate editing UX.
 					<div className="flex h-full min-h-0 flex-col">
+						{showPlateSizeWarning && (
+							<Alert className="m-4 mb-0 shrink-0">
+								<FileText className="size-4" />
+								<AlertDescription>
+									{isOverPlateLimit
+										? `This document is ${formatBytes(activeMarkdownSizeBytes)}, above the rich editor limit of ${formatBytes(plateMaxBytes)}. You can save, but it will reopen in raw markdown mode.`
+										: `This document is approaching the rich editor limit (${formatBytes(activeMarkdownSizeBytes)} of ${formatBytes(plateMaxBytes)}).`}
+								</AlertDescription>
+							</Alert>
+						)}
 						<div className="flex-1 min-h-0 overflow-hidden">
 							<PlateEditor
 								key={`${

surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx

@@ -2,9 +2,10 @@
 
 import { Download, FileQuestionMark, FileText, Pencil, RefreshCw } from "lucide-react";
 import { useRouter } from "next/navigation";
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { toast } from "sonner";
 import { PlateEditor } from "@/components/editor/plate-editor";
+import { SourceCodeEditor } from "@/components/editor/source-code-editor";
 import { MarkdownViewer } from "@/components/markdown-viewer";
 import { Alert, AlertDescription } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
@@ -21,7 +22,8 @@ interface DocumentContent {
 	source_markdown: string;
 	content_size_bytes?: number;
 	chunk_count?: number;
-	truncated?: boolean;
+	viewer_mode?: ViewerMode;
+	editor_plate_max_bytes?: number;
 }
 
 function DocumentSkeleton() {
@@ -51,6 +53,21 @@ interface DocumentTabContentProps {
 }
 
 const EDITABLE_DOCUMENT_TYPES = new Set(["FILE", "NOTE"]);
+type ViewerMode = "plate" | "monaco";
+
+function getUtf8ByteSize(value: string): number {
+	return new TextEncoder().encode(value).byteLength;
+}
+
+function formatBytes(bytes: number): string {
+	if (bytes >= 1024 * 1024) {
+		return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
+	}
+	if (bytes >= 1024) {
+		return `${Math.round(bytes / 1024)}KB`;
+	}
+	return `${bytes}B`;
+}
 
 export function DocumentTabContent({ documentId, searchSpaceId, title }: DocumentTabContentProps) {
 	const [doc, setDoc] = useState<DocumentContent | null>(null);
@@ -65,7 +82,13 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 	const changeCountRef = useRef(0);
 	const router = useRouter();
 
-	const isLargeDocument = (doc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD;
+	const plateMaxBytes = doc?.editor_plate_max_bytes ?? LARGE_DOCUMENT_THRESHOLD;
+	const isLargeDocument = (doc?.content_size_bytes ?? 0) > plateMaxBytes;
+	const viewerMode: ViewerMode = doc?.viewer_mode ?? (isLargeDocument ? "monaco" : "plate");
+	const activeMarkdown = editedMarkdown ?? doc?.source_markdown ?? "";
+	const activeMarkdownSizeBytes = useMemo(() => getUtf8ByteSize(activeMarkdown), [activeMarkdown]);
+	const isNearPlateLimit = activeMarkdownSizeBytes >= plateMaxBytes * 0.9;
+	const isOverPlateLimit = activeMarkdownSizeBytes > plateMaxBytes;
 
 	useEffect(() => {
 		const controller = new AbortController();
@@ -88,8 +111,6 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 				const url = new URL(
 					`${BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`
 				);
-				url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD));
-
 				const response = await authenticatedFetch(url.toString(), { method: "GET" });
 
 				if (controller.signal.aborted) return;
@@ -161,14 +182,19 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 
 			setDoc((prev) => (prev ? { ...prev, source_markdown: markdownRef.current } : prev));
 			setEditedMarkdown(null);
-			toast.success("Document saved! Reindexing in background...");
+			const savedSizeBytes = getUtf8ByteSize(markdownRef.current);
+			if (savedSizeBytes > plateMaxBytes) {
+				toast.success("Document saved. It will reopen in raw markdown mode.");
+			} else {
+				toast.success("Document saved! Reindexing in background...");
+			}
 		} catch (err) {
 			console.error("Error saving document:", err);
 			toast.error(err instanceof Error ? err.message : "Failed to save document");
 		} finally {
 			setSaving(false);
 		}
-	}, [documentId, searchSpaceId]);
+	}, [documentId, plateMaxBytes, searchSpaceId]);
 
 	if (isLoading) return <DocumentSkeleton />;
 
@@ -204,9 +230,9 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 		);
 	}
 
-	const isEditable = EDITABLE_DOCUMENT_TYPES.has(doc.document_type ?? "") && !isLargeDocument;
+	const isEditable = viewerMode === "plate" && EDITABLE_DOCUMENT_TYPES.has(doc.document_type ?? "");
 
-	if (isEditing && !isLargeDocument) {
+	if (isEditing && viewerMode === "plate") {
 		return (
 			<div className="flex flex-col h-full overflow-hidden">
 				<div className="flex items-center justify-between px-6 py-3 border-b shrink-0">
@@ -228,20 +254,32 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 						Done editing
 					</Button>
 				</div>
-				<div className="flex-1 overflow-hidden">
+				<div className="flex-1 min-h-0 overflow-hidden flex flex-col">
-					<PlateEditor
+					{isNearPlateLimit && (
-						key={`edit-${documentId}`}
+						<Alert className="m-4 mb-0 shrink-0">
-						preset="full"
+							<FileText className="size-4" />
-						markdown={doc.source_markdown}
+							<AlertDescription>
-						onMarkdownChange={handleMarkdownChange}
+								{isOverPlateLimit
-						readOnly={false}
+									? `This document is ${formatBytes(activeMarkdownSizeBytes)}, above the rich editor limit of ${formatBytes(plateMaxBytes)}. You can save, but it will reopen in raw markdown mode.`
-						placeholder="Start writing..."
+									: `This document is approaching the rich editor limit (${formatBytes(activeMarkdownSizeBytes)} of ${formatBytes(plateMaxBytes)}).`}
-						editorVariant="default"
+							</AlertDescription>
-						onSave={handleSave}
+						</Alert>
-						hasUnsavedChanges={editedMarkdown !== null}
+					)}
-						isSaving={saving}
+					<div className="min-h-0 flex-1 overflow-hidden">
-						defaultEditing={true}
+						<PlateEditor
-					/>
+							key={`edit-${documentId}`}
+							preset="full"
+							markdown={doc.source_markdown}
+							onMarkdownChange={handleMarkdownChange}
+							readOnly={false}
+							placeholder="Start writing..."
+							editorVariant="default"
+							onSave={handleSave}
+							hasUnsavedChanges={editedMarkdown !== null}
+							isSaving={saving}
+							defaultEditing={true}
+						/>
+					</div>
 				</div>
 			</div>
 		);
@@ -265,64 +303,74 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 					</Button>
 				)}
 			</div>
-			<div className="flex-1 overflow-auto">
+			<div className="flex-1 min-h-0 overflow-hidden">
-				<div className="max-w-4xl mx-auto px-6 py-6">
+				{viewerMode === "monaco" ? (
-					{isLargeDocument ? (
+					<div className="flex h-full min-h-0 flex-col">
-						<>
+						<Alert className="m-4 shrink-0">
-							<Alert className="mb-4">
+							<FileText className="size-4" />
-								<FileText className="size-4" />
+							<AlertDescription className="flex items-center justify-between gap-4">
-								<AlertDescription className="flex items-center justify-between gap-4">
+								<span>
-									<span>
+									This document is too large for the editor (
-										This document is too large for the editor (
+									{Math.round((doc.content_size_bytes ?? 0) / 1024 / 1024)}MB,{" "}
-										{Math.round((doc.content_size_bytes ?? 0) / 1024 / 1024)}MB,{" "}
+									{doc.chunk_count ?? 0} chunks). Showing raw markdown below.
-										{doc.chunk_count ?? 0} chunks). Showing a preview below.
+								</span>
+								<Button
+									variant="outline"
+									size="sm"
+									className="relative shrink-0"
+									disabled={downloading}
+									onClick={async () => {
+										setDownloading(true);
+										try {
+											const response = await authenticatedFetch(
+												`${BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/download-markdown`,
+												{ method: "GET" }
+											);
+											if (!response.ok) throw new Error("Download failed");
+											const blob = await response.blob();
+											const url = URL.createObjectURL(blob);
+											const a = document.createElement("a");
+											a.href = url;
+											const disposition = response.headers.get("content-disposition");
+											const match = disposition?.match(/filename="(.+)"/);
+											a.download = match?.[1] ?? `${doc.title || "document"}.md`;
+											document.body.appendChild(a);
+											a.click();
+											a.remove();
+											URL.revokeObjectURL(url);
+											toast.success("Download started");
+										} catch {
+											toast.error("Failed to download document");
+										} finally {
+											setDownloading(false);
+										}
+									}}
+								>
+									<span className={`flex items-center gap-1.5 ${downloading ? "opacity-0" : ""}`}>
+										<Download className="size-3.5" />
+										Download .md
 									</span>
-									<Button
+									{downloading && <Spinner size="sm" className="absolute" />}
-										variant="outline"
+								</Button>
-										size="sm"
+							</AlertDescription>
-										className="relative shrink-0"
+						</Alert>
-										disabled={downloading}
+						<div className="min-h-0 flex-1 overflow-hidden">
-										onClick={async () => {
+							<SourceCodeEditor
-											setDownloading(true);
+								path={`${doc.title || "document"}.md`}
-											try {
+								language="markdown"
-												const response = await authenticatedFetch(
+								value={doc.source_markdown}
-													`${BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/download-markdown`,
+								readOnly
-													{ method: "GET" }
+								onChange={() => {}}
-												);
+							/>
-												if (!response.ok) throw new Error("Download failed");
+						</div>
-												const blob = await response.blob();
+					</div>
-												const url = URL.createObjectURL(blob);
+				) : (
-												const a = document.createElement("a");
+					<div className="h-full overflow-auto">
-												a.href = url;
+						<div className="max-w-4xl mx-auto px-6 py-6">
-												const disposition = response.headers.get("content-disposition");
-												const match = disposition?.match(/filename="(.+)"/);
-												a.download = match?.[1] ?? `${doc.title || "document"}.md`;
-												document.body.appendChild(a);
-												a.click();
-												a.remove();
-												URL.revokeObjectURL(url);
-												toast.success("Download started");
-											} catch {
-												toast.error("Failed to download document");
-											} finally {
-												setDownloading(false);
-											}
-										}}
-									>
-										<span className={`flex items-center gap-1.5 ${downloading ? "opacity-0" : ""}`}>
-											<Download className="size-3.5" />
-											Download .md
-										</span>
-										{downloading && <Spinner size="sm" className="absolute" />}
-									</Button>
-								</AlertDescription>
-							</Alert>
 							<MarkdownViewer content={doc.source_markdown} enableCitations />
-						</>
+						</div>
-					) : (
+					</div>
-						<MarkdownViewer content={doc.source_markdown} enableCitations />
+				)}
-					)}
-				</div>
 			</div>
 		</div>
 	);

surfsense_web/tests/helpers/api/documents.ts

@@ -43,7 +43,8 @@ export type EditorContent = {
 	source_markdown: string;
 	content_size_bytes: number;
 	chunk_count: number;
-	truncated: boolean;
+	viewer_mode?: "plate" | "monaco";
+	editor_plate_max_bytes?: number;
 };
 
 // Same endpoint the UI hits when a user opens a document in the dashboard.