feat(editor): increase editor byte limit to 5MB and implement UTF-8 truncation for document content

surfsense_backend/app/routes/editor_routes.py

@@ -38,7 +38,14 @@ logger = logging.getLogger(__name__)
 
 router = APIRouter()
 
-EDITOR_PLATE_MAX_BYTES = 2 * 1024 * 1024
+EDITOR_PLATE_MAX_BYTES = 5 * 1024 * 1024
+
+
+def _truncate_utf8(text: str, max_bytes: int) -> str:
+    encoded = text.encode("utf-8")
+    if len(encoded) <= max_bytes:
+        return text
+    return encoded[:max_bytes].decode("utf-8", errors="ignore")
 
 
 @router.get("/search-spaces/{search_space_id}/documents/{document_id}/editor-content")
@@ -46,7 +53,8 @@ async def get_editor_content(
     search_space_id: int,
     document_id: int,
     max_length: int | None = Query(
-        None, description="Truncate source_markdown to this many characters"
+        None,
+        description="Truncate source_markdown to this many UTF-8 bytes. Defaults to the Plate editor byte limit.",
     ),
     session: AsyncSession = Depends(get_async_session),
     user: User = Depends(current_active_user),
@@ -87,10 +95,11 @@ async def get_editor_content(
     def _build_response(md: str) -> dict:
         size_bytes = len(md.encode("utf-8"))
         viewer_mode = "monaco" if size_bytes > EDITOR_PLATE_MAX_BYTES else "plate"
+        content_limit = max_length if max_length is not None else EDITOR_PLATE_MAX_BYTES
         truncated = False
         output_md = md
-        if max_length is not None and size_bytes > max_length:
-            output_md = md[:max_length]
+        if size_bytes > content_limit:
+            output_md = _truncate_utf8(md, content_limit)
             truncated = True
         return {
             "document_id": document.id,
@@ -101,6 +110,7 @@ async def get_editor_content(
             "chunk_count": chunk_count,
             "truncated": truncated,
             "viewer_mode": viewer_mode,
+            "editor_plate_max_bytes": EDITOR_PLATE_MAX_BYTES,
             "updated_at": document.updated_at.isoformat()
             if document.updated_at
             else None,

surfsense_web/components/editor-panel/editor-panel.tsx

@@ -12,7 +12,7 @@ import {
 	XIcon,
 } from "lucide-react";
 import dynamic from "next/dynamic";
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { toast } from "sonner";
 import { closeEditorPanelAtom, editorPanelAtom } from "@/atoms/editor/editor-panel.atom";
 import { DownloadOriginalButton } from "@/components/documents/download-original-button";
@@ -52,6 +52,7 @@ interface EditorContent {
 	chunk_count?: number;
 	truncated?: boolean;
 	viewer_mode?: ViewerMode;
+	editor_plate_max_bytes?: number;
 }
 
 const EDITABLE_DOCUMENT_TYPES = new Set(["FILE", "NOTE"]);
@@ -114,6 +115,20 @@ function EditorPanelSkeleton() {
 	);
 }
 
+function getUtf8ByteSize(value: string): number {
+	return new TextEncoder().encode(value).byteLength;
+}
+
+function formatBytes(bytes: number): string {
+	if (bytes >= 1024 * 1024) {
+		return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
+	}
+	if (bytes >= 1024) {
+		return `${Math.round(bytes / 1024)}KB`;
+	}
+	return `${bytes}B`;
+}
+
 export function EditorPanelContent({
 	kind = "document",
 	documentId,
@@ -169,7 +184,8 @@ export function EditorPanelContent({
 		[electronAPI, searchSpaceId]
 	);
 
-	const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD;
+	const plateMaxBytes = editorDoc?.editor_plate_max_bytes ?? LARGE_DOCUMENT_THRESHOLD;
+	const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > plateMaxBytes;
 	const viewerMode: ViewerMode = isMemoryMode
 		? "plate"
 		: (editorDoc?.viewer_mode ?? (isLargeDocument ? "monaco" : "plate"));
@@ -248,8 +264,6 @@ export function EditorPanelContent({
 				const url = new URL(
 					`${BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`
 				);
-				url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD));
-
 				const response = await authenticatedFetch(url.toString(), { method: "GET" });
 
 				if (controller.signal.aborted) return;
@@ -407,7 +421,12 @@ export function EditorPanelContent({
 				setEditorDoc((prev) => (prev ? { ...prev, source_markdown: markdownRef.current } : prev));
 				setEditedMarkdown(null);
 				if (!options?.silent) {
-					toast.success("Document saved! Reindexing in background...");
+					const savedSizeBytes = getUtf8ByteSize(markdownRef.current);
+					if (savedSizeBytes > plateMaxBytes) {
+						toast.success("Document saved. It will reopen in raw markdown mode.");
+					} else {
+						toast.success("Document saved! Reindexing in background...");
+					}
 				}
 				return true;
 			} catch (err) {
@@ -428,6 +447,7 @@ export function EditorPanelContent({
 			localFilePath,
 			memoryLimits,
 			memoryScope,
+			plateMaxBytes,
 			resolveLocalVirtualPath,
 			searchSpaceId,
 		]
@@ -447,6 +467,11 @@ export function EditorPanelContent({
 	const showEditingActions = isEditableType && isEditing;
 	const localFileLanguage = inferMonacoLanguageFromPath(localFilePath);
 	const activeMarkdown = editedMarkdown ?? editorDoc?.source_markdown ?? "";
+	const activeMarkdownSizeBytes = useMemo(() => getUtf8ByteSize(activeMarkdown), [activeMarkdown]);
+	const isNearPlateLimit = activeMarkdownSizeBytes >= plateMaxBytes * 0.9;
+	const isOverPlateLimit = activeMarkdownSizeBytes > plateMaxBytes;
+	const showPlateSizeWarning =
+		showEditingActions && !isMemoryMode && !isLocalFileMode && isNearPlateLimit;
 	const memoryLimitState = isMemoryMode
 		? getMemoryLimitState(activeMarkdown.length, memoryLimits)
 		: null;
@@ -773,6 +798,16 @@ export function EditorPanelContent({
 				) : renderInPlateEditor ? (
 					// Editable doc (FILE/NOTE) — Plate editing UX.
 					<div className="flex h-full min-h-0 flex-col">
+						{showPlateSizeWarning && (
+							<Alert className="m-4 mb-0 shrink-0">
+								<FileText className="size-4" />
+								<AlertDescription>
+									{isOverPlateLimit
+										? `This document is ${formatBytes(activeMarkdownSizeBytes)}, above the rich editor limit of ${formatBytes(plateMaxBytes)}. You can save, but it will reopen in raw markdown mode.`
+										: `This document is approaching the rich editor limit (${formatBytes(activeMarkdownSizeBytes)} of ${formatBytes(plateMaxBytes)}).`}
+								</AlertDescription>
+							</Alert>
+						)}
 						<div className="flex-1 min-h-0 overflow-hidden">
 							<PlateEditor
 								key={`${

surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx

@@ -2,7 +2,7 @@
 
 import { Download, FileQuestionMark, FileText, Pencil, RefreshCw } from "lucide-react";
 import { useRouter } from "next/navigation";
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { toast } from "sonner";
 import { PlateEditor } from "@/components/editor/plate-editor";
 import { SourceCodeEditor } from "@/components/editor/source-code-editor";
@@ -24,6 +24,7 @@ interface DocumentContent {
 	chunk_count?: number;
 	truncated?: boolean;
 	viewer_mode?: ViewerMode;
+	editor_plate_max_bytes?: number;
 }
 
 function DocumentSkeleton() {
@@ -55,6 +56,20 @@ interface DocumentTabContentProps {
 const EDITABLE_DOCUMENT_TYPES = new Set(["FILE", "NOTE"]);
 type ViewerMode = "plate" | "monaco";
 
+function getUtf8ByteSize(value: string): number {
+	return new TextEncoder().encode(value).byteLength;
+}
+
+function formatBytes(bytes: number): string {
+	if (bytes >= 1024 * 1024) {
+		return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
+	}
+	if (bytes >= 1024) {
+		return `${Math.round(bytes / 1024)}KB`;
+	}
+	return `${bytes}B`;
+}
+
 export function DocumentTabContent({ documentId, searchSpaceId, title }: DocumentTabContentProps) {
 	const [doc, setDoc] = useState<DocumentContent | null>(null);
 	const [isLoading, setIsLoading] = useState(true);
@@ -68,8 +83,13 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 	const changeCountRef = useRef(0);
 	const router = useRouter();
 
-	const isLargeDocument = (doc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD;
+	const plateMaxBytes = doc?.editor_plate_max_bytes ?? LARGE_DOCUMENT_THRESHOLD;
+	const isLargeDocument = (doc?.content_size_bytes ?? 0) > plateMaxBytes;
 	const viewerMode: ViewerMode = doc?.viewer_mode ?? (isLargeDocument ? "monaco" : "plate");
+	const activeMarkdown = editedMarkdown ?? doc?.source_markdown ?? "";
+	const activeMarkdownSizeBytes = useMemo(() => getUtf8ByteSize(activeMarkdown), [activeMarkdown]);
+	const isNearPlateLimit = activeMarkdownSizeBytes >= plateMaxBytes * 0.9;
+	const isOverPlateLimit = activeMarkdownSizeBytes > plateMaxBytes;
 
 	useEffect(() => {
 		const controller = new AbortController();
@@ -92,8 +112,6 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 				const url = new URL(
 					`${BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`
 				);
-				url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD));
-
 				const response = await authenticatedFetch(url.toString(), { method: "GET" });
 
 				if (controller.signal.aborted) return;
@@ -165,14 +183,19 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 
 			setDoc((prev) => (prev ? { ...prev, source_markdown: markdownRef.current } : prev));
 			setEditedMarkdown(null);
-			toast.success("Document saved! Reindexing in background...");
+			const savedSizeBytes = getUtf8ByteSize(markdownRef.current);
+			if (savedSizeBytes > plateMaxBytes) {
+				toast.success("Document saved. It will reopen in raw markdown mode.");
+			} else {
+				toast.success("Document saved! Reindexing in background...");
+			}
 		} catch (err) {
 			console.error("Error saving document:", err);
 			toast.error(err instanceof Error ? err.message : "Failed to save document");
 		} finally {
 			setSaving(false);
 		}
-	}, [documentId, searchSpaceId]);
+	}, [documentId, plateMaxBytes, searchSpaceId]);
 
 	if (isLoading) return <DocumentSkeleton />;
 
@@ -232,20 +255,32 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
 						Done editing
 					</Button>
 				</div>
-				<div className="flex-1 overflow-hidden">
-					<PlateEditor
-						key={`edit-${documentId}`}
-						preset="full"
-						markdown={doc.source_markdown}
-						onMarkdownChange={handleMarkdownChange}
-						readOnly={false}
-						placeholder="Start writing..."
-						editorVariant="default"
-						onSave={handleSave}
-						hasUnsavedChanges={editedMarkdown !== null}
-						isSaving={saving}
-						defaultEditing={true}
-					/>
+				<div className="flex-1 min-h-0 overflow-hidden flex flex-col">
+					{isNearPlateLimit && (
+						<Alert className="m-4 mb-0 shrink-0">
+							<FileText className="size-4" />
+							<AlertDescription>
+								{isOverPlateLimit
+									? `This document is ${formatBytes(activeMarkdownSizeBytes)}, above the rich editor limit of ${formatBytes(plateMaxBytes)}. You can save, but it will reopen in raw markdown mode.`
+									: `This document is approaching the rich editor limit (${formatBytes(activeMarkdownSizeBytes)} of ${formatBytes(plateMaxBytes)}).`}
+							</AlertDescription>
+						</Alert>
+					)}
+					<div className="min-h-0 flex-1 overflow-hidden">
+						<PlateEditor
+							key={`edit-${documentId}`}
+							preset="full"
+							markdown={doc.source_markdown}
+							onMarkdownChange={handleMarkdownChange}
+							readOnly={false}
+							placeholder="Start writing..."
+							editorVariant="default"
+							onSave={handleSave}
+							hasUnsavedChanges={editedMarkdown !== null}
+							isSaving={saving}
+							defaultEditing={true}
+						/>
+					</div>
 				</div>
 			</div>
 		);

surfsense_web/tests/helpers/api/documents.ts

@@ -45,6 +45,7 @@ export type EditorContent = {
 	chunk_count: number;
 	truncated: boolean;
 	viewer_mode?: "plate" | "monaco";
+	editor_plate_max_bytes?: number;
 };
 
 // Same endpoint the UI hits when a user opens a document in the dashboard.