From 640238e1af4db5f851d02273e8097226a3a07392 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:57:52 +0530 Subject: [PATCH] feat(editor): increase editor byte limit to 5MB and implement UTF-8 truncation for document content --- surfsense_backend/app/routes/editor_routes.py | 18 ++++- .../components/editor-panel/editor-panel.tsx | 45 +++++++++-- .../layout/ui/tabs/DocumentTabContent.tsx | 75 ++++++++++++++----- surfsense_web/tests/helpers/api/documents.ts | 1 + 4 files changed, 110 insertions(+), 29 deletions(-) diff --git a/surfsense_backend/app/routes/editor_routes.py b/surfsense_backend/app/routes/editor_routes.py index 7abbdd31d..fb33076c4 100644 --- a/surfsense_backend/app/routes/editor_routes.py +++ b/surfsense_backend/app/routes/editor_routes.py @@ -38,7 +38,14 @@ logger = logging.getLogger(__name__) router = APIRouter() -EDITOR_PLATE_MAX_BYTES = 2 * 1024 * 1024 +EDITOR_PLATE_MAX_BYTES = 5 * 1024 * 1024 + + +def _truncate_utf8(text: str, max_bytes: int) -> str: + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text + return encoded[:max_bytes].decode("utf-8", errors="ignore") @router.get("/search-spaces/{search_space_id}/documents/{document_id}/editor-content") @@ -46,7 +53,8 @@ async def get_editor_content( search_space_id: int, document_id: int, max_length: int | None = Query( - None, description="Truncate source_markdown to this many characters" + None, + description="Truncate source_markdown to this many UTF-8 bytes. Defaults to the Plate editor byte limit.", ), session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), @@ -87,10 +95,11 @@ async def get_editor_content( def _build_response(md: str) -> dict: size_bytes = len(md.encode("utf-8")) viewer_mode = "monaco" if size_bytes > EDITOR_PLATE_MAX_BYTES else "plate" + content_limit = max_length if max_length is not None else EDITOR_PLATE_MAX_BYTES truncated = False output_md = md - if max_length is not None and size_bytes > max_length: - output_md = md[:max_length] + if size_bytes > content_limit: + output_md = _truncate_utf8(md, content_limit) truncated = True return { "document_id": document.id, @@ -101,6 +110,7 @@ async def get_editor_content( "chunk_count": chunk_count, "truncated": truncated, "viewer_mode": viewer_mode, + "editor_plate_max_bytes": EDITOR_PLATE_MAX_BYTES, "updated_at": document.updated_at.isoformat() if document.updated_at else None, diff --git a/surfsense_web/components/editor-panel/editor-panel.tsx b/surfsense_web/components/editor-panel/editor-panel.tsx index 2c0d316d9..c06d782e7 100644 --- a/surfsense_web/components/editor-panel/editor-panel.tsx +++ b/surfsense_web/components/editor-panel/editor-panel.tsx @@ -12,7 +12,7 @@ import { XIcon, } from "lucide-react"; import dynamic from "next/dynamic"; -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { closeEditorPanelAtom, editorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { DownloadOriginalButton } from "@/components/documents/download-original-button"; @@ -52,6 +52,7 @@ interface EditorContent { chunk_count?: number; truncated?: boolean; viewer_mode?: ViewerMode; + editor_plate_max_bytes?: number; } const EDITABLE_DOCUMENT_TYPES = new Set(["FILE", "NOTE"]); @@ -114,6 +115,20 @@ function EditorPanelSkeleton() { ); } +function getUtf8ByteSize(value: string): number { + return new TextEncoder().encode(value).byteLength; +} + +function formatBytes(bytes: number): string { + if (bytes >= 1024 * 1024) { + return `${(bytes / 1024 / 1024).toFixed(1)}MB`; + } + if (bytes >= 1024) { + return `${Math.round(bytes / 1024)}KB`; + } + return `${bytes}B`; +} + export function EditorPanelContent({ kind = "document", documentId, @@ -169,7 +184,8 @@ export function EditorPanelContent({ [electronAPI, searchSpaceId] ); - const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD; + const plateMaxBytes = editorDoc?.editor_plate_max_bytes ?? LARGE_DOCUMENT_THRESHOLD; + const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > plateMaxBytes; const viewerMode: ViewerMode = isMemoryMode ? "plate" : (editorDoc?.viewer_mode ?? (isLargeDocument ? "monaco" : "plate")); @@ -248,8 +264,6 @@ export function EditorPanelContent({ const url = new URL( `${BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` ); - url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD)); - const response = await authenticatedFetch(url.toString(), { method: "GET" }); if (controller.signal.aborted) return; @@ -407,7 +421,12 @@ export function EditorPanelContent({ setEditorDoc((prev) => (prev ? { ...prev, source_markdown: markdownRef.current } : prev)); setEditedMarkdown(null); if (!options?.silent) { - toast.success("Document saved! Reindexing in background..."); + const savedSizeBytes = getUtf8ByteSize(markdownRef.current); + if (savedSizeBytes > plateMaxBytes) { + toast.success("Document saved. It will reopen in raw markdown mode."); + } else { + toast.success("Document saved! Reindexing in background..."); + } } return true; } catch (err) { @@ -428,6 +447,7 @@ export function EditorPanelContent({ localFilePath, memoryLimits, memoryScope, + plateMaxBytes, resolveLocalVirtualPath, searchSpaceId, ] @@ -447,6 +467,11 @@ export function EditorPanelContent({ const showEditingActions = isEditableType && isEditing; const localFileLanguage = inferMonacoLanguageFromPath(localFilePath); const activeMarkdown = editedMarkdown ?? editorDoc?.source_markdown ?? ""; + const activeMarkdownSizeBytes = useMemo(() => getUtf8ByteSize(activeMarkdown), [activeMarkdown]); + const isNearPlateLimit = activeMarkdownSizeBytes >= plateMaxBytes * 0.9; + const isOverPlateLimit = activeMarkdownSizeBytes > plateMaxBytes; + const showPlateSizeWarning = + showEditingActions && !isMemoryMode && !isLocalFileMode && isNearPlateLimit; const memoryLimitState = isMemoryMode ? getMemoryLimitState(activeMarkdown.length, memoryLimits) : null; @@ -773,6 +798,16 @@ export function EditorPanelContent({ ) : renderInPlateEditor ? ( // Editable doc (FILE/NOTE) — Plate editing UX.
+ {showPlateSizeWarning && ( + + + + {isOverPlateLimit + ? `This document is ${formatBytes(activeMarkdownSizeBytes)}, above the rich editor limit of ${formatBytes(plateMaxBytes)}. You can save, but it will reopen in raw markdown mode.` + : `This document is approaching the rich editor limit (${formatBytes(activeMarkdownSizeBytes)} of ${formatBytes(plateMaxBytes)}).`} + + + )}
= 1024 * 1024) { + return `${(bytes / 1024 / 1024).toFixed(1)}MB`; + } + if (bytes >= 1024) { + return `${Math.round(bytes / 1024)}KB`; + } + return `${bytes}B`; +} + export function DocumentTabContent({ documentId, searchSpaceId, title }: DocumentTabContentProps) { const [doc, setDoc] = useState(null); const [isLoading, setIsLoading] = useState(true); @@ -68,8 +83,13 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen const changeCountRef = useRef(0); const router = useRouter(); - const isLargeDocument = (doc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD; + const plateMaxBytes = doc?.editor_plate_max_bytes ?? LARGE_DOCUMENT_THRESHOLD; + const isLargeDocument = (doc?.content_size_bytes ?? 0) > plateMaxBytes; const viewerMode: ViewerMode = doc?.viewer_mode ?? (isLargeDocument ? "monaco" : "plate"); + const activeMarkdown = editedMarkdown ?? doc?.source_markdown ?? ""; + const activeMarkdownSizeBytes = useMemo(() => getUtf8ByteSize(activeMarkdown), [activeMarkdown]); + const isNearPlateLimit = activeMarkdownSizeBytes >= plateMaxBytes * 0.9; + const isOverPlateLimit = activeMarkdownSizeBytes > plateMaxBytes; useEffect(() => { const controller = new AbortController(); @@ -92,8 +112,6 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen const url = new URL( `${BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` ); - url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD)); - const response = await authenticatedFetch(url.toString(), { method: "GET" }); if (controller.signal.aborted) return; @@ -165,14 +183,19 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen setDoc((prev) => (prev ? { ...prev, source_markdown: markdownRef.current } : prev)); setEditedMarkdown(null); - toast.success("Document saved! Reindexing in background..."); + const savedSizeBytes = getUtf8ByteSize(markdownRef.current); + if (savedSizeBytes > plateMaxBytes) { + toast.success("Document saved. It will reopen in raw markdown mode."); + } else { + toast.success("Document saved! Reindexing in background..."); + } } catch (err) { console.error("Error saving document:", err); toast.error(err instanceof Error ? err.message : "Failed to save document"); } finally { setSaving(false); } - }, [documentId, searchSpaceId]); + }, [documentId, plateMaxBytes, searchSpaceId]); if (isLoading) return ; @@ -232,20 +255,32 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen Done editing
-
- +
+ {isNearPlateLimit && ( + + + + {isOverPlateLimit + ? `This document is ${formatBytes(activeMarkdownSizeBytes)}, above the rich editor limit of ${formatBytes(plateMaxBytes)}. You can save, but it will reopen in raw markdown mode.` + : `This document is approaching the rich editor limit (${formatBytes(activeMarkdownSizeBytes)} of ${formatBytes(plateMaxBytes)}).`} + + + )} +
+ +
); diff --git a/surfsense_web/tests/helpers/api/documents.ts b/surfsense_web/tests/helpers/api/documents.ts index 0d8cc8839..a6cdad72a 100644 --- a/surfsense_web/tests/helpers/api/documents.ts +++ b/surfsense_web/tests/helpers/api/documents.ts @@ -45,6 +45,7 @@ export type EditorContent = { chunk_count: number; truncated: boolean; viewer_mode?: "plate" | "monaco"; + editor_plate_max_bytes?: number; }; // Same endpoint the UI hits when a user opens a document in the dashboard.