From bf805fd81ef651df1a8d288d311b2d633c05a016 Mon Sep 17 00:00:00 2001
From: DhruvTilva <tilvadhruv8@gmail.com>
Date: Fri, 26 Jun 2026 00:09:11 +0530
Subject: [PATCH] fix: use token-safe truncation for document embeddings

The document saving logic used a hardcoded character slice ([:4000]) for
the document summary content fed to the embedder. For UTF-8 documents
(e.g., Arabic, Chinese, Japanese), characters above U+007F take multiple
bytes but count as 1 character in a slice, potentially producing text
that exceeds the token limit of the embedding model.

Replaced the arbitrary slice with 	runcate_for_embedding(), which safely
bounds the text using the embedding model's actual tokenizer.
---
 surfsense_backend/app/tasks/document_processors/_save.py     | 5 ++++-
 .../app/tasks/document_processors/markdown_processor.py      | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/surfsense_backend/app/tasks/document_processors/_save.py b/surfsense_backend/app/tasks/document_processors/_save.py
index 3b9616cbd..9456e2f1c 100644
--- a/surfsense_backend/app/tasks/document_processors/_save.py
+++ b/surfsense_backend/app/tasks/document_processors/_save.py
@@ -10,6 +10,7 @@ from app.utils.document_converters import (
     create_document_chunks,
     embed_text,
     generate_content_hash,
+    truncate_for_embedding,
 )
 
 from ._helpers import (
@@ -73,7 +74,9 @@ async def save_file_document(
             if should_skip:
                 return doc
 
-        document_content = f"File: {file_name}\n\n{markdown_content[:4000]}"
+        document_content = (
+            f"File: {file_name}\n\n{truncate_for_embedding(markdown_content)}"
+        )
         document_embedding = embed_text(document_content)
         chunks = await create_document_chunks(markdown_content)
         doc_metadata = {"FILE_NAME": file_name, "ETL_SERVICE": etl_service}
diff --git a/surfsense_backend/app/tasks/document_processors/markdown_processor.py b/surfsense_backend/app/tasks/document_processors/markdown_processor.py
index 19a4df87d..463951a64 100644
--- a/surfsense_backend/app/tasks/document_processors/markdown_processor.py
+++ b/surfsense_backend/app/tasks/document_processors/markdown_processor.py
@@ -13,6 +13,7 @@ from app.utils.document_converters import (
     create_document_chunks,
     embed_text,
     generate_content_hash,
+    truncate_for_embedding,
 )
 
 from ._helpers import (
@@ -182,7 +183,9 @@ async def add_received_markdown_file_document(
                 return doc
             # Content changed - continue to update
 
-        summary_content = f"File: {file_name}\n\n{file_in_markdown[:4000]}"
+        summary_content = (
+            f"File: {file_name}\n\n{truncate_for_embedding(file_in_markdown)}"
+        )
         summary_embedding = embed_text(summary_content)
 
         # Process chunks