fix: use token-safe truncation for document embeddings

The document saving logic used a hardcoded character slice ([:4000]) for
the document summary content fed to the embedder. For UTF-8 documents
(e.g., Arabic, Chinese, Japanese), characters above U+007F take multiple
bytes but count as 1 character in a slice, potentially producing text
that exceeds the token limit of the embedding model.

Replaced the arbitrary slice with 	runcate_for_embedding(), which safely
bounds the text using the embedding model's actual tokenizer.

surfsense_backend/app/tasks/document_processors/_save.py

@@ -10,6 +10,7 @@ from app.utils.document_converters import (
     create_document_chunks,
     embed_text,
     generate_content_hash,
+    truncate_for_embedding,
 )
 
 from ._helpers import (
@@ -73,7 +74,9 @@ async def save_file_document(
             if should_skip:
                 return doc
 
-        document_content = f"File: {file_name}\n\n{markdown_content[:4000]}"
+        document_content = (
+            f"File: {file_name}\n\n{truncate_for_embedding(markdown_content)}"
+        )
         document_embedding = embed_text(document_content)
         chunks = await create_document_chunks(markdown_content)
         doc_metadata = {"FILE_NAME": file_name, "ETL_SERVICE": etl_service}

surfsense_backend/app/tasks/document_processors/markdown_processor.py

@@ -13,6 +13,7 @@ from app.utils.document_converters import (
     create_document_chunks,
     embed_text,
     generate_content_hash,
+    truncate_for_embedding,
 )
 
 from ._helpers import (
@@ -182,7 +183,9 @@ async def add_received_markdown_file_document(
                 return doc
             # Content changed - continue to update
 
-        summary_content = f"File: {file_name}\n\n{file_in_markdown[:4000]}"
+        summary_content = (
+            f"File: {file_name}\n\n{truncate_for_embedding(file_in_markdown)}"
+        )
         summary_embedding = embed_text(summary_content)
 
         # Process chunks