apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-04 22:02:16 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #1542 from DhruvTilva/fix/multilingual-embedding-truncation

Browse source 
fix: use token-safe truncation for document embeddings
This commit is contained in:
Rohan Verma 2026-06-29 17:44:55 -07:00 • committed by GitHub
commit d22131185f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 8 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
surfsense_backend/app/tasks/document_processors/_save.py
View file

@ -10,6 +10,7 @@ from app.utils.document_converters import (
create_document_chunks,
embed_text,
generate_content_hash,
truncate_for_embedding,
)
from ._helpers import (
@ -73,7 +74,9 @@ async def save_file_document(
if should_skip:
return doc
document_content = f"File: {file_name}\n\n{markdown_content[:4000]}"
document_content = (
f"File: {file_name}\n\n{truncate_for_embedding(markdown_content)}"
)
document_embedding = embed_text(document_content)
chunks = await create_document_chunks(markdown_content)
doc_metadata = {"FILE_NAME": file_name, "ETL_SERVICE": etl_service}

5
surfsense_backend/app/tasks/document_processors/markdown_processor.py
View file

@ -13,6 +13,7 @@ from app.utils.document_converters import (
create_document_chunks,
embed_text,
generate_content_hash,
truncate_for_embedding,
)
from ._helpers import (
@ -182,7 +183,9 @@ async def add_received_markdown_file_document(
return doc
# Content changed - continue to update
summary_content = f"File: {file_name}\n\n{file_in_markdown[:4000]}"
summary_content = (
f"File: {file_name}\n\n{truncate_for_embedding(file_in_markdown)}"
)
summary_embedding = embed_text(summary_content)
# Process chunks