diff --git a/surfsense_backend/app/tasks/document_processors/_save.py b/surfsense_backend/app/tasks/document_processors/_save.py index 3b9616cbd..9456e2f1c 100644 --- a/surfsense_backend/app/tasks/document_processors/_save.py +++ b/surfsense_backend/app/tasks/document_processors/_save.py @@ -10,6 +10,7 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, + truncate_for_embedding, ) from ._helpers import ( @@ -73,7 +74,9 @@ async def save_file_document( if should_skip: return doc - document_content = f"File: {file_name}\n\n{markdown_content[:4000]}" + document_content = ( + f"File: {file_name}\n\n{truncate_for_embedding(markdown_content)}" + ) document_embedding = embed_text(document_content) chunks = await create_document_chunks(markdown_content) doc_metadata = {"FILE_NAME": file_name, "ETL_SERVICE": etl_service} diff --git a/surfsense_backend/app/tasks/document_processors/markdown_processor.py b/surfsense_backend/app/tasks/document_processors/markdown_processor.py index 19a4df87d..463951a64 100644 --- a/surfsense_backend/app/tasks/document_processors/markdown_processor.py +++ b/surfsense_backend/app/tasks/document_processors/markdown_processor.py @@ -13,6 +13,7 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, + truncate_for_embedding, ) from ._helpers import ( @@ -182,7 +183,9 @@ async def add_received_markdown_file_document( return doc # Content changed - continue to update - summary_content = f"File: {file_name}\n\n{file_in_markdown[:4000]}" + summary_content = ( + f"File: {file_name}\n\n{truncate_for_embedding(file_in_markdown)}" + ) summary_embedding = embed_text(summary_content) # Process chunks