apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-06 22:12:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge commit '683a7d3bc2' into feat/rename-searchspace-to-workspace

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-06-29 17:55:15 -07:00
commit 2836654701
27 changed files with 543 additions and 323 deletions
Download patch file Download diff file Expand all files Collapse all files

5
surfsense_backend/app/tasks/document_processors/_save.py
View file

@ -10,6 +10,7 @@ from app.utils.document_converters import (
create_document_chunks,
embed_text,
generate_content_hash,
truncate_for_embedding,
)
from ._helpers import (
@ -73,7 +74,9 @@ async def save_file_document(
if should_skip:
return doc
document_content = f"File: {file_name}\n\n{markdown_content[:4000]}"
document_content = (
f"File: {file_name}\n\n{truncate_for_embedding(markdown_content)}"
)
document_embedding = embed_text(document_content)
chunks = await create_document_chunks(markdown_content)
doc_metadata = {"FILE_NAME": file_name, "ETL_SERVICE": etl_service}

5
surfsense_backend/app/tasks/document_processors/markdown_processor.py
View file

@ -13,6 +13,7 @@ from app.utils.document_converters import (
create_document_chunks,
embed_text,
generate_content_hash,
truncate_for_embedding,
)
from ._helpers import (
@ -182,7 +183,9 @@ async def add_received_markdown_file_document(
return doc
# Content changed - continue to update
summary_content = f"File: {file_name}\n\n{file_in_markdown[:4000]}"
summary_content = (
f"File: {file_name}\n\n{truncate_for_embedding(file_in_markdown)}"
)
summary_embedding = embed_text(summary_content)
# Process chunks