apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-14 22:52:15 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: ran linting

Browse source
This commit is contained in:
Anish Sarkar 2026-04-03 13:14:40 +05:30
parent 6ace8850bb
commit 746c730b2e
31 changed files with 801 additions and 660 deletions
Download patch file Download diff file Expand all files Collapse all files

63
surfsense_backend/app/routes/documents_routes.py
View file

@ -977,15 +977,19 @@ async def get_watched_folders(
)
folders = (
await session.execute(
select(Folder).where(
Folder.search_space_id == search_space_id,
Folder.parent_id.is_(None),
Folder.folder_metadata.isnot(None),
Folder.folder_metadata["watched"].astext == "true",
(
await session.execute(
select(Folder).where(
Folder.search_space_id == search_space_id,
Folder.parent_id.is_(None),
Folder.folder_metadata.isnot(None),
Folder.folder_metadata["watched"].astext == "true",
)
)
)
).scalars().all()
.scalars()
.all()
)
return folders
@ -1265,15 +1269,21 @@ async def list_document_versions(
if not document:
raise HTTPException(status_code=404, detail="Document not found")
await check_permission(session, user, document.search_space_id, Permission.DOCUMENTS_READ.value)
await check_permission(
session, user, document.search_space_id, Permission.DOCUMENTS_READ.value
)
versions = (
await session.execute(
select(DocumentVersion)
.where(DocumentVersion.document_id == document_id)
.order_by(DocumentVersion.version_number.desc())
(
await session.execute(
select(DocumentVersion)
.where(DocumentVersion.document_id == document_id)
.order_by(DocumentVersion.version_number.desc())
)
)
).scalars().all()
.scalars()
.all()
)
return [
{
@ -1300,7 +1310,9 @@ async def get_document_version(
if not document:
raise HTTPException(status_code=404, detail="Document not found")
await check_permission(session, user, document.search_space_id, Permission.DOCUMENTS_READ.value)
await check_permission(
session, user, document.search_space_id, Permission.DOCUMENTS_READ.value
)
version = (
await session.execute(
@ -1331,14 +1343,14 @@ async def restore_document_version(
):
"""Restore a previous version: snapshot current state, then overwrite document content."""
document = (
await session.execute(
select(Document).where(Document.id == document_id)
)
await session.execute(select(Document).where(Document.id == document_id))
).scalar_one_or_none()
if not document:
raise HTTPException(status_code=404, detail="Document not found")
await check_permission(session, user, document.search_space_id, Permission.DOCUMENTS_UPDATE.value)
await check_permission(
session, user, document.search_space_id, Permission.DOCUMENTS_UPDATE.value
)
version = (
await session.execute(
@ -1363,6 +1375,7 @@ async def restore_document_version(
await session.commit()
from app.tasks.celery_tasks.document_reindex_tasks import reindex_document_task
reindex_document_task.delay(document_id, str(user.id))
return {
@ -1430,9 +1443,7 @@ async def folder_index(
root_folder_id = request.root_folder_id
if root_folder_id:
existing = (
await session.execute(
select(Folder).where(Folder.id == root_folder_id)
)
await session.execute(select(Folder).where(Folder.id == root_folder_id))
).scalar_one_or_none()
if not existing:
root_folder_id = None
@ -1492,7 +1503,9 @@ async def folder_index_files(
)
if not request.target_file_paths:
raise HTTPException(status_code=400, detail="target_file_paths must not be empty")
raise HTTPException(
status_code=400, detail="target_file_paths must not be empty"
)
await check_permission(
session,
@ -1507,11 +1520,11 @@ async def folder_index_files(
for fp in request.target_file_paths:
try:
Path(fp).relative_to(request.folder_path)
except ValueError:
except ValueError as err:
raise HTTPException(
status_code=400,
detail=f"target_file_path {fp} must be inside folder_path",
)
) from err
from app.tasks.celery_tasks.document_tasks import index_local_folder_task
@ -1530,5 +1543,3 @@ async def folder_index_files(
"status": "processing",
"file_count": len(request.target_file_paths),
}

6
surfsense_backend/app/routes/editor_routes.py
View file

@ -129,7 +129,11 @@ async def get_editor_content(
if not chunk_contents:
doc_status = document.status or {}
state = doc_status.get("state", "ready") if isinstance(doc_status, dict) else "ready"
state = (
doc_status.get("state", "ready")
if isinstance(doc_status, dict)
else "ready"
)
if state in ("pending", "processing"):
raise HTTPException(
status_code=409,

1
surfsense_backend/app/routes/search_source_connectors_routes.py
View file

@ -20,7 +20,6 @@ Non-OAuth connectors (BookStack, GitHub, etc.) are limited to one per search spa
import asyncio
import logging
import os
from contextlib import suppress
from datetime import UTC, datetime, timedelta
from typing import Any

7
surfsense_backend/app/schemas/folders.py
View file

@ -1,9 +1,8 @@
"""Pydantic schemas for folder CRUD, move, and reorder operations."""
from datetime import datetime
from uuid import UUID
from typing import Any
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
@ -36,7 +35,9 @@ class FolderRead(BaseModel):
created_by_id: UUID | None
created_at: datetime
updated_at: datetime
metadata: dict[str, Any] | None = Field(default=None, validation_alias="folder_metadata")
metadata: dict[str, Any] | None = Field(
default=None, validation_alias="folder_metadata"
)
model_config = ConfigDict(from_attributes=True)

15
surfsense_backend/app/tasks/celery_tasks/document_tasks.py
View file

@ -1,6 +1,7 @@
"""Celery tasks for document processing."""
import asyncio
import contextlib
import logging
import os
from uuid import UUID
@ -1337,9 +1338,7 @@ async def _index_local_folder_async(
)
notification_id = notification.id
_start_heartbeat(notification_id)
heartbeat_task = asyncio.create_task(
_run_heartbeat_loop(notification_id)
)
heartbeat_task = asyncio.create_task(_run_heartbeat_loop(notification_id))
except Exception:
logger.warning(
"Failed to create notification for local folder indexing",
@ -1349,18 +1348,16 @@ async def _index_local_folder_async(
async def _heartbeat_progress(completed_count: int) -> None:
"""Refresh heartbeat and optionally update notification progress."""
if notification:
try:
with contextlib.suppress(Exception):
await NotificationService.document_processing.notify_processing_progress(
session=session,
notification=notification,
stage="indexing",
stage_message=f"Syncing files ({completed_count}/{file_count or '?'})",
)
except Exception:
pass
try:
indexed, skipped_or_failed, _rfid, err = await index_local_folder(
_indexed, _skipped_or_failed, _rfid, err = await index_local_folder(
session=session,
search_space_id=search_space_id,
user_id=user_id,
@ -1371,7 +1368,9 @@ async def _index_local_folder_async(
root_folder_id=root_folder_id,
enable_summary=enable_summary,
target_file_paths=target_file_paths,
on_heartbeat_callback=_heartbeat_progress if (is_batch or is_full_scan) else None,
on_heartbeat_callback=_heartbeat_progress
if (is_batch or is_full_scan)
else None,
)
if notification:

215
surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py
View file

@ -43,30 +43,110 @@ from .base import (
logger,
)
PLAINTEXT_EXTENSIONS = frozenset({
".md", ".markdown", ".txt", ".text", ".csv", ".tsv",
".json", ".jsonl", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf",
".xml", ".html", ".htm", ".css", ".scss", ".less", ".sass",
".py", ".pyw", ".pyi", ".pyx",
".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs",
".java", ".kt", ".kts", ".scala", ".groovy",
".c", ".h", ".cpp", ".cxx", ".cc", ".hpp", ".hxx",
".cs", ".fs", ".fsx",
".go", ".rs", ".rb", ".php", ".pl", ".pm", ".lua",
".swift", ".m", ".mm",
".r", ".R", ".jl",
".sh", ".bash", ".zsh", ".fish", ".bat", ".cmd", ".ps1",
".sql", ".graphql", ".gql",
".env", ".gitignore", ".dockerignore", ".editorconfig",
".makefile", ".cmake",
".log", ".rst", ".tex", ".bib", ".org", ".adoc", ".asciidoc",
".vue", ".svelte", ".astro",
".tf", ".hcl", ".proto",
})
PLAINTEXT_EXTENSIONS = frozenset(
{
".md",
".markdown",
".txt",
".text",
".csv",
".tsv",
".json",
".jsonl",
".yaml",
".yml",
".toml",
".ini",
".cfg",
".conf",
".xml",
".html",
".htm",
".css",
".scss",
".less",
".sass",
".py",
".pyw",
".pyi",
".pyx",
".js",
".jsx",
".ts",
".tsx",
".mjs",
".cjs",
".java",
".kt",
".kts",
".scala",
".groovy",
".c",
".h",
".cpp",
".cxx",
".cc",
".hpp",
".hxx",
".cs",
".fs",
".fsx",
".go",
".rs",
".rb",
".php",
".pl",
".pm",
".lua",
".swift",
".m",
".mm",
".r",
".R",
".jl",
".sh",
".bash",
".zsh",
".fish",
".bat",
".cmd",
".ps1",
".sql",
".graphql",
".gql",
".env",
".gitignore",
".dockerignore",
".editorconfig",
".makefile",
".cmake",
".log",
".rst",
".tex",
".bib",
".org",
".adoc",
".asciidoc",
".vue",
".svelte",
".astro",
".tf",
".hcl",
".proto",
}
)
AUDIO_EXTENSIONS = frozenset({
".mp3", ".mp4", ".mpeg", ".mpga", ".m4a", ".wav", ".webm",
})
AUDIO_EXTENSIONS = frozenset(
{
".mp3",
".mp4",
".mpeg",
".mpga",
".m4a",
".wav",
".webm",
}
)
def _is_plaintext_file(filename: str) -> bool:
@ -81,6 +161,7 @@ def _needs_etl(filename: str) -> bool:
"""File is not plaintext and not audio — requires ETL service to parse."""
return not _is_plaintext_file(filename) and not _is_audio_file(filename)
HeartbeatCallbackType = Callable[[int], Awaitable[None]]
DEFAULT_EXCLUDE_PATTERNS = [
@ -121,9 +202,7 @@ def scan_folder(
for dirpath, dirnames, filenames in os.walk(root):
rel_dir = Path(dirpath).relative_to(root)
dirnames[:] = [
d for d in dirnames if d not in exclude_patterns
]
dirnames[:] = [d for d in dirnames if d not in exclude_patterns]
if any(part in exclude_patterns for part in rel_dir.parts):
continue
@ -134,9 +213,11 @@ def scan_folder(
full = Path(dirpath) / fname
if file_extensions is not None:
if full.suffix.lower() not in file_extensions:
continue
if (
file_extensions is not None
and full.suffix.lower() not in file_extensions
):
continue
try:
stat = full.stat()
@ -209,11 +290,14 @@ def _content_hash(content: str, search_space_id: int) -> str:
pipeline so that dedup checks are consistent.
"""
import hashlib
return hashlib.sha256(f"{search_space_id}:{content}".encode("utf-8")).hexdigest()
return hashlib.sha256(f"{search_space_id}:{content}".encode()).hexdigest()
async def _compute_file_content_hash(
file_path: str, filename: str, search_space_id: int,
file_path: str,
filename: str,
search_space_id: int,
) -> tuple[str, str]:
"""Read a file (via ETL if needed) and compute its content hash.
@ -257,9 +341,7 @@ async def _mirror_folder_structure(
if root_folder_id:
existing = (
await session.execute(
select(Folder).where(Folder.id == root_folder_id)
)
await session.execute(select(Folder).where(Folder.id == root_folder_id))
).scalar_one_or_none()
if existing:
mapping[""] = existing.id
@ -412,13 +494,17 @@ async def _cleanup_empty_folders(
id_to_rel: dict[int, str] = {fid: rel for rel, fid in folder_mapping.items() if rel}
all_folders = (
await session.execute(
select(Folder).where(
Folder.search_space_id == search_space_id,
Folder.id != root_folder_id,
(
await session.execute(
select(Folder).where(
Folder.search_space_id == search_space_id,
Folder.id != root_folder_id,
)
)
)
).scalars().all()
.scalars()
.all()
)
candidates: list[Folder] = []
for folder in all_folders:
@ -520,7 +606,9 @@ async def index_local_folder(
metadata={
"folder_path": folder_path,
"user_id": str(user_id),
"target_file_paths_count": len(target_file_paths) if target_file_paths else None,
"target_file_paths_count": len(target_file_paths)
if target_file_paths
else None,
},
)
@ -532,7 +620,12 @@ async def index_local_folder(
"Folder not found",
{},
)
return 0, 0, root_folder_id, f"Folder path missing or does not exist: {folder_path}"
return (
0,
0,
root_folder_id,
f"Folder path missing or does not exist: {folder_path}",
)
if exclude_patterns is None:
exclude_patterns = DEFAULT_EXCLUDE_PATTERNS
@ -639,7 +732,9 @@ async def index_local_folder(
)
if existing_document:
stored_mtime = (existing_document.document_metadata or {}).get("mtime")
stored_mtime = (existing_document.document_metadata or {}).get(
"mtime"
)
current_mtime = file_info["modified_at"].timestamp()
if stored_mtime and abs(current_mtime - stored_mtime) < 1.0:
@ -709,23 +804,31 @@ async def index_local_folder(
# ================================================================
all_root_folder_ids = set(folder_mapping.values())
all_db_folders = (
await session.execute(
select(Folder.id).where(
Folder.search_space_id == search_space_id,
(
await session.execute(
select(Folder.id).where(
Folder.search_space_id == search_space_id,
)
)
)
).scalars().all()
.scalars()
.all()
)
all_root_folder_ids.update(all_db_folders)
all_folder_docs = (
await session.execute(
select(Document).where(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == search_space_id,
Document.folder_id.in_(list(all_root_folder_ids)),
(
await session.execute(
select(Document).where(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == search_space_id,
Document.folder_id.in_(list(all_root_folder_ids)),
)
)
)
).scalars().all()
.scalars()
.all()
)
for doc in all_folder_docs:
if doc.unique_identifier_hash not in seen_unique_hashes:
@ -742,9 +845,7 @@ async def index_local_folder(
)
pipeline = IndexingPipelineService(session)
doc_map = {
compute_unique_identifier_hash(cd): cd for cd in connector_docs
}
doc_map = {compute_unique_identifier_hash(cd): cd for cd in connector_docs}
documents = await pipeline.prepare_for_indexing(connector_docs)
# Assign folder_id immediately so docs appear in the correct
@ -1033,7 +1134,9 @@ async def _index_single_file(
db_doc.document_metadata = doc_meta
await session.commit()
indexed = 1 if DocumentStatus.is_state(db_doc.status, DocumentStatus.READY) else 0
indexed = (
1 if DocumentStatus.is_state(db_doc.status, DocumentStatus.READY) else 0
)
failed_msg = None if indexed else "Indexing failed"
if indexed:

6
surfsense_backend/app/utils/document_versioning.py
View file

@ -83,9 +83,9 @@ async def create_version_snapshot(
# Cleanup: cap at MAX_VERSIONS_PER_DOCUMENT
count = (
await session.execute(
select(func.count()).select_from(DocumentVersion).where(
DocumentVersion.document_id == document.id
)
select(func.count())
.select_from(DocumentVersion)
.where(DocumentVersion.document_id == document.id)
)
).scalar_one()