apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-20 23:21:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: ran linting

Browse source
This commit is contained in:
Anish Sarkar 2026-04-03 13:14:40 +05:30
parent 6ace8850bb
commit 746c730b2e
31 changed files with 801 additions and 660 deletions
Download patch file Download diff file Expand all files Collapse all files

63
surfsense_backend/app/routes/documents_routes.py
View file

@ -977,15 +977,19 @@ async def get_watched_folders(
) )
folders = ( folders = (
await session.execute( (
select(Folder).where( await session.execute(
Folder.search_space_id == search_space_id, select(Folder).where(
Folder.parent_id.is_(None), Folder.search_space_id == search_space_id,
Folder.folder_metadata.isnot(None), Folder.parent_id.is_(None),
Folder.folder_metadata["watched"].astext == "true", Folder.folder_metadata.isnot(None),
Folder.folder_metadata["watched"].astext == "true",
)
) )
) )
).scalars().all() .scalars()
.all()
)
return folders return folders
@ -1265,15 +1269,21 @@ async def list_document_versions(
if not document: if not document:
raise HTTPException(status_code=404, detail="Document not found") raise HTTPException(status_code=404, detail="Document not found")
await check_permission(session, user, document.search_space_id, Permission.DOCUMENTS_READ.value) await check_permission(
session, user, document.search_space_id, Permission.DOCUMENTS_READ.value
)
versions = ( versions = (
await session.execute( (
select(DocumentVersion) await session.execute(
.where(DocumentVersion.document_id == document_id) select(DocumentVersion)
.order_by(DocumentVersion.version_number.desc()) .where(DocumentVersion.document_id == document_id)
.order_by(DocumentVersion.version_number.desc())
)
) )
).scalars().all() .scalars()
.all()
)
return [ return [
{ {
@ -1300,7 +1310,9 @@ async def get_document_version(
if not document: if not document:
raise HTTPException(status_code=404, detail="Document not found") raise HTTPException(status_code=404, detail="Document not found")
await check_permission(session, user, document.search_space_id, Permission.DOCUMENTS_READ.value) await check_permission(
session, user, document.search_space_id, Permission.DOCUMENTS_READ.value
)
version = ( version = (
await session.execute( await session.execute(
@ -1331,14 +1343,14 @@ async def restore_document_version(
): ):
"""Restore a previous version: snapshot current state, then overwrite document content.""" """Restore a previous version: snapshot current state, then overwrite document content."""
document = ( document = (
await session.execute( await session.execute(select(Document).where(Document.id == document_id))
select(Document).where(Document.id == document_id)
)
).scalar_one_or_none() ).scalar_one_or_none()
if not document: if not document:
raise HTTPException(status_code=404, detail="Document not found") raise HTTPException(status_code=404, detail="Document not found")
await check_permission(session, user, document.search_space_id, Permission.DOCUMENTS_UPDATE.value) await check_permission(
session, user, document.search_space_id, Permission.DOCUMENTS_UPDATE.value
)
version = ( version = (
await session.execute( await session.execute(
@ -1363,6 +1375,7 @@ async def restore_document_version(
await session.commit() await session.commit()
from app.tasks.celery_tasks.document_reindex_tasks import reindex_document_task from app.tasks.celery_tasks.document_reindex_tasks import reindex_document_task
reindex_document_task.delay(document_id, str(user.id)) reindex_document_task.delay(document_id, str(user.id))
return { return {
@ -1430,9 +1443,7 @@ async def folder_index(
root_folder_id = request.root_folder_id root_folder_id = request.root_folder_id
if root_folder_id: if root_folder_id:
existing = ( existing = (
await session.execute( await session.execute(select(Folder).where(Folder.id == root_folder_id))
select(Folder).where(Folder.id == root_folder_id)
)
).scalar_one_or_none() ).scalar_one_or_none()
if not existing: if not existing:
root_folder_id = None root_folder_id = None
@ -1492,7 +1503,9 @@ async def folder_index_files(
) )
if not request.target_file_paths: if not request.target_file_paths:
raise HTTPException(status_code=400, detail="target_file_paths must not be empty") raise HTTPException(
status_code=400, detail="target_file_paths must not be empty"
)
await check_permission( await check_permission(
session, session,
@ -1507,11 +1520,11 @@ async def folder_index_files(
for fp in request.target_file_paths: for fp in request.target_file_paths:
try: try:
Path(fp).relative_to(request.folder_path) Path(fp).relative_to(request.folder_path)
except ValueError: except ValueError as err:
raise HTTPException( raise HTTPException(
status_code=400, status_code=400,
detail=f"target_file_path {fp} must be inside folder_path", detail=f"target_file_path {fp} must be inside folder_path",
) ) from err
from app.tasks.celery_tasks.document_tasks import index_local_folder_task from app.tasks.celery_tasks.document_tasks import index_local_folder_task
@ -1530,5 +1543,3 @@ async def folder_index_files(
"status": "processing", "status": "processing",
"file_count": len(request.target_file_paths), "file_count": len(request.target_file_paths),
} }

6
surfsense_backend/app/routes/editor_routes.py
View file

@ -129,7 +129,11 @@ async def get_editor_content(
if not chunk_contents: if not chunk_contents:
doc_status = document.status or {} doc_status = document.status or {}
state = doc_status.get("state", "ready") if isinstance(doc_status, dict) else "ready" state = (
doc_status.get("state", "ready")
if isinstance(doc_status, dict)
else "ready"
)
if state in ("pending", "processing"): if state in ("pending", "processing"):
raise HTTPException( raise HTTPException(
status_code=409, status_code=409,

1
surfsense_backend/app/routes/search_source_connectors_routes.py
View file

@ -20,7 +20,6 @@ Non-OAuth connectors (BookStack, GitHub, etc.) are limited to one per search spa
import asyncio import asyncio
import logging import logging
import os
from contextlib import suppress from contextlib import suppress
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from typing import Any from typing import Any

7
surfsense_backend/app/schemas/folders.py
View file

@ -1,9 +1,8 @@
"""Pydantic schemas for folder CRUD, move, and reorder operations.""" """Pydantic schemas for folder CRUD, move, and reorder operations."""
from datetime import datetime from datetime import datetime
from uuid import UUID
from typing import Any from typing import Any
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field from pydantic import BaseModel, ConfigDict, Field
@ -36,7 +35,9 @@ class FolderRead(BaseModel):
created_by_id: UUID | None created_by_id: UUID | None
created_at: datetime created_at: datetime
updated_at: datetime updated_at: datetime
metadata: dict[str, Any] | None = Field(default=None, validation_alias="folder_metadata") metadata: dict[str, Any] | None = Field(
default=None, validation_alias="folder_metadata"
)
model_config = ConfigDict(from_attributes=True) model_config = ConfigDict(from_attributes=True)

15
surfsense_backend/app/tasks/celery_tasks/document_tasks.py
View file

@ -1,6 +1,7 @@
"""Celery tasks for document processing.""" """Celery tasks for document processing."""
import asyncio import asyncio
import contextlib
import logging import logging
import os import os
from uuid import UUID from uuid import UUID
@ -1337,9 +1338,7 @@ async def _index_local_folder_async(
) )
notification_id = notification.id notification_id = notification.id
_start_heartbeat(notification_id) _start_heartbeat(notification_id)
heartbeat_task = asyncio.create_task( heartbeat_task = asyncio.create_task(_run_heartbeat_loop(notification_id))
_run_heartbeat_loop(notification_id)
)
except Exception: except Exception:
logger.warning( logger.warning(
"Failed to create notification for local folder indexing", "Failed to create notification for local folder indexing",
@ -1349,18 +1348,16 @@ async def _index_local_folder_async(
async def _heartbeat_progress(completed_count: int) -> None: async def _heartbeat_progress(completed_count: int) -> None:
"""Refresh heartbeat and optionally update notification progress.""" """Refresh heartbeat and optionally update notification progress."""
if notification: if notification:
try: with contextlib.suppress(Exception):
await NotificationService.document_processing.notify_processing_progress( await NotificationService.document_processing.notify_processing_progress(
session=session, session=session,
notification=notification, notification=notification,
stage="indexing", stage="indexing",
stage_message=f"Syncing files ({completed_count}/{file_count or '?'})", stage_message=f"Syncing files ({completed_count}/{file_count or '?'})",
) )
except Exception:
pass
try: try:
indexed, skipped_or_failed, _rfid, err = await index_local_folder( _indexed, _skipped_or_failed, _rfid, err = await index_local_folder(
session=session, session=session,
search_space_id=search_space_id, search_space_id=search_space_id,
user_id=user_id, user_id=user_id,
@ -1371,7 +1368,9 @@ async def _index_local_folder_async(
root_folder_id=root_folder_id, root_folder_id=root_folder_id,
enable_summary=enable_summary, enable_summary=enable_summary,
target_file_paths=target_file_paths, target_file_paths=target_file_paths,
on_heartbeat_callback=_heartbeat_progress if (is_batch or is_full_scan) else None, on_heartbeat_callback=_heartbeat_progress
if (is_batch or is_full_scan)
else None,
) )
if notification: if notification:

215
surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py
View file

@ -43,30 +43,110 @@ from .base import (
logger, logger,
) )
PLAINTEXT_EXTENSIONS = frozenset({ PLAINTEXT_EXTENSIONS = frozenset(
".md", ".markdown", ".txt", ".text", ".csv", ".tsv", {
".json", ".jsonl", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf", ".md",
".xml", ".html", ".htm", ".css", ".scss", ".less", ".sass", ".markdown",
".py", ".pyw", ".pyi", ".pyx", ".txt",
".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".text",
".java", ".kt", ".kts", ".scala", ".groovy", ".csv",
".c", ".h", ".cpp", ".cxx", ".cc", ".hpp", ".hxx", ".tsv",
".cs", ".fs", ".fsx", ".json",
".go", ".rs", ".rb", ".php", ".pl", ".pm", ".lua", ".jsonl",
".swift", ".m", ".mm", ".yaml",
".r", ".R", ".jl", ".yml",
".sh", ".bash", ".zsh", ".fish", ".bat", ".cmd", ".ps1", ".toml",
".sql", ".graphql", ".gql", ".ini",
".env", ".gitignore", ".dockerignore", ".editorconfig", ".cfg",
".makefile", ".cmake", ".conf",
".log", ".rst", ".tex", ".bib", ".org", ".adoc", ".asciidoc", ".xml",
".vue", ".svelte", ".astro", ".html",
".tf", ".hcl", ".proto", ".htm",
}) ".css",
".scss",
".less",
".sass",
".py",
".pyw",
".pyi",
".pyx",
".js",
".jsx",
".ts",
".tsx",
".mjs",
".cjs",
".java",
".kt",
".kts",
".scala",
".groovy",
".c",
".h",
".cpp",
".cxx",
".cc",
".hpp",
".hxx",
".cs",
".fs",
".fsx",
".go",
".rs",
".rb",
".php",
".pl",
".pm",
".lua",
".swift",
".m",
".mm",
".r",
".R",
".jl",
".sh",
".bash",
".zsh",
".fish",
".bat",
".cmd",
".ps1",
".sql",
".graphql",
".gql",
".env",
".gitignore",
".dockerignore",
".editorconfig",
".makefile",
".cmake",
".log",
".rst",
".tex",
".bib",
".org",
".adoc",
".asciidoc",
".vue",
".svelte",
".astro",
".tf",
".hcl",
".proto",
}
)
AUDIO_EXTENSIONS = frozenset({ AUDIO_EXTENSIONS = frozenset(
".mp3", ".mp4", ".mpeg", ".mpga", ".m4a", ".wav", ".webm", {
}) ".mp3",
".mp4",
".mpeg",
".mpga",
".m4a",
".wav",
".webm",
}
)
def _is_plaintext_file(filename: str) -> bool: def _is_plaintext_file(filename: str) -> bool:
@ -81,6 +161,7 @@ def _needs_etl(filename: str) -> bool:
"""File is not plaintext and not audio — requires ETL service to parse.""" """File is not plaintext and not audio — requires ETL service to parse."""
return not _is_plaintext_file(filename) and not _is_audio_file(filename) return not _is_plaintext_file(filename) and not _is_audio_file(filename)
HeartbeatCallbackType = Callable[[int], Awaitable[None]] HeartbeatCallbackType = Callable[[int], Awaitable[None]]
DEFAULT_EXCLUDE_PATTERNS = [ DEFAULT_EXCLUDE_PATTERNS = [
@ -121,9 +202,7 @@ def scan_folder(
for dirpath, dirnames, filenames in os.walk(root): for dirpath, dirnames, filenames in os.walk(root):
rel_dir = Path(dirpath).relative_to(root) rel_dir = Path(dirpath).relative_to(root)
dirnames[:] = [ dirnames[:] = [d for d in dirnames if d not in exclude_patterns]
d for d in dirnames if d not in exclude_patterns
]
if any(part in exclude_patterns for part in rel_dir.parts): if any(part in exclude_patterns for part in rel_dir.parts):
continue continue
@ -134,9 +213,11 @@ def scan_folder(
full = Path(dirpath) / fname full = Path(dirpath) / fname
if file_extensions is not None: if (
if full.suffix.lower() not in file_extensions: file_extensions is not None
continue and full.suffix.lower() not in file_extensions
):
continue
try: try:
stat = full.stat() stat = full.stat()
@ -209,11 +290,14 @@ def _content_hash(content: str, search_space_id: int) -> str:
pipeline so that dedup checks are consistent. pipeline so that dedup checks are consistent.
""" """
import hashlib import hashlib
return hashlib.sha256(f"{search_space_id}:{content}".encode("utf-8")).hexdigest()
return hashlib.sha256(f"{search_space_id}:{content}".encode()).hexdigest()
async def _compute_file_content_hash( async def _compute_file_content_hash(
file_path: str, filename: str, search_space_id: int, file_path: str,
filename: str,
search_space_id: int,
) -> tuple[str, str]: ) -> tuple[str, str]:
"""Read a file (via ETL if needed) and compute its content hash. """Read a file (via ETL if needed) and compute its content hash.
@ -257,9 +341,7 @@ async def _mirror_folder_structure(
if root_folder_id: if root_folder_id:
existing = ( existing = (
await session.execute( await session.execute(select(Folder).where(Folder.id == root_folder_id))
select(Folder).where(Folder.id == root_folder_id)
)
).scalar_one_or_none() ).scalar_one_or_none()
if existing: if existing:
mapping[""] = existing.id mapping[""] = existing.id
@ -412,13 +494,17 @@ async def _cleanup_empty_folders(
id_to_rel: dict[int, str] = {fid: rel for rel, fid in folder_mapping.items() if rel} id_to_rel: dict[int, str] = {fid: rel for rel, fid in folder_mapping.items() if rel}
all_folders = ( all_folders = (
await session.execute( (
select(Folder).where( await session.execute(
Folder.search_space_id == search_space_id, select(Folder).where(
Folder.id != root_folder_id, Folder.search_space_id == search_space_id,
Folder.id != root_folder_id,
)
) )
) )
).scalars().all() .scalars()
.all()
)
candidates: list[Folder] = [] candidates: list[Folder] = []
for folder in all_folders: for folder in all_folders:
@ -520,7 +606,9 @@ async def index_local_folder(
metadata={ metadata={
"folder_path": folder_path, "folder_path": folder_path,
"user_id": str(user_id), "user_id": str(user_id),
"target_file_paths_count": len(target_file_paths) if target_file_paths else None, "target_file_paths_count": len(target_file_paths)
if target_file_paths
else None,
}, },
) )
@ -532,7 +620,12 @@ async def index_local_folder(
"Folder not found", "Folder not found",
{}, {},
) )
return 0, 0, root_folder_id, f"Folder path missing or does not exist: {folder_path}" return (
0,
0,
root_folder_id,
f"Folder path missing or does not exist: {folder_path}",
)
if exclude_patterns is None: if exclude_patterns is None:
exclude_patterns = DEFAULT_EXCLUDE_PATTERNS exclude_patterns = DEFAULT_EXCLUDE_PATTERNS
@ -639,7 +732,9 @@ async def index_local_folder(
) )
if existing_document: if existing_document:
stored_mtime = (existing_document.document_metadata or {}).get("mtime") stored_mtime = (existing_document.document_metadata or {}).get(
"mtime"
)
current_mtime = file_info["modified_at"].timestamp() current_mtime = file_info["modified_at"].timestamp()
if stored_mtime and abs(current_mtime - stored_mtime) < 1.0: if stored_mtime and abs(current_mtime - stored_mtime) < 1.0:
@ -709,23 +804,31 @@ async def index_local_folder(
# ================================================================ # ================================================================
all_root_folder_ids = set(folder_mapping.values()) all_root_folder_ids = set(folder_mapping.values())
all_db_folders = ( all_db_folders = (
await session.execute( (
select(Folder.id).where( await session.execute(
Folder.search_space_id == search_space_id, select(Folder.id).where(
Folder.search_space_id == search_space_id,
)
) )
) )
).scalars().all() .scalars()
.all()
)
all_root_folder_ids.update(all_db_folders) all_root_folder_ids.update(all_db_folders)
all_folder_docs = ( all_folder_docs = (
await session.execute( (
select(Document).where( await session.execute(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE, select(Document).where(
Document.search_space_id == search_space_id, Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.folder_id.in_(list(all_root_folder_ids)), Document.search_space_id == search_space_id,
Document.folder_id.in_(list(all_root_folder_ids)),
)
) )
) )
).scalars().all() .scalars()
.all()
)
for doc in all_folder_docs: for doc in all_folder_docs:
if doc.unique_identifier_hash not in seen_unique_hashes: if doc.unique_identifier_hash not in seen_unique_hashes:
@ -742,9 +845,7 @@ async def index_local_folder(
) )
pipeline = IndexingPipelineService(session) pipeline = IndexingPipelineService(session)
doc_map = { doc_map = {compute_unique_identifier_hash(cd): cd for cd in connector_docs}
compute_unique_identifier_hash(cd): cd for cd in connector_docs
}
documents = await pipeline.prepare_for_indexing(connector_docs) documents = await pipeline.prepare_for_indexing(connector_docs)
# Assign folder_id immediately so docs appear in the correct # Assign folder_id immediately so docs appear in the correct
@ -1033,7 +1134,9 @@ async def _index_single_file(
db_doc.document_metadata = doc_meta db_doc.document_metadata = doc_meta
await session.commit() await session.commit()
indexed = 1 if DocumentStatus.is_state(db_doc.status, DocumentStatus.READY) else 0 indexed = (
1 if DocumentStatus.is_state(db_doc.status, DocumentStatus.READY) else 0
)
failed_msg = None if indexed else "Indexing failed" failed_msg = None if indexed else "Indexing failed"
if indexed: if indexed:

6
surfsense_backend/app/utils/document_versioning.py
View file

@ -83,9 +83,9 @@ async def create_version_snapshot(
# Cleanup: cap at MAX_VERSIONS_PER_DOCUMENT # Cleanup: cap at MAX_VERSIONS_PER_DOCUMENT
count = ( count = (
await session.execute( await session.execute(
select(func.count()).select_from(DocumentVersion).where( select(func.count())
DocumentVersion.document_id == document.id .select_from(DocumentVersion)
) .where(DocumentVersion.document_id == document.id)
) )
).scalar_one() ).scalar_one()

2
surfsense_backend/tests/integration/conftest.py
View file

@ -166,5 +166,3 @@ def make_connector_document(db_connector, db_user):
return ConnectorDocument(**defaults) return ConnectorDocument(**defaults)
return _make return _make

198
surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py
View file

@ -21,7 +21,9 @@ from app.db import (
pytestmark = pytest.mark.integration pytestmark = pytest.mark.integration
UNIFIED_FIXTURES = ( UNIFIED_FIXTURES = (
"patched_summarize", "patched_embed_texts", "patched_chunk_text", "patched_summarize",
"patched_embed_texts",
"patched_chunk_text",
) )
@ -37,6 +39,7 @@ class _FakeSessionMaker:
@asynccontextmanager @asynccontextmanager
async def _ctx(): async def _ctx():
yield self._session yield self._session
return _ctx() return _ctx()
@ -59,7 +62,6 @@ def patched_batch_sessions(monkeypatch, db_session):
class TestFullIndexer: class TestFullIndexer:
@pytest.mark.usefixtures(*UNIFIED_FIXTURES) @pytest.mark.usefixtures(*UNIFIED_FIXTURES)
async def test_i1_new_file_indexed( async def test_i1_new_file_indexed(
self, self,
@ -73,7 +75,7 @@ class TestFullIndexer:
(tmp_path / "note.md").write_text("# Hello World\n\nContent here.") (tmp_path / "note.md").write_text("# Hello World\n\nContent here.")
count, skipped, root_folder_id, err = await index_local_folder( count, _skipped, _root_folder_id, err = await index_local_folder(
session=db_session, session=db_session,
search_space_id=db_search_space.id, search_space_id=db_search_space.id,
user_id=str(db_user.id), user_id=str(db_user.id),
@ -85,13 +87,17 @@ class TestFullIndexer:
assert count == 1 assert count == 1
docs = ( docs = (
await db_session.execute( (
select(Document).where( await db_session.execute(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE, select(Document).where(
Document.search_space_id == db_search_space.id, Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == db_search_space.id,
)
) )
) )
).scalars().all() .scalars()
.all()
)
assert len(docs) == 1 assert len(docs) == 1
assert docs[0].document_type == DocumentType.LOCAL_FOLDER_FILE assert docs[0].document_type == DocumentType.LOCAL_FOLDER_FILE
assert DocumentStatus.is_state(docs[0].status, DocumentStatus.READY) assert DocumentStatus.is_state(docs[0].status, DocumentStatus.READY)
@ -130,7 +136,9 @@ class TestFullIndexer:
total = ( total = (
await db_session.execute( await db_session.execute(
select(func.count()).select_from(Document).where( select(func.count())
.select_from(Document)
.where(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE, Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == db_search_space.id, Document.search_space_id == db_search_space.id,
) )
@ -174,13 +182,19 @@ class TestFullIndexer:
assert count == 1 assert count == 1
versions = ( versions = (
await db_session.execute( (
select(DocumentVersion).join(Document).where( await db_session.execute(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE, select(DocumentVersion)
Document.search_space_id == db_search_space.id, .join(Document)
.where(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == db_search_space.id,
)
) )
) )
).scalars().all() .scalars()
.all()
)
assert len(versions) >= 1 assert len(versions) >= 1
@pytest.mark.usefixtures(*UNIFIED_FIXTURES) @pytest.mark.usefixtures(*UNIFIED_FIXTURES)
@ -207,7 +221,9 @@ class TestFullIndexer:
docs_before = ( docs_before = (
await db_session.execute( await db_session.execute(
select(func.count()).select_from(Document).where( select(func.count())
.select_from(Document)
.where(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE, Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == db_search_space.id, Document.search_space_id == db_search_space.id,
) )
@ -228,7 +244,9 @@ class TestFullIndexer:
docs_after = ( docs_after = (
await db_session.execute( await db_session.execute(
select(func.count()).select_from(Document).where( select(func.count())
.select_from(Document)
.where(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE, Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == db_search_space.id, Document.search_space_id == db_search_space.id,
) )
@ -262,13 +280,17 @@ class TestFullIndexer:
assert count == 1 assert count == 1
docs = ( docs = (
await db_session.execute( (
select(Document).where( await db_session.execute(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE, select(Document).where(
Document.search_space_id == db_search_space.id, Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == db_search_space.id,
)
) )
) )
).scalars().all() .scalars()
.all()
)
assert len(docs) == 1 assert len(docs) == 1
assert docs[0].title == "b.md" assert docs[0].title == "b.md"
@ -279,7 +301,6 @@ class TestFullIndexer:
class TestFolderMirroring: class TestFolderMirroring:
@pytest.mark.usefixtures(*UNIFIED_FIXTURES) @pytest.mark.usefixtures(*UNIFIED_FIXTURES)
async def test_f1_root_folder_created( async def test_f1_root_folder_created(
self, self,
@ -335,10 +356,14 @@ class TestFolderMirroring:
) )
folders = ( folders = (
await db_session.execute( (
select(Folder).where(Folder.search_space_id == db_search_space.id) await db_session.execute(
select(Folder).where(Folder.search_space_id == db_search_space.id)
)
) )
).scalars().all() .scalars()
.all()
)
folder_names = {f.name for f in folders} folder_names = {f.name for f in folders}
assert "notes" in folder_names assert "notes" in folder_names
@ -376,10 +401,14 @@ class TestFolderMirroring:
) )
folders_before = ( folders_before = (
await db_session.execute( (
select(Folder).where(Folder.search_space_id == db_search_space.id) await db_session.execute(
select(Folder).where(Folder.search_space_id == db_search_space.id)
)
) )
).scalars().all() .scalars()
.all()
)
ids_before = {f.id for f in folders_before} ids_before = {f.id for f in folders_before}
await index_local_folder( await index_local_folder(
@ -392,10 +421,14 @@ class TestFolderMirroring:
) )
folders_after = ( folders_after = (
await db_session.execute( (
select(Folder).where(Folder.search_space_id == db_search_space.id) await db_session.execute(
select(Folder).where(Folder.search_space_id == db_search_space.id)
)
) )
).scalars().all() .scalars()
.all()
)
ids_after = {f.id for f in folders_after} ids_after = {f.id for f in folders_after}
assert ids_before == ids_after assert ids_before == ids_after
@ -425,21 +458,23 @@ class TestFolderMirroring:
) )
docs = ( docs = (
await db_session.execute( (
select(Document).where( await db_session.execute(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE, select(Document).where(
Document.search_space_id == db_search_space.id, Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == db_search_space.id,
)
) )
) )
).scalars().all() .scalars()
.all()
)
today_doc = next(d for d in docs if d.title == "today.md") today_doc = next(d for d in docs if d.title == "today.md")
root_doc = next(d for d in docs if d.title == "root.md") root_doc = next(d for d in docs if d.title == "root.md")
daily_folder = ( daily_folder = (
await db_session.execute( await db_session.execute(select(Folder).where(Folder.name == "daily"))
select(Folder).where(Folder.name == "daily")
)
).scalar_one() ).scalar_one()
assert today_doc.folder_id == daily_folder.id assert today_doc.folder_id == daily_folder.id
@ -455,9 +490,10 @@ class TestFolderMirroring:
tmp_path: Path, tmp_path: Path,
): ):
"""F5: Deleted dir's empty Folder row is cleaned up on re-sync.""" """F5: Deleted dir's empty Folder row is cleaned up on re-sync."""
from app.tasks.connector_indexers.local_folder_indexer import index_local_folder
import shutil import shutil
from app.tasks.connector_indexers.local_folder_indexer import index_local_folder
daily = tmp_path / "notes" / "daily" daily = tmp_path / "notes" / "daily"
daily.mkdir(parents=True) daily.mkdir(parents=True)
weekly = tmp_path / "notes" / "weekly" weekly = tmp_path / "notes" / "weekly"
@ -474,9 +510,7 @@ class TestFolderMirroring:
) )
weekly_folder = ( weekly_folder = (
await db_session.execute( await db_session.execute(select(Folder).where(Folder.name == "weekly"))
select(Folder).where(Folder.name == "weekly")
)
).scalar_one_or_none() ).scalar_one_or_none()
assert weekly_folder is not None assert weekly_folder is not None
@ -492,16 +526,12 @@ class TestFolderMirroring:
) )
weekly_after = ( weekly_after = (
await db_session.execute( await db_session.execute(select(Folder).where(Folder.name == "weekly"))
select(Folder).where(Folder.name == "weekly")
)
).scalar_one_or_none() ).scalar_one_or_none()
assert weekly_after is None assert weekly_after is None
daily_after = ( daily_after = (
await db_session.execute( await db_session.execute(select(Folder).where(Folder.name == "daily"))
select(Folder).where(Folder.name == "daily")
)
).scalar_one_or_none() ).scalar_one_or_none()
assert daily_after is not None assert daily_after is not None
@ -551,18 +581,14 @@ class TestFolderMirroring:
).scalar_one() ).scalar_one()
daily_folder = ( daily_folder = (
await db_session.execute( await db_session.execute(select(Folder).where(Folder.name == "daily"))
select(Folder).where(Folder.name == "daily")
)
).scalar_one() ).scalar_one()
assert doc.folder_id == daily_folder.id assert doc.folder_id == daily_folder.id
assert daily_folder.parent_id is not None assert daily_folder.parent_id is not None
notes_folder = ( notes_folder = (
await db_session.execute( await db_session.execute(select(Folder).where(Folder.name == "notes"))
select(Folder).where(Folder.name == "notes")
)
).scalar_one() ).scalar_one()
assert daily_folder.parent_id == notes_folder.id assert daily_folder.parent_id == notes_folder.id
assert notes_folder.parent_id == root_folder_id assert notes_folder.parent_id == root_folder_id
@ -592,9 +618,7 @@ class TestFolderMirroring:
) )
eph_folder = ( eph_folder = (
await db_session.execute( await db_session.execute(select(Folder).where(Folder.name == "ephemeral"))
select(Folder).where(Folder.name == "ephemeral")
)
).scalar_one_or_none() ).scalar_one_or_none()
assert eph_folder is not None assert eph_folder is not None
@ -612,16 +636,12 @@ class TestFolderMirroring:
) )
eph_after = ( eph_after = (
await db_session.execute( await db_session.execute(select(Folder).where(Folder.name == "ephemeral"))
select(Folder).where(Folder.name == "ephemeral")
)
).scalar_one_or_none() ).scalar_one_or_none()
assert eph_after is None assert eph_after is None
notes_after = ( notes_after = (
await db_session.execute( await db_session.execute(select(Folder).where(Folder.name == "notes"))
select(Folder).where(Folder.name == "notes")
)
).scalar_one_or_none() ).scalar_one_or_none()
assert notes_after is None assert notes_after is None
@ -632,7 +652,6 @@ class TestFolderMirroring:
class TestBatchMode: class TestBatchMode:
@pytest.mark.usefixtures(*UNIFIED_FIXTURES) @pytest.mark.usefixtures(*UNIFIED_FIXTURES)
async def test_b1_batch_indexes_multiple_files( async def test_b1_batch_indexes_multiple_files(
self, self,
@ -649,7 +668,7 @@ class TestBatchMode:
(tmp_path / "b.md").write_text("File B content") (tmp_path / "b.md").write_text("File B content")
(tmp_path / "c.md").write_text("File C content") (tmp_path / "c.md").write_text("File C content")
count, failed, root_folder_id, err = await index_local_folder( count, failed, _root_folder_id, err = await index_local_folder(
session=db_session, session=db_session,
search_space_id=db_search_space.id, search_space_id=db_search_space.id,
user_id=str(db_user.id), user_id=str(db_user.id),
@ -667,13 +686,17 @@ class TestBatchMode:
assert err is None assert err is None
docs = ( docs = (
await db_session.execute( (
select(Document).where( await db_session.execute(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE, select(Document).where(
Document.search_space_id == db_search_space.id, Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == db_search_space.id,
)
) )
) )
).scalars().all() .scalars()
.all()
)
assert len(docs) == 3 assert len(docs) == 3
assert {d.title for d in docs} == {"a.md", "b.md", "c.md"} assert {d.title for d in docs} == {"a.md", "b.md", "c.md"}
assert all( assert all(
@ -714,13 +737,17 @@ class TestBatchMode:
assert err is not None assert err is not None
docs = ( docs = (
await db_session.execute( (
select(Document).where( await db_session.execute(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE, select(Document).where(
Document.search_space_id == db_search_space.id, Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == db_search_space.id,
)
) )
) )
).scalars().all() .scalars()
.all()
)
assert len(docs) == 2 assert len(docs) == 2
assert {d.title for d in docs} == {"good1.md", "good2.md"} assert {d.title for d in docs} == {"good1.md", "good2.md"}
@ -731,7 +758,6 @@ class TestBatchMode:
class TestPipelineIntegration: class TestPipelineIntegration:
@pytest.mark.usefixtures(*UNIFIED_FIXTURES) @pytest.mark.usefixtures(*UNIFIED_FIXTURES)
async def test_p1_local_folder_file_through_pipeline( async def test_p1_local_folder_file_through_pipeline(
self, self,
@ -742,7 +768,9 @@ class TestPipelineIntegration:
): ):
"""P1: LOCAL_FOLDER_FILE ConnectorDocument through prepare+index to READY.""" """P1: LOCAL_FOLDER_FILE ConnectorDocument through prepare+index to READY."""
from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.connector_document import ConnectorDocument
from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService from app.indexing_pipeline.indexing_pipeline_service import (
IndexingPipelineService,
)
doc = ConnectorDocument( doc = ConnectorDocument(
title="Test Local File", title="Test Local File",
@ -763,12 +791,16 @@ class TestPipelineIntegration:
assert result is not None assert result is not None
docs = ( docs = (
await db_session.execute( (
select(Document).where( await db_session.execute(
Document.document_type == DocumentType.LOCAL_FOLDER_FILE, select(Document).where(
Document.search_space_id == db_search_space.id, Document.document_type == DocumentType.LOCAL_FOLDER_FILE,
Document.search_space_id == db_search_space.id,
)
) )
) )
).scalars().all() .scalars()
.all()
)
assert len(docs) == 1 assert len(docs) == 1
assert DocumentStatus.is_state(docs[0].status, DocumentStatus.READY) assert DocumentStatus.is_state(docs[0].status, DocumentStatus.READY)

55
surfsense_backend/tests/integration/test_document_versioning.py
View file

@ -34,14 +34,16 @@ async def db_document(
async def _version_count(session: AsyncSession, document_id: int) -> int: async def _version_count(session: AsyncSession, document_id: int) -> int:
result = await session.execute( result = await session.execute(
select(func.count()).select_from(DocumentVersion).where( select(func.count())
DocumentVersion.document_id == document_id .select_from(DocumentVersion)
) .where(DocumentVersion.document_id == document_id)
) )
return result.scalar_one() return result.scalar_one()
async def _get_versions(session: AsyncSession, document_id: int) -> list[DocumentVersion]: async def _get_versions(
session: AsyncSession, document_id: int
) -> list[DocumentVersion]:
result = await session.execute( result = await session.execute(
select(DocumentVersion) select(DocumentVersion)
.where(DocumentVersion.document_id == document_id) .where(DocumentVersion.document_id == document_id)
@ -74,18 +76,14 @@ class TestCreateVersionSnapshot:
from app.utils.document_versioning import create_version_snapshot from app.utils.document_versioning import create_version_snapshot
t0 = datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC) t0 = datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC)
monkeypatch.setattr( monkeypatch.setattr("app.utils.document_versioning._now", lambda: t0)
"app.utils.document_versioning._now", lambda: t0
)
await create_version_snapshot(db_session, db_document) await create_version_snapshot(db_session, db_document)
# Simulate content change and time passing # Simulate content change and time passing
db_document.source_markdown = "# Test\n\nUpdated content." db_document.source_markdown = "# Test\n\nUpdated content."
db_document.content_hash = "def456" db_document.content_hash = "def456"
t1 = t0 + timedelta(minutes=31) t1 = t0 + timedelta(minutes=31)
monkeypatch.setattr( monkeypatch.setattr("app.utils.document_versioning._now", lambda: t1)
"app.utils.document_versioning._now", lambda: t1
)
await create_version_snapshot(db_session, db_document) await create_version_snapshot(db_session, db_document)
versions = await _get_versions(db_session, db_document.id) versions = await _get_versions(db_session, db_document.id)
@ -101,9 +99,7 @@ class TestCreateVersionSnapshot:
from app.utils.document_versioning import create_version_snapshot from app.utils.document_versioning import create_version_snapshot
t0 = datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC) t0 = datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC)
monkeypatch.setattr( monkeypatch.setattr("app.utils.document_versioning._now", lambda: t0)
"app.utils.document_versioning._now", lambda: t0
)
await create_version_snapshot(db_session, db_document) await create_version_snapshot(db_session, db_document)
count_after_first = await _version_count(db_session, db_document.id) count_after_first = await _version_count(db_session, db_document.id)
assert count_after_first == 1 assert count_after_first == 1
@ -112,9 +108,7 @@ class TestCreateVersionSnapshot:
db_document.source_markdown = "# Test\n\nQuick edit." db_document.source_markdown = "# Test\n\nQuick edit."
db_document.content_hash = "quick123" db_document.content_hash = "quick123"
t1 = t0 + timedelta(minutes=10) t1 = t0 + timedelta(minutes=10)
monkeypatch.setattr( monkeypatch.setattr("app.utils.document_versioning._now", lambda: t1)
"app.utils.document_versioning._now", lambda: t1
)
await create_version_snapshot(db_session, db_document) await create_version_snapshot(db_session, db_document)
count_after_second = await _version_count(db_session, db_document.id) count_after_second = await _version_count(db_session, db_document.id)
@ -134,22 +128,15 @@ class TestCreateVersionSnapshot:
# Create 5 versions spread across time: 3 older than 90 days, 2 recent # Create 5 versions spread across time: 3 older than 90 days, 2 recent
for i in range(5): for i in range(5):
db_document.source_markdown = f"Content v{i+1}" db_document.source_markdown = f"Content v{i + 1}"
db_document.content_hash = f"hash_{i+1}" db_document.content_hash = f"hash_{i + 1}"
if i < 3: t = base + timedelta(days=i) if i < 3 else base + timedelta(days=100 + i)
t = base + timedelta(days=i) # old monkeypatch.setattr("app.utils.document_versioning._now", lambda _t=t: _t)
else:
t = base + timedelta(days=100 + i) # recent
monkeypatch.setattr(
"app.utils.document_versioning._now", lambda _t=t: _t
)
await create_version_snapshot(db_session, db_document) await create_version_snapshot(db_session, db_document)
# Now trigger cleanup from a "current" time that makes the first 3 versions > 90 days old # Now trigger cleanup from a "current" time that makes the first 3 versions > 90 days old
now = base + timedelta(days=200) now = base + timedelta(days=200)
monkeypatch.setattr( monkeypatch.setattr("app.utils.document_versioning._now", lambda: now)
"app.utils.document_versioning._now", lambda: now
)
db_document.source_markdown = "Content v6" db_document.source_markdown = "Content v6"
db_document.content_hash = "hash_6" db_document.content_hash = "hash_6"
await create_version_snapshot(db_session, db_document) await create_version_snapshot(db_session, db_document)
@ -160,9 +147,7 @@ class TestCreateVersionSnapshot:
age = now - v.created_at.replace(tzinfo=UTC) age = now - v.created_at.replace(tzinfo=UTC)
assert age <= timedelta(days=90), f"Version {v.version_number} is too old" assert age <= timedelta(days=90), f"Version {v.version_number} is too old"
async def test_v5_cap_at_20_versions( async def test_v5_cap_at_20_versions(self, db_session, db_document, monkeypatch):
self, db_session, db_document, monkeypatch
):
"""V5: More than 20 versions triggers cap — oldest gets deleted.""" """V5: More than 20 versions triggers cap — oldest gets deleted."""
from app.utils.document_versioning import create_version_snapshot from app.utils.document_versioning import create_version_snapshot
@ -170,12 +155,10 @@ class TestCreateVersionSnapshot:
# Create 21 versions (all within 90 days, each 31 min apart) # Create 21 versions (all within 90 days, each 31 min apart)
for i in range(21): for i in range(21):
db_document.source_markdown = f"Content v{i+1}" db_document.source_markdown = f"Content v{i + 1}"
db_document.content_hash = f"hash_{i+1}" db_document.content_hash = f"hash_{i + 1}"
t = base + timedelta(minutes=31 * i) t = base + timedelta(minutes=31 * i)
monkeypatch.setattr( monkeypatch.setattr("app.utils.document_versioning._now", lambda _t=t: _t)
"app.utils.document_versioning._now", lambda _t=t: _t
)
await create_version_snapshot(db_session, db_document) await create_version_snapshot(db_session, db_document)
versions = await _get_versions(db_session, db_document.id) versions = await _get_versions(db_session, db_document.id)

4
surfsense_backend/tests/unit/connector_indexers/test_local_folder_scan.py
View file

@ -51,9 +51,7 @@ class TestScanFolder:
git.mkdir() git.mkdir()
(git / "config").write_text("gitconfig") (git / "config").write_text("gitconfig")
results = scan_folder( results = scan_folder(str(tmp_path), exclude_patterns=["node_modules", ".git"])
str(tmp_path), exclude_patterns=["node_modules", ".git"]
)
names = {r["relative_path"] for r in results} names = {r["relative_path"] for r in results}
assert "good.md" in names assert "good.md" in names

20
surfsense_web/app/(home)/login/LocalLoginForm.tsx
View file

@ -160,11 +160,11 @@ export function LocalLoginForm() {
placeholder="you@example.com" placeholder="you@example.com"
value={username} value={username}
onChange={(e) => setUsername(e.target.value)} onChange={(e) => setUsername(e.target.value)}
className={`mt-1 block w-full rounded-md border px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${ className={`mt-1 block w-full rounded-md border px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${
error.title error.title
? "border-destructive focus:border-destructive focus:ring-destructive/40" ? "border-destructive focus:border-destructive focus:ring-destructive/40"
: "border-border focus:border-primary focus:ring-primary/40" : "border-border focus:border-primary focus:ring-primary/40"
}`} }`}
disabled={isLoggingIn} disabled={isLoggingIn}
/> />
</div> </div>
@ -181,11 +181,11 @@ export function LocalLoginForm() {
placeholder="Enter your password" placeholder="Enter your password"
value={password} value={password}
onChange={(e) => setPassword(e.target.value)} onChange={(e) => setPassword(e.target.value)}
className={`mt-1 block w-full rounded-md border pr-10 px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${ className={`mt-1 block w-full rounded-md border pr-10 px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${
error.title error.title
? "border-destructive focus:border-destructive focus:ring-destructive/40" ? "border-destructive focus:border-destructive focus:ring-destructive/40"
: "border-border focus:border-primary focus:ring-primary/40" : "border-border focus:border-primary focus:ring-primary/40"
}`} }`}
disabled={isLoggingIn} disabled={isLoggingIn}
/> />
<button <button

131
surfsense_web/app/(home)/register/page.tsx
View file

@ -229,72 +229,66 @@ export default function RegisterPage() {
</AnimatePresence> </AnimatePresence>
<div> <div>
<label <label htmlFor="email" className="block text-sm font-medium text-foreground">
htmlFor="email" {t("email")}
className="block text-sm font-medium text-foreground" </label>
> <input
{t("email")} id="email"
</label> type="email"
<input required
id="email" placeholder="you@example.com"
type="email" value={email}
required onChange={(e) => setEmail(e.target.value)}
placeholder="you@example.com" className={`mt-1 block w-full rounded-md border px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${
value={email} error.title
onChange={(e) => setEmail(e.target.value)} ? "border-destructive focus:border-destructive focus:ring-destructive/40"
className={`mt-1 block w-full rounded-md border px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${ : "border-border focus:border-primary focus:ring-primary/40"
error.title }`}
? "border-destructive focus:border-destructive focus:ring-destructive/40" disabled={isRegistering}
: "border-border focus:border-primary focus:ring-primary/40" />
}`} </div>
disabled={isRegistering}
/>
</div>
<div> <div>
<label <label htmlFor="password" className="block text-sm font-medium text-foreground">
htmlFor="password" {t("password")}
className="block text-sm font-medium text-foreground" </label>
> <input
{t("password")} id="password"
</label> type="password"
<input required
id="password" placeholder="Enter your password"
type="password" value={password}
required onChange={(e) => setPassword(e.target.value)}
placeholder="Enter your password" className={`mt-1 block w-full rounded-md border px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${
value={password} error.title
onChange={(e) => setPassword(e.target.value)} ? "border-destructive focus:border-destructive focus:ring-destructive/40"
className={`mt-1 block w-full rounded-md border px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${ : "border-border focus:border-primary focus:ring-primary/40"
error.title }`}
? "border-destructive focus:border-destructive focus:ring-destructive/40" disabled={isRegistering}
: "border-border focus:border-primary focus:ring-primary/40" />
}`} </div>
disabled={isRegistering}
/>
</div>
<div> <div>
<label <label
htmlFor="confirmPassword" htmlFor="confirmPassword"
className="block text-sm font-medium text-foreground" className="block text-sm font-medium text-foreground"
> >
{t("confirm_password")} {t("confirm_password")}
</label> </label>
<input <input
id="confirmPassword" id="confirmPassword"
type="password" type="password"
required required
placeholder="Confirm your password" placeholder="Confirm your password"
value={confirmPassword} value={confirmPassword}
onChange={(e) => setConfirmPassword(e.target.value)} onChange={(e) => setConfirmPassword(e.target.value)}
className={`mt-1 block w-full rounded-md border px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${ className={`mt-1 block w-full rounded-md border px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${
error.title error.title
? "border-destructive focus:border-destructive focus:ring-destructive/40" ? "border-destructive focus:border-destructive focus:ring-destructive/40"
: "border-border focus:border-primary focus:ring-primary/40" : "border-border focus:border-primary focus:ring-primary/40"
}`} }`}
disabled={isRegistering} disabled={isRegistering}
/> />
</div> </div>
<button <button
@ -312,12 +306,9 @@ export default function RegisterPage() {
</form> </form>
<div className="mt-4 text-center text-sm"> <div className="mt-4 text-center text-sm">
<p className="text-muted-foreground"> <p className="text-muted-foreground">
{t("already_have_account")}{" "} {t("already_have_account")}{" "}
<Link <Link href="/login" className="font-medium text-primary hover:text-primary/90">
href="/login"
className="font-medium text-primary hover:text-primary/90"
>
{t("sign_in")} {t("sign_in")}
</Link> </Link>
</p> </p>

22
surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx
View file

@ -214,17 +214,17 @@ export function DocumentsFilters({
</Tooltip> </Tooltip>
)} )}
{/* Upload Button */} {/* Upload Button */}
<Button <Button
data-joyride="upload-button" data-joyride="upload-button"
onClick={openUploadDialog} onClick={openUploadDialog}
variant="outline" variant="outline"
size="sm" size="sm"
className="h-9 shrink-0 gap-1.5 bg-white text-gray-700 border-white hover:bg-gray-50 dark:bg-white dark:text-gray-800 dark:hover:bg-gray-100" className="h-9 shrink-0 gap-1.5 bg-white text-gray-700 border-white hover:bg-gray-50 dark:bg-white dark:text-gray-800 dark:hover:bg-gray-100"
> >
<Upload size={14} /> <Upload size={14} />
<span>Upload</span> <span>Upload</span>
</Button> </Button>
</div> </div>
</div> </div>
); );

7
surfsense_web/app/dashboard/[search_space_id]/user-settings/components/PromptsContent.tsx
View file

@ -2,7 +2,6 @@
import { useAtomValue } from "jotai"; import { useAtomValue } from "jotai";
import { AlertTriangle, Globe, Lock, PenLine, Sparkles, Trash2 } from "lucide-react"; import { AlertTriangle, Globe, Lock, PenLine, Sparkles, Trash2 } from "lucide-react";
import { ShortcutKbd } from "@/components/ui/shortcut-kbd";
import { useCallback, useState } from "react"; import { useCallback, useState } from "react";
import { toast } from "sonner"; import { toast } from "sonner";
import { import {
@ -24,6 +23,7 @@ import {
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input"; import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label"; import { Label } from "@/components/ui/label";
import { ShortcutKbd } from "@/components/ui/shortcut-kbd";
import { Spinner } from "@/components/ui/spinner"; import { Spinner } from "@/components/ui/spinner";
import { Switch } from "@/components/ui/switch"; import { Switch } from "@/components/ui/switch";
import type { PromptRead } from "@/contracts/types/prompts.types"; import type { PromptRead } from "@/contracts/types/prompts.types";
@ -145,9 +145,8 @@ export function PromptsContent() {
<div className="space-y-6 min-w-0 overflow-hidden"> <div className="space-y-6 min-w-0 overflow-hidden">
<div className="flex items-center justify-between"> <div className="flex items-center justify-between">
<p className="text-sm text-muted-foreground"> <p className="text-sm text-muted-foreground">
Create prompt templates triggered with{" "} Create prompt templates triggered with <ShortcutKbd keys={["/"]} className="ml-0" /> in
<ShortcutKbd keys={["/"]} className="ml-0" /> in the the chat composer.
chat composer.
</p> </p>
{!showForm && ( {!showForm && (
<Button <Button

5
surfsense_web/components/assistant-ui/connector-popup.tsx
View file

@ -374,7 +374,10 @@ export const ConnectorIndicator = forwardRef<ConnectorIndicatorHandle, Connector
<div className="px-4 sm:px-12 py-4 sm:py-8 pb-12 sm:pb-16"> <div className="px-4 sm:px-12 py-4 sm:py-8 pb-12 sm:pb-16">
{/* LLM Configuration Warning */} {/* LLM Configuration Warning */}
{!llmConfigLoading && !hasDocumentSummaryLLM && ( {!llmConfigLoading && !hasDocumentSummaryLLM && (
<Alert variant="destructive" className="mb-6 bg-muted/50 rounded-xl border-destructive/30"> <Alert
variant="destructive"
className="mb-6 bg-muted/50 rounded-xl border-destructive/30"
>
<AlertTriangle className="h-4 w-4" /> <AlertTriangle className="h-4 w-4" />
<AlertTitle>LLM Configuration Required</AlertTitle> <AlertTitle>LLM Configuration Required</AlertTitle>
<AlertDescription className="mt-2"> <AlertDescription className="mt-2">

58
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx
View file

@ -294,36 +294,36 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
)} )}
{(() => { {(() => {
const isGoogleDrive = connector.connector_type === "GOOGLE_DRIVE_CONNECTOR"; const isGoogleDrive = connector.connector_type === "GOOGLE_DRIVE_CONNECTOR";
const isComposioGoogleDrive = const isComposioGoogleDrive =
connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR"; connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR";
const requiresFolderSelection = isGoogleDrive || isComposioGoogleDrive; const requiresFolderSelection = isGoogleDrive || isComposioGoogleDrive;
const selectedFolders = const selectedFolders =
(connector.config?.selected_folders as (connector.config?.selected_folders as
| Array<{ id: string; name: string }> | Array<{ id: string; name: string }>
| undefined) || []; | undefined) || [];
const selectedFiles = const selectedFiles =
(connector.config?.selected_files as (connector.config?.selected_files as
| Array<{ id: string; name: string }> | Array<{ id: string; name: string }>
| undefined) || []; | undefined) || [];
const hasItemsSelected = selectedFolders.length > 0 || selectedFiles.length > 0; const hasItemsSelected = selectedFolders.length > 0 || selectedFiles.length > 0;
const isDisabled = requiresFolderSelection && !hasItemsSelected; const isDisabled = requiresFolderSelection && !hasItemsSelected;
return ( return (
<PeriodicSyncConfig <PeriodicSyncConfig
enabled={periodicEnabled} enabled={periodicEnabled}
frequencyMinutes={frequencyMinutes} frequencyMinutes={frequencyMinutes}
onEnabledChange={onPeriodicEnabledChange} onEnabledChange={onPeriodicEnabledChange}
onFrequencyChange={onFrequencyChange} onFrequencyChange={onFrequencyChange}
disabled={isDisabled} disabled={isDisabled}
disabledMessage={ disabledMessage={
isDisabled isDisabled
? "Select at least one folder or file above to enable periodic sync" ? "Select at least one folder or file above to enable periodic sync"
: undefined : undefined
} }
/> />
); );
})()} })()}
</> </>
)} )}

5
surfsense_web/components/assistant-ui/document-upload-popup.tsx
View file

@ -143,7 +143,10 @@ const DocumentUploadPopupContent: FC<{
<div className="px-4 sm:px-6 pb-4 sm:pb-6"> <div className="px-4 sm:px-6 pb-4 sm:pb-6">
{!isLoading && !hasDocumentSummaryLLM ? ( {!isLoading && !hasDocumentSummaryLLM ? (
<Alert variant="destructive" className="mb-4 bg-muted/50 rounded-xl border-destructive/30"> <Alert
variant="destructive"
className="mb-4 bg-muted/50 rounded-xl border-destructive/30"
>
<AlertTriangle className="h-4 w-4" /> <AlertTriangle className="h-4 w-4" />
<AlertTitle>LLM Configuration Required</AlertTitle> <AlertTitle>LLM Configuration Required</AlertTitle>
<AlertDescription className="mt-2"> <AlertDescription className="mt-2">

3
surfsense_web/components/assistant-ui/inline-citation.tsx
View file

@ -32,7 +32,8 @@ export const InlineCitation: FC<InlineCitationProps> = ({ chunkId, isDocsChunk =
<button <button
type="button" type="button"
onClick={() => setIsOpen(true)} onClick={() => setIsOpen(true)}
className="ml-0.5 inline-flex h-5 min-w-5 cursor-pointer items-center justify-center rounded-md bg-muted/60 px-1.5 text-[11px] font-medium text-muted-foreground align-baseline shadow-sm transition-colors hover:bg-muted hover:text-foreground focus-visible:ring-ring focus-visible:ring-2 focus-visible:outline-none" title={`View source chunk #${chunkId}`} className="ml-0.5 inline-flex h-5 min-w-5 cursor-pointer items-center justify-center rounded-md bg-muted/60 px-1.5 text-[11px] font-medium text-muted-foreground align-baseline shadow-sm transition-colors hover:bg-muted hover:text-foreground focus-visible:ring-ring focus-visible:ring-2 focus-visible:outline-none"
title={`View source chunk #${chunkId}`}
> >
{chunkId} {chunkId}
</button> </button>

17
surfsense_web/components/documents/DocumentNode.tsx
View file

@ -39,8 +39,8 @@ import { Spinner } from "@/components/ui/spinner";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import type { DocumentTypeEnum } from "@/contracts/types/document.types"; import type { DocumentTypeEnum } from "@/contracts/types/document.types";
import { cn } from "@/lib/utils"; import { cn } from "@/lib/utils";
import { isVersionableType } from "./version-history";
import { DND_TYPES } from "./FolderNode"; import { DND_TYPES } from "./FolderNode";
import { isVersionableType } from "./version-history";
const EDITABLE_DOCUMENT_TYPES = new Set(["FILE", "NOTE"]); const EDITABLE_DOCUMENT_TYPES = new Set(["FILE", "NOTE"]);
@ -199,7 +199,10 @@ export const DocumentNode = React.memo(function DocumentNode({
<span className="flex-1 min-w-0 truncate">{doc.title}</span> <span className="flex-1 min-w-0 truncate">{doc.title}</span>
{getDocumentTypeIcon(doc.document_type as DocumentTypeEnum, "h-3.5 w-3.5 text-muted-foreground") && ( {getDocumentTypeIcon(
doc.document_type as DocumentTypeEnum,
"h-3.5 w-3.5 text-muted-foreground"
) && (
<span className="shrink-0"> <span className="shrink-0">
{getDocumentTypeIcon( {getDocumentTypeIcon(
doc.document_type as DocumentTypeEnum, doc.document_type as DocumentTypeEnum,
@ -251,10 +254,7 @@ export const DocumentNode = React.memo(function DocumentNode({
</DropdownMenuSub> </DropdownMenuSub>
)} )}
{onVersionHistory && isVersionableType(doc.document_type) && ( {onVersionHistory && isVersionableType(doc.document_type) && (
<DropdownMenuItem <DropdownMenuItem disabled={isProcessing} onClick={() => onVersionHistory(doc)}>
disabled={isProcessing}
onClick={() => onVersionHistory(doc)}
>
<History className="mr-2 h-4 w-4" /> <History className="mr-2 h-4 w-4" />
Versions Versions
</DropdownMenuItem> </DropdownMenuItem>
@ -300,10 +300,7 @@ export const DocumentNode = React.memo(function DocumentNode({
</ContextMenuSub> </ContextMenuSub>
)} )}
{onVersionHistory && isVersionableType(doc.document_type) && ( {onVersionHistory && isVersionableType(doc.document_type) && (
<ContextMenuItem <ContextMenuItem disabled={isProcessing} onClick={() => onVersionHistory(doc)}>
disabled={isProcessing}
onClick={() => onVersionHistory(doc)}
>
<History className="mr-2 h-4 w-4" /> <History className="mr-2 h-4 w-4" />
Versions Versions
</ContextMenuItem> </ContextMenuItem>

186
surfsense_web/components/documents/FolderNode.tsx
View file

@ -256,15 +256,15 @@ export const FolderNode = React.memo(function FolderNode({
isOver && !canDrop && "cursor-not-allowed" isOver && !canDrop && "cursor-not-allowed"
)} )}
style={{ paddingLeft: `${depth * 16 + 4}px` }} style={{ paddingLeft: `${depth * 16 + 4}px` }}
onClick={() => { onClick={() => {
onToggleExpand(folder.id);
}}
onKeyDown={(e) => {
if (e.key === "Enter" || e.key === " ") {
e.preventDefault();
onToggleExpand(folder.id); onToggleExpand(folder.id);
} }}
}} onKeyDown={(e) => {
if (e.key === "Enter" || e.key === " ") {
e.preventDefault();
onToggleExpand(folder.id);
}
}}
onDoubleClick={(e) => { onDoubleClick={(e) => {
e.stopPropagation(); e.stopPropagation();
startRename(); startRename();
@ -306,7 +306,11 @@ export const FolderNode = React.memo(function FolderNode({
) : ( ) : (
<Checkbox <Checkbox
checked={ checked={
selectionState === "all" ? true : selectionState === "some" ? "indeterminate" : false selectionState === "all"
? true
: selectionState === "some"
? "indeterminate"
: false
} }
onCheckedChange={handleCheckChange} onCheckedChange={handleCheckChange}
onClick={(e) => e.stopPropagation()} onClick={(e) => e.stopPropagation()}
@ -350,107 +354,107 @@ export const FolderNode = React.memo(function FolderNode({
<MoreHorizontal className="h-3.5 w-3.5" /> <MoreHorizontal className="h-3.5 w-3.5" />
</Button> </Button>
</DropdownMenuTrigger> </DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-40"> <DropdownMenuContent align="end" className="w-40">
{isWatched && onRescan && ( {isWatched && onRescan && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onRescan(folder);
}}
>
<RefreshCw className="mr-2 h-4 w-4" />
Re-scan
</DropdownMenuItem>
)}
{isWatched && onStopWatching && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onStopWatching(folder);
}}
>
<EyeOff className="mr-2 h-4 w-4" />
Stop watching
</DropdownMenuItem>
)}
<DropdownMenuItem <DropdownMenuItem
onClick={(e) => { onClick={(e) => {
e.stopPropagation(); e.stopPropagation();
onRescan(folder); onCreateSubfolder(folder.id);
}} }}
> >
<RefreshCw className="mr-2 h-4 w-4" /> <FolderPlus className="mr-2 h-4 w-4" />
Re-scan New subfolder
</DropdownMenuItem> </DropdownMenuItem>
)}
{isWatched && onStopWatching && (
<DropdownMenuItem <DropdownMenuItem
onClick={(e) => { onClick={(e) => {
e.stopPropagation(); e.stopPropagation();
onStopWatching(folder); startRename();
}} }}
> >
<EyeOff className="mr-2 h-4 w-4" /> <PenLine className="mr-2 h-4 w-4" />
Stop watching Rename
</DropdownMenuItem> </DropdownMenuItem>
)} <DropdownMenuItem
<DropdownMenuItem onClick={(e) => {
onClick={(e) => { e.stopPropagation();
e.stopPropagation(); onMove(folder);
onCreateSubfolder(folder.id); }}
}} >
> <Move className="mr-2 h-4 w-4" />
<FolderPlus className="mr-2 h-4 w-4" /> Move to...
New subfolder </DropdownMenuItem>
</DropdownMenuItem> <DropdownMenuItem
<DropdownMenuItem className="text-destructive focus:text-destructive"
onClick={(e) => { onClick={(e) => {
e.stopPropagation(); e.stopPropagation();
startRename(); onDelete(folder);
}} }}
> >
<PenLine className="mr-2 h-4 w-4" /> <Trash2 className="mr-2 h-4 w-4" />
Rename Delete
</DropdownMenuItem> </DropdownMenuItem>
<DropdownMenuItem </DropdownMenuContent>
onClick={(e) => {
e.stopPropagation();
onMove(folder);
}}
>
<Move className="mr-2 h-4 w-4" />
Move to...
</DropdownMenuItem>
<DropdownMenuItem
className="text-destructive focus:text-destructive"
onClick={(e) => {
e.stopPropagation();
onDelete(folder);
}}
>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu> </DropdownMenu>
)} )}
</div> </div>
</ContextMenuTrigger> </ContextMenuTrigger>
{!isRenaming && contextMenuOpen && ( {!isRenaming && contextMenuOpen && (
<ContextMenuContent className="w-40"> <ContextMenuContent className="w-40">
{isWatched && onRescan && ( {isWatched && onRescan && (
<ContextMenuItem onClick={() => onRescan(folder)}> <ContextMenuItem onClick={() => onRescan(folder)}>
<RefreshCw className="mr-2 h-4 w-4" /> <RefreshCw className="mr-2 h-4 w-4" />
Re-scan Re-scan
</ContextMenuItem>
)}
{isWatched && onStopWatching && (
<ContextMenuItem onClick={() => onStopWatching(folder)}>
<EyeOff className="mr-2 h-4 w-4" />
Stop watching
</ContextMenuItem>
)}
<ContextMenuItem onClick={() => onCreateSubfolder(folder.id)}>
<FolderPlus className="mr-2 h-4 w-4" />
New subfolder
</ContextMenuItem> </ContextMenuItem>
)} <ContextMenuItem onClick={() => startRename()}>
{isWatched && onStopWatching && ( <PenLine className="mr-2 h-4 w-4" />
<ContextMenuItem onClick={() => onStopWatching(folder)}> Rename
<EyeOff className="mr-2 h-4 w-4" />
Stop watching
</ContextMenuItem> </ContextMenuItem>
)} <ContextMenuItem onClick={() => onMove(folder)}>
<ContextMenuItem onClick={() => onCreateSubfolder(folder.id)}> <Move className="mr-2 h-4 w-4" />
<FolderPlus className="mr-2 h-4 w-4" /> Move to...
New subfolder </ContextMenuItem>
</ContextMenuItem> <ContextMenuItem
<ContextMenuItem onClick={() => startRename()}> className="text-destructive focus:text-destructive"
<PenLine className="mr-2 h-4 w-4" /> onClick={() => onDelete(folder)}
Rename >
</ContextMenuItem> <Trash2 className="mr-2 h-4 w-4" />
<ContextMenuItem onClick={() => onMove(folder)}> Delete
<Move className="mr-2 h-4 w-4" /> </ContextMenuItem>
Move to... </ContextMenuContent>
</ContextMenuItem> )}
<ContextMenuItem
className="text-destructive focus:text-destructive"
onClick={() => onDelete(folder)}
>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</ContextMenuItem>
</ContextMenuContent>
)}
</ContextMenu> </ContextMenu>
); );
}); });

8
surfsense_web/components/documents/FolderTreeView.tsx
View file

@ -242,10 +242,10 @@ export function FolderTreeView({
siblingPositions={siblingPositions} siblingPositions={siblingPositions}
contextMenuOpen={openContextMenuId === `folder-${f.id}`} contextMenuOpen={openContextMenuId === `folder-${f.id}`}
onContextMenuOpenChange={(open) => setOpenContextMenuId(open ? `folder-${f.id}` : null)} onContextMenuOpenChange={(open) => setOpenContextMenuId(open ? `folder-${f.id}` : null)}
isWatched={watchedFolderIds?.has(f.id)} isWatched={watchedFolderIds?.has(f.id)}
onRescan={onRescanFolder} onRescan={onRescanFolder}
onStopWatching={onStopWatchingFolder} onStopWatching={onStopWatchingFolder}
/> />
); );
if (isExpanded) { if (isExpanded) {

42
surfsense_web/components/documents/version-history.tsx
View file

@ -1,19 +1,14 @@
"use client"; "use client";
import { useCallback, useEffect, useState } from "react";
import { Check, ChevronRight, Clock, Copy, RotateCcw } from "lucide-react"; import { Check, ChevronRight, Clock, Copy, RotateCcw } from "lucide-react";
import { useCallback, useEffect, useState } from "react";
import { toast } from "sonner";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { import { Dialog, DialogContent, DialogTitle, DialogTrigger } from "@/components/ui/dialog";
Dialog,
DialogContent,
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog";
import { Separator } from "@/components/ui/separator"; import { Separator } from "@/components/ui/separator";
import { Spinner } from "@/components/ui/spinner"; import { Spinner } from "@/components/ui/spinner";
import { cn } from "@/lib/utils";
import { documentsApiService } from "@/lib/apis/documents-api.service"; import { documentsApiService } from "@/lib/apis/documents-api.service";
import { toast } from "sonner"; import { cn } from "@/lib/utils";
interface DocumentVersionSummary { interface DocumentVersionSummary {
version_number: number; version_number: number;
@ -123,10 +118,9 @@ function VersionHistoryPanel({ documentId }: { documentId: number }) {
setSelectedVersion(versionNumber); setSelectedVersion(versionNumber);
setContentLoading(true); setContentLoading(true);
try { try {
const data = (await documentsApiService.getDocumentVersion( const data = (await documentsApiService.getDocumentVersion(documentId, versionNumber)) as {
documentId, source_markdown: string;
versionNumber };
)) as { source_markdown: string };
setVersionContent(data.source_markdown || ""); setVersionContent(data.source_markdown || "");
} catch { } catch {
toast.error("Failed to load version content"); toast.error("Failed to load version content");
@ -196,13 +190,11 @@ function VersionHistoryPanel({ documentId }: { documentId: number }) {
> >
<div className="flex-1 min-w-0 space-y-0.5"> <div className="flex-1 min-w-0 space-y-0.5">
<p className="text-sm font-medium truncate"> <p className="text-sm font-medium truncate">
{v.created_at ? formatRelativeTime(v.created_at) : `Version ${v.version_number}`} {v.created_at
? formatRelativeTime(v.created_at)
: `Version ${v.version_number}`}
</p> </p>
{v.title && ( {v.title && <p className="text-xs text-muted-foreground truncate">{v.title}</p>}
<p className="text-xs text-muted-foreground truncate">
{v.title}
</p>
)}
</div> </div>
<ChevronRight className="h-3.5 w-3.5 shrink-0 opacity-50" /> <ChevronRight className="h-3.5 w-3.5 shrink-0 opacity-50" />
</button> </button>
@ -227,11 +219,7 @@ function VersionHistoryPanel({ documentId }: { documentId: number }) {
onClick={handleCopy} onClick={handleCopy}
disabled={contentLoading || copied} disabled={contentLoading || copied}
> >
{copied ? ( {copied ? <Check className="h-3 w-3" /> : <Copy className="h-3 w-3" />}
<Check className="h-3 w-3" />
) : (
<Copy className="h-3 w-3" />
)}
{copied ? "Copied" : "Copy"} {copied ? "Copied" : "Copy"}
</Button> </Button>
<Button <Button
@ -241,11 +229,7 @@ function VersionHistoryPanel({ documentId }: { documentId: number }) {
disabled={restoring || contentLoading} disabled={restoring || contentLoading}
onClick={() => handleRestore(selectedVersion)} onClick={() => handleRestore(selectedVersion)}
> >
{restoring ? ( {restoring ? <Spinner size="xs" /> : <RotateCcw className="h-3 w-3" />}
<Spinner size="xs" />
) : (
<RotateCcw className="h-3 w-3" />
)}
Restore Restore
</Button> </Button>
</div> </div>

39
surfsense_web/components/editor-panel/editor-panel.tsx
View file

@ -54,7 +54,6 @@ function EditorPanelSkeleton() {
); );
} }
export function EditorPanelContent({ export function EditorPanelContent({
documentId, documentId,
searchSpaceId, searchSpaceId,
@ -194,24 +193,24 @@ export function EditorPanelContent({
return ( return (
<> <>
<div className="flex items-center justify-between px-4 py-2 shrink-0 border-b"> <div className="flex items-center justify-between px-4 py-2 shrink-0 border-b">
<div className="flex-1 min-w-0"> <div className="flex-1 min-w-0">
<h2 className="text-sm font-semibold truncate">{displayTitle}</h2> <h2 className="text-sm font-semibold truncate">{displayTitle}</h2>
{isEditableType && editedMarkdown !== null && ( {isEditableType && editedMarkdown !== null && (
<p className="text-[10px] text-muted-foreground">Unsaved changes</p> <p className="text-[10px] text-muted-foreground">Unsaved changes</p>
)} )}
</div>
<div className="flex items-center gap-1 shrink-0">
{editorDoc?.document_type && (
<VersionHistoryButton documentId={documentId} documentType={editorDoc.document_type} />
)}
{onClose && (
<Button variant="ghost" size="icon" onClick={onClose} className="size-7 shrink-0">
<XIcon className="size-4" />
<span className="sr-only">Close editor panel</span>
</Button>
)}
</div>
</div> </div>
<div className="flex items-center gap-1 shrink-0">
{editorDoc?.document_type && (
<VersionHistoryButton documentId={documentId} documentType={editorDoc.document_type} />
)}
{onClose && (
<Button variant="ghost" size="icon" onClick={onClose} className="size-7 shrink-0">
<XIcon className="size-4" />
<span className="sr-only">Close editor panel</span>
</Button>
)}
</div>
</div>
<div className="flex-1 overflow-hidden"> <div className="flex-1 overflow-hidden">
{isLoading ? ( {isLoading ? (
@ -233,7 +232,9 @@ export function EditorPanelContent({
? "Document is processing" ? "Document is processing"
: "Document unavailable"} : "Document unavailable"}
</p> </p>
<p className="text-sm text-muted-foreground">{error || "An unknown error occurred"}</p> <p className="text-sm text-muted-foreground">
{error || "An unknown error occurred"}
</p>
</div> </div>
</div> </div>
) : isLargeDocument ? ( ) : isLargeDocument ? (

185
surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx
View file

@ -121,9 +121,7 @@ export function DocumentsSidebar({
} }
const recovered = await api!.getWatchedFolders(); const recovered = await api!.getWatchedFolders();
const ids = new Set( const ids = new Set(
recovered recovered.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number)
.filter((f) => f.rootFolderId != null)
.map((f) => f.rootFolderId as number)
); );
setWatchedFolderIds(ids); setWatchedFolderIds(ids);
return; return;
@ -133,9 +131,7 @@ export function DocumentsSidebar({
} }
const ids = new Set( const ids = new Set(
folders folders.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number)
.filter((f) => f.rootFolderId != null)
.map((f) => f.rootFolderId as number)
); );
setWatchedFolderIds(ids); setWatchedFolderIds(ids);
} }
@ -305,28 +301,25 @@ export function DocumentsSidebar({
[searchSpaceId] [searchSpaceId]
); );
const handleStopWatching = useCallback( const handleStopWatching = useCallback(async (folder: FolderDisplay) => {
async (folder: FolderDisplay) => { const api = window.electronAPI;
const api = window.electronAPI; if (!api) return;
if (!api) return;
const watchedFolders = await api.getWatchedFolders(); const watchedFolders = await api.getWatchedFolders();
const matched = watchedFolders.find((wf) => wf.rootFolderId === folder.id); const matched = watchedFolders.find((wf) => wf.rootFolderId === folder.id);
if (!matched) { if (!matched) {
toast.error("This folder is not being watched"); toast.error("This folder is not being watched");
return; return;
} }
await api.removeWatchedFolder(matched.path); await api.removeWatchedFolder(matched.path);
try { try {
await foldersApiService.stopWatching(folder.id); await foldersApiService.stopWatching(folder.id);
} catch (err) { } catch (err) {
console.error("[DocumentsSidebar] Failed to clear watched metadata:", err); console.error("[DocumentsSidebar] Failed to clear watched metadata:", err);
} }
toast.success(`Stopped watching: ${matched.name}`); toast.success(`Stopped watching: ${matched.name}`);
}, }, []);
[]
);
const handleRenameFolder = useCallback(async (folder: FolderDisplay, newName: string) => { const handleRenameFolder = useCallback(async (folder: FolderDisplay, newName: string) => {
try { try {
@ -755,81 +748,83 @@ export function DocumentsSidebar({
<div className="flex-1 min-h-0 overflow-x-hidden pt-0 flex flex-col"> <div className="flex-1 min-h-0 overflow-x-hidden pt-0 flex flex-col">
<div className="px-4 pb-2"> <div className="px-4 pb-2">
<DocumentsFilters <DocumentsFilters
typeCounts={typeCounts} typeCounts={typeCounts}
onSearch={setSearch} onSearch={setSearch}
searchValue={search} searchValue={search}
onToggleType={onToggleType} onToggleType={onToggleType}
activeTypes={activeTypes} activeTypes={activeTypes}
onCreateFolder={() => handleCreateFolder(null)} onCreateFolder={() => handleCreateFolder(null)}
/> />
</div> </div>
<div className="relative flex-1 min-h-0 overflow-auto"> <div className="relative flex-1 min-h-0 overflow-auto">
{deletableSelectedIds.length > 0 && ( {deletableSelectedIds.length > 0 && (
<div className="absolute inset-x-0 top-0 z-10 flex items-center justify-center px-4 py-1.5 animate-in fade-in duration-150 pointer-events-none"> <div className="absolute inset-x-0 top-0 z-10 flex items-center justify-center px-4 py-1.5 animate-in fade-in duration-150 pointer-events-none">
<button <button
type="button" type="button"
onClick={() => setBulkDeleteConfirmOpen(true)} onClick={() => setBulkDeleteConfirmOpen(true)}
className="pointer-events-auto flex items-center gap-1.5 px-3 py-1 rounded-md bg-destructive text-destructive-foreground shadow-lg text-xs font-medium hover:bg-destructive/90 transition-colors" className="pointer-events-auto flex items-center gap-1.5 px-3 py-1 rounded-md bg-destructive text-destructive-foreground shadow-lg text-xs font-medium hover:bg-destructive/90 transition-colors"
> >
<Trash2 size={12} /> <Trash2 size={12} />
Delete {deletableSelectedIds.length}{" "} Delete {deletableSelectedIds.length}{" "}
{deletableSelectedIds.length === 1 ? "item" : "items"} {deletableSelectedIds.length === 1 ? "item" : "items"}
</button> </button>
</div> </div>
)} )}
<FolderTreeView <FolderTreeView
folders={treeFolders} folders={treeFolders}
documents={searchFilteredDocuments} documents={searchFilteredDocuments}
expandedIds={expandedIds} expandedIds={expandedIds}
onToggleExpand={toggleFolderExpand} onToggleExpand={toggleFolderExpand}
mentionedDocIds={mentionedDocIds} mentionedDocIds={mentionedDocIds}
onToggleChatMention={handleToggleChatMention} onToggleChatMention={handleToggleChatMention}
onToggleFolderSelect={handleToggleFolderSelect} onToggleFolderSelect={handleToggleFolderSelect}
onRenameFolder={handleRenameFolder} onRenameFolder={handleRenameFolder}
onDeleteFolder={handleDeleteFolder} onDeleteFolder={handleDeleteFolder}
onMoveFolder={handleMoveFolder} onMoveFolder={handleMoveFolder}
onCreateFolder={handleCreateFolder} onCreateFolder={handleCreateFolder}
searchQuery={debouncedSearch.trim() || undefined} searchQuery={debouncedSearch.trim() || undefined}
onPreviewDocument={(doc) => { onPreviewDocument={(doc) => {
openEditorPanel({ openEditorPanel({
documentId: doc.id, documentId: doc.id,
searchSpaceId, searchSpaceId,
title: doc.title, title: doc.title,
}); });
}} }}
onEditDocument={(doc) => { onEditDocument={(doc) => {
openEditorPanel({ openEditorPanel({
documentId: doc.id, documentId: doc.id,
searchSpaceId, searchSpaceId,
title: doc.title, title: doc.title,
}); });
}} }}
onDeleteDocument={(doc) => handleDeleteDocument(doc.id)} onDeleteDocument={(doc) => handleDeleteDocument(doc.id)}
onMoveDocument={handleMoveDocument} onMoveDocument={handleMoveDocument}
onExportDocument={handleExportDocument} onExportDocument={handleExportDocument}
onVersionHistory={(doc) => setVersionDocId(doc.id)} onVersionHistory={(doc) => setVersionDocId(doc.id)}
activeTypes={activeTypes} activeTypes={activeTypes}
onDropIntoFolder={handleDropIntoFolder} onDropIntoFolder={handleDropIntoFolder}
onReorderFolder={handleReorderFolder} onReorderFolder={handleReorderFolder}
watchedFolderIds={watchedFolderIds} watchedFolderIds={watchedFolderIds}
onRescanFolder={handleRescanFolder} onRescanFolder={handleRescanFolder}
onStopWatchingFolder={handleStopWatching} onStopWatchingFolder={handleStopWatching}
/> />
</div>
</div> </div>
</div>
{versionDocId !== null && ( {versionDocId !== null && (
<VersionHistoryDialog <VersionHistoryDialog
open open
onOpenChange={(open) => { if (!open) setVersionDocId(null); }} onOpenChange={(open) => {
documentId={versionDocId} if (!open) setVersionDocId(null);
/> }}
)} documentId={versionDocId}
/>
)}
<FolderPickerDialog <FolderPickerDialog
open={folderPickerOpen} open={folderPickerOpen}
onOpenChange={setFolderPickerOpen} onOpenChange={setFolderPickerOpen}
folders={treeFolders} folders={treeFolders}

4
surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx
View file

@ -185,9 +185,7 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
<p className="font-semibold text-foreground text-lg"> <p className="font-semibold text-foreground text-lg">
{isProcessing ? "Document is processing" : "Document unavailable"} {isProcessing ? "Document is processing" : "Document unavailable"}
</p> </p>
<p className="text-sm text-muted-foreground"> <p className="text-sm text-muted-foreground">{error || "An unknown error occurred"}</p>
{error || "An unknown error occurred"}
</p>
</div> </div>
{!isProcessing && ( {!isProcessing && (
<Button <Button

4
surfsense_web/components/new-chat/source-detail-panel.tsx
View file

@ -480,9 +480,7 @@ export function SourceDetailPanel({
<FileQuestionMark className="h-10 w-10 text-muted-foreground" /> <FileQuestionMark className="h-10 w-10 text-muted-foreground" />
</div> </div>
<div> <div>
<p className="font-semibold text-foreground text-lg"> <p className="font-semibold text-foreground text-lg">Document unavailable</p>
Document unavailable
</p>
<p className="text-sm text-muted-foreground mt-2 max-w-md"> <p className="text-sm text-muted-foreground mt-2 max-w-md">
{documentByChunkFetchingError.message || {documentByChunkFetchingError.message ||
"An unexpected error occurred. Please try again."} "An unexpected error occurred. Please try again."}

41
surfsense_web/components/settings/llm-role-manager.tsx
View file

@ -134,24 +134,27 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
preferences?.image_generation_config_id, preferences?.image_generation_config_id,
]); ]);
const handleRoleAssignment = useCallback(async (prefKey: string, configId: string) => { const handleRoleAssignment = useCallback(
const value = configId === "unassigned" ? "" : parseInt(configId); async (prefKey: string, configId: string) => {
const value = configId === "unassigned" ? "" : parseInt(configId);
setAssignments((prev) => ({ ...prev, [prefKey]: value })); setAssignments((prev) => ({ ...prev, [prefKey]: value }));
setSavingRole(prefKey); setSavingRole(prefKey);
savingRef.current = true; savingRef.current = true;
try { try {
await updatePreferences({ await updatePreferences({
search_space_id: searchSpaceId, search_space_id: searchSpaceId,
data: { [prefKey]: value || undefined }, data: { [prefKey]: value || undefined },
}); });
toast.success("Role assignment updated"); toast.success("Role assignment updated");
} finally { } finally {
setSavingRole(null); setSavingRole(null);
savingRef.current = false; savingRef.current = false;
} }
}, [updatePreferences, searchSpaceId]); },
[updatePreferences, searchSpaceId]
);
// Combine global and custom LLM configs // Combine global and custom LLM configs
const allLLMConfigs = [ const allLLMConfigs = [
@ -199,10 +202,7 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
Refresh Refresh
</Button> </Button>
{isAssignmentComplete && !isLoading && !hasError && ( {isAssignmentComplete && !isLoading && !hasError && (
<Badge <Badge variant="outline" className="text-xs gap-1.5 text-muted-foreground">
variant="outline"
className="text-xs gap-1.5 text-muted-foreground"
>
<CircleCheck className="h-3 w-3" /> <CircleCheck className="h-3 w-3" />
All roles assigned All roles assigned
</Badge> </Badge>
@ -483,7 +483,6 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
})} })}
</div> </div>
)} )}
</div> </div>
); );
} }

73
surfsense_web/components/sources/DocumentUploadTab.tsx
View file

@ -128,7 +128,8 @@ const MAX_TOTAL_SIZE_BYTES = MAX_TOTAL_SIZE_MB * 1024 * 1024;
const MAX_FILE_SIZE_MB = 500; const MAX_FILE_SIZE_MB = 500;
const MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024; const MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024;
const toggleRowClass = "flex items-center justify-between rounded-lg bg-slate-400/5 dark:bg-white/5 p-3"; const toggleRowClass =
"flex items-center justify-between rounded-lg bg-slate-400/5 dark:bg-white/5 p-3";
export function DocumentUploadTab({ export function DocumentUploadTab({
searchSpaceId, searchSpaceId,
@ -326,7 +327,14 @@ export function DocumentUploadTab({
await api.addWatchedFolder({ await api.addWatchedFolder({
path: selectedFolder.path, path: selectedFolder.path,
name: selectedFolder.name, name: selectedFolder.name,
excludePatterns: [".git", "node_modules", "__pycache__", ".DS_Store", ".obsidian", ".trash"], excludePatterns: [
".git",
"node_modules",
"__pycache__",
".DS_Store",
".obsidian",
".trash",
],
fileExtensions: null, fileExtensions: null,
rootFolderId, rootFolderId,
searchSpaceId: Number(searchSpaceId), searchSpaceId: Number(searchSpaceId),
@ -393,12 +401,20 @@ export function DocumentUploadTab({
return ( return (
<DropdownMenu> <DropdownMenu>
<DropdownMenuTrigger asChild onClick={(e) => e.stopPropagation()}> <DropdownMenuTrigger asChild onClick={(e) => e.stopPropagation()}>
<Button variant="ghost" size="sm" className={`text-xs gap-1 bg-neutral-700/50 hover:bg-neutral-600/50 ${sizeClass} ${widthClass}`}> <Button
variant="ghost"
size="sm"
className={`text-xs gap-1 bg-neutral-700/50 hover:bg-neutral-600/50 ${sizeClass} ${widthClass}`}
>
Browse Browse
<ChevronDown className="h-3 w-3 opacity-60" /> <ChevronDown className="h-3 w-3 opacity-60" />
</Button> </Button>
</DropdownMenuTrigger> </DropdownMenuTrigger>
<DropdownMenuContent align="center" className="dark:bg-neutral-800" onClick={(e) => e.stopPropagation()}> <DropdownMenuContent
align="center"
className="dark:bg-neutral-800"
onClick={(e) => e.stopPropagation()}
>
<DropdownMenuItem onClick={handleBrowseFiles}> <DropdownMenuItem onClick={handleBrowseFiles}>
<FileIcon className="h-4 w-4 mr-2" /> <FileIcon className="h-4 w-4 mr-2" />
Files Files
@ -415,7 +431,11 @@ export function DocumentUploadTab({
return ( return (
<DropdownMenu> <DropdownMenu>
<DropdownMenuTrigger asChild onClick={(e) => e.stopPropagation()}> <DropdownMenuTrigger asChild onClick={(e) => e.stopPropagation()}>
<Button variant="secondary" size="sm" className={`text-xs gap-1 ${sizeClass} ${widthClass}`}> <Button
variant="secondary"
size="sm"
className={`text-xs gap-1 ${sizeClass} ${widthClass}`}
>
Browse Browse
<ChevronDown className="h-3 w-3 opacity-60" /> <ChevronDown className="h-3 w-3 opacity-60" />
</Button> </Button>
@ -457,21 +477,19 @@ export function DocumentUploadTab({
{/* MOBILE DROP ZONE */} {/* MOBILE DROP ZONE */}
<div className="sm:hidden"> <div className="sm:hidden">
{hasContent ? ( {hasContent ? (
!selectedFolder && !isFileCountLimitReached && ( !selectedFolder &&
isElectron ? ( !isFileCountLimitReached &&
<div className="w-full"> (isElectron ? (
{renderBrowseButton({ compact: true, fullWidth: true })} <div className="w-full">{renderBrowseButton({ compact: true, fullWidth: true })}</div>
</div> ) : (
) : ( <button
<button type="button"
type="button" className="w-full text-xs h-8 flex items-center justify-center gap-1.5 rounded-md border border-dashed border-muted-foreground/30 text-muted-foreground hover:text-foreground hover:border-foreground/50 transition-colors"
className="w-full text-xs h-8 flex items-center justify-center gap-1.5 rounded-md border border-dashed border-muted-foreground/30 text-muted-foreground hover:text-foreground hover:border-foreground/50 transition-colors" onClick={() => fileInputRef.current?.click()}
onClick={() => fileInputRef.current?.click()} >
> Add more files
Add more files </button>
</button> ))
)
)
) : ( ) : (
<div <div
className="flex flex-col items-center gap-4 py-12 px-4 cursor-pointer" className="flex flex-col items-center gap-4 py-12 px-4 cursor-pointer"
@ -487,7 +505,9 @@ export function DocumentUploadTab({
<p className="text-sm text-muted-foreground inline-flex items-center flex-wrap justify-center"> <p className="text-sm text-muted-foreground inline-flex items-center flex-wrap justify-center">
<span>{t("file_size_limit")}</span> <span>{t("file_size_limit")}</span>
<Dot className="h-4 w-4 shrink-0" /> <Dot className="h-4 w-4 shrink-0" />
<span>{t("upload_limits", { maxFiles: MAX_FILES, maxSizeMB: MAX_TOTAL_SIZE_MB })}</span> <span>
{t("upload_limits", { maxFiles: MAX_FILES, maxSizeMB: MAX_TOTAL_SIZE_MB })}
</span>
</p> </p>
</div> </div>
<div className="w-full mt-1" onClick={(e) => e.stopPropagation()}> <div className="w-full mt-1" onClick={(e) => e.stopPropagation()}>
@ -538,7 +558,9 @@ export function DocumentUploadTab({
<p className="text-xs text-muted-foreground text-center inline-flex items-center flex-wrap justify-center"> <p className="text-xs text-muted-foreground text-center inline-flex items-center flex-wrap justify-center">
<span>{t("file_size_limit")}</span> <span>{t("file_size_limit")}</span>
<Dot className="h-4 w-4 shrink-0" /> <Dot className="h-4 w-4 shrink-0" />
<span>{t("upload_limits", { maxFiles: MAX_FILES, maxSizeMB: MAX_TOTAL_SIZE_MB })}</span> <span>
{t("upload_limits", { maxFiles: MAX_FILES, maxSizeMB: MAX_TOTAL_SIZE_MB })}
</span>
</p> </p>
<div className="mt-1">{renderBrowseButton()}</div> <div className="mt-1">{renderBrowseButton()}</div>
</div> </div>
@ -569,9 +591,7 @@ export function DocumentUploadTab({
<div className="flex items-center justify-between p-3"> <div className="flex items-center justify-between p-3">
<div className="space-y-0.5"> <div className="space-y-0.5">
<p className="font-medium text-sm">Watch folder</p> <p className="font-medium text-sm">Watch folder</p>
<p className="text-xs text-muted-foreground"> <p className="text-xs text-muted-foreground">Auto-sync when files change</p>
Auto-sync when files change
</p>
</div> </div>
<Switch <Switch
id="watch-folder-toggle" id="watch-folder-toggle"
@ -612,7 +632,8 @@ export function DocumentUploadTab({
<div className="rounded-lg border border-border p-3 space-y-2"> <div className="rounded-lg border border-border p-3 space-y-2">
<div className="flex items-center justify-between"> <div className="flex items-center justify-between">
<p className="text-sm font-medium"> <p className="text-sm font-medium">
{t("selected_files", { count: files.length })} &middot; {formatFileSize(totalFileSize)} {t("selected_files", { count: files.length })} &middot;{" "}
{formatFileSize(totalFileSize)}
</p> </p>
<Button <Button
variant="ghost" variant="ghost"

1
surfsense_web/lib/apis/connectors-api.service.ts
View file

@ -404,7 +404,6 @@ class ConnectorsApiService {
listDiscordChannelsResponse listDiscordChannelsResponse
); );
}; };
} }
export type { SlackChannel, DiscordChannel }; export type { SlackChannel, DiscordChannel };

38
surfsense_web/lib/apis/documents-api.service.ts
View file

@ -417,27 +417,47 @@ class DocumentsApiService {
}; };
getDocumentVersion = async (documentId: number, versionNumber: number) => { getDocumentVersion = async (documentId: number, versionNumber: number) => {
return baseApiService.get( return baseApiService.get(`/api/v1/documents/${documentId}/versions/${versionNumber}`);
`/api/v1/documents/${documentId}/versions/${versionNumber}`
);
}; };
restoreDocumentVersion = async (documentId: number, versionNumber: number) => { restoreDocumentVersion = async (documentId: number, versionNumber: number) => {
return baseApiService.post( return baseApiService.post(`/api/v1/documents/${documentId}/versions/${versionNumber}/restore`);
`/api/v1/documents/${documentId}/versions/${versionNumber}/restore`
);
}; };
folderIndex = async (searchSpaceId: number, body: { folder_path: string; folder_name: string; search_space_id: number; exclude_patterns?: string[]; file_extensions?: string[]; root_folder_id?: number; enable_summary?: boolean }) => { folderIndex = async (
searchSpaceId: number,
body: {
folder_path: string;
folder_name: string;
search_space_id: number;
exclude_patterns?: string[];
file_extensions?: string[];
root_folder_id?: number;
enable_summary?: boolean;
}
) => {
return baseApiService.post(`/api/v1/documents/folder-index`, undefined, { body }); return baseApiService.post(`/api/v1/documents/folder-index`, undefined, { body });
}; };
folderIndexFiles = async (searchSpaceId: number, body: { folder_path: string; folder_name: string; search_space_id: number; target_file_paths: string[]; root_folder_id?: number | null; enable_summary?: boolean }) => { folderIndexFiles = async (
searchSpaceId: number,
body: {
folder_path: string;
folder_name: string;
search_space_id: number;
target_file_paths: string[];
root_folder_id?: number | null;
enable_summary?: boolean;
}
) => {
return baseApiService.post(`/api/v1/documents/folder-index-files`, undefined, { body }); return baseApiService.post(`/api/v1/documents/folder-index-files`, undefined, { body });
}; };
getWatchedFolders = async (searchSpaceId: number) => { getWatchedFolders = async (searchSpaceId: number) => {
return baseApiService.get(`/api/v1/documents/watched-folders?search_space_id=${searchSpaceId}`, folderListResponse); return baseApiService.get(
`/api/v1/documents/watched-folders?search_space_id=${searchSpaceId}`,
folderListResponse
);
}; };
/** /**