diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 01a6bbda0..77a001a0d 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -25,7 +25,7 @@ from sqlalchemy import ( ) from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine -from sqlalchemy.orm import DeclarativeBase, Mapped, backref, declared_attr, relationship +from sqlalchemy.orm import DeclarativeBase, Mapped, declared_attr, relationship from app.config import config @@ -1086,9 +1086,7 @@ class DocumentVersion(BaseModel, TimestampMixin): content_hash = Column(String, nullable=False) title = Column(String, nullable=True) - document = relationship( - "Document", backref=backref("versions", passive_deletes=True) - ) + document = relationship("Document", backref="versions") class Chunk(BaseModel, TimestampMixin): diff --git a/surfsense_backend/app/indexing_pipeline/connector_document.py b/surfsense_backend/app/indexing_pipeline/connector_document.py index 4f5d6e2e0..019efe287 100644 --- a/surfsense_backend/app/indexing_pipeline/connector_document.py +++ b/surfsense_backend/app/indexing_pipeline/connector_document.py @@ -17,7 +17,6 @@ class ConnectorDocument(BaseModel): metadata: dict = {} connector_id: int | None = None created_by_id: str - folder_id: int | None = None @field_validator("title", "source_markdown", "unique_id", "created_by_id") @classmethod diff --git a/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py b/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py index 22c552e5c..0fa4006f5 100644 --- a/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py +++ b/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py @@ -268,8 +268,6 @@ class IndexingPipelineService: ): existing.status = DocumentStatus.pending() existing.updated_at = datetime.now(UTC) - if connector_doc.folder_id is not None: - existing.folder_id = connector_doc.folder_id documents.append(existing) log_document_requeued(ctx) continue @@ -296,8 +294,6 @@ class IndexingPipelineService: existing.document_metadata = connector_doc.metadata existing.updated_at = datetime.now(UTC) existing.status = DocumentStatus.pending() - if connector_doc.folder_id is not None: - existing.folder_id = connector_doc.folder_id documents.append(existing) log_document_updated(ctx) continue @@ -321,7 +317,6 @@ class IndexingPipelineService: created_by_id=connector_doc.created_by_id, updated_at=datetime.now(UTC), status=DocumentStatus.pending(), - folder_id=connector_doc.folder_id, ) self.session.add(document) documents.append(document) diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index 53312c647..5008b1a10 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -1385,48 +1385,45 @@ async def restore_document_version( } -# ===== Upload-based local folder indexing endpoints ===== -# These work for ALL deployment modes (cloud, self-hosted remote, self-hosted local). -# The desktop app reads files locally and uploads them here. +# ===== Local folder indexing endpoints ===== -class FolderMtimeCheckFile(PydanticBaseModel): - relative_path: str - mtime: float - - -class FolderMtimeCheckRequest(PydanticBaseModel): - folder_name: str - search_space_id: int - files: list[FolderMtimeCheckFile] - - -class FolderUnlinkRequest(PydanticBaseModel): +class FolderIndexRequest(PydanticBaseModel): + folder_path: str folder_name: str search_space_id: int + exclude_patterns: list[str] | None = None + file_extensions: list[str] | None = None root_folder_id: int | None = None - relative_paths: list[str] + enable_summary: bool = False -class FolderSyncFinalizeRequest(PydanticBaseModel): +class FolderIndexFilesRequest(PydanticBaseModel): + folder_path: str folder_name: str search_space_id: int + target_file_paths: list[str] root_folder_id: int | None = None - all_relative_paths: list[str] + enable_summary: bool = False -@router.post("/documents/folder-mtime-check") -async def folder_mtime_check( - request: FolderMtimeCheckRequest, +@router.post("/documents/folder-index") +async def folder_index( + request: FolderIndexRequest, session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): - """Pre-upload optimization: check which files need uploading based on mtime. - - Returns the subset of relative paths where the file is new or has a - different mtime, so the client can skip reading/uploading unchanged files. + """Full-scan index of a local folder. Creates the root Folder row synchronously + and dispatches the heavy indexing work to a Celery task. + Returns the root_folder_id so the desktop can persist it. """ - from app.indexing_pipeline.document_hashing import compute_identifier_hash + from app.config import config as app_config + + if not app_config.is_self_hosted(): + raise HTTPException( + status_code=400, + detail="Local folder indexing is only available in self-hosted mode", + ) await check_permission( session, @@ -1436,309 +1433,113 @@ async def folder_mtime_check( "You don't have permission to create documents in this search space", ) - uid_hashes = {} - for f in request.files: - uid = f"{request.folder_name}:{f.relative_path}" - uid_hash = compute_identifier_hash( - DocumentType.LOCAL_FOLDER_FILE.value, uid, request.search_space_id - ) - uid_hashes[uid_hash] = f + watched_metadata = { + "watched": True, + "folder_path": request.folder_path, + "exclude_patterns": request.exclude_patterns, + "file_extensions": request.file_extensions, + } - existing_docs = ( - ( - await session.execute( - select(Document).where( - Document.unique_identifier_hash.in_(list(uid_hashes.keys())), - Document.document_type == DocumentType.LOCAL_FOLDER_FILE, - ) - ) - ) - .scalars() - .all() - ) - - existing_by_hash = {doc.unique_identifier_hash: doc for doc in existing_docs} - - mtime_tolerance = 1.0 - files_to_upload: list[str] = [] - - for uid_hash, file_info in uid_hashes.items(): - doc = existing_by_hash.get(uid_hash) - if doc is None: - files_to_upload.append(file_info.relative_path) - continue - - stored_mtime = (doc.document_metadata or {}).get("mtime") - if stored_mtime is None: - files_to_upload.append(file_info.relative_path) - continue - - if abs(file_info.mtime - stored_mtime) >= mtime_tolerance: - files_to_upload.append(file_info.relative_path) - - return {"files_to_upload": files_to_upload} - - -@router.post("/documents/folder-upload") -async def folder_upload( - files: list[UploadFile], - folder_name: str = Form(...), - search_space_id: int = Form(...), - relative_paths: str = Form(...), - root_folder_id: int | None = Form(None), - enable_summary: bool = Form(False), - session: AsyncSession = Depends(get_async_session), - user: User = Depends(current_active_user), -): - """Upload files from the desktop app for folder indexing. - - Files are written to temp storage and dispatched to a Celery task. - Works for all deployment modes (no is_self_hosted guard). - """ - import json - import tempfile - - await check_permission( - session, - user, - search_space_id, - Permission.DOCUMENTS_CREATE.value, - "You don't have permission to create documents in this search space", - ) - - if not files: - raise HTTPException(status_code=400, detail="No files provided") - - try: - rel_paths: list[str] = json.loads(relative_paths) - except (json.JSONDecodeError, TypeError) as e: - raise HTTPException( - status_code=400, detail=f"Invalid relative_paths JSON: {e}" - ) from e - - if len(rel_paths) != len(files): - raise HTTPException( - status_code=400, - detail=f"Mismatch: {len(files)} files but {len(rel_paths)} relative_paths", - ) - - for file in files: - file_size = file.size or 0 - if file_size > MAX_FILE_SIZE_BYTES: - raise HTTPException( - status_code=413, - detail=f"File '{file.filename}' ({file_size / (1024 * 1024):.1f} MB) " - f"exceeds the {MAX_FILE_SIZE_BYTES // (1024 * 1024)} MB per-file limit.", - ) + root_folder_id = request.root_folder_id + if root_folder_id: + existing = ( + await session.execute(select(Folder).where(Folder.id == root_folder_id)) + ).scalar_one_or_none() + if not existing: + root_folder_id = None + else: + existing.folder_metadata = watched_metadata + await session.commit() if not root_folder_id: - watched_metadata = { - "watched": True, - "folder_path": folder_name, - } - existing_root = ( - await session.execute( - select(Folder).where( - Folder.name == folder_name, - Folder.parent_id.is_(None), - Folder.search_space_id == search_space_id, - ) - ) - ).scalar_one_or_none() - - if existing_root: - root_folder_id = existing_root.id - existing_root.folder_metadata = watched_metadata - else: - root_folder = Folder( - name=folder_name, - search_space_id=search_space_id, - created_by_id=str(user.id), - position="a0", - folder_metadata=watched_metadata, - ) - session.add(root_folder) - await session.flush() - root_folder_id = root_folder.id - + root_folder = Folder( + name=request.folder_name, + search_space_id=request.search_space_id, + created_by_id=str(user.id), + position="a0", + folder_metadata=watched_metadata, + ) + session.add(root_folder) + await session.flush() + root_folder_id = root_folder.id await session.commit() - async def _read_and_save(file: UploadFile, idx: int) -> dict: - content = await file.read() - filename = file.filename or rel_paths[idx].split("/")[-1] + from app.tasks.celery_tasks.document_tasks import index_local_folder_task - def _write_temp() -> str: - with tempfile.NamedTemporaryFile( - delete=False, suffix=os.path.splitext(filename)[1] - ) as tmp: - tmp.write(content) - return tmp.name - - temp_path = await asyncio.to_thread(_write_temp) - return { - "temp_path": temp_path, - "relative_path": rel_paths[idx], - "filename": filename, - } - - file_mappings = await asyncio.gather( - *(_read_and_save(f, i) for i, f in enumerate(files)) - ) - - from app.tasks.celery_tasks.document_tasks import ( - index_uploaded_folder_files_task, - ) - - index_uploaded_folder_files_task.delay( - search_space_id=search_space_id, + index_local_folder_task.delay( + search_space_id=request.search_space_id, user_id=str(user.id), - folder_name=folder_name, + folder_path=request.folder_path, + folder_name=request.folder_name, + exclude_patterns=request.exclude_patterns, + file_extensions=request.file_extensions, root_folder_id=root_folder_id, - enable_summary=enable_summary, - file_mappings=list(file_mappings), + enable_summary=request.enable_summary, ) return { - "message": f"Folder upload started for {len(files)} file(s)", + "message": "Folder indexing started", "status": "processing", "root_folder_id": root_folder_id, - "file_count": len(files), } -@router.post("/documents/folder-unlink") -async def folder_unlink( - request: FolderUnlinkRequest, +@router.post("/documents/folder-index-files") +async def folder_index_files( + request: FolderIndexFilesRequest, session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): - """Handle file deletion events from the desktop watcher. - - For each relative path, find the matching document and delete it. + """Index multiple files within a watched folder (batched chokidar trigger). + Validates that all target_file_paths are under folder_path. + Dispatches a single Celery task that processes them in parallel. """ - from app.indexing_pipeline.document_hashing import compute_identifier_hash - from app.tasks.connector_indexers.local_folder_indexer import ( - _cleanup_empty_folder_chain, - ) + from app.config import config as app_config + + if not app_config.is_self_hosted(): + raise HTTPException( + status_code=400, + detail="Local folder indexing is only available in self-hosted mode", + ) + + if not request.target_file_paths: + raise HTTPException( + status_code=400, detail="target_file_paths must not be empty" + ) await check_permission( session, user, request.search_space_id, - Permission.DOCUMENTS_DELETE.value, - "You don't have permission to delete documents in this search space", + Permission.DOCUMENTS_CREATE.value, + "You don't have permission to create documents in this search space", ) - deleted_count = 0 + from pathlib import Path - for rel_path in request.relative_paths: - unique_id = f"{request.folder_name}:{rel_path}" - uid_hash = compute_identifier_hash( - DocumentType.LOCAL_FOLDER_FILE.value, - unique_id, - request.search_space_id, - ) + for fp in request.target_file_paths: + try: + Path(fp).relative_to(request.folder_path) + except ValueError as err: + raise HTTPException( + status_code=400, + detail=f"target_file_path {fp} must be inside folder_path", + ) from err - existing = ( - await session.execute( - select(Document).where(Document.unique_identifier_hash == uid_hash) - ) - ).scalar_one_or_none() + from app.tasks.celery_tasks.document_tasks import index_local_folder_task - if existing: - deleted_folder_id = existing.folder_id - await session.delete(existing) - await session.flush() - - if deleted_folder_id and request.root_folder_id: - await _cleanup_empty_folder_chain( - session, deleted_folder_id, request.root_folder_id - ) - deleted_count += 1 - - await session.commit() - return {"deleted_count": deleted_count} - - -@router.post("/documents/folder-sync-finalize") -async def folder_sync_finalize( - request: FolderSyncFinalizeRequest, - session: AsyncSession = Depends(get_async_session), - user: User = Depends(current_active_user), -): - """Finalize a full folder scan by deleting orphaned documents. - - The client sends the complete list of relative paths currently in the - folder. Any document in the DB for this folder that is NOT in the list - gets deleted. - """ - from app.indexing_pipeline.document_hashing import compute_identifier_hash - from app.services.folder_service import get_folder_subtree_ids - from app.tasks.connector_indexers.local_folder_indexer import ( - _cleanup_empty_folders, + index_local_folder_task.delay( + search_space_id=request.search_space_id, + user_id=str(user.id), + folder_path=request.folder_path, + folder_name=request.folder_name, + target_file_paths=request.target_file_paths, + root_folder_id=request.root_folder_id, + enable_summary=request.enable_summary, ) - await check_permission( - session, - user, - request.search_space_id, - Permission.DOCUMENTS_DELETE.value, - "You don't have permission to delete documents in this search space", - ) - - if not request.root_folder_id: - return {"deleted_count": 0} - - subtree_ids = await get_folder_subtree_ids(session, request.root_folder_id) - - seen_hashes: set[str] = set() - for rel_path in request.all_relative_paths: - unique_id = f"{request.folder_name}:{rel_path}" - uid_hash = compute_identifier_hash( - DocumentType.LOCAL_FOLDER_FILE.value, - unique_id, - request.search_space_id, - ) - seen_hashes.add(uid_hash) - - all_folder_docs = ( - ( - await session.execute( - select(Document).where( - Document.document_type == DocumentType.LOCAL_FOLDER_FILE, - Document.search_space_id == request.search_space_id, - Document.folder_id.in_(subtree_ids), - ) - ) - ) - .scalars() - .all() - ) - - deleted_count = 0 - for doc in all_folder_docs: - if doc.unique_identifier_hash not in seen_hashes: - await session.delete(doc) - deleted_count += 1 - - await session.flush() - - existing_dirs: set[str] = set() - for rel_path in request.all_relative_paths: - parent = str(os.path.dirname(rel_path)) - if parent and parent != ".": - existing_dirs.add(parent) - - folder_mapping: dict[str, int] = {"": request.root_folder_id} - - await _cleanup_empty_folders( - session, - request.root_folder_id, - request.search_space_id, - existing_dirs, - folder_mapping, - subtree_ids=subtree_ids, - ) - - await session.commit() - return {"deleted_count": deleted_count} + return { + "message": f"Batch indexing started for {len(request.target_file_paths)} file(s)", + "status": "processing", + "file_count": len(request.target_file_paths), + } diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index 62720826f..4e9249d34 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -11,10 +11,7 @@ from app.config import config from app.services.notification_service import NotificationService from app.services.task_logging_service import TaskLoggingService from app.tasks.celery_tasks import get_celery_session_maker -from app.tasks.connector_indexers.local_folder_indexer import ( - index_local_folder, - index_uploaded_files, -) +from app.tasks.connector_indexers.local_folder_indexer import index_local_folder from app.tasks.document_processors import ( add_extension_received_document, add_youtube_video_document, @@ -1414,132 +1411,3 @@ async def _index_local_folder_async( heartbeat_task.cancel() if notification_id is not None: _stop_heartbeat(notification_id) - - -# ===== Upload-based folder indexing task ===== - - -@celery_app.task(name="index_uploaded_folder_files", bind=True) -def index_uploaded_folder_files_task( - self, - search_space_id: int, - user_id: str, - folder_name: str, - root_folder_id: int, - enable_summary: bool, - file_mappings: list[dict], -): - """Celery task to index files uploaded from the desktop app.""" - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - loop.run_until_complete( - _index_uploaded_folder_files_async( - search_space_id=search_space_id, - user_id=user_id, - folder_name=folder_name, - root_folder_id=root_folder_id, - enable_summary=enable_summary, - file_mappings=file_mappings, - ) - ) - finally: - loop.close() - - -async def _index_uploaded_folder_files_async( - search_space_id: int, - user_id: str, - folder_name: str, - root_folder_id: int, - enable_summary: bool, - file_mappings: list[dict], -): - """Run upload-based folder indexing with notification + heartbeat.""" - file_count = len(file_mappings) - doc_name = f"{folder_name} ({file_count} file{'s' if file_count != 1 else ''})" - - notification = None - notification_id: int | None = None - heartbeat_task = None - - async with get_celery_session_maker()() as session: - try: - notification = ( - await NotificationService.document_processing.notify_processing_started( - session=session, - user_id=UUID(user_id), - document_type="LOCAL_FOLDER_FILE", - document_name=doc_name, - search_space_id=search_space_id, - ) - ) - notification_id = notification.id - _start_heartbeat(notification_id) - heartbeat_task = asyncio.create_task(_run_heartbeat_loop(notification_id)) - except Exception: - logger.warning( - "Failed to create notification for uploaded folder indexing", - exc_info=True, - ) - - async def _heartbeat_progress(completed_count: int) -> None: - if notification: - with contextlib.suppress(Exception): - await NotificationService.document_processing.notify_processing_progress( - session=session, - notification=notification, - stage="indexing", - stage_message=f"Syncing files ({completed_count}/{file_count})", - ) - - try: - _indexed, _failed, err = await index_uploaded_files( - session=session, - search_space_id=search_space_id, - user_id=user_id, - folder_name=folder_name, - root_folder_id=root_folder_id, - enable_summary=enable_summary, - file_mappings=file_mappings, - on_heartbeat_callback=_heartbeat_progress, - ) - - if notification: - try: - await session.refresh(notification) - if err: - await NotificationService.document_processing.notify_processing_completed( - session=session, - notification=notification, - error_message=err, - ) - else: - await NotificationService.document_processing.notify_processing_completed( - session=session, - notification=notification, - ) - except Exception: - logger.warning( - "Failed to update notification after uploaded folder indexing", - exc_info=True, - ) - - except Exception as e: - logger.exception(f"Uploaded folder indexing failed: {e}") - if notification: - try: - await session.refresh(notification) - await NotificationService.document_processing.notify_processing_completed( - session=session, - notification=notification, - error_message=str(e)[:200], - ) - except Exception: - pass - raise - finally: - if heartbeat_task: - heartbeat_task.cancel() - if notification_id is not None: - _stop_heartbeat(notification_id) diff --git a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py index f503ff864..7f42f4638 100644 --- a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py @@ -14,14 +14,13 @@ no connector row is read. """ import asyncio -import contextlib import os from collections.abc import Awaitable, Callable from datetime import UTC, datetime from pathlib import Path from sqlalchemy import select -from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.exc import IntegrityError, SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession from app.db import ( @@ -179,22 +178,6 @@ def _content_hash(content: str, search_space_id: int) -> str: return hashlib.sha256(f"{search_space_id}:{content}".encode()).hexdigest() -def _compute_raw_file_hash(file_path: str) -> str: - """SHA-256 hash of the raw file bytes. - - Much cheaper than ETL/OCR extraction -- only performs sequential I/O. - Used as a pre-filter to skip expensive content extraction when the - underlying file hasn't changed at all. - """ - import hashlib - - h = hashlib.sha256() - with open(file_path, "rb") as f: - for chunk in iter(lambda: f.read(8192), b""): - h.update(chunk) - return h.hexdigest() - - async def _compute_file_content_hash( file_path: str, filename: str, @@ -345,27 +328,6 @@ async def _resolve_folder_for_file( return current_parent_id -async def _set_indexing_flag(session: AsyncSession, folder_id: int) -> None: - folder = await session.get(Folder, folder_id) - if folder: - meta = dict(folder.folder_metadata or {}) - meta["indexing_in_progress"] = True - folder.folder_metadata = meta - await session.commit() - - -async def _clear_indexing_flag(session: AsyncSession, folder_id: int) -> None: - try: - folder = await session.get(Folder, folder_id) - if folder: - meta = dict(folder.folder_metadata or {}) - meta.pop("indexing_in_progress", None) - folder.folder_metadata = meta - await session.commit() - except Exception: - pass - - async def _cleanup_empty_folder_chain( session: AsyncSession, folder_id: int, @@ -409,21 +371,24 @@ async def _cleanup_empty_folders( search_space_id: int, existing_dirs_on_disk: set[str], folder_mapping: dict[str, int], - subtree_ids: list[int] | None = None, ) -> None: """Delete Folder rows that are empty (no docs, no children) and no longer on disk.""" from sqlalchemy import delete as sa_delete id_to_rel: dict[int, str] = {fid: rel for rel, fid in folder_mapping.items() if rel} - query = select(Folder).where( - Folder.search_space_id == search_space_id, - Folder.id != root_folder_id, + all_folders = ( + ( + await session.execute( + select(Folder).where( + Folder.search_space_id == search_space_id, + Folder.id != root_folder_id, + ) + ) + ) + .scalars() + .all() ) - if subtree_ids is not None: - query = query.where(Folder.id.in_(subtree_ids)) - - all_folders = (await session.execute(query)).scalars().all() candidates: list[Folder] = [] for folder in all_folders: @@ -553,50 +518,44 @@ async def index_local_folder( # BATCH MODE (1..N files) # ==================================================================== if target_file_paths: - if root_folder_id: - await _set_indexing_flag(session, root_folder_id) - try: - if len(target_file_paths) == 1: - indexed, skipped, err = await _index_single_file( - session=session, - search_space_id=search_space_id, - user_id=user_id, - folder_path=folder_path, - folder_name=folder_name, - target_file_path=target_file_paths[0], - enable_summary=enable_summary, - root_folder_id=root_folder_id, - task_logger=task_logger, - log_entry=log_entry, - ) - return indexed, skipped, root_folder_id, err - - indexed, failed, err = await _index_batch_files( + if len(target_file_paths) == 1: + indexed, skipped, err = await _index_single_file( + session=session, search_space_id=search_space_id, user_id=user_id, folder_path=folder_path, folder_name=folder_name, - target_file_paths=target_file_paths, + target_file_path=target_file_paths[0], enable_summary=enable_summary, root_folder_id=root_folder_id, - on_progress_callback=on_heartbeat_callback, + task_logger=task_logger, + log_entry=log_entry, ) - if err: - await task_logger.log_task_success( - log_entry, - f"Batch indexing: {indexed} indexed, {failed} failed", - {"indexed": indexed, "failed": failed}, - ) - else: - await task_logger.log_task_success( - log_entry, - f"Batch indexing complete: {indexed} indexed", - {"indexed": indexed, "failed": failed}, - ) - return indexed, failed, root_folder_id, err - finally: - if root_folder_id: - await _clear_indexing_flag(session, root_folder_id) + return indexed, skipped, root_folder_id, err + + indexed, failed, err = await _index_batch_files( + search_space_id=search_space_id, + user_id=user_id, + folder_path=folder_path, + folder_name=folder_name, + target_file_paths=target_file_paths, + enable_summary=enable_summary, + root_folder_id=root_folder_id, + on_progress_callback=on_heartbeat_callback, + ) + if err: + await task_logger.log_task_success( + log_entry, + f"Batch indexing: {indexed} indexed, {failed} failed", + {"indexed": indexed, "failed": failed}, + ) + else: + await task_logger.log_task_success( + log_entry, + f"Batch indexing complete: {indexed} indexed", + {"indexed": indexed, "failed": failed}, + ) + return indexed, failed, root_folder_id, err # ==================================================================== # FULL-SCAN MODE @@ -616,7 +575,6 @@ async def index_local_folder( exclude_patterns=exclude_patterns, ) await session.flush() - await _set_indexing_flag(session, root_folder_id) try: files = scan_folder(folder_path, file_extensions, exclude_patterns) @@ -624,7 +582,6 @@ async def index_local_folder( await task_logger.log_task_failure( log_entry, f"Failed to scan folder: {e}", "Scan error", {} ) - await _clear_indexing_flag(session, root_folder_id) return 0, 0, root_folder_id, f"Failed to scan folder: {e}" logger.info(f"Found {len(files)} files in folder") @@ -673,24 +630,6 @@ async def index_local_folder( skipped_count += 1 continue - raw_hash = await asyncio.to_thread( - _compute_raw_file_hash, file_path_abs - ) - - stored_raw_hash = (existing_document.document_metadata or {}).get( - "raw_file_hash" - ) - if stored_raw_hash and stored_raw_hash == raw_hash: - meta = dict(existing_document.document_metadata or {}) - meta["mtime"] = current_mtime - existing_document.document_metadata = meta - if not DocumentStatus.is_state( - existing_document.status, DocumentStatus.READY - ): - existing_document.status = DocumentStatus.ready() - skipped_count += 1 - continue - try: estimated_pages = await _check_page_limit_or_skip( page_limit_service, user_id, file_path_abs @@ -714,7 +653,6 @@ async def index_local_folder( if existing_document.content_hash == content_hash: meta = dict(existing_document.document_metadata or {}) meta["mtime"] = current_mtime - meta["raw_file_hash"] = raw_hash existing_document.document_metadata = meta if not DocumentStatus.is_state( existing_document.status, DocumentStatus.READY @@ -749,10 +687,6 @@ async def index_local_folder( skipped_count += 1 continue - raw_hash = await asyncio.to_thread( - _compute_raw_file_hash, file_path_abs - ) - doc = _build_connector_doc( title=file_info["name"], content=content, @@ -768,7 +702,6 @@ async def index_local_folder( "mtime": file_info["modified_at"].timestamp(), "estimated_pages": estimated_pages, "content_length": len(content), - "raw_file_hash": raw_hash, } except Exception as e: @@ -820,16 +753,29 @@ async def index_local_folder( compute_unique_identifier_hash, ) - for cd in connector_docs: + pipeline = IndexingPipelineService(session) + doc_map = {compute_unique_identifier_hash(cd): cd for cd in connector_docs} + documents = await pipeline.prepare_for_indexing(connector_docs) + + # Assign folder_id immediately so docs appear in the correct + # folder while still pending/processing (visible via Zero sync). + for document in documents: + cd = doc_map.get(document.unique_identifier_hash) + if cd is None: + continue rel_path = (cd.metadata or {}).get("file_path", "") parent_dir = str(Path(rel_path).parent) if rel_path else "" if parent_dir == ".": parent_dir = "" - cd.folder_id = folder_mapping.get(parent_dir, folder_mapping.get("")) - - pipeline = IndexingPipelineService(session) - doc_map = {compute_unique_identifier_hash(cd): cd for cd in connector_docs} - documents = await pipeline.prepare_for_indexing(connector_docs) + document.folder_id = folder_mapping.get( + parent_dir, folder_mapping.get("") + ) + try: + await session.commit() + except IntegrityError: + await session.rollback() + for document in documents: + await session.refresh(document) llm = await get_user_long_context_llm(session, user_id, search_space_id) @@ -849,7 +795,6 @@ async def index_local_folder( doc_meta = dict(result.document_metadata or {}) doc_meta["mtime"] = mtime_info.get("mtime") - doc_meta["raw_file_hash"] = mtime_info.get("raw_file_hash") result.document_metadata = doc_meta est = mtime_info.get("estimated_pages", 1) @@ -878,16 +823,8 @@ async def index_local_folder( root_fid = folder_mapping.get("") if root_fid: - from app.services.folder_service import get_folder_subtree_ids - - subtree_ids = await get_folder_subtree_ids(session, root_fid) await _cleanup_empty_folders( - session, - root_fid, - search_space_id, - existing_dirs, - folder_mapping, - subtree_ids=subtree_ids, + session, root_fid, search_space_id, existing_dirs, folder_mapping ) try: @@ -914,7 +851,6 @@ async def index_local_folder( }, ) - await _clear_indexing_flag(session, root_folder_id) return indexed_count, skipped_count, root_folder_id, warning_message except SQLAlchemyError as e: @@ -923,8 +859,6 @@ async def index_local_folder( await task_logger.log_task_failure( log_entry, f"DB error: {e}", "Database error", {} ) - if root_folder_id: - await _clear_indexing_flag(session, root_folder_id) return 0, 0, root_folder_id, f"Database error: {e}" except Exception as e: @@ -932,8 +866,6 @@ async def index_local_folder( await task_logger.log_task_failure( log_entry, f"Error: {e}", "Unexpected error", {} ) - if root_folder_id: - await _clear_indexing_flag(session, root_folder_id) return 0, 0, root_folder_id, str(e) @@ -1056,22 +988,6 @@ async def _index_single_file( DocumentType.LOCAL_FOLDER_FILE.value, unique_id, search_space_id ) - raw_hash = await asyncio.to_thread(_compute_raw_file_hash, str(full_path)) - - existing = await check_document_by_unique_identifier(session, uid_hash) - - if existing: - stored_raw_hash = (existing.document_metadata or {}).get("raw_file_hash") - if stored_raw_hash and stored_raw_hash == raw_hash: - mtime = full_path.stat().st_mtime - meta = dict(existing.document_metadata or {}) - meta["mtime"] = mtime - existing.document_metadata = meta - if not DocumentStatus.is_state(existing.status, DocumentStatus.READY): - existing.status = DocumentStatus.ready() - await session.commit() - return 0, 0, None - page_limit_service = PageLimitService(session) try: estimated_pages = await _check_page_limit_or_skip( @@ -1090,12 +1006,13 @@ async def _index_single_file( if not content.strip(): return 0, 1, None + existing = await check_document_by_unique_identifier(session, uid_hash) + if existing: if existing.content_hash == content_hash: mtime = full_path.stat().st_mtime meta = dict(existing.document_metadata or {}) meta["mtime"] = mtime - meta["raw_file_hash"] = raw_hash existing.document_metadata = meta await session.commit() return 0, 1, None @@ -1114,11 +1031,6 @@ async def _index_single_file( enable_summary=enable_summary, ) - if root_folder_id: - connector_doc.folder_id = await _resolve_folder_for_file( - session, rel_path, root_folder_id, search_space_id, user_id - ) - pipeline = IndexingPipelineService(session) llm = await get_user_long_context_llm(session, user_id, search_space_id) documents = await pipeline.prepare_for_indexing([connector_doc]) @@ -1128,12 +1040,21 @@ async def _index_single_file( db_doc = documents[0] + if root_folder_id: + try: + db_doc.folder_id = await _resolve_folder_for_file( + session, rel_path, root_folder_id, search_space_id, user_id + ) + await session.commit() + except IntegrityError: + await session.rollback() + await session.refresh(db_doc) + await pipeline.index(db_doc, connector_doc, llm) await session.refresh(db_doc) doc_meta = dict(db_doc.document_metadata or {}) doc_meta["mtime"] = mtime - doc_meta["raw_file_hash"] = raw_hash db_doc.document_metadata = doc_meta await session.commit() @@ -1160,305 +1081,3 @@ async def _index_single_file( logger.exception(f"Error indexing single file {target_file_path}: {e}") await session.rollback() return 0, 0, str(e) - - -# ======================================================================== -# Upload-based folder indexing (works for all deployment modes) -# ======================================================================== - - -async def _mirror_folder_structure_from_paths( - session: AsyncSession, - relative_paths: list[str], - folder_name: str, - search_space_id: int, - user_id: str, - root_folder_id: int | None = None, -) -> tuple[dict[str, int], int]: - """Create DB Folder rows from a list of relative file paths. - - Unlike ``_mirror_folder_structure`` this does not walk the filesystem; - it derives the directory tree from the paths provided by the client. - - Returns (mapping, root_folder_id) where mapping is - relative_dir_path -> folder_id. The empty-string key maps to root. - """ - dir_set: set[str] = set() - for rp in relative_paths: - parent = str(Path(rp).parent) - if parent == ".": - continue - parts = Path(parent).parts - for i in range(len(parts)): - dir_set.add(str(Path(*parts[: i + 1]))) - - subdirs = sorted(dir_set, key=lambda p: p.count(os.sep)) - - mapping: dict[str, int] = {} - - if root_folder_id: - existing = ( - await session.execute(select(Folder).where(Folder.id == root_folder_id)) - ).scalar_one_or_none() - if existing: - mapping[""] = existing.id - else: - root_folder_id = None - - if not root_folder_id: - root_folder = Folder( - name=folder_name, - search_space_id=search_space_id, - created_by_id=user_id, - position="a0", - ) - session.add(root_folder) - await session.flush() - mapping[""] = root_folder.id - root_folder_id = root_folder.id - - for rel_dir in subdirs: - dir_parts = Path(rel_dir).parts - dir_name = dir_parts[-1] - parent_rel = str(Path(*dir_parts[:-1])) if len(dir_parts) > 1 else "" - - parent_id = mapping.get(parent_rel, mapping[""]) - - existing_folder = ( - await session.execute( - select(Folder).where( - Folder.name == dir_name, - Folder.parent_id == parent_id, - Folder.search_space_id == search_space_id, - ) - ) - ).scalar_one_or_none() - - if existing_folder: - mapping[rel_dir] = existing_folder.id - else: - new_folder = Folder( - name=dir_name, - parent_id=parent_id, - search_space_id=search_space_id, - created_by_id=user_id, - position="a0", - ) - session.add(new_folder) - await session.flush() - mapping[rel_dir] = new_folder.id - - await session.flush() - return mapping, root_folder_id - - -UPLOAD_BATCH_CONCURRENCY = 5 - - -async def index_uploaded_files( - session: AsyncSession, - search_space_id: int, - user_id: str, - folder_name: str, - root_folder_id: int, - enable_summary: bool, - file_mappings: list[dict], - on_heartbeat_callback: HeartbeatCallbackType | None = None, -) -> tuple[int, int, str | None]: - """Index files uploaded from the desktop app via temp paths. - - Each entry in *file_mappings* is ``{temp_path, relative_path, filename}``. - This function mirrors the folder structure from the provided relative - paths, then indexes each file exactly like ``_index_single_file`` but - reads from the temp path. Temp files are cleaned up after processing. - - Returns ``(indexed_count, failed_count, error_summary_or_none)``. - """ - task_logger = TaskLoggingService(session, search_space_id) - log_entry = await task_logger.log_task_start( - task_name="local_folder_indexing", - source="uploaded_folder_indexing", - message=f"Indexing {len(file_mappings)} uploaded file(s) for {folder_name}", - metadata={"file_count": len(file_mappings)}, - ) - - try: - all_relative_paths = [m["relative_path"] for m in file_mappings] - _folder_mapping, root_folder_id = await _mirror_folder_structure_from_paths( - session=session, - relative_paths=all_relative_paths, - folder_name=folder_name, - search_space_id=search_space_id, - user_id=user_id, - root_folder_id=root_folder_id, - ) - await session.flush() - - await _set_indexing_flag(session, root_folder_id) - - page_limit_service = PageLimitService(session) - pipeline = IndexingPipelineService(session) - llm = await get_user_long_context_llm(session, user_id, search_space_id) - - indexed_count = 0 - failed_count = 0 - errors: list[str] = [] - - for i, mapping in enumerate(file_mappings): - temp_path = mapping["temp_path"] - relative_path = mapping["relative_path"] - filename = mapping["filename"] - - try: - unique_id = f"{folder_name}:{relative_path}" - uid_hash = compute_identifier_hash( - DocumentType.LOCAL_FOLDER_FILE.value, - unique_id, - search_space_id, - ) - - raw_hash = await asyncio.to_thread(_compute_raw_file_hash, temp_path) - - existing = await check_document_by_unique_identifier(session, uid_hash) - - if existing: - stored_raw_hash = (existing.document_metadata or {}).get( - "raw_file_hash" - ) - if stored_raw_hash and stored_raw_hash == raw_hash: - meta = dict(existing.document_metadata or {}) - meta["mtime"] = datetime.now(UTC).timestamp() - existing.document_metadata = meta - if not DocumentStatus.is_state( - existing.status, DocumentStatus.READY - ): - existing.status = DocumentStatus.ready() - await session.commit() - continue - - try: - estimated_pages = await _check_page_limit_or_skip( - page_limit_service, user_id, temp_path - ) - except PageLimitExceededError: - logger.warning(f"Page limit exceeded, skipping: {relative_path}") - failed_count += 1 - continue - - try: - content, content_hash = await _compute_file_content_hash( - temp_path, filename, search_space_id - ) - except Exception as e: - logger.warning(f"Could not read {relative_path}: {e}") - failed_count += 1 - errors.append(f"{filename}: {e}") - continue - - if not content.strip(): - failed_count += 1 - continue - - if existing: - if existing.content_hash == content_hash: - meta = dict(existing.document_metadata or {}) - meta["mtime"] = datetime.now(UTC).timestamp() - meta["raw_file_hash"] = raw_hash - existing.document_metadata = meta - if not DocumentStatus.is_state( - existing.status, DocumentStatus.READY - ): - existing.status = DocumentStatus.ready() - await session.commit() - continue - - await create_version_snapshot(session, existing) - - connector_doc = _build_connector_doc( - title=filename, - content=content, - relative_path=relative_path, - folder_name=folder_name, - search_space_id=search_space_id, - user_id=user_id, - enable_summary=enable_summary, - ) - - connector_doc.folder_id = await _resolve_folder_for_file( - session, - relative_path, - root_folder_id, - search_space_id, - user_id, - ) - - documents = await pipeline.prepare_for_indexing([connector_doc]) - if not documents: - failed_count += 1 - continue - - db_doc = documents[0] - - await pipeline.index(db_doc, connector_doc, llm) - - await session.refresh(db_doc) - doc_meta = dict(db_doc.document_metadata or {}) - doc_meta["mtime"] = datetime.now(UTC).timestamp() - doc_meta["raw_file_hash"] = raw_hash - db_doc.document_metadata = doc_meta - await session.commit() - - if DocumentStatus.is_state(db_doc.status, DocumentStatus.READY): - indexed_count += 1 - final_pages = _compute_final_pages( - page_limit_service, estimated_pages, len(content) - ) - await page_limit_service.update_page_usage( - user_id, final_pages, allow_exceed=True - ) - else: - failed_count += 1 - - if on_heartbeat_callback and (i + 1) % 5 == 0: - await on_heartbeat_callback(i + 1) - - except Exception as e: - logger.exception(f"Error indexing uploaded file {relative_path}: {e}") - await session.rollback() - failed_count += 1 - errors.append(f"{filename}: {e}") - finally: - with contextlib.suppress(OSError): - os.unlink(temp_path) - - error_summary = None - if errors: - error_summary = f"{failed_count} file(s) failed: " + "; ".join(errors[:5]) - if len(errors) > 5: - error_summary += f" ... and {len(errors) - 5} more" - - await task_logger.log_task_success( - log_entry, - f"Upload indexing complete: {indexed_count} indexed, {failed_count} failed", - {"indexed": indexed_count, "failed": failed_count}, - ) - - return indexed_count, failed_count, error_summary - - except SQLAlchemyError as e: - logger.exception(f"Database error during uploaded file indexing: {e}") - await session.rollback() - await task_logger.log_task_failure( - log_entry, f"DB error: {e}", "Database error", {} - ) - return 0, 0, f"Database error: {e}" - - except Exception as e: - logger.exception(f"Error during uploaded file indexing: {e}") - await task_logger.log_task_failure( - log_entry, f"Error: {e}", "Unexpected error", {} - ) - return 0, 0, str(e) - - finally: - await _clear_indexing_flag(session, root_folder_id) diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py index 4dc5742f7..000f43aa8 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py @@ -1,4 +1,4 @@ -"""Integration tests for local folder indexer — Tier 3 (I1-I5), Tier 4 (F1-F7), Tier 5 (P1), Tier 6 (B1-B2), Tier 7 (IP1-IP3).""" +"""Integration tests for local folder indexer — Tier 3 (I1-I5), Tier 4 (F1-F7), Tier 5 (P1), Tier 6 (B1-B2).""" import os from contextlib import asynccontextmanager @@ -1178,131 +1178,3 @@ class TestPageLimits: await db_session.refresh(db_user) assert db_user.pages_used > 0 assert db_user.pages_used <= db_user.pages_limit + 1 - - -# ==================================================================== -# Tier 7: Indexing Progress Flag (IP1-IP3) -# ==================================================================== - - -class TestIndexingProgressFlag: - @pytest.mark.usefixtures(*UNIFIED_FIXTURES) - async def test_ip1_full_scan_clears_flag( - self, - db_session: AsyncSession, - db_user: User, - db_search_space: SearchSpace, - tmp_path: Path, - ): - """IP1: Full-scan mode clears indexing_in_progress after completion.""" - from app.tasks.connector_indexers.local_folder_indexer import index_local_folder - - (tmp_path / "note.md").write_text("# Hello\n\nContent.") - - _, _, root_folder_id, _ = await index_local_folder( - session=db_session, - search_space_id=db_search_space.id, - user_id=str(db_user.id), - folder_path=str(tmp_path), - folder_name="test-folder", - ) - - assert root_folder_id is not None - root_folder = ( - await db_session.execute(select(Folder).where(Folder.id == root_folder_id)) - ).scalar_one() - meta = root_folder.folder_metadata or {} - assert "indexing_in_progress" not in meta - - @pytest.mark.usefixtures(*UNIFIED_FIXTURES) - async def test_ip2_single_file_clears_flag( - self, - db_session: AsyncSession, - db_user: User, - db_search_space: SearchSpace, - tmp_path: Path, - ): - """IP2: Single-file (Chokidar) mode clears indexing_in_progress after completion.""" - from app.tasks.connector_indexers.local_folder_indexer import index_local_folder - - (tmp_path / "root.md").write_text("root") - _, _, root_folder_id, _ = await index_local_folder( - session=db_session, - search_space_id=db_search_space.id, - user_id=str(db_user.id), - folder_path=str(tmp_path), - folder_name="test-folder", - ) - - (tmp_path / "new.md").write_text("new file content") - - await index_local_folder( - session=db_session, - search_space_id=db_search_space.id, - user_id=str(db_user.id), - folder_path=str(tmp_path), - folder_name="test-folder", - target_file_paths=[str(tmp_path / "new.md")], - root_folder_id=root_folder_id, - ) - - root_folder = ( - await db_session.execute(select(Folder).where(Folder.id == root_folder_id)) - ).scalar_one() - meta = root_folder.folder_metadata or {} - assert "indexing_in_progress" not in meta - - @pytest.mark.usefixtures(*UNIFIED_FIXTURES) - async def test_ip3_flag_set_during_indexing( - self, - db_session: AsyncSession, - db_user: User, - db_search_space: SearchSpace, - tmp_path: Path, - ): - """IP3: indexing_in_progress is True on the root folder while indexing is running.""" - from app.tasks.connector_indexers.local_folder_indexer import index_local_folder - - (tmp_path / "note.md").write_text("# Check flag\n\nDuring indexing.") - - from app.indexing_pipeline.indexing_pipeline_service import ( - IndexingPipelineService, - ) - - original_index = IndexingPipelineService.index - flag_observed = [] - - async def patched_index(self_pipe, document, connector_doc, llm): - folder = ( - await db_session.execute( - select(Folder).where( - Folder.search_space_id == db_search_space.id, - Folder.parent_id.is_(None), - ) - ) - ).scalar_one_or_none() - if folder: - meta = folder.folder_metadata or {} - flag_observed.append(meta.get("indexing_in_progress", False)) - return await original_index(self_pipe, document, connector_doc, llm) - - IndexingPipelineService.index = patched_index - try: - _, _, root_folder_id, _ = await index_local_folder( - session=db_session, - search_space_id=db_search_space.id, - user_id=str(db_user.id), - folder_path=str(tmp_path), - folder_name="test-folder", - ) - finally: - IndexingPipelineService.index = original_index - - assert len(flag_observed) > 0, "index() should have been called at least once" - assert all(flag_observed), "indexing_in_progress should be True during indexing" - - root_folder = ( - await db_session.execute(select(Folder).where(Folder.id == root_folder_id)) - ).scalar_one() - meta = root_folder.folder_metadata or {} - assert "indexing_in_progress" not in meta diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts index 61213eb46..39e75f046 100644 --- a/surfsense_desktop/src/ipc/channels.ts +++ b/surfsense_desktop/src/ipc/channels.ts @@ -30,8 +30,6 @@ export const IPC_CHANNELS = { FOLDER_SYNC_RENDERER_READY: 'folder-sync:renderer-ready', FOLDER_SYNC_GET_PENDING_EVENTS: 'folder-sync:get-pending-events', FOLDER_SYNC_ACK_EVENTS: 'folder-sync:ack-events', - FOLDER_SYNC_LIST_FILES: 'folder-sync:list-files', - FOLDER_SYNC_SEED_MTIMES: 'folder-sync:seed-mtimes', BROWSE_FILES: 'browse:files', READ_LOCAL_FILES: 'browse:read-local-files', // Auth token sync across windows diff --git a/surfsense_desktop/src/ipc/handlers.ts b/surfsense_desktop/src/ipc/handlers.ts index afb2ba038..200fa75bd 100644 --- a/surfsense_desktop/src/ipc/handlers.ts +++ b/surfsense_desktop/src/ipc/handlers.ts @@ -19,9 +19,6 @@ import { markRendererReady, browseFiles, readLocalFiles, - listFolderFiles, - seedFolderMtimes, - type WatchedFolderConfig, } from '../modules/folder-watcher'; import { getShortcuts, setShortcuts, type ShortcutConfig } from '../modules/shortcuts'; import { getActiveSearchSpaceId, setActiveSearchSpaceId } from '../modules/active-search-space'; @@ -94,16 +91,6 @@ export function registerIpcHandlers(): void { acknowledgeFileEvents(eventIds) ); - ipcMain.handle(IPC_CHANNELS.FOLDER_SYNC_LIST_FILES, (_event, config: WatchedFolderConfig) => - listFolderFiles(config) - ); - - ipcMain.handle( - IPC_CHANNELS.FOLDER_SYNC_SEED_MTIMES, - (_event, folderPath: string, mtimes: Record) => - seedFolderMtimes(folderPath, mtimes), - ); - ipcMain.handle(IPC_CHANNELS.BROWSE_FILES, () => browseFiles()); ipcMain.handle(IPC_CHANNELS.READ_LOCAL_FILES, (_event, paths: string[]) => diff --git a/surfsense_desktop/src/modules/folder-watcher.ts b/surfsense_desktop/src/modules/folder-watcher.ts index 96b490d7b..969dabe97 100644 --- a/surfsense_desktop/src/modules/folder-watcher.ts +++ b/surfsense_desktop/src/modules/folder-watcher.ts @@ -188,31 +188,6 @@ function walkFolderMtimes(config: WatchedFolderConfig): MtimeMap { return result; } -export interface FolderFileEntry { - relativePath: string; - fullPath: string; - size: number; - mtimeMs: number; -} - -export function listFolderFiles(config: WatchedFolderConfig): FolderFileEntry[] { - const root = config.path; - const mtimeMap = walkFolderMtimes(config); - const entries: FolderFileEntry[] = []; - - for (const [relativePath, mtimeMs] of Object.entries(mtimeMap)) { - const fullPath = path.join(root, relativePath); - try { - const stat = fs.statSync(fullPath); - entries.push({ relativePath, fullPath, size: stat.size, mtimeMs }); - } catch { - // File may have been removed between walk and stat - } - } - - return entries; -} - function getMainWindow(): BrowserWindow | null { const windows = BrowserWindow.getAllWindows(); return windows.length > 0 ? windows[0] : null; @@ -449,30 +424,14 @@ export async function acknowledgeFileEvents(eventIds: string[]): Promise<{ ackno const ackSet = new Set(eventIds); let acknowledged = 0; - const foldersToUpdate = new Set(); for (const [key, event] of outboxEvents.entries()) { if (ackSet.has(event.id)) { - if (event.action !== 'unlink') { - const map = mtimeMaps.get(event.folderPath); - if (map) { - try { - map[event.relativePath] = fs.statSync(event.fullPath).mtimeMs; - foldersToUpdate.add(event.folderPath); - } catch { - // File may have been removed - } - } - } outboxEvents.delete(key); acknowledged += 1; } } - for (const fp of foldersToUpdate) { - persistMtimeMap(fp); - } - if (acknowledged > 0) { persistOutbox(); } @@ -480,17 +439,6 @@ export async function acknowledgeFileEvents(eventIds: string[]): Promise<{ ackno return { acknowledged }; } -export async function seedFolderMtimes( - folderPath: string, - mtimes: Record, -): Promise { - const ms = await getMtimeStore(); - const existing: MtimeMap = ms.get(folderPath) ?? {}; - const merged = { ...existing, ...mtimes }; - mtimeMaps.set(folderPath, merged); - ms.set(folderPath, merged); -} - export async function pauseWatcher(): Promise { for (const [, entry] of watchers) { if (entry.watcher) { diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index e3d12c5e6..4d9537c91 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -64,9 +64,6 @@ contextBridge.exposeInMainWorld('electronAPI', { signalRendererReady: () => ipcRenderer.invoke(IPC_CHANNELS.FOLDER_SYNC_RENDERER_READY), getPendingFileEvents: () => ipcRenderer.invoke(IPC_CHANNELS.FOLDER_SYNC_GET_PENDING_EVENTS), acknowledgeFileEvents: (eventIds: string[]) => ipcRenderer.invoke(IPC_CHANNELS.FOLDER_SYNC_ACK_EVENTS, eventIds), - listFolderFiles: (config: any) => ipcRenderer.invoke(IPC_CHANNELS.FOLDER_SYNC_LIST_FILES, config), - seedFolderMtimes: (folderPath: string, mtimes: Record) => - ipcRenderer.invoke(IPC_CHANNELS.FOLDER_SYNC_SEED_MTIMES, folderPath, mtimes), // Browse files via native dialog browseFiles: () => ipcRenderer.invoke(IPC_CHANNELS.BROWSE_FILES), diff --git a/surfsense_web/app/(home)/login/page.tsx b/surfsense_web/app/(home)/login/page.tsx index 3dbbf21a9..161951d97 100644 --- a/surfsense_web/app/(home)/login/page.tsx +++ b/surfsense_web/app/(home)/login/page.tsx @@ -1,7 +1,8 @@ "use client"; import { AnimatePresence, motion } from "motion/react"; -import { useRouter, useSearchParams } from "next/navigation"; +import { useRouter } from "next/navigation"; +import { useSearchParams } from "next/navigation"; import { useTranslations } from "next-intl"; import { Suspense, useEffect, useState } from "react"; import { toast } from "sonner"; diff --git a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx index c5be2b590..443c964cf 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx @@ -46,6 +46,7 @@ import { useParams } from "next/navigation"; import { useTranslations } from "next-intl"; import React, { useCallback, useContext, useEffect, useId, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; +import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { createLogMutationAtom, deleteLogMutationAtom, @@ -95,7 +96,6 @@ import { TableRow, } from "@/components/ui/table"; import type { CreateLogRequest, Log, UpdateLogRequest } from "@/contracts/types/log.types"; -import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { type LogLevel, type LogStatus, useLogs, useLogsSummary } from "@/hooks/use-logs"; import { cn } from "@/lib/utils"; @@ -728,7 +728,10 @@ function LogsFilters({ setFilterInput(e.target.value)} placeholder={t("filter_by_message")} diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index b545a6be2..7bef1fff2 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -38,29 +38,18 @@ import { removeChatTabAtom, updateChatTabTitleAtom } from "@/atoms/tabs/tabs.ato import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { ThinkingStepsDataUI } from "@/components/assistant-ui/thinking-steps"; import { Thread } from "@/components/assistant-ui/thread"; -import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; -import { useMessagesSync } from "@/hooks/use-messages-sync"; import Loading from "../loading"; const MobileEditorPanel = dynamic( - () => - import("@/components/editor-panel/editor-panel").then((m) => ({ - default: m.MobileEditorPanel, - })), + () => import("@/components/editor-panel/editor-panel").then((m) => ({ default: m.MobileEditorPanel })), { ssr: false } ); const MobileHitlEditPanel = dynamic( - () => - import("@/components/hitl-edit-panel/hitl-edit-panel").then((m) => ({ - default: m.MobileHitlEditPanel, - })), + () => import("@/components/hitl-edit-panel/hitl-edit-panel").then((m) => ({ default: m.MobileHitlEditPanel })), { ssr: false } ); const MobileReportPanel = dynamic( - () => - import("@/components/report-panel/report-panel").then((m) => ({ - default: m.MobileReportPanel, - })), + () => import("@/components/report-panel/report-panel").then((m) => ({ default: m.MobileReportPanel })), { ssr: false } ); diff --git a/surfsense_web/components/assistant-ui/assistant-message.tsx b/surfsense_web/components/assistant-ui/assistant-message.tsx index 097197107..626e6237f 100644 --- a/surfsense_web/components/assistant-ui/assistant-message.tsx +++ b/surfsense_web/components/assistant-ui/assistant-message.tsx @@ -51,172 +51,131 @@ const IS_QUICK_ASSIST_WINDOW = // Dynamically import tool UI components to avoid loading them in main bundle const GenerateReportToolUI = dynamic( - () => - import("@/components/tool-ui/generate-report").then((m) => ({ - default: m.GenerateReportToolUI, - })), + () => import("@/components/tool-ui/generate-report").then(m => ({ default: m.GenerateReportToolUI })), { ssr: false } ); const GeneratePodcastToolUI = dynamic( - () => - import("@/components/tool-ui/generate-podcast").then((m) => ({ - default: m.GeneratePodcastToolUI, - })), + () => import("@/components/tool-ui/generate-podcast").then(m => ({ default: m.GeneratePodcastToolUI })), { ssr: false } ); const GenerateVideoPresentationToolUI = dynamic( - () => - import("@/components/tool-ui/video-presentation").then((m) => ({ - default: m.GenerateVideoPresentationToolUI, - })), + () => import("@/components/tool-ui/video-presentation").then(m => ({ default: m.GenerateVideoPresentationToolUI })), { ssr: false } ); const GenerateImageToolUI = dynamic( - () => - import("@/components/tool-ui/generate-image").then((m) => ({ default: m.GenerateImageToolUI })), + () => import("@/components/tool-ui/generate-image").then(m => ({ default: m.GenerateImageToolUI })), { ssr: false } ); const SaveMemoryToolUI = dynamic( - () => import("@/components/tool-ui/user-memory").then((m) => ({ default: m.SaveMemoryToolUI })), + () => import("@/components/tool-ui/user-memory").then(m => ({ default: m.SaveMemoryToolUI })), { ssr: false } ); const RecallMemoryToolUI = dynamic( - () => import("@/components/tool-ui/user-memory").then((m) => ({ default: m.RecallMemoryToolUI })), + () => import("@/components/tool-ui/user-memory").then(m => ({ default: m.RecallMemoryToolUI })), { ssr: false } ); const SandboxExecuteToolUI = dynamic( - () => - import("@/components/tool-ui/sandbox-execute").then((m) => ({ - default: m.SandboxExecuteToolUI, - })), + () => import("@/components/tool-ui/sandbox-execute").then(m => ({ default: m.SandboxExecuteToolUI })), { ssr: false } ); const CreateNotionPageToolUI = dynamic( - () => import("@/components/tool-ui/notion").then((m) => ({ default: m.CreateNotionPageToolUI })), + () => import("@/components/tool-ui/notion").then(m => ({ default: m.CreateNotionPageToolUI })), { ssr: false } ); const UpdateNotionPageToolUI = dynamic( - () => import("@/components/tool-ui/notion").then((m) => ({ default: m.UpdateNotionPageToolUI })), + () => import("@/components/tool-ui/notion").then(m => ({ default: m.UpdateNotionPageToolUI })), { ssr: false } ); const DeleteNotionPageToolUI = dynamic( - () => import("@/components/tool-ui/notion").then((m) => ({ default: m.DeleteNotionPageToolUI })), + () => import("@/components/tool-ui/notion").then(m => ({ default: m.DeleteNotionPageToolUI })), { ssr: false } ); const CreateLinearIssueToolUI = dynamic( - () => import("@/components/tool-ui/linear").then((m) => ({ default: m.CreateLinearIssueToolUI })), + () => import("@/components/tool-ui/linear").then(m => ({ default: m.CreateLinearIssueToolUI })), { ssr: false } ); const UpdateLinearIssueToolUI = dynamic( - () => import("@/components/tool-ui/linear").then((m) => ({ default: m.UpdateLinearIssueToolUI })), + () => import("@/components/tool-ui/linear").then(m => ({ default: m.UpdateLinearIssueToolUI })), { ssr: false } ); const DeleteLinearIssueToolUI = dynamic( - () => import("@/components/tool-ui/linear").then((m) => ({ default: m.DeleteLinearIssueToolUI })), + () => import("@/components/tool-ui/linear").then(m => ({ default: m.DeleteLinearIssueToolUI })), { ssr: false } ); const CreateGoogleDriveFileToolUI = dynamic( - () => - import("@/components/tool-ui/google-drive").then((m) => ({ - default: m.CreateGoogleDriveFileToolUI, - })), + () => import("@/components/tool-ui/google-drive").then(m => ({ default: m.CreateGoogleDriveFileToolUI })), { ssr: false } ); const DeleteGoogleDriveFileToolUI = dynamic( - () => - import("@/components/tool-ui/google-drive").then((m) => ({ - default: m.DeleteGoogleDriveFileToolUI, - })), + () => import("@/components/tool-ui/google-drive").then(m => ({ default: m.DeleteGoogleDriveFileToolUI })), { ssr: false } ); const CreateOneDriveFileToolUI = dynamic( - () => - import("@/components/tool-ui/onedrive").then((m) => ({ default: m.CreateOneDriveFileToolUI })), + () => import("@/components/tool-ui/onedrive").then(m => ({ default: m.CreateOneDriveFileToolUI })), { ssr: false } ); const DeleteOneDriveFileToolUI = dynamic( - () => - import("@/components/tool-ui/onedrive").then((m) => ({ default: m.DeleteOneDriveFileToolUI })), + () => import("@/components/tool-ui/onedrive").then(m => ({ default: m.DeleteOneDriveFileToolUI })), { ssr: false } ); const CreateDropboxFileToolUI = dynamic( - () => - import("@/components/tool-ui/dropbox").then((m) => ({ default: m.CreateDropboxFileToolUI })), + () => import("@/components/tool-ui/dropbox").then(m => ({ default: m.CreateDropboxFileToolUI })), { ssr: false } ); const DeleteDropboxFileToolUI = dynamic( - () => - import("@/components/tool-ui/dropbox").then((m) => ({ default: m.DeleteDropboxFileToolUI })), + () => import("@/components/tool-ui/dropbox").then(m => ({ default: m.DeleteDropboxFileToolUI })), { ssr: false } ); const CreateCalendarEventToolUI = dynamic( - () => - import("@/components/tool-ui/google-calendar").then((m) => ({ - default: m.CreateCalendarEventToolUI, - })), + () => import("@/components/tool-ui/google-calendar").then(m => ({ default: m.CreateCalendarEventToolUI })), { ssr: false } ); const UpdateCalendarEventToolUI = dynamic( - () => - import("@/components/tool-ui/google-calendar").then((m) => ({ - default: m.UpdateCalendarEventToolUI, - })), + () => import("@/components/tool-ui/google-calendar").then(m => ({ default: m.UpdateCalendarEventToolUI })), { ssr: false } ); const DeleteCalendarEventToolUI = dynamic( - () => - import("@/components/tool-ui/google-calendar").then((m) => ({ - default: m.DeleteCalendarEventToolUI, - })), + () => import("@/components/tool-ui/google-calendar").then(m => ({ default: m.DeleteCalendarEventToolUI })), { ssr: false } ); const CreateGmailDraftToolUI = dynamic( - () => import("@/components/tool-ui/gmail").then((m) => ({ default: m.CreateGmailDraftToolUI })), + () => import("@/components/tool-ui/gmail").then(m => ({ default: m.CreateGmailDraftToolUI })), { ssr: false } ); const UpdateGmailDraftToolUI = dynamic( - () => import("@/components/tool-ui/gmail").then((m) => ({ default: m.UpdateGmailDraftToolUI })), + () => import("@/components/tool-ui/gmail").then(m => ({ default: m.UpdateGmailDraftToolUI })), { ssr: false } ); const SendGmailEmailToolUI = dynamic( - () => import("@/components/tool-ui/gmail").then((m) => ({ default: m.SendGmailEmailToolUI })), + () => import("@/components/tool-ui/gmail").then(m => ({ default: m.SendGmailEmailToolUI })), { ssr: false } ); const TrashGmailEmailToolUI = dynamic( - () => import("@/components/tool-ui/gmail").then((m) => ({ default: m.TrashGmailEmailToolUI })), + () => import("@/components/tool-ui/gmail").then(m => ({ default: m.TrashGmailEmailToolUI })), { ssr: false } ); const CreateJiraIssueToolUI = dynamic( - () => import("@/components/tool-ui/jira").then((m) => ({ default: m.CreateJiraIssueToolUI })), + () => import("@/components/tool-ui/jira").then(m => ({ default: m.CreateJiraIssueToolUI })), { ssr: false } ); const UpdateJiraIssueToolUI = dynamic( - () => import("@/components/tool-ui/jira").then((m) => ({ default: m.UpdateJiraIssueToolUI })), + () => import("@/components/tool-ui/jira").then(m => ({ default: m.UpdateJiraIssueToolUI })), { ssr: false } ); const DeleteJiraIssueToolUI = dynamic( - () => import("@/components/tool-ui/jira").then((m) => ({ default: m.DeleteJiraIssueToolUI })), + () => import("@/components/tool-ui/jira").then(m => ({ default: m.DeleteJiraIssueToolUI })), { ssr: false } ); const CreateConfluencePageToolUI = dynamic( - () => - import("@/components/tool-ui/confluence").then((m) => ({ - default: m.CreateConfluencePageToolUI, - })), + () => import("@/components/tool-ui/confluence").then(m => ({ default: m.CreateConfluencePageToolUI })), { ssr: false } ); const UpdateConfluencePageToolUI = dynamic( - () => - import("@/components/tool-ui/confluence").then((m) => ({ - default: m.UpdateConfluencePageToolUI, - })), + () => import("@/components/tool-ui/confluence").then(m => ({ default: m.UpdateConfluencePageToolUI })), { ssr: false } ); const DeleteConfluencePageToolUI = dynamic( - () => - import("@/components/tool-ui/confluence").then((m) => ({ - default: m.DeleteConfluencePageToolUI, - })), + () => import("@/components/tool-ui/confluence").then(m => ({ default: m.DeleteConfluencePageToolUI })), { ssr: false } ); diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx index 24ead1552..20762c1b3 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx @@ -25,38 +25,16 @@ export interface ConnectFormProps { export type ConnectFormComponent = FC; const formMap: Record Promise<{ default: FC }>> = { - TAVILY_API: () => - import("./components/tavily-api-connect-form").then((m) => ({ - default: m.TavilyApiConnectForm, - })), - LINKUP_API: () => - import("./components/linkup-api-connect-form").then((m) => ({ - default: m.LinkupApiConnectForm, - })), - BAIDU_SEARCH_API: () => - import("./components/baidu-search-api-connect-form").then((m) => ({ - default: m.BaiduSearchApiConnectForm, - })), - ELASTICSEARCH_CONNECTOR: () => - import("./components/elasticsearch-connect-form").then((m) => ({ - default: m.ElasticsearchConnectForm, - })), - BOOKSTACK_CONNECTOR: () => - import("./components/bookstack-connect-form").then((m) => ({ - default: m.BookStackConnectForm, - })), - GITHUB_CONNECTOR: () => - import("./components/github-connect-form").then((m) => ({ default: m.GithubConnectForm })), - LUMA_CONNECTOR: () => - import("./components/luma-connect-form").then((m) => ({ default: m.LumaConnectForm })), - CIRCLEBACK_CONNECTOR: () => - import("./components/circleback-connect-form").then((m) => ({ - default: m.CirclebackConnectForm, - })), - MCP_CONNECTOR: () => - import("./components/mcp-connect-form").then((m) => ({ default: m.MCPConnectForm })), - OBSIDIAN_CONNECTOR: () => - import("./components/obsidian-connect-form").then((m) => ({ default: m.ObsidianConnectForm })), + TAVILY_API: () => import("./components/tavily-api-connect-form").then(m => ({ default: m.TavilyApiConnectForm })), + LINKUP_API: () => import("./components/linkup-api-connect-form").then(m => ({ default: m.LinkupApiConnectForm })), + BAIDU_SEARCH_API: () => import("./components/baidu-search-api-connect-form").then(m => ({ default: m.BaiduSearchApiConnectForm })), + ELASTICSEARCH_CONNECTOR: () => import("./components/elasticsearch-connect-form").then(m => ({ default: m.ElasticsearchConnectForm })), + BOOKSTACK_CONNECTOR: () => import("./components/bookstack-connect-form").then(m => ({ default: m.BookStackConnectForm })), + GITHUB_CONNECTOR: () => import("./components/github-connect-form").then(m => ({ default: m.GithubConnectForm })), + LUMA_CONNECTOR: () => import("./components/luma-connect-form").then(m => ({ default: m.LumaConnectForm })), + CIRCLEBACK_CONNECTOR: () => import("./components/circleback-connect-form").then(m => ({ default: m.CirclebackConnectForm })), + MCP_CONNECTOR: () => import("./components/mcp-connect-form").then(m => ({ default: m.MCPConnectForm })), + OBSIDIAN_CONNECTOR: () => import("./components/obsidian-connect-form").then(m => ({ default: m.ObsidianConnectForm })), }; const componentCache = new Map(); diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx index e3d6641fc..fc378612f 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx @@ -14,53 +14,29 @@ export interface ConnectorConfigProps { export type ConnectorConfigComponent = FC; const configMap: Record Promise<{ default: FC }>> = { - GOOGLE_DRIVE_CONNECTOR: () => - import("./components/google-drive-config").then((m) => ({ default: m.GoogleDriveConfig })), - TAVILY_API: () => - import("./components/tavily-api-config").then((m) => ({ default: m.TavilyApiConfig })), - LINKUP_API: () => - import("./components/linkup-api-config").then((m) => ({ default: m.LinkupApiConfig })), - BAIDU_SEARCH_API: () => - import("./components/baidu-search-api-config").then((m) => ({ - default: m.BaiduSearchApiConfig, - })), - WEBCRAWLER_CONNECTOR: () => - import("./components/webcrawler-config").then((m) => ({ default: m.WebcrawlerConfig })), - ELASTICSEARCH_CONNECTOR: () => - import("./components/elasticsearch-config").then((m) => ({ default: m.ElasticsearchConfig })), - SLACK_CONNECTOR: () => - import("./components/slack-config").then((m) => ({ default: m.SlackConfig })), - DISCORD_CONNECTOR: () => - import("./components/discord-config").then((m) => ({ default: m.DiscordConfig })), - TEAMS_CONNECTOR: () => - import("./components/teams-config").then((m) => ({ default: m.TeamsConfig })), - DROPBOX_CONNECTOR: () => - import("./components/dropbox-config").then((m) => ({ default: m.DropboxConfig })), - ONEDRIVE_CONNECTOR: () => - import("./components/onedrive-config").then((m) => ({ default: m.OneDriveConfig })), - CONFLUENCE_CONNECTOR: () => - import("./components/confluence-config").then((m) => ({ default: m.ConfluenceConfig })), - BOOKSTACK_CONNECTOR: () => - import("./components/bookstack-config").then((m) => ({ default: m.BookStackConfig })), - GITHUB_CONNECTOR: () => - import("./components/github-config").then((m) => ({ default: m.GithubConfig })), - JIRA_CONNECTOR: () => import("./components/jira-config").then((m) => ({ default: m.JiraConfig })), - CLICKUP_CONNECTOR: () => - import("./components/clickup-config").then((m) => ({ default: m.ClickUpConfig })), - LUMA_CONNECTOR: () => import("./components/luma-config").then((m) => ({ default: m.LumaConfig })), - CIRCLEBACK_CONNECTOR: () => - import("./components/circleback-config").then((m) => ({ default: m.CirclebackConfig })), - MCP_CONNECTOR: () => import("./components/mcp-config").then((m) => ({ default: m.MCPConfig })), - OBSIDIAN_CONNECTOR: () => - import("./components/obsidian-config").then((m) => ({ default: m.ObsidianConfig })), - COMPOSIO_GOOGLE_DRIVE_CONNECTOR: () => - import("./components/composio-drive-config").then((m) => ({ default: m.ComposioDriveConfig })), - COMPOSIO_GMAIL_CONNECTOR: () => - import("./components/composio-gmail-config").then((m) => ({ default: m.ComposioGmailConfig })), - COMPOSIO_GOOGLE_CALENDAR_CONNECTOR: () => - import("./components/composio-calendar-config").then((m) => ({ - default: m.ComposioCalendarConfig, - })), + GOOGLE_DRIVE_CONNECTOR: () => import("./components/google-drive-config").then(m => ({ default: m.GoogleDriveConfig })), + TAVILY_API: () => import("./components/tavily-api-config").then(m => ({ default: m.TavilyApiConfig })), + LINKUP_API: () => import("./components/linkup-api-config").then(m => ({ default: m.LinkupApiConfig })), + BAIDU_SEARCH_API: () => import("./components/baidu-search-api-config").then(m => ({ default: m.BaiduSearchApiConfig })), + WEBCRAWLER_CONNECTOR: () => import("./components/webcrawler-config").then(m => ({ default: m.WebcrawlerConfig })), + ELASTICSEARCH_CONNECTOR: () => import("./components/elasticsearch-config").then(m => ({ default: m.ElasticsearchConfig })), + SLACK_CONNECTOR: () => import("./components/slack-config").then(m => ({ default: m.SlackConfig })), + DISCORD_CONNECTOR: () => import("./components/discord-config").then(m => ({ default: m.DiscordConfig })), + TEAMS_CONNECTOR: () => import("./components/teams-config").then(m => ({ default: m.TeamsConfig })), + DROPBOX_CONNECTOR: () => import("./components/dropbox-config").then(m => ({ default: m.DropboxConfig })), + ONEDRIVE_CONNECTOR: () => import("./components/onedrive-config").then(m => ({ default: m.OneDriveConfig })), + CONFLUENCE_CONNECTOR: () => import("./components/confluence-config").then(m => ({ default: m.ConfluenceConfig })), + BOOKSTACK_CONNECTOR: () => import("./components/bookstack-config").then(m => ({ default: m.BookStackConfig })), + GITHUB_CONNECTOR: () => import("./components/github-config").then(m => ({ default: m.GithubConfig })), + JIRA_CONNECTOR: () => import("./components/jira-config").then(m => ({ default: m.JiraConfig })), + CLICKUP_CONNECTOR: () => import("./components/clickup-config").then(m => ({ default: m.ClickUpConfig })), + LUMA_CONNECTOR: () => import("./components/luma-config").then(m => ({ default: m.LumaConfig })), + CIRCLEBACK_CONNECTOR: () => import("./components/circleback-config").then(m => ({ default: m.CirclebackConfig })), + MCP_CONNECTOR: () => import("./components/mcp-config").then(m => ({ default: m.MCPConfig })), + OBSIDIAN_CONNECTOR: () => import("./components/obsidian-config").then(m => ({ default: m.ObsidianConfig })), + COMPOSIO_GOOGLE_DRIVE_CONNECTOR: () => import("./components/composio-drive-config").then(m => ({ default: m.ComposioDriveConfig })), + COMPOSIO_GMAIL_CONNECTOR: () => import("./components/composio-gmail-config").then(m => ({ default: m.ComposioGmailConfig })), + COMPOSIO_GOOGLE_CALENDAR_CONNECTOR: () => import("./components/composio-calendar-config").then(m => ({ default: m.ComposioCalendarConfig })), }; const componentCache = new Map(); diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index 814959ec4..bbbf6dd57 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -302,12 +302,12 @@ export const AllConnectorsTab: FC = ({ return (
- {/* File Storage Integrations */} + {/* Document/Files Connectors */} {hasDocumentFileConnectors && (

- File Storage Integrations + Document/Files Connectors

diff --git a/surfsense_web/components/assistant-ui/document-upload-popup.tsx b/surfsense_web/components/assistant-ui/document-upload-popup.tsx index 3c1d64a6a..0b38979a5 100644 --- a/surfsense_web/components/assistant-ui/document-upload-popup.tsx +++ b/surfsense_web/components/assistant-ui/document-upload-popup.tsx @@ -20,13 +20,7 @@ import { searchSpaceSettingsDialogAtom } from "@/atoms/settings/settings-dialog. import { DocumentUploadTab } from "@/components/sources/DocumentUploadTab"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogHeader, - DialogTitle, -} from "@/components/ui/dialog"; +import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog"; // Context for opening the dialog from anywhere interface DocumentUploadDialogContextType { @@ -133,15 +127,17 @@ const DocumentUploadPopupContent: FC<{ onEscapeKeyDown={(e) => e.preventDefault()} className="select-none max-w-2xl w-[95vw] sm:w-[640px] h-[min(440px,75dvh)] sm:h-[min(520px,80vh)] flex flex-col p-0 gap-0 overflow-hidden border border-border ring-0 bg-muted dark:bg-muted text-foreground [&>button]:right-3 sm:[&>button]:right-6 [&>button]:top-5 sm:[&>button]:top-8 [&>button]:opacity-80 [&>button]:hover:opacity-100 [&>button]:hover:bg-foreground/10 [&>button]:z-[100] [&>button>svg]:size-4 sm:[&>button>svg]:size-5" > + Upload Document +
- - - Upload Documents - - +
+
+

Upload Documents

+
+

Upload and sync your documents to your search space - - +

+
{!isLoading && !hasDocumentSummaryLLM ? ( diff --git a/surfsense_web/components/assistant-ui/tooltip-icon-button.tsx b/surfsense_web/components/assistant-ui/tooltip-icon-button.tsx index 3dfeb9196..3db00e990 100644 --- a/surfsense_web/components/assistant-ui/tooltip-icon-button.tsx +++ b/surfsense_web/components/assistant-ui/tooltip-icon-button.tsx @@ -1,7 +1,7 @@ "use client"; import { Slottable } from "@radix-ui/react-slot"; -import { type ComponentPropsWithRef, forwardRef, type ReactNode, useState } from "react"; +import { type ComponentPropsWithRef, forwardRef, type ReactNode } from "react"; import { Button } from "@/components/ui/button"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { useMediaQuery } from "@/hooks/use-media-query"; @@ -17,13 +17,9 @@ export const TooltipIconButton = forwardRef { const isTouchDevice = useMediaQuery("(pointer: coarse)"); const suppressTooltip = disableTooltip || isTouchDevice; - const [tooltipOpen, setTooltipOpen] = useState(false); return ( - + diff --git a/surfsense_web/components/documents/FolderTreeView.tsx b/surfsense_web/components/documents/FolderTreeView.tsx index 6eb53da50..47cd17596 100644 --- a/surfsense_web/components/documents/FolderTreeView.tsx +++ b/surfsense_web/components/documents/FolderTreeView.tsx @@ -168,12 +168,6 @@ export function FolderTreeView({ return states; }, [folders, docsByFolder, foldersByParent, mentionedDocIds]); - const folderMap = useMemo(() => { - const map: Record = {}; - for (const f of folders) map[f.id] = f; - return map; - }, [folders]); - const folderProcessingStates = useMemo(() => { const states: Record = {}; @@ -184,11 +178,6 @@ export function FolderTreeView({ ); let hasFailed = directDocs.some((d) => d.status?.state === "failed"); - const folder = folderMap[folderId]; - if (folder?.metadata?.indexing_in_progress) { - hasProcessing = true; - } - for (const child of foldersByParent[folderId] ?? []) { const sub = compute(child.id); hasProcessing = hasProcessing || sub.hasProcessing; @@ -206,7 +195,7 @@ export function FolderTreeView({ if (states[f.id] === undefined) compute(f.id); } return states; - }, [folders, docsByFolder, foldersByParent, folderMap]); + }, [folders, docsByFolder, foldersByParent]); function renderLevel(parentId: number | null, depth: number): React.ReactNode[] { const key = parentId ?? "root"; @@ -294,7 +283,7 @@ export function FolderTreeView({ if (treeNodes.length === 0 && folders.length === 0 && documents.length === 0) { return ( -
+

No documents found

Use the upload button or connect a source above diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index ba5a9d9ac..1e31c41da 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -59,15 +59,13 @@ const TAB_ITEMS = [ }, { title: "Extreme Assist", - description: - "Get inline writing suggestions powered by your knowledge base as you type in any app.", + description: "Get inline writing suggestions powered by your knowledge base as you type in any app.", src: "/homepage/hero_tutorial/extreme_assist.mp4", featured: true, }, { title: "Watch Local Folder", - description: - "Watch a local folder and automatically sync file changes to your knowledge base. Works great with Obsidian vaults.", + description: "Watch a local folder and automatically sync file changes to your knowledge base. Works great with Obsidian vaults.", src: "/homepage/hero_tutorial/folder_watch.mp4", featured: true, }, @@ -86,8 +84,7 @@ const TAB_ITEMS = [ // }, { title: "Video & Presentations", - description: - "Create short videos and editable presentations with AI-generated visuals and narration from your sources.", + description: "Create short videos and editable presentations with AI-generated visuals and narration from your sources.", src: "/homepage/hero_tutorial/video_gen_surf.mp4", featured: false, }, @@ -346,12 +343,7 @@ function DownloadButton() { ))} - + All downloads @@ -506,3 +498,4 @@ const TabVideo = memo(function TabVideo({ src }: { src: string }) { }); const GITHUB_RELEASES_URL = "https://github.com/MODSetter/SurfSense/releases/latest"; + diff --git a/surfsense_web/components/homepage/navbar.tsx b/surfsense_web/components/homepage/navbar.tsx index 8ab44b50b..7faf45dab 100644 --- a/surfsense_web/components/homepage/navbar.tsx +++ b/surfsense_web/components/homepage/navbar.tsx @@ -144,13 +144,13 @@ const MobileNav = ({ navItems, isScrolled, scrolledBgClassName }: any) => { ref={navRef} animate={{ borderRadius: open ? "4px" : "2rem" }} key={String(open)} - className={cn( - "relative mx-auto flex w-full max-w-[calc(100vw-2rem)] flex-col items-center justify-between px-4 py-2 lg:hidden transition-[background-color,border-color,box-shadow] duration-300", - isScrolled - ? (scrolledBgClassName ?? - "bg-white/80 backdrop-blur-md border border-white/20 shadow-lg dark:bg-neutral-950/80 dark:border-neutral-800/50") - : "bg-transparent border border-transparent" - )} + className={cn( + "relative mx-auto flex w-full max-w-[calc(100vw-2rem)] flex-col items-center justify-between px-4 py-2 lg:hidden transition-[background-color,border-color,box-shadow] duration-300", + isScrolled + ? (scrolledBgClassName ?? + "bg-white/80 backdrop-blur-md border border-white/20 shadow-lg dark:bg-neutral-950/80 dark:border-neutral-800/50") + : "bg-transparent border border-transparent" + )} className={cn( "relative mx-auto flex w-full max-w-[calc(100vw-2rem)] flex-col items-center justify-between px-4 py-2 lg:hidden transition-all duration-300", isScrolled diff --git a/surfsense_web/components/homepage/why-surfsense.tsx b/surfsense_web/components/homepage/why-surfsense.tsx index c50c3604a..5133a4269 100644 --- a/surfsense_web/components/homepage/why-surfsense.tsx +++ b/surfsense_web/components/homepage/why-surfsense.tsx @@ -1,9 +1,9 @@ "use client"; +import { useRef, useState } from "react"; +import { motion, useInView } from "motion/react"; import { IconPointerFilled } from "@tabler/icons-react"; import { Check, X } from "lucide-react"; -import { motion, useInView } from "motion/react"; -import { useRef, useState } from "react"; import { Badge } from "@/components/ui/badge"; import { Separator } from "@/components/ui/separator"; import { cn } from "@/lib/utils"; @@ -40,8 +40,8 @@ export function WhySurfSense() { Everything NotebookLM should have been

- Open source. No data limits. No vendor lock-in. Built for teams that care about privacy - and flexibility. + Open source. No data limits. No vendor lock-in. Built for teams that + care about privacy and flexibility.

@@ -68,7 +68,10 @@ function UnlimitedSkeleton({ className }: { className?: string }) { ]; return ( -
+
{items.map((item, index) => ( {item.icon} - {item.label} + + {item.label} +
{item.notebookLm} @@ -120,7 +125,10 @@ function LLMFlexibilitySkeleton({ className }: { className?: string }) { return (
-

{model.name}

-

{model.provider}

+

+ {model.name} +

+

+ {model.provider} +

{selected === index && ( -
+
))} @@ -275,7 +295,9 @@ function MultiplayerSkeleton({ className }: { className?: string }) {
{collaborator.name[0]}
- {collaborator.name} + + {collaborator.name} + {collaborator.role} @@ -299,7 +321,9 @@ function FeatureCard({
{skeleton}
-

{title}

+

+ {title} +

{description}

@@ -384,7 +408,9 @@ function ComparisonStrip() { transition={{ duration: 0.3, delay: 0.15 + index * 0.06 }} >
- {row.feature} + + {row.feature} + {typeof row.notebookLm === "boolean" ? ( row.notebookLm ? ( @@ -393,7 +419,9 @@ function ComparisonStrip() { ) ) : ( - {row.notebookLm} + + {row.notebookLm} + )} @@ -408,7 +436,9 @@ function ComparisonStrip() { )}
- {index !== comparisonRows.length - 1 && } + {index !== comparisonRows.length - 1 && ( + + )} ))} diff --git a/surfsense_web/components/layout/ui/dialogs/CreateSearchSpaceDialog.tsx b/surfsense_web/components/layout/ui/dialogs/CreateSearchSpaceDialog.tsx index 6f385b465..3f01ab24a 100644 --- a/surfsense_web/components/layout/ui/dialogs/CreateSearchSpaceDialog.tsx +++ b/surfsense_web/components/layout/ui/dialogs/CreateSearchSpaceDialog.tsx @@ -152,10 +152,16 @@ export function CreateSearchSpaceDialog({ open, onOpenChange }: CreateSearchSpac diff --git a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx index 8b3a119ae..c806c61d8 100644 --- a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx @@ -23,11 +23,7 @@ import { FolderPickerDialog } from "@/components/documents/FolderPickerDialog"; import { FolderTreeView } from "@/components/documents/FolderTreeView"; import { VersionHistoryDialog } from "@/components/documents/version-history"; import { EXPORT_FILE_EXTENSIONS } from "@/components/shared/ExportMenuItems"; -import { - DEFAULT_EXCLUDE_PATTERNS, - FolderWatchDialog, - type SelectedFolder, -} from "@/components/sources/FolderWatchDialog"; +import { FolderWatchDialog, type SelectedFolder } from "@/components/sources/FolderWatchDialog"; import { AlertDialog, AlertDialogAction, @@ -50,8 +46,6 @@ import { useElectronAPI } from "@/hooks/use-platform"; import { documentsApiService } from "@/lib/apis/documents-api.service"; import { foldersApiService } from "@/lib/apis/folders-api.service"; import { authenticatedFetch } from "@/lib/auth-utils"; -import { uploadFolderScan } from "@/lib/folder-sync-upload"; -import { getSupportedExtensionsSet } from "@/lib/supported-extensions"; import { queries } from "@/zero/queries/index"; import { SidebarSlideOutPanel } from "./SidebarSlideOutPanel"; @@ -120,48 +114,48 @@ export function DocumentsSidebar({ setFolderWatchOpen(true); }, []); - const refreshWatchedIds = useCallback(async () => { + useEffect(() => { if (!electronAPI?.getWatchedFolders) return; const api = electronAPI; - const folders = await api.getWatchedFolders(); + async function loadWatchedIds() { + const folders = await api.getWatchedFolders(); - if (folders.length === 0) { - try { - const backendFolders = await documentsApiService.getWatchedFolders(searchSpaceId); - for (const bf of backendFolders) { - const meta = bf.metadata as Record | null; - if (!meta?.watched || !meta.folder_path) continue; - await api.addWatchedFolder({ - path: meta.folder_path as string, - name: bf.name, - rootFolderId: bf.id, - searchSpaceId: bf.search_space_id, - excludePatterns: (meta.exclude_patterns as string[]) ?? [], - fileExtensions: (meta.file_extensions as string[] | null) ?? null, - active: true, - }); + if (folders.length === 0) { + try { + const backendFolders = await documentsApiService.getWatchedFolders(searchSpaceId); + for (const bf of backendFolders) { + const meta = bf.metadata as Record | null; + if (!meta?.watched || !meta.folder_path) continue; + await api.addWatchedFolder({ + path: meta.folder_path as string, + name: bf.name, + rootFolderId: bf.id, + searchSpaceId: bf.search_space_id, + excludePatterns: (meta.exclude_patterns as string[]) ?? [], + fileExtensions: (meta.file_extensions as string[] | null) ?? null, + active: true, + }); + } + const recovered = await api.getWatchedFolders(); + const ids = new Set( + recovered.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number) + ); + setWatchedFolderIds(ids); + return; + } catch (err) { + console.error("[DocumentsSidebar] Recovery from backend failed:", err); } - const recovered = await api.getWatchedFolders(); - const ids = new Set( - recovered.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number) - ); - setWatchedFolderIds(ids); - return; - } catch (err) { - console.error("[DocumentsSidebar] Recovery from backend failed:", err); } + + const ids = new Set( + folders.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number) + ); + setWatchedFolderIds(ids); } - const ids = new Set( - folders.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number) - ); - setWatchedFolderIds(ids); + loadWatchedIds(); }, [searchSpaceId, electronAPI]); - - useEffect(() => { - refreshWatchedIds(); - }, [refreshWatchedIds]); const { mutateAsync: deleteDocumentMutation } = useAtomValue(deleteDocumentMutationAtom); const [sidebarDocs, setSidebarDocs] = useAtom(sidebarSelectedDocumentsAtom); @@ -198,7 +192,6 @@ export function DocumentsSidebar({ position: f.position, parentId: f.parentId ?? null, searchSpaceId: f.searchSpaceId, - metadata: f.metadata as Record | null | undefined, })), [zeroFolders] ); @@ -311,17 +304,14 @@ export function DocumentsSidebar({ } try { - toast.info(`Re-scanning folder: ${matched.name}`); - await uploadFolderScan({ - folderPath: matched.path, - folderName: matched.name, - searchSpaceId, - excludePatterns: matched.excludePatterns ?? DEFAULT_EXCLUDE_PATTERNS, - fileExtensions: matched.fileExtensions ?? Array.from(getSupportedExtensionsSet()), - enableSummary: false, - rootFolderId: folder.id, + await documentsApiService.folderIndex(searchSpaceId, { + folder_path: matched.path, + folder_name: matched.name, + search_space_id: searchSpaceId, + root_folder_id: folder.id, + file_extensions: matched.fileExtensions ?? undefined, }); - toast.success(`Re-scan complete: ${matched.name}`); + toast.success(`Re-scanning folder: ${matched.name}`); } catch (err) { toast.error((err as Error)?.message || "Failed to re-scan folder"); } @@ -347,9 +337,8 @@ export function DocumentsSidebar({ console.error("[DocumentsSidebar] Failed to clear watched metadata:", err); } toast.success(`Stopped watching: ${matched.name}`); - refreshWatchedIds(); }, - [electronAPI, refreshWatchedIds] + [electronAPI] ); const handleRenameFolder = useCallback(async (folder: FolderDisplay, newName: string) => { @@ -878,7 +867,6 @@ export function DocumentsSidebar({ }} searchSpaceId={searchSpaceId} initialFolder={watchInitialFolder} - onSuccess={refreshWatchedIds} /> )} diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarSlideOutPanel.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarSlideOutPanel.tsx index 5195082cd..47c6c2b4a 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarSlideOutPanel.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarSlideOutPanel.tsx @@ -91,12 +91,13 @@ export function SidebarSlideOutPanel({ {/* Panel extending from sidebar's right edge, flush with the wrapper border */}
- import("@/components/tool-ui/video-presentation").then((m) => ({ - default: m.GenerateVideoPresentationToolUI, - })), + () => import("@/components/tool-ui/video-presentation").then((m) => ({ default: m.GenerateVideoPresentationToolUI })), { ssr: false } ); diff --git a/surfsense_web/components/settings/search-space-settings-dialog.tsx b/surfsense_web/components/settings/search-space-settings-dialog.tsx index 34d28eb2a..dc0627305 100644 --- a/surfsense_web/components/settings/search-space-settings-dialog.tsx +++ b/surfsense_web/components/settings/search-space-settings-dialog.tsx @@ -1,62 +1,43 @@ "use client"; +import dynamic from "next/dynamic"; import { useAtom } from "jotai"; import { Bot, Brain, Eye, FileText, Globe, ImageIcon, MessageSquare, Shield } from "lucide-react"; -import dynamic from "next/dynamic"; import { useTranslations } from "next-intl"; import type React from "react"; import { searchSpaceSettingsDialogAtom } from "@/atoms/settings/settings-dialog.atoms"; import { SettingsDialog } from "@/components/settings/settings-dialog"; const GeneralSettingsManager = dynamic( - () => - import("@/components/settings/general-settings-manager").then((m) => ({ - default: m.GeneralSettingsManager, - })), + () => import("@/components/settings/general-settings-manager").then(m => ({ default: m.GeneralSettingsManager })), { ssr: false } ); const ModelConfigManager = dynamic( - () => - import("@/components/settings/model-config-manager").then((m) => ({ - default: m.ModelConfigManager, - })), + () => import("@/components/settings/model-config-manager").then(m => ({ default: m.ModelConfigManager })), { ssr: false } ); const LLMRoleManager = dynamic( - () => - import("@/components/settings/llm-role-manager").then((m) => ({ default: m.LLMRoleManager })), + () => import("@/components/settings/llm-role-manager").then(m => ({ default: m.LLMRoleManager })), { ssr: false } ); const ImageModelManager = dynamic( - () => - import("@/components/settings/image-model-manager").then((m) => ({ - default: m.ImageModelManager, - })), + () => import("@/components/settings/image-model-manager").then(m => ({ default: m.ImageModelManager })), { ssr: false } ); const VisionModelManager = dynamic( - () => - import("@/components/settings/vision-model-manager").then((m) => ({ - default: m.VisionModelManager, - })), + () => import("@/components/settings/vision-model-manager").then(m => ({ default: m.VisionModelManager })), { ssr: false } ); const RolesManager = dynamic( - () => import("@/components/settings/roles-manager").then((m) => ({ default: m.RolesManager })), + () => import("@/components/settings/roles-manager").then(m => ({ default: m.RolesManager })), { ssr: false } ); const PromptConfigManager = dynamic( - () => - import("@/components/settings/prompt-config-manager").then((m) => ({ - default: m.PromptConfigManager, - })), + () => import("@/components/settings/prompt-config-manager").then(m => ({ default: m.PromptConfigManager })), { ssr: false } ); const PublicChatSnapshotsManager = dynamic( - () => - import("@/components/public-chat-snapshots/public-chat-snapshots-manager").then((m) => ({ - default: m.PublicChatSnapshotsManager, - })), + () => import("@/components/public-chat-snapshots/public-chat-snapshots-manager").then(m => ({ default: m.PublicChatSnapshotsManager })), { ssr: false } ); diff --git a/surfsense_web/components/settings/user-settings-dialog.tsx b/surfsense_web/components/settings/user-settings-dialog.tsx index e755da197..ee0f7e62d 100644 --- a/surfsense_web/components/settings/user-settings-dialog.tsx +++ b/surfsense_web/components/settings/user-settings-dialog.tsx @@ -1,8 +1,8 @@ "use client"; +import dynamic from "next/dynamic"; import { useAtom } from "jotai"; import { Globe, KeyRound, Monitor, Receipt, Sparkles, User } from "lucide-react"; -import dynamic from "next/dynamic"; import { useTranslations } from "next-intl"; import { useMemo } from "react"; import { userSettingsDialogAtom } from "@/atoms/settings/settings-dialog.atoms"; @@ -10,45 +10,27 @@ import { SettingsDialog } from "@/components/settings/settings-dialog"; import { usePlatform } from "@/hooks/use-platform"; const ProfileContent = dynamic( - () => - import("@/app/dashboard/[search_space_id]/user-settings/components/ProfileContent").then( - (m) => ({ default: m.ProfileContent }) - ), + () => import("@/app/dashboard/[search_space_id]/user-settings/components/ProfileContent").then(m => ({ default: m.ProfileContent })), { ssr: false } ); const ApiKeyContent = dynamic( - () => - import("@/app/dashboard/[search_space_id]/user-settings/components/ApiKeyContent").then( - (m) => ({ default: m.ApiKeyContent }) - ), + () => import("@/app/dashboard/[search_space_id]/user-settings/components/ApiKeyContent").then(m => ({ default: m.ApiKeyContent })), { ssr: false } ); const PromptsContent = dynamic( - () => - import("@/app/dashboard/[search_space_id]/user-settings/components/PromptsContent").then( - (m) => ({ default: m.PromptsContent }) - ), + () => import("@/app/dashboard/[search_space_id]/user-settings/components/PromptsContent").then(m => ({ default: m.PromptsContent })), { ssr: false } ); const CommunityPromptsContent = dynamic( - () => - import( - "@/app/dashboard/[search_space_id]/user-settings/components/CommunityPromptsContent" - ).then((m) => ({ default: m.CommunityPromptsContent })), + () => import("@/app/dashboard/[search_space_id]/user-settings/components/CommunityPromptsContent").then(m => ({ default: m.CommunityPromptsContent })), { ssr: false } ); const PurchaseHistoryContent = dynamic( - () => - import( - "@/app/dashboard/[search_space_id]/user-settings/components/PurchaseHistoryContent" - ).then((m) => ({ default: m.PurchaseHistoryContent })), + () => import("@/app/dashboard/[search_space_id]/user-settings/components/PurchaseHistoryContent").then(m => ({ default: m.PurchaseHistoryContent })), { ssr: false } ); const DesktopContent = dynamic( - () => - import("@/app/dashboard/[search_space_id]/user-settings/components/DesktopContent").then( - (m) => ({ default: m.DesktopContent }) - ), + () => import("@/app/dashboard/[search_space_id]/user-settings/components/DesktopContent").then(m => ({ default: m.DesktopContent })), { ssr: false } ); diff --git a/surfsense_web/components/sources/DocumentUploadTab.tsx b/surfsense_web/components/sources/DocumentUploadTab.tsx index 2802dbe93..636b2bb35 100644 --- a/surfsense_web/components/sources/DocumentUploadTab.tsx +++ b/surfsense_web/components/sources/DocumentUploadTab.tsx @@ -341,36 +341,36 @@ export function DocumentUploadTab({ ) ) : ( -
{ +
{ + if (!isElectron) fileInputRef.current?.click(); + }} + onKeyDown={(e) => { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); if (!isElectron) fileInputRef.current?.click(); - }} - onKeyDown={(e) => { - if (e.key === "Enter" || e.key === " ") { - e.preventDefault(); - if (!isElectron) fileInputRef.current?.click(); - } - }} - > - -
-

- {isElectron ? "Select files or folder" : "Tap to select files or folder"} -

-

{t("file_size_limit")}

-
-
e.stopPropagation()} - onKeyDown={(e) => e.stopPropagation()} - role="group" - > - {renderBrowseButton({ fullWidth: true })} -
+ } + }} + > + +
+

+ {isElectron ? "Select files or folder" : "Tap to select files or folder"} +

+

{t("file_size_limit")}

+
e.stopPropagation()} + onKeyDown={(e) => e.stopPropagation()} + role="group" + > + {renderBrowseButton({ fullWidth: true })} +
+
)}
diff --git a/surfsense_web/components/sources/FolderWatchDialog.tsx b/surfsense_web/components/sources/FolderWatchDialog.tsx index 3590279ce..1c66ea6b9 100644 --- a/surfsense_web/components/sources/FolderWatchDialog.tsx +++ b/surfsense_web/components/sources/FolderWatchDialog.tsx @@ -1,7 +1,7 @@ "use client"; import { X } from "lucide-react"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { toast } from "sonner"; import { Button } from "@/components/ui/button"; import { @@ -13,7 +13,7 @@ import { } from "@/components/ui/dialog"; import { Spinner } from "@/components/ui/spinner"; import { Switch } from "@/components/ui/switch"; -import { type FolderSyncProgress, uploadFolderScan } from "@/lib/folder-sync-upload"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; import { getSupportedExtensionsSet } from "@/lib/supported-extensions"; export interface SelectedFolder { @@ -29,7 +29,7 @@ interface FolderWatchDialogProps { initialFolder?: SelectedFolder | null; } -export const DEFAULT_EXCLUDE_PATTERNS = [ +const DEFAULT_EXCLUDE_PATTERNS = [ ".git", "node_modules", "__pycache__", @@ -48,8 +48,6 @@ export function FolderWatchDialog({ const [selectedFolder, setSelectedFolder] = useState(null); const [shouldSummarize, setShouldSummarize] = useState(false); const [submitting, setSubmitting] = useState(false); - const [progress, setProgress] = useState(null); - const abortRef = useRef(null); useEffect(() => { if (open && initialFolder) { @@ -66,42 +64,33 @@ export function FolderWatchDialog({ const folderPath = await api.selectFolder(); if (!folderPath) return; - const folderName = folderPath.split(/[/\\]/).pop() || folderPath; + const folderName = folderPath.split("/").pop() || folderPath.split("\\").pop() || folderPath; setSelectedFolder({ path: folderPath, name: folderName }); }, []); - const handleCancel = useCallback(() => { - abortRef.current?.abort(); - }, []); - const handleSubmit = useCallback(async () => { if (!selectedFolder) return; const api = window.electronAPI; if (!api) return; - const controller = new AbortController(); - abortRef.current = controller; setSubmitting(true); - setProgress(null); - try { - const rootFolderId = await uploadFolderScan({ - folderPath: selectedFolder.path, - folderName: selectedFolder.name, - searchSpaceId, - excludePatterns: DEFAULT_EXCLUDE_PATTERNS, - fileExtensions: supportedExtensions, - enableSummary: shouldSummarize, - onProgress: setProgress, - signal: controller.signal, + const result = await documentsApiService.folderIndex(searchSpaceId, { + folder_path: selectedFolder.path, + folder_name: selectedFolder.name, + search_space_id: searchSpaceId, + enable_summary: shouldSummarize, + file_extensions: supportedExtensions, }); + const rootFolderId = (result as { root_folder_id?: number })?.root_folder_id ?? null; + await api.addWatchedFolder({ path: selectedFolder.path, name: selectedFolder.name, excludePatterns: DEFAULT_EXCLUDE_PATTERNS, fileExtensions: supportedExtensions, - rootFolderId: rootFolderId ?? null, + rootFolderId, searchSpaceId, active: true, }); @@ -109,19 +98,12 @@ export function FolderWatchDialog({ toast.success(`Watching folder: ${selectedFolder.name}`); setSelectedFolder(null); setShouldSummarize(false); - setProgress(null); onOpenChange(false); onSuccess?.(); } catch (err) { - if ((err as Error)?.name === "AbortError") { - toast.info("Folder sync cancelled. Partial progress was saved."); - } else { - toast.error((err as Error)?.message || "Failed to watch folder"); - } + toast.error((err as Error)?.message || "Failed to watch folder"); } finally { - abortRef.current = null; setSubmitting(false); - setProgress(null); } }, [ selectedFolder, @@ -137,44 +119,21 @@ export function FolderWatchDialog({ if (!nextOpen && !submitting) { setSelectedFolder(null); setShouldSummarize(false); - setProgress(null); } onOpenChange(nextOpen); }, [onOpenChange, submitting] ); - const progressLabel = useMemo(() => { - if (!progress) return null; - switch (progress.phase) { - case "listing": - return "Scanning folder..."; - case "checking": - return `Checking ${progress.total} file(s)...`; - case "uploading": - return `Uploading ${progress.uploaded}/${progress.total} file(s)...`; - case "finalizing": - return "Finalizing..."; - case "done": - return "Done!"; - default: - return null; - } - }, [progress]); - return ( - - - - Watch Local Folder - - - Select a folder to sync and watch for changes - + + + Watch Local Folder + Select a folder to sync and watch for changes. -
+
{selectedFolder ? (
@@ -197,7 +156,7 @@ export function FolderWatchDialog({ @@ -215,41 +174,14 @@ export function FolderWatchDialog({
- {progressLabel && ( -
-

{progressLabel}

- {progress && progress.phase === "uploading" && progress.total > 0 && ( -
-
-
- )} -
- )} - -
- {submitting ? ( - <> - - - - ) : ( - +
+ )}
diff --git a/surfsense_web/contracts/types/folder.types.ts b/surfsense_web/contracts/types/folder.types.ts index 8a9696a1f..60919bf46 100644 --- a/surfsense_web/contracts/types/folder.types.ts +++ b/surfsense_web/contracts/types/folder.types.ts @@ -9,7 +9,7 @@ export const folder = z.object({ created_by_id: z.string().nullable().optional(), created_at: z.string(), updated_at: z.string(), - metadata: z.record(z.string(), z.any()).nullable().optional(), + metadata: z.record(z.unknown()).nullable().optional(), }); export const folderCreateRequest = z.object({ diff --git a/surfsense_web/hooks/use-folder-sync.ts b/surfsense_web/hooks/use-folder-sync.ts index 8e0d0ebdc..847d0081b 100644 --- a/surfsense_web/hooks/use-folder-sync.ts +++ b/surfsense_web/hooks/use-folder-sync.ts @@ -20,18 +20,12 @@ const DEBOUNCE_MS = 2000; const MAX_WAIT_MS = 10_000; const MAX_BATCH_SIZE = 50; -interface FileEntry { - fullPath: string; - relativePath: string; - action: string; -} - interface BatchItem { folderPath: string; folderName: string; searchSpaceId: number; rootFolderId: number | null; - files: FileEntry[]; + filePaths: string[]; ackIds: string[]; } @@ -50,42 +44,18 @@ export function useFolderSync() { while (queueRef.current.length > 0) { const batch = queueRef.current.shift()!; try { - const addChangeFiles = batch.files.filter( - (f) => f.action === "add" || f.action === "change" - ); - const unlinkFiles = batch.files.filter((f) => f.action === "unlink"); - - if (addChangeFiles.length > 0 && electronAPI?.readLocalFiles) { - const fullPaths = addChangeFiles.map((f) => f.fullPath); - const fileDataArr = await electronAPI.readLocalFiles(fullPaths); - - const files: File[] = fileDataArr.map((fd) => { - const blob = new Blob([fd.data], { type: fd.mimeType || "application/octet-stream" }); - return new File([blob], fd.name, { type: blob.type }); - }); - - await documentsApiService.folderUploadFiles(files, { - folder_name: batch.folderName, - search_space_id: batch.searchSpaceId, - relative_paths: addChangeFiles.map((f) => f.relativePath), - root_folder_id: batch.rootFolderId, - }); - } - - if (unlinkFiles.length > 0) { - await documentsApiService.folderNotifyUnlinked({ - folder_name: batch.folderName, - search_space_id: batch.searchSpaceId, - root_folder_id: batch.rootFolderId, - relative_paths: unlinkFiles.map((f) => f.relativePath), - }); - } - + await documentsApiService.folderIndexFiles(batch.searchSpaceId, { + folder_path: batch.folderPath, + folder_name: batch.folderName, + search_space_id: batch.searchSpaceId, + target_file_paths: batch.filePaths, + root_folder_id: batch.rootFolderId, + }); if (electronAPI?.acknowledgeFileEvents && batch.ackIds.length > 0) { await electronAPI.acknowledgeFileEvents(batch.ackIds); } } catch (err) { - console.error("[FolderSync] Failed to process batch:", err); + console.error("[FolderSync] Failed to trigger batch re-index:", err); } } processingRef.current = false; @@ -98,10 +68,10 @@ export function useFolderSync() { if (!pending) return; pendingByFolder.current.delete(folderKey); - for (let i = 0; i < pending.files.length; i += MAX_BATCH_SIZE) { + for (let i = 0; i < pending.filePaths.length; i += MAX_BATCH_SIZE) { queueRef.current.push({ ...pending, - files: pending.files.slice(i, i + MAX_BATCH_SIZE), + filePaths: pending.filePaths.slice(i, i + MAX_BATCH_SIZE), ackIds: i === 0 ? pending.ackIds : [], }); } @@ -113,14 +83,9 @@ export function useFolderSync() { const existing = pendingByFolder.current.get(folderKey); if (existing) { - const pathSet = new Set(existing.files.map((f) => f.fullPath)); - if (!pathSet.has(event.fullPath)) { - existing.files.push({ - fullPath: event.fullPath, - relativePath: event.relativePath, - action: event.action, - }); - } + const pathSet = new Set(existing.filePaths); + pathSet.add(event.fullPath); + existing.filePaths = Array.from(pathSet); if (!existing.ackIds.includes(event.id)) { existing.ackIds.push(event.id); } @@ -130,13 +95,7 @@ export function useFolderSync() { folderName: event.folderName, searchSpaceId: event.searchSpaceId, rootFolderId: event.rootFolderId, - files: [ - { - fullPath: event.fullPath, - relativePath: event.relativePath, - action: event.action, - }, - ], + filePaths: [event.fullPath], ackIds: [event.id], }); firstEventTime.current.set(folderKey, Date.now()); diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts index 584f2e212..5961522ec 100644 --- a/surfsense_web/lib/apis/documents-api.service.ts +++ b/surfsense_web/lib/apis/documents-api.service.ts @@ -424,79 +424,33 @@ class DocumentsApiService { return baseApiService.post(`/api/v1/documents/${documentId}/versions/${versionNumber}/restore`); }; - folderMtimeCheck = async (body: { - folder_name: string; - search_space_id: number; - files: { relative_path: string; mtime: number }[]; - }): Promise<{ files_to_upload: string[] }> => { - return baseApiService.post(`/api/v1/documents/folder-mtime-check`, undefined, { - body, - }) as unknown as { files_to_upload: string[] }; - }; - - folderUploadFiles = async ( - files: File[], - metadata: { + folderIndex = async ( + searchSpaceId: number, + body: { + folder_path: string; folder_name: string; search_space_id: number; - relative_paths: string[]; + exclude_patterns?: string[]; + file_extensions?: string[]; + root_folder_id?: number; + enable_summary?: boolean; + } + ) => { + return baseApiService.post(`/api/v1/documents/folder-index`, undefined, { body }); + }; + + folderIndexFiles = async ( + searchSpaceId: number, + body: { + folder_path: string; + folder_name: string; + search_space_id: number; + target_file_paths: string[]; root_folder_id?: number | null; enable_summary?: boolean; - }, - signal?: AbortSignal - ): Promise<{ message: string; status: string; root_folder_id: number; file_count: number }> => { - const formData = new FormData(); - for (const file of files) { - formData.append("files", file); } - formData.append("folder_name", metadata.folder_name); - formData.append("search_space_id", String(metadata.search_space_id)); - formData.append("relative_paths", JSON.stringify(metadata.relative_paths)); - if (metadata.root_folder_id != null) { - formData.append("root_folder_id", String(metadata.root_folder_id)); - } - formData.append("enable_summary", String(metadata.enable_summary ?? false)); - - const totalSize = files.reduce((acc, f) => acc + f.size, 0); - const timeoutMs = Math.min(Math.max((totalSize / (1024 * 1024)) * 5000, 30_000), 600_000); - - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeoutMs); - - if (signal) { - signal.addEventListener("abort", () => controller.abort(), { once: true }); - } - - try { - return (await baseApiService.postFormData(`/api/v1/documents/folder-upload`, undefined, { - body: formData, - signal: controller.signal, - })) as { message: string; status: string; root_folder_id: number; file_count: number }; - } finally { - clearTimeout(timeoutId); - } - }; - - folderNotifyUnlinked = async (body: { - folder_name: string; - search_space_id: number; - root_folder_id: number | null; - relative_paths: string[]; - }): Promise<{ deleted_count: number }> => { - return baseApiService.post(`/api/v1/documents/folder-unlink`, undefined, { - body, - }) as unknown as { deleted_count: number }; - }; - - folderSyncFinalize = async (body: { - folder_name: string; - search_space_id: number; - root_folder_id: number | null; - all_relative_paths: string[]; - }): Promise<{ deleted_count: number }> => { - return baseApiService.post(`/api/v1/documents/folder-sync-finalize`, undefined, { - body, - }) as unknown as { deleted_count: number }; + ) => { + return baseApiService.post(`/api/v1/documents/folder-index-files`, undefined, { body }); }; getWatchedFolders = async (searchSpaceId: number) => { diff --git a/surfsense_web/lib/folder-sync-upload.ts b/surfsense_web/lib/folder-sync-upload.ts deleted file mode 100644 index 7a9810d76..000000000 --- a/surfsense_web/lib/folder-sync-upload.ts +++ /dev/null @@ -1,239 +0,0 @@ -import { documentsApiService } from "@/lib/apis/documents-api.service"; - -const MAX_BATCH_SIZE_BYTES = 20 * 1024 * 1024; // 20 MB -const MAX_BATCH_FILES = 10; -const UPLOAD_CONCURRENCY = 3; - -export interface FolderSyncProgress { - phase: "listing" | "checking" | "uploading" | "finalizing" | "done"; - uploaded: number; - total: number; -} - -export interface FolderSyncParams { - folderPath: string; - folderName: string; - searchSpaceId: number; - excludePatterns: string[]; - fileExtensions: string[]; - enableSummary: boolean; - rootFolderId?: number | null; - onProgress?: (progress: FolderSyncProgress) => void; - signal?: AbortSignal; -} - -function buildBatches(entries: FolderFileEntry[]): FolderFileEntry[][] { - const batches: FolderFileEntry[][] = []; - let currentBatch: FolderFileEntry[] = []; - let currentSize = 0; - - for (const entry of entries) { - if (entry.size >= MAX_BATCH_SIZE_BYTES) { - if (currentBatch.length > 0) { - batches.push(currentBatch); - currentBatch = []; - currentSize = 0; - } - batches.push([entry]); - continue; - } - - if (currentBatch.length >= MAX_BATCH_FILES || currentSize + entry.size > MAX_BATCH_SIZE_BYTES) { - batches.push(currentBatch); - currentBatch = []; - currentSize = 0; - } - - currentBatch.push(entry); - currentSize += entry.size; - } - - if (currentBatch.length > 0) { - batches.push(currentBatch); - } - - return batches; -} - -async function uploadBatchesWithConcurrency( - batches: FolderFileEntry[][], - params: { - folderName: string; - searchSpaceId: number; - rootFolderId: number | null; - enableSummary: boolean; - signal?: AbortSignal; - onBatchComplete?: (filesInBatch: number) => void; - } -): Promise { - const api = window.electronAPI; - if (!api) throw new Error("Electron API not available"); - - let batchIdx = 0; - let resolvedRootFolderId = params.rootFolderId; - const errors: string[] = []; - - async function processNext(): Promise { - while (true) { - if (params.signal?.aborted) return; - - const idx = batchIdx++; - if (idx >= batches.length) return; - - const batch = batches[idx]; - const fullPaths = batch.map((e) => e.fullPath); - - try { - const fileDataArr = await api.readLocalFiles(fullPaths); - - const files: File[] = fileDataArr.map((fd) => { - const blob = new Blob([fd.data], { type: fd.mimeType || "application/octet-stream" }); - return new File([blob], fd.name, { type: blob.type }); - }); - - const result = await documentsApiService.folderUploadFiles( - files, - { - folder_name: params.folderName, - search_space_id: params.searchSpaceId, - relative_paths: batch.map((e) => e.relativePath), - root_folder_id: resolvedRootFolderId, - enable_summary: params.enableSummary, - }, - params.signal - ); - - if (result.root_folder_id && !resolvedRootFolderId) { - resolvedRootFolderId = result.root_folder_id; - } - - params.onBatchComplete?.(batch.length); - } catch (err) { - if (params.signal?.aborted) return; - const msg = (err as Error)?.message || "Upload failed"; - errors.push(`Batch ${idx}: ${msg}`); - } - } - } - - const workers = Array.from({ length: Math.min(UPLOAD_CONCURRENCY, batches.length) }, () => - processNext() - ); - await Promise.all(workers); - - if (errors.length > 0 && !params.signal?.aborted) { - console.error("Some batches failed:", errors); - } - - return resolvedRootFolderId; -} - -/** - * Run a full upload-based folder scan: list files, mtime-check, upload - * changed files in parallel batches, and finalize (delete orphans). - * - * Returns the root_folder_id to pass to addWatchedFolder. - */ -export async function uploadFolderScan(params: FolderSyncParams): Promise { - const api = window.electronAPI; - if (!api) throw new Error("Electron API not available"); - - const { - folderPath, - folderName, - searchSpaceId, - excludePatterns, - fileExtensions, - enableSummary, - signal, - } = params; - let rootFolderId = params.rootFolderId ?? null; - - params.onProgress?.({ phase: "listing", uploaded: 0, total: 0 }); - - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - - const allFiles = await api.listFolderFiles({ - path: folderPath, - name: folderName, - excludePatterns, - fileExtensions, - rootFolderId: rootFolderId ?? null, - searchSpaceId, - active: true, - }); - - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - - params.onProgress?.({ phase: "checking", uploaded: 0, total: allFiles.length }); - - const mtimeCheckResult = await documentsApiService.folderMtimeCheck({ - folder_name: folderName, - search_space_id: searchSpaceId, - files: allFiles.map((f) => ({ relative_path: f.relativePath, mtime: f.mtimeMs / 1000 })), - }); - - const filesToUpload = mtimeCheckResult.files_to_upload; - const uploadSet = new Set(filesToUpload); - const entriesToUpload = allFiles.filter((f) => uploadSet.has(f.relativePath)); - - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - - if (entriesToUpload.length > 0) { - const batches = buildBatches(entriesToUpload); - - let uploaded = 0; - params.onProgress?.({ phase: "uploading", uploaded: 0, total: entriesToUpload.length }); - - const uploadedRootId = await uploadBatchesWithConcurrency(batches, { - folderName, - searchSpaceId, - rootFolderId: rootFolderId ?? null, - enableSummary, - signal, - onBatchComplete: (count) => { - uploaded += count; - params.onProgress?.({ phase: "uploading", uploaded, total: entriesToUpload.length }); - }, - }); - - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - - if (uploadedRootId) { - rootFolderId = uploadedRootId; - } - } - - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - - params.onProgress?.({ - phase: "finalizing", - uploaded: entriesToUpload.length, - total: entriesToUpload.length, - }); - - await documentsApiService.folderSyncFinalize({ - folder_name: folderName, - search_space_id: searchSpaceId, - root_folder_id: rootFolderId ?? null, - all_relative_paths: allFiles.map((f) => f.relativePath), - }); - - params.onProgress?.({ - phase: "done", - uploaded: entriesToUpload.length, - total: entriesToUpload.length, - }); - - // Seed the Electron mtime store so the reconciliation scan in - // startWatcher won't re-emit events for files we just indexed. - if (api.seedFolderMtimes) { - const mtimes: Record = {}; - for (const f of allFiles) { - mtimes[f.relativePath] = f.mtimeMs; - } - await api.seedFolderMtimes(folderPath, mtimes); - } - - return rootFolderId; -} diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index 004aefcd5..eeef51b0b 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -34,13 +34,6 @@ interface LocalFileData { size: number; } -interface FolderFileEntry { - relativePath: string; - fullPath: string; - size: number; - mtimeMs: number; -} - interface ElectronAPI { versions: { electron: string; @@ -89,8 +82,6 @@ interface ElectronAPI { signalRendererReady: () => Promise; getPendingFileEvents: () => Promise; acknowledgeFileEvents: (eventIds: string[]) => Promise<{ acknowledged: number }>; - listFolderFiles: (config: WatchedFolderConfig) => Promise; - seedFolderMtimes: (folderPath: string, mtimes: Record) => Promise; // Browse files/folders via native dialogs browseFiles: () => Promise; readLocalFiles: (paths: string[]) => Promise; diff --git a/surfsense_web/zero/schema/folders.ts b/surfsense_web/zero/schema/folders.ts index c5b192942..2313506ab 100644 --- a/surfsense_web/zero/schema/folders.ts +++ b/surfsense_web/zero/schema/folders.ts @@ -1,4 +1,4 @@ -import { json, number, string, table } from "@rocicorp/zero"; +import { number, string, table } from "@rocicorp/zero"; export const folderTable = table("folders") .columns({ @@ -10,6 +10,5 @@ export const folderTable = table("folders") createdById: string().optional().from("created_by_id"), createdAt: number().from("created_at"), updatedAt: number().from("updated_at"), - metadata: json>().optional().from("metadata"), }) .primaryKey("id");