diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index c28fddfe0..39458dc5f 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -1385,166 +1385,6 @@ async def restore_document_version( } -# ===== Local folder indexing endpoints ===== - - -class FolderIndexRequest(PydanticBaseModel): - folder_path: str - folder_name: str - search_space_id: int - exclude_patterns: list[str] | None = None - file_extensions: list[str] | None = None - root_folder_id: int | None = None - enable_summary: bool = False - - -class FolderIndexFilesRequest(PydanticBaseModel): - folder_path: str - folder_name: str - search_space_id: int - target_file_paths: list[str] - root_folder_id: int | None = None - enable_summary: bool = False - - -@router.post("/documents/folder-index") -async def folder_index( - request: FolderIndexRequest, - session: AsyncSession = Depends(get_async_session), - user: User = Depends(current_active_user), -): - """Full-scan index of a local folder. Creates the root Folder row synchronously - and dispatches the heavy indexing work to a Celery task. - Returns the root_folder_id so the desktop can persist it. - """ - from app.config import config as app_config - - if not app_config.is_self_hosted(): - raise HTTPException( - status_code=400, - detail="Local folder indexing is only available in self-hosted mode", - ) - - await check_permission( - session, - user, - request.search_space_id, - Permission.DOCUMENTS_CREATE.value, - "You don't have permission to create documents in this search space", - ) - - watched_metadata = { - "watched": True, - "folder_path": request.folder_path, - "exclude_patterns": request.exclude_patterns, - "file_extensions": request.file_extensions, - } - - root_folder_id = request.root_folder_id - if root_folder_id: - existing = ( - await session.execute(select(Folder).where(Folder.id == root_folder_id)) - ).scalar_one_or_none() - if not existing: - root_folder_id = None - else: - existing.folder_metadata = watched_metadata - await session.commit() - - if not root_folder_id: - root_folder = Folder( - name=request.folder_name, - search_space_id=request.search_space_id, - created_by_id=str(user.id), - position="a0", - folder_metadata=watched_metadata, - ) - session.add(root_folder) - await session.flush() - root_folder_id = root_folder.id - await session.commit() - - from app.tasks.celery_tasks.document_tasks import index_local_folder_task - - index_local_folder_task.delay( - search_space_id=request.search_space_id, - user_id=str(user.id), - folder_path=request.folder_path, - folder_name=request.folder_name, - exclude_patterns=request.exclude_patterns, - file_extensions=request.file_extensions, - root_folder_id=root_folder_id, - enable_summary=request.enable_summary, - ) - - return { - "message": "Folder indexing started", - "status": "processing", - "root_folder_id": root_folder_id, - } - - -@router.post("/documents/folder-index-files") -async def folder_index_files( - request: FolderIndexFilesRequest, - session: AsyncSession = Depends(get_async_session), - user: User = Depends(current_active_user), -): - """Index multiple files within a watched folder (batched chokidar trigger). - Validates that all target_file_paths are under folder_path. - Dispatches a single Celery task that processes them in parallel. - """ - from app.config import config as app_config - - if not app_config.is_self_hosted(): - raise HTTPException( - status_code=400, - detail="Local folder indexing is only available in self-hosted mode", - ) - - if not request.target_file_paths: - raise HTTPException( - status_code=400, detail="target_file_paths must not be empty" - ) - - await check_permission( - session, - user, - request.search_space_id, - Permission.DOCUMENTS_CREATE.value, - "You don't have permission to create documents in this search space", - ) - - from pathlib import Path - - for fp in request.target_file_paths: - try: - Path(fp).relative_to(request.folder_path) - except ValueError as err: - raise HTTPException( - status_code=400, - detail=f"target_file_path {fp} must be inside folder_path", - ) from err - - from app.tasks.celery_tasks.document_tasks import index_local_folder_task - - index_local_folder_task.delay( - search_space_id=request.search_space_id, - user_id=str(user.id), - folder_path=request.folder_path, - folder_name=request.folder_name, - target_file_paths=request.target_file_paths, - root_folder_id=request.root_folder_id, - enable_summary=request.enable_summary, - ) - - return { - "message": f"Batch indexing started for {len(request.target_file_paths)} file(s)", - "status": "processing", - "file_count": len(request.target_file_paths), - } - - # ===== Upload-based local folder indexing endpoints ===== # These work for ALL deployment modes (cloud, self-hosted remote, self-hosted local). # The desktop app reads files locally and uploads them here. diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts index 3018cbc34..34a0b6dce 100644 --- a/surfsense_web/lib/apis/documents-api.service.ts +++ b/surfsense_web/lib/apis/documents-api.service.ts @@ -424,35 +424,6 @@ class DocumentsApiService { return baseApiService.post(`/api/v1/documents/${documentId}/versions/${versionNumber}/restore`); }; - folderIndex = async ( - searchSpaceId: number, - body: { - folder_path: string; - folder_name: string; - search_space_id: number; - exclude_patterns?: string[]; - file_extensions?: string[]; - root_folder_id?: number; - enable_summary?: boolean; - } - ) => { - return baseApiService.post(`/api/v1/documents/folder-index`, undefined, { body }); - }; - - folderIndexFiles = async ( - searchSpaceId: number, - body: { - folder_path: string; - folder_name: string; - search_space_id: number; - target_file_paths: string[]; - root_folder_id?: number | null; - enable_summary?: boolean; - } - ) => { - return baseApiService.post(`/api/v1/documents/folder-index-files`, undefined, { body }); - }; - folderMtimeCheck = async (body: { folder_name: string; search_space_id: number; diff --git a/surfsense_web/lib/folder-sync-upload.ts b/surfsense_web/lib/folder-sync-upload.ts index ef53c6b29..28f38ced4 100644 --- a/surfsense_web/lib/folder-sync-upload.ts +++ b/surfsense_web/lib/folder-sync-upload.ts @@ -70,11 +70,12 @@ async function uploadBatchesWithConcurrency( signal?: AbortSignal; onBatchComplete?: (filesInBatch: number) => void; }, -) { +): Promise { const api = window.electronAPI; if (!api) throw new Error("Electron API not available"); let batchIdx = 0; + let resolvedRootFolderId = params.rootFolderId; const errors: string[] = []; async function processNext(): Promise { @@ -95,18 +96,22 @@ async function uploadBatchesWithConcurrency( return new File([blob], fd.name, { type: blob.type }); }); - await documentsApiService.folderUploadFiles( + const result = await documentsApiService.folderUploadFiles( files, { folder_name: params.folderName, search_space_id: params.searchSpaceId, relative_paths: batch.map((e) => e.relativePath), - root_folder_id: params.rootFolderId, + root_folder_id: resolvedRootFolderId, enable_summary: params.enableSummary, }, params.signal, ); + if (result.root_folder_id && !resolvedRootFolderId) { + resolvedRootFolderId = result.root_folder_id; + } + params.onBatchComplete?.(batch.length); } catch (err) { if (params.signal?.aborted) return; @@ -122,6 +127,8 @@ async function uploadBatchesWithConcurrency( if (errors.length > 0 && !params.signal?.aborted) { console.error("Some batches failed:", errors); } + + return resolvedRootFolderId; } /** @@ -173,7 +180,7 @@ export async function uploadFolderScan(params: FolderSyncParams): Promise | undefined; - const matched = folderList?.find((f) => f.name === folderName); - if (matched?.id) { - rootFolderId = matched.id; - } + if (uploadedRootId) { + rootFolderId = uploadedRootId; } }