diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index b20f8cd9c..226d511cc 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -178,9 +178,26 @@ async def create_documents_file_upload( session, unique_identifier_hash ) if existing: - # Clean up temp file for duplicates - os.unlink(temp_path) - skipped_duplicates += 1 + if DocumentStatus.is_state(existing.status, DocumentStatus.READY): + # True duplicate — content already indexed, skip + os.unlink(temp_path) + skipped_duplicates += 1 + continue + + # Existing document is stuck (failed/pending/processing) + # Reset it to pending and re-dispatch for processing + existing.status = DocumentStatus.pending() + existing.content = "Processing..." + existing.document_metadata = { + **(existing.document_metadata or {}), + "file_size": file_size, + "upload_time": datetime.now().isoformat(), + } + existing.updated_at = get_current_timestamp() + created_documents.append(existing) + files_to_process.append( + (existing, temp_path, file.filename or "unknown") + ) continue # Create pending document (visible immediately in UI via ElectricSQL) diff --git a/surfsense_backend/app/routes/notifications_routes.py b/surfsense_backend/app/routes/notifications_routes.py index e8e89e6c4..4f80c6529 100644 --- a/surfsense_backend/app/routes/notifications_routes.py +++ b/surfsense_backend/app/routes/notifications_routes.py @@ -144,6 +144,9 @@ async def list_notifications( before_date: str | None = Query( None, description="Get notifications before this ISO date (for pagination)" ), + search: str | None = Query( + None, description="Search notifications by title or message (case-insensitive)" + ), limit: int = Query(50, ge=1, le=100, description="Number of items to return"), offset: int = Query(0, ge=0, description="Number of items to skip"), user: User = Depends(current_active_user), @@ -191,6 +194,15 @@ async def list_notifications( detail="Invalid date format. Use ISO format (e.g., 2024-01-15T00:00:00Z)", ) from None + # Filter by search query (case-insensitive title/message search) + if search: + search_term = f"%{search}%" + search_filter = Notification.title.ilike( + search_term + ) | Notification.message.ilike(search_term) + query = query.where(search_filter) + count_query = count_query.where(search_filter) + # Get total count total_result = await session.execute(count_query) total = total_result.scalar() or 0 diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index 6dfcbff46..81c5dbba2 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -1,6 +1,8 @@ """Celery tasks for document processing.""" +import asyncio import logging +import os from uuid import UUID from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine @@ -17,6 +19,79 @@ from app.tasks.document_processors import ( logger = logging.getLogger(__name__) +# ===== Redis heartbeat for document processing tasks ===== +# Same mechanism as connector indexing heartbeats (search_source_connectors_routes.py). +# A background coroutine refreshes a Redis key every 60s with a 2-min TTL. +# If the Celery worker crashes, the coroutine dies, the key expires, and the +# stale_notification_cleanup_task detects the missing key and marks the +# notification + document as failed. +_doc_heartbeat_redis = None +HEARTBEAT_TTL_SECONDS = 120 # 2 minutes — same as connector indexing +HEARTBEAT_REFRESH_INTERVAL = 60 # Refresh every 60 seconds + + +def _get_doc_heartbeat_redis(): + """Get Redis client for document processing heartbeat.""" + import redis + + global _doc_heartbeat_redis + if _doc_heartbeat_redis is None: + redis_url = os.getenv( + "REDIS_APP_URL", + os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0"), + ) + _doc_heartbeat_redis = redis.from_url(redis_url, decode_responses=True) + return _doc_heartbeat_redis + + +def _get_heartbeat_key(notification_id: int) -> str: + """Generate Redis key for document processing heartbeat. + + Uses same key pattern as connector indexing: indexing:heartbeat:{notification_id} + """ + return f"indexing:heartbeat:{notification_id}" + + +def _start_heartbeat(notification_id: int) -> None: + """Set initial Redis heartbeat key for a document processing task.""" + try: + key = _get_heartbeat_key(notification_id) + _get_doc_heartbeat_redis().setex(key, HEARTBEAT_TTL_SECONDS, "started") + except Exception as e: + logger.warning( + f"Failed to set initial heartbeat for notification {notification_id}: {e}" + ) + + +def _stop_heartbeat(notification_id: int) -> None: + """Delete Redis heartbeat key when task completes (success or failure).""" + try: + key = _get_heartbeat_key(notification_id) + _get_doc_heartbeat_redis().delete(key) + except Exception: + pass # Key will expire on its own + + +async def _run_heartbeat_loop(notification_id: int): + """Background coroutine that refreshes Redis heartbeat every 60 seconds. + + This keeps the heartbeat alive while the task is running. + When the task finishes, this coroutine is cancelled via heartbeat_task.cancel(). + When the worker crashes, this coroutine dies with it and the key expires. + """ + key = _get_heartbeat_key(notification_id) + try: + while True: + await asyncio.sleep(HEARTBEAT_REFRESH_INTERVAL) + try: + _get_doc_heartbeat_redis().setex(key, HEARTBEAT_TTL_SECONDS, "alive") + except Exception as e: + logger.warning( + f"Failed to refresh heartbeat for notification {notification_id}: {e}" + ) + except asyncio.CancelledError: + pass # Normal cancellation when task completes + def get_celery_session_maker(): """ @@ -44,8 +119,6 @@ def process_extension_document_task( search_space_id: ID of the search space user_id: ID of the user """ - import asyncio - # Create a new event loop for this task loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -196,8 +269,6 @@ def process_youtube_video_task(self, url: str, search_space_id: int, user_id: st search_space_id: ID of the search space user_id: ID of the user """ - import asyncio - loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -226,6 +297,10 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str): ) ) + # Start Redis heartbeat for stale task detection + _start_heartbeat(notification.id) + heartbeat_task = asyncio.create_task(_run_heartbeat_loop(notification.id)) + log_entry = await task_logger.log_task_start( task_name="process_youtube_video", source="document_processor", @@ -243,7 +318,7 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str): ) result = await add_youtube_video_document( - session, url, search_space_id, user_id + session, url, search_space_id, user_id, notification=notification ) if result: @@ -307,6 +382,10 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str): logger.error(f"Error processing YouTube video: {e!s}") raise + finally: + # Stop heartbeat — key deleted on success, expires on crash + heartbeat_task.cancel() + _stop_heartbeat(notification.id) @celery_app.task(name="process_file_upload", bind=True) @@ -322,8 +401,6 @@ def process_file_upload_task( search_space_id: ID of the search space user_id: ID of the user """ - import asyncio - import os import traceback logger.info( @@ -336,7 +413,7 @@ def process_file_upload_task( if not os.path.exists(file_path): logger.error( f"[process_file_upload] File does not exist: {file_path}. " - "The temp file may have been cleaned up before the task ran." + "File may have been removed before syncing could start." ) return @@ -370,8 +447,6 @@ async def _process_file_upload( file_path: str, filename: str, search_space_id: int, user_id: str ): """Process file upload with new session.""" - import os - from app.tasks.document_processors.file_processors import process_file_in_background logger.info(f"[_process_file_upload] Starting async processing for: {filename}") @@ -404,6 +479,10 @@ async def _process_file_upload( f"[_process_file_upload] Notification created with ID: {notification.id if notification else 'None'}" ) + # Start Redis heartbeat for stale task detection + _start_heartbeat(notification.id) + heartbeat_task = asyncio.create_task(_run_heartbeat_loop(notification.id)) + log_entry = await task_logger.log_task_start( task_name="process_file_upload", source="document_processor", @@ -535,6 +614,10 @@ async def _process_file_upload( ) logger.error(error_message) raise + finally: + # Stop heartbeat — key deleted on success, expires on crash + heartbeat_task.cancel() + _stop_heartbeat(notification.id) @celery_app.task(name="process_file_upload_with_document", bind=True) @@ -560,8 +643,6 @@ def process_file_upload_with_document_task( search_space_id: ID of the search space user_id: ID of the user """ - import asyncio - import os import traceback logger.info( @@ -573,7 +654,7 @@ def process_file_upload_with_document_task( if not os.path.exists(temp_path): logger.error( f"[process_file_upload_with_document] File does not exist: {temp_path}. " - "The temp file may have been cleaned up before the task ran." + "File may have been removed before syncing could start." ) # Mark document as failed since file is missing loop = asyncio.new_event_loop() @@ -582,7 +663,7 @@ def process_file_upload_with_document_task( loop.run_until_complete( _mark_document_failed( document_id, - "File not found - temp file may have been cleaned up", + "File not found. Please re-upload the file.", ) ) finally: @@ -640,8 +721,6 @@ async def _process_file_with_document( - Processes the file (parsing, embedding, chunking) - Updates document to 'ready' on success or 'failed' on error """ - import os - from app.db import Document, DocumentStatus from app.tasks.document_processors.base import get_current_timestamp from app.tasks.document_processors.file_processors import ( @@ -689,6 +768,19 @@ async def _process_file_with_document( ) ) + # Store document_id in notification metadata so cleanup task can find the document + if notification and notification.notification_metadata is not None: + notification.notification_metadata["document_id"] = document_id + from sqlalchemy.orm.attributes import flag_modified + + flag_modified(notification, "notification_metadata") + await session.commit() + await session.refresh(notification) + + # Start Redis heartbeat for stale task detection + _start_heartbeat(notification.id) + heartbeat_task = asyncio.create_task(_run_heartbeat_loop(notification.id)) + log_entry = await task_logger.log_task_start( task_name="process_file_upload_with_document", source="document_processor", @@ -822,6 +914,10 @@ async def _process_file_with_document( raise finally: + # Stop heartbeat — key deleted on success, expires on crash + heartbeat_task.cancel() + _stop_heartbeat(notification.id) + # Clean up temp file if os.path.exists(temp_path): try: @@ -856,8 +952,6 @@ def process_circleback_meeting_task( search_space_id: ID of the search space connector_id: ID of the Circleback connector (for deletion support) """ - import asyncio - loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -897,6 +991,7 @@ async def _process_circleback_meeting( # Create notification if user_id is available notification = None + heartbeat_task = None if user_id: notification = ( await NotificationService.document_processing.notify_processing_started( @@ -908,6 +1003,10 @@ async def _process_circleback_meeting( ) ) + # Start Redis heartbeat for stale task detection + _start_heartbeat(notification.id) + heartbeat_task = asyncio.create_task(_run_heartbeat_loop(notification.id)) + log_entry = await task_logger.log_task_start( task_name="process_circleback_meeting", source="circleback_webhook", @@ -1000,3 +1099,9 @@ async def _process_circleback_meeting( logger.error(f"Error processing Circleback meeting: {e!s}") raise + finally: + # Stop heartbeat — key deleted on success, expires on crash + if heartbeat_task: + heartbeat_task.cancel() + if notification: + _stop_heartbeat(notification.id) diff --git a/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py b/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py index ef3a30e43..f3bbddee0 100644 --- a/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py +++ b/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py @@ -1,18 +1,25 @@ -"""Celery task to detect and mark stale connector indexing notifications as failed. +"""Celery task to detect and mark stale notifications as failed. This task runs periodically (every 5 minutes by default) to find notifications that are stuck in "in_progress" status but don't have an active Redis heartbeat key. -These are marked as "failed" to prevent the frontend from showing a perpetual "syncing" state. +These are marked as "failed" to prevent the frontend from showing a perpetual +"syncing" or "processing" state. -Additionally, it cleans up documents stuck in pending/processing state that belong -to connectors with stale notifications. +It handles two notification types: +1. **connector_indexing** — connector sync tasks (Google Calendar, etc.) +2. **document_processing** — manual file uploads, YouTube videos, etc. + +Additionally, it cleans up documents stuck in pending/processing state: +- For connectors: by connector_id +- For non-connector documents (FILE uploads, YouTube): by document_id from notification metadata Detection mechanism: -- Active indexing tasks set a Redis key with TTL (2 minutes) as a heartbeat -- If the task crashes, the Redis key expires automatically +- Active tasks set a Redis key with TTL (2 minutes) as a heartbeat +- A background coroutine refreshes the key every 60 seconds +- If the task/worker crashes, the Redis key expires automatically - This cleanup task checks for in-progress notifications without a Redis heartbeat key - Such notifications are marked as failed with O(1) batch UPDATE -- Documents with pending/processing status for those connectors are also marked as failed +- Associated documents are also marked as failed """ import contextlib @@ -36,8 +43,9 @@ logger = logging.getLogger(__name__) # Redis client for checking heartbeats _redis_client: redis.Redis | None = None -# Error message shown to users when sync is interrupted +# Error messages shown to users when tasks are interrupted STALE_SYNC_ERROR_MESSAGE = "Sync was interrupted unexpectedly. Please retry." +STALE_PROCESSING_ERROR_MESSAGE = "Syncing was interrupted unexpectedly. Please retry." def get_redis_client() -> redis.Redis: @@ -70,14 +78,13 @@ def get_celery_session_maker(): @celery_app.task(name="cleanup_stale_indexing_notifications") def cleanup_stale_indexing_notifications_task(): """ - Check for stale connector indexing notifications and mark them as failed. + Check for stale notifications and mark them as failed. - This task finds notifications that: - - Have type = 'connector_indexing' - - Have metadata.status = 'in_progress' - - Do NOT have a corresponding Redis heartbeat key (meaning task crashed) + Handles two notification types: + 1. connector_indexing — connector sync tasks + 2. document_processing — manual file uploads, YouTube videos, etc. - And marks them as failed with O(1) batch UPDATE. + Detection: Redis heartbeat key with 2-min TTL. Missing key = stale task. Also marks associated pending/processing documents as failed. """ import asyncio @@ -87,6 +94,7 @@ def cleanup_stale_indexing_notifications_task(): try: loop.run_until_complete(_cleanup_stale_notifications()) + loop.run_until_complete(_cleanup_stale_document_processing_notifications()) finally: loop.close() @@ -269,3 +277,186 @@ async def _cleanup_stuck_documents(session, connector_ids: list[int]): exc_info=True, ) # Don't raise - let the notification cleanup continue even if document cleanup fails + + +# ===== Document Processing Cleanup (FILE uploads, YouTube, etc.) ===== + + +async def _cleanup_stale_document_processing_notifications(): + """Find and mark stale document processing notifications as failed. + + Same Redis heartbeat mechanism as connector indexing cleanup, but for + document_processing type notifications (FILE uploads, YouTube videos, etc.). + + For each stale notification that contains a document_id in its metadata, + the associated document is also marked as failed. + """ + async with get_celery_session_maker()() as session: + try: + # Find all in-progress document processing notifications + result = await session.execute( + select( + Notification.id, + Notification.notification_metadata, + ).where( + and_( + Notification.type == "document_processing", + Notification.notification_metadata["status"].astext + == "in_progress", + ) + ) + ) + in_progress_rows = result.fetchall() + + if not in_progress_rows: + logger.debug("No in-progress document processing notifications found") + return + + # Check which ones are missing heartbeat keys in Redis + redis_client = get_redis_client() + stale_notification_ids = [] + stale_document_ids = [] + + for row in in_progress_rows: + notification_id = row[0] + metadata = row[1] # Full metadata dict + heartbeat_key = _get_heartbeat_key(notification_id) + if not redis_client.exists(heartbeat_key): + stale_notification_ids.append(notification_id) + # Extract document_id from metadata for document cleanup + if metadata and isinstance(metadata, dict): + doc_id = metadata.get("document_id") + if doc_id is not None: + with contextlib.suppress(ValueError, TypeError): + stale_document_ids.append(int(doc_id)) + + if not stale_notification_ids: + logger.debug( + f"All {len(in_progress_rows)} in-progress document processing " + "notifications have active Redis heartbeats" + ) + return + + logger.warning( + f"Found {len(stale_notification_ids)} stale document processing " + f"notifications (no Redis heartbeat): {stale_notification_ids}" + ) + + # O(1) Batch UPDATE: Mark stale notifications as failed + update_data = { + "status": "failed", + "completed_at": datetime.now(UTC).isoformat(), + "error_message": STALE_PROCESSING_ERROR_MESSAGE, + "processing_stage": "failed", + } + + await session.execute( + text(""" + UPDATE notifications + SET metadata = metadata || CAST(:update_json AS jsonb), + title = 'Failed: ' || COALESCE(metadata->>'document_name', 'Document'), + message = :display_message + WHERE id = ANY(:ids) + """), + { + "update_json": json.dumps(update_data), + "display_message": STALE_PROCESSING_ERROR_MESSAGE, + "ids": stale_notification_ids, + }, + ) + + logger.info( + f"Successfully marked {len(stale_notification_ids)} stale document " + "processing notifications as failed" + ) + + # Clean up stuck documents by document_id from notification metadata + if stale_document_ids: + await _cleanup_stuck_non_connector_documents( + session, stale_document_ids + ) + + await session.commit() + + except Exception as e: + logger.error( + f"Error cleaning up stale document processing notifications: {e!s}", + exc_info=True, + ) + await session.rollback() + + +async def _cleanup_stuck_non_connector_documents(session, document_ids: list[int]): + """ + Mark specific non-connector documents stuck in pending/processing as failed. + + These are documents (FILE uploads, YouTube, etc.) identified from stale + notification metadata. Only documents that are still in pending/processing + state are updated — already-completed documents are left untouched. + + Args: + session: Database session + document_ids: List of document IDs to check and potentially mark as failed + """ + if not document_ids: + return + + try: + # Find which of these documents are actually stuck + count_result = await session.execute( + select(Document.id).where( + and_( + Document.id.in_(document_ids), + or_( + Document.status["state"].astext == DocumentStatus.PENDING, + Document.status["state"].astext == DocumentStatus.PROCESSING, + ), + ) + ) + ) + stuck_doc_ids = [row[0] for row in count_result.fetchall()] + + if not stuck_doc_ids: + logger.debug( + f"No stuck non-connector documents found for IDs: {document_ids}" + ) + return + + logger.warning( + f"Found {len(stuck_doc_ids)} stuck non-connector documents " + f"(pending/processing): {stuck_doc_ids}" + ) + + failed_status = DocumentStatus.failed(STALE_PROCESSING_ERROR_MESSAGE) + + await session.execute( + text(""" + UPDATE documents + SET status = CAST(:failed_status AS jsonb), + updated_at = :now + WHERE id = ANY(:doc_ids) + AND ( + status->>'state' = :pending_state + OR status->>'state' = :processing_state + ) + """), + { + "failed_status": json.dumps(failed_status), + "now": datetime.now(UTC), + "doc_ids": stuck_doc_ids, + "pending_state": DocumentStatus.PENDING, + "processing_state": DocumentStatus.PROCESSING, + }, + ) + + logger.info( + f"Successfully marked {len(stuck_doc_ids)} stuck non-connector " + "documents as failed" + ) + + except Exception as e: + logger.error( + f"Error cleaning up stuck non-connector documents {document_ids}: {e!s}", + exc_info=True, + ) + # Don't raise — let the rest of the cleanup continue diff --git a/surfsense_backend/app/tasks/document_processors/youtube_processor.py b/surfsense_backend/app/tasks/document_processors/youtube_processor.py index 6d087b6d0..80cdaae4d 100644 --- a/surfsense_backend/app/tasks/document_processors/youtube_processor.py +++ b/surfsense_backend/app/tasks/document_processors/youtube_processor.py @@ -61,7 +61,11 @@ def get_youtube_video_id(url: str) -> str | None: async def add_youtube_video_document( - session: AsyncSession, url: str, search_space_id: int, user_id: str + session: AsyncSession, + url: str, + search_space_id: int, + user_id: str, + notification=None, ) -> Document: """ Process a YouTube video URL, extract transcripts, and store as a document. @@ -75,6 +79,9 @@ async def add_youtube_video_document( url: YouTube video URL (supports standard, shortened, and embed formats) search_space_id: ID of the search space to add the document to user_id: ID of the user + notification: Optional notification object — if provided, the document_id + is stored in its metadata right after document creation so the stale + cleanup task can identify stuck documents. Returns: Document: The created document object @@ -182,6 +189,15 @@ async def add_youtube_video_document( await session.commit() # Document visible in UI now with pending status! is_new_document = True + # Store document_id in notification metadata so stale cleanup task + # can identify this document if the worker crashes. + if notification and notification.notification_metadata is not None: + from sqlalchemy.orm.attributes import flag_modified + + notification.notification_metadata["document_id"] = document.id + flag_modified(notification, "notification_metadata") + await session.commit() + logging.info(f"Created pending document for YouTube video {video_id}") # ======================================================================= diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx index b214c96be..92ddb0057 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx @@ -35,9 +35,9 @@ export function DocumentTypeChip({ type, className }: { type: string; className? const chip = ( - {icon} + {icon} {fullLabel} diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx index ebdf431e4..c954474c6 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx @@ -178,7 +178,7 @@ export function DocumentsFilters({
setTypeSearchQuery(e.target.value)} className="h-6 pl-6 text-sm bg-transparent border-0 focus-visible:ring-0" diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx index d5ee00dfb..a44295ec0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx @@ -3,6 +3,7 @@ import { formatDistanceToNow } from "date-fns"; import { AlertCircle, + BadgeInfo, Calendar, CheckCircle2, ChevronDown, @@ -372,7 +373,7 @@ export function DocumentsTableShell({ )} {columnVisibility.status && ( - + )} @@ -414,7 +415,7 @@ export function DocumentsTableShell({ )} {columnVisibility.status && ( - + )} @@ -466,7 +467,7 @@ export function DocumentsTableShell({ className="flex flex-col items-center gap-4 max-w-md px-4 text-center" >
- +

{t("no_documents")}

@@ -544,8 +545,11 @@ export function DocumentsTableShell({ )} {columnVisibility.status && ( - - Status + + + + Status + )} @@ -644,7 +648,7 @@ export function DocumentsTableShell({ )} {columnVisibility.status && ( - + )} diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx index ec355f576..eb44d114a 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx @@ -9,6 +9,7 @@ import { AlertDialogAction, AlertDialogCancel, AlertDialogContent, + AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, AlertDialogTitle, @@ -205,7 +206,10 @@ export function RowActions({ - Are you sure? + Delete document? + + This action cannot be undone. This will permanently delete this document from your search space. + Cancel diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index 8cf2fe8da..585a09c43 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -140,6 +140,9 @@ export default function DocumentsTable() { } }); setPageIndex(0); + // Clear selections when filter changes — selected IDs from the previous + // filter view are no longer visible and would cause misleading bulk actions + setSelectedIds(new Set()); }; const onBulkDelete = async () => { diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 1a535539d..22085e064 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -844,7 +844,12 @@ export default function NewChatPage() { }); // Invalidate thread detail for breadcrumb update queryClient.invalidateQueries({ - queryKey: ["threads", String(searchSpaceId), "detail", String(titleData.threadId)], + queryKey: [ + "threads", + String(searchSpaceId), + "detail", + String(titleData.threadId), + ], }); } break; @@ -1403,7 +1408,7 @@ export default function NewChatPage() { // Show loading state only when loading an existing thread if (isInitializing) { return ( -
+
{/* User message */}
@@ -1444,7 +1449,7 @@ export default function NewChatPage() { // For new chats (urlChatId === 0), threadId being null is expected (lazy creation) if (!threadId && urlChatId > 0) { return ( -
+
Failed to load chat
); + const menuItems = ( + <> + {onSettings && ( + + + {tCommon("settings")} + + )} + {onSettings && onDelete && } + {onDelete && isOwner && ( + + + {tCommon("delete")} + + )} + {onDelete && !isOwner && ( + + + {t("leave")} + + )} + + ); + // If delete or settings handlers are provided, wrap with context menu if (onDelete || onSettings) { + // Mobile: use long-press triggered DropdownMenu + if (disableTooltip) { + return ( + + +
+ {avatarButton} +
+ + {menuItems} + + ); + } + + // Desktop: use right-click ContextMenu + Tooltip return ( @@ -150,6 +234,10 @@ export function SearchSpaceAvatar({ } // No context menu needed + if (disableTooltip) { + return avatarButton; + } + return ( {avatarButton} diff --git a/surfsense_web/components/layout/ui/sidebar/AllPrivateChatsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/AllPrivateChatsSidebar.tsx index d39d88d61..1d4f590bd 100644 --- a/surfsense_web/components/layout/ui/sidebar/AllPrivateChatsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/AllPrivateChatsSidebar.tsx @@ -27,10 +27,12 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { Input } from "@/components/ui/input"; +import { Skeleton } from "@/components/ui/skeleton"; import { Spinner } from "@/components/ui/spinner"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; +import { useIsMobile } from "@/hooks/use-mobile"; import { deleteThread, fetchThreads, @@ -56,6 +58,7 @@ export function AllPrivateChatsSidebar({ const router = useRouter(); const params = useParams(); const queryClient = useQueryClient(); + const isMobile = useIsMobile(); const currentChatId = Array.isArray(params.chat_id) ? Number(params.chat_id[0]) @@ -303,8 +306,16 @@ export function AllPrivateChatsSidebar({
{isLoading ? ( -
- +
+ {[75, 90, 55, 80, 65, 85].map((titleWidth, i) => ( +
+ + +
+ ))}
) : error ? (
@@ -329,25 +340,37 @@ export function AllPrivateChatsSidebar({ isBusy && "opacity-50 pointer-events-none" )} > - - - - - -

- {t("updated") || "Updated"}:{" "} - {format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")} -

- - + {isMobile ? ( + + ) : ( + + + + + +

+ {t("updated") || "Updated"}:{" "} + {format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")} +

+ + + )} {isLoading ? ( -
- +
+ {[75, 90, 55, 80, 65, 85].map((titleWidth, i) => ( +
+ + +
+ ))}
) : error ? (
@@ -329,25 +340,37 @@ export function AllSharedChatsSidebar({ isBusy && "opacity-50 pointer-events-none" )} > - - - - - -

- {t("updated") || "Updated"}:{" "} - {format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")} -

- - + {isMobile ? ( + + ) : ( + + + + + +

+ {t("updated") || "Updated"}:{" "} + {format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")} +

+ + + )} {t("more_options")} - - {onRename && ( - { - e.stopPropagation(); - onRename(); - }} - > - - {t("rename") || "Rename"} - - )} - {onArchive && ( - { - e.stopPropagation(); - onArchive(); - }} - > - {archived ? ( + + {onRename && ( + { + e.stopPropagation(); + onRename(); + }} + > + + {t("rename") || "Rename"} + + )} + {onArchive && ( + { + e.stopPropagation(); + onArchive(); + }} + > + {archived ? ( <> {t("unarchive") || "Restore"} diff --git a/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx index f313dd6f9..b6caed330 100644 --- a/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx @@ -1,5 +1,6 @@ "use client"; +import { useQuery } from "@tanstack/react-query"; import { useAtom } from "jotai"; import { AlertCircle, @@ -19,7 +20,7 @@ import { X, } from "lucide-react"; import { AnimatePresence, motion } from "motion/react"; -import { useRouter } from "next/navigation"; +import { useParams, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { setCommentsCollapsedAtom, setTargetCommentIdAtom } from "@/atoms/chat/current-thread.atom"; @@ -41,6 +42,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { Input } from "@/components/ui/input"; +import { Skeleton } from "@/components/ui/skeleton"; import { Spinner } from "@/components/ui/spinner"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; @@ -52,7 +54,10 @@ import { isPageLimitExceededMetadata, } from "@/contracts/types/inbox.types"; import type { InboxItem } from "@/hooks/use-inbox"; +import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useMediaQuery } from "@/hooks/use-media-query"; +import { notificationsApiService } from "@/lib/apis/notifications-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; import { cn } from "@/lib/utils"; import { useSidebarContextSafe } from "../../hooks"; @@ -179,7 +184,9 @@ export function InboxSidebar({ }: InboxSidebarProps) { const t = useTranslations("sidebar"); const router = useRouter(); + const params = useParams(); const isMobile = !useMediaQuery("(min-width: 640px)"); + const searchSpaceId = params?.search_space_id ? Number(params.search_space_id) : null; // Comments collapsed state (desktop only, when docked) const [, setCommentsCollapsed] = useAtom(setCommentsCollapsedAtom); @@ -187,12 +194,22 @@ export function InboxSidebar({ const [, setTargetCommentId] = useAtom(setTargetCommentIdAtom); const [searchQuery, setSearchQuery] = useState(""); + const debouncedSearch = useDebouncedValue(searchQuery, 300); + const isSearchMode = !!debouncedSearch.trim(); const [activeTab, setActiveTab] = useState("comments"); const [activeFilter, setActiveFilter] = useState("all"); const [selectedConnector, setSelectedConnector] = useState(null); const [mounted, setMounted] = useState(false); // Dropdown state for filter menu (desktop only) const [openDropdown, setOpenDropdown] = useState<"filter" | null>(null); + // Scroll shadow state for connector list + const [connectorScrollPos, setConnectorScrollPos] = useState<"top" | "middle" | "bottom">("top"); + const handleConnectorScroll = useCallback((e: React.UIEvent) => { + const el = e.currentTarget; + const atTop = el.scrollTop <= 2; + const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight <= 2; + setConnectorScrollPos(atTop ? "top" : atBottom ? "bottom" : "middle"); + }, []); // Drawer state for filter menu (mobile only) const [filterDrawerOpen, setFilterDrawerOpen] = useState(false); const [markingAsReadId, setMarkingAsReadId] = useState(null); @@ -200,6 +217,24 @@ export function InboxSidebar({ // Prefetch trigger ref - placed on item near the end const prefetchTriggerRef = useRef(null); + // Server-side search query (enabled only when user is typing a search) + // Determines which notification types to search based on active tab + const searchTypeFilter = activeTab === "comments" ? ("new_mention" as const) : undefined; + const { data: searchResponse, isLoading: isSearchLoading } = useQuery({ + queryKey: cacheKeys.notifications.search(searchSpaceId, debouncedSearch.trim(), activeTab), + queryFn: () => + notificationsApiService.getNotifications({ + queryParams: { + search_space_id: searchSpaceId ?? undefined, + type: searchTypeFilter, + search: debouncedSearch.trim(), + limit: 50, + }, + }), + staleTime: 30 * 1000, // 30 seconds (search results don't need to be super fresh) + enabled: isSearchMode && open, + }); + useEffect(() => { setMounted(true); }, []); @@ -234,17 +269,11 @@ export function InboxSidebar({ } }, [activeTab]); - // Both tabs now derive items from status (all types), so use status for pagination - const { loading, loadingMore = false, hasMore = false, loadMore } = status; + // Each tab uses its own data source for independent pagination + // Comments tab: uses mentions data source (fetches only mention/reply types from server) + const commentsItems = mentions.items; - // Comments tab: mentions and comment replies - const commentsItems = useMemo( - () => - status.items.filter((item) => item.type === "new_mention" || item.type === "comment_reply"), - [status.items] - ); - - // Status tab: connector indexing, document processing, page limit exceeded, connector deletion + // Status tab: filters status data source (fetches all types) to status-specific types const statusItems = useMemo( () => status.items.filter( @@ -257,6 +286,14 @@ export function InboxSidebar({ [status.items] ); + // Pagination switches based on active tab + const loading = activeTab === "comments" ? mentions.loading : status.loading; + const loadingMore = + activeTab === "comments" ? (mentions.loadingMore ?? false) : (status.loadingMore ?? false); + const hasMore = + activeTab === "comments" ? (mentions.hasMore ?? false) : (status.hasMore ?? false); + const loadMore = activeTab === "comments" ? mentions.loadMore : status.loadMore; + // Get unique connector types from status items for filtering const uniqueConnectorTypes = useMemo(() => { const connectorTypes = new Set(); @@ -279,9 +316,23 @@ export function InboxSidebar({ // Get items for current tab const displayItems = activeTab === "comments" ? commentsItems : statusItems; - // Filter items based on filter type, connector filter, and search query + // Filter items based on filter type, connector filter, and search mode + // When searching: use server-side API results (searches ALL notifications) + // When not searching: use Electric real-time items (fast, local) const filteredItems = useMemo(() => { - let items = displayItems; + // In search mode, use API results + let items: InboxItem[] = isSearchMode ? (searchResponse?.items ?? []) : displayItems; + + // For status tab search results, filter to status-specific types + if (isSearchMode && activeTab === "status") { + items = items.filter( + (item) => + item.type === "connector_indexing" || + item.type === "document_processing" || + item.type === "page_limit_exceeded" || + item.type === "connector_deletion" + ); + } // Apply read/unread filter if (activeFilter === "unread") { @@ -302,22 +353,14 @@ export function InboxSidebar({ }); } - // Apply search query - if (searchQuery.trim()) { - const query = searchQuery.toLowerCase(); - items = items.filter( - (item) => - item.title.toLowerCase().includes(query) || item.message.toLowerCase().includes(query) - ); - } - return items; - }, [displayItems, activeFilter, activeTab, selectedConnector, searchQuery]); + }, [displayItems, searchResponse, isSearchMode, activeFilter, activeTab, selectedConnector]); // Intersection Observer for infinite scroll with prefetching - // Only active when not searching (search results are client-side filtered) + // Re-runs when active tab changes so each tab gets its own pagination + // Disabled during server-side search (search results are not paginated via infinite scroll) useEffect(() => { - if (!loadMore || !hasMore || loadingMore || !open || searchQuery.trim()) return; + if (!loadMore || !hasMore || loadingMore || !open || isSearchMode) return; const observer = new IntersectionObserver( (entries) => { @@ -338,17 +381,11 @@ export function InboxSidebar({ } return () => observer.disconnect(); - }, [loadMore, hasMore, loadingMore, open, searchQuery]); + }, [loadMore, hasMore, loadingMore, open, isSearchMode, activeTab]); - // Unread counts derived from filtered items - const unreadCommentsCount = useMemo( - () => commentsItems.filter((item) => !item.read).length, - [commentsItems] - ); - const unreadStatusCount = useMemo( - () => statusItems.filter((item) => !item.read).length, - [statusItems] - ); + // Unread counts from server-side accurate totals (passed via props) + const unreadCommentsCount = mentions.unreadCount; + const unreadStatusCount = status.unreadCount; const handleItemClick = useCallback( async (item: InboxItem) => { @@ -539,26 +576,33 @@ export function InboxSidebar({
+ {/* Back button - mobile only */} + {isMobile && ( + + )}

{t("inbox") || "Inbox"}

{/* Mobile: Button that opens bottom drawer */} {isMobile ? ( <> - - - - - {t("filter") || "Filter"} - + {t("connectors") || "Connectors"} - setSelectedConnector(null)} - className="flex items-center justify-between" +
- - - {t("all_connectors") || "All connectors"} - - {selectedConnector === null && } - - {uniqueConnectorTypes.map((connector) => ( setSelectedConnector(connector.type)} + onClick={() => setSelectedConnector(null)} className="flex items-center justify-between" > - {getConnectorIcon(connector.type, "h-4 w-4")} - {connector.displayName} + + {t("all_connectors") || "All connectors"} - {selectedConnector === connector.type && } + {selectedConnector === null && } - ))} + {uniqueConnectorTypes.map((connector) => ( + setSelectedConnector(connector.type)} + className="flex items-center justify-between" + > + + {getConnectorIcon(connector.type, "h-4 w-4")} + {connector.displayName} + + {selectedConnector === connector.type && } + + ))} +
)} )} - - - - - - {t("mark_all_read") || "Mark all as read"} - - - {/* Close button - mobile only */} - {isMobile && ( + {isMobile ? ( + + ) : ( - {t("close") || "Close"} + + {t("mark_all_read") || "Mark all as read"} + )} {/* Dock/Undock button - desktop only */} @@ -881,17 +930,48 @@ export function InboxSidebar({
- {loading ? ( -
- + {(isSearchMode ? isSearchLoading : loading) ? ( +
+ {activeTab === "comments" + ? /* Comments skeleton: avatar + two-line text + time */ + [85, 60, 90, 70, 50, 75].map((titleWidth, i) => ( +
+ +
+ + +
+ +
+ )) + : /* Status skeleton: status icon circle + two-line text + time */ + [75, 90, 55, 80, 65, 85].map((titleWidth, i) => ( +
+ +
+ + +
+
+ + +
+
+ ))}
) : filteredItems.length > 0 ? (
{filteredItems.map((item, index) => { const isMarkingAsRead = markingAsReadId === item.id; - // Place prefetch trigger on 5th item from end (only if not searching) + // Place prefetch trigger on 5th item from end (only when not searching) const isPrefetchTrigger = - !searchQuery && hasMore && index === filteredItems.length - 5; + !isSearchMode && hasMore && index === filteredItems.length - 5; return (
- - - - - -

{item.title}

-

- {convertRenderedToDisplay(item.message)} -

- - + {isMobile ? ( + + ) : ( + + + + + +

{item.title}

+

+ {convertRenderedToDisplay(item.message)} +

+ + + )} {/* Time and unread dot - fixed width to prevent content shift */}
@@ -947,11 +1051,43 @@ export function InboxSidebar({ ); })} {/* Fallback trigger at the very end if less than 5 items and not searching */} - {!searchQuery && filteredItems.length < 5 && hasMore && ( + {!isSearchMode && filteredItems.length < 5 && hasMore && (
)} + {/* Loading more skeletons at the bottom during infinite scroll */} + {loadingMore && + (activeTab === "comments" + ? [80, 60, 90].map((titleWidth, i) => ( +
+ +
+ + +
+ +
+ )) + : [70, 85, 55].map((titleWidth, i) => ( +
+ +
+ + +
+
+ + +
+
+ )))}
- ) : searchQuery ? ( + ) : isSearchMode ? (

diff --git a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx index 71d85f600..377bf65f5 100644 --- a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx @@ -1,7 +1,8 @@ "use client"; -import { Menu, Plus } from "lucide-react"; +import { PanelRightClose, Plus } from "lucide-react"; import { Button } from "@/components/ui/button"; +import { ScrollArea } from "@/components/ui/scroll-area"; import { Sheet, SheetContent, SheetTitle } from "@/components/ui/sheet"; import type { ChatItem, NavItem, PageUsage, SearchSpace, User } from "../../types/layout.types"; import { SearchSpaceAvatar } from "../icon-rail/SearchSpaceAvatar"; @@ -43,7 +44,7 @@ interface MobileSidebarProps { export function MobileSidebarTrigger({ onClick }: { onClick: () => void }) { return ( ); @@ -97,15 +98,16 @@ export function MobileSidebar({ return ( - + Navigation - {/* Horizontal Search Spaces Rail */} -

-
- {searchSpaces.map((space) => ( -
+ {/* Vertical Search Spaces Rail - left side */} +
+ +
+ {searchSpaces.map((space) => ( 1} @@ -116,26 +118,28 @@ export function MobileSidebar({ onSearchSpaceSettings ? () => onSearchSpaceSettings(space) : undefined } size="md" + disableTooltip /> -
- ))} - -
+ ))} + +
+
- {/* Sidebar Content */} -
+ {/* Sidebar Content - right side */} +
onOpenChange(false)} navItems={navItems} onNavItemClick={handleNavItemClick} chats={chats} @@ -149,8 +153,22 @@ export function MobileSidebar({ onChatRename={onChatRename} onChatDelete={onChatDelete} onChatArchive={onChatArchive} - onViewAllSharedChats={onViewAllSharedChats} - onViewAllPrivateChats={onViewAllPrivateChats} + onViewAllSharedChats={ + onViewAllSharedChats + ? () => { + onOpenChange(false); + onViewAllSharedChats(); + } + : undefined + } + onViewAllPrivateChats={ + onViewAllPrivateChats + ? () => { + onOpenChange(false); + onViewAllPrivateChats(); + } + : undefined + } user={user} onSettings={onSettings} onManageMembers={onManageMembers} @@ -161,6 +179,7 @@ export function MobileSidebar({ setTheme={setTheme} className="w-full border-none" isLoadingChats={isLoadingChats} + disableTooltips />
diff --git a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx index 7b53fdc6a..883fa5890 100644 --- a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx @@ -50,6 +50,7 @@ interface SidebarProps { setTheme?: (theme: "light" | "dark" | "system") => void; className?: string; isLoadingChats?: boolean; + disableTooltips?: boolean; } export function Sidebar({ @@ -78,6 +79,7 @@ export function Sidebar({ setTheme, className, isLoadingChats = false, + disableTooltips = false, }: SidebarProps) { const t = useTranslations("sidebar"); @@ -95,20 +97,22 @@ export function Sidebar({ {})} + disableTooltip={disableTooltips} />
) : ( -
+
-
+
{})} + disableTooltip={disableTooltips} />
@@ -138,7 +142,7 @@ export function Sidebar({ {isCollapsed ? (
) : ( -
+
{/* Shared Chats Section - takes only space needed, max 50% */} - - - - - {t("view_all_shared_chats") || "View all shared chats"} - - + disableTooltips ? ( + + ) : ( + + + + + + {t("view_all_shared_chats") || "View all shared chats"} + + + ) ) : undefined } > @@ -208,21 +223,32 @@ export function Sidebar({ fillHeight={true} action={ onViewAllPrivateChats ? ( - - - - - - {t("view_all_private_chats") || "View all private chats"} - - + disableTooltips ? ( + + ) : ( + + + + + + {t("view_all_private_chats") || "View all private chats"} + + + ) ) : undefined } > diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarCollapseButton.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarCollapseButton.tsx index 3eaa87070..44f05249c 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarCollapseButton.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarCollapseButton.tsx @@ -8,21 +8,30 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip interface SidebarCollapseButtonProps { isCollapsed: boolean; onToggle: () => void; + disableTooltip?: boolean; } -export function SidebarCollapseButton({ isCollapsed, onToggle }: SidebarCollapseButtonProps) { +export function SidebarCollapseButton({ + isCollapsed, + onToggle, + disableTooltip = false, +}: SidebarCollapseButtonProps) { const t = useTranslations("sidebar"); + const button = ( + + ); + + if (disableTooltip) { + return button; + } + return ( - - - + {button} {isCollapsed ? `${t("expand_sidebar")} (⌘B)` : `${t("collapse_sidebar")} (⌘B)`} diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx index 28c359e64..cdcf2384e 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx @@ -1,6 +1,6 @@ "use client"; -import { ChevronsUpDown, Logs, Settings, Users } from "lucide-react"; +import { ChevronsUpDown, Settings, Users } from "lucide-react"; import { useParams, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { Button } from "@/components/ui/button"; @@ -35,14 +35,14 @@ export function SidebarHeader({ const searchSpaceId = params.search_space_id as string; return ( -
+
-
@@ -379,7 +410,9 @@ export function ImageConfigSidebar({ setFormData((p) => ({ ...p, description: e.target.value }))} + onChange={(e) => + setFormData((p) => ({ ...p, description: e.target.value })) + } />
@@ -390,7 +423,9 @@ export function ImageConfigSidebar({ setFormData((p) => ({ ...p, model_name: e.target.value }))} + onChange={(e) => + setFormData((p) => ({ ...p, model_name: e.target.value })) + } /> )}
@@ -489,14 +543,20 @@ export function ImageConfigSidebar({ setFormData((p) => ({ ...p, api_version: e.target.value }))} + onChange={(e) => + setFormData((p) => ({ ...p, api_version: e.target.value })) + } />
)} {/* Actions */}
-
- {filteredGlobal.map((config) => { - const isSelected = currentConfig?.id === config.id; - const isAuto = "is_auto_mode" in config && config.is_auto_mode; - return ( - handleSelect(config.id)} - className={cn( - "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50", - isSelected && "bg-accent/80", - isAuto && "border border-violet-200 dark:border-violet-800/50" - )} - > -
-
- {isAuto ? ( - - ) : ( - + {filteredGlobal.map((config) => { + const isSelected = currentConfig?.id === config.id; + const isAuto = "is_auto_mode" in config && config.is_auto_mode; + return ( + handleSelect(config.id)} + className={cn( + "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50", + isSelected && "bg-accent/80", + isAuto && "border border-violet-200 dark:border-violet-800/50" + )} + > +
+
+ {isAuto ? ( + + ) : ( + + )} +
+
+
+ {config.name} + {isAuto && ( + + Recommended + + )} + {isSelected && } +
+ + {isAuto ? "Auto load balancing" : config.model_name} + +
+ {onEdit && ( + { + e.stopPropagation(); + setOpen(false); + onEdit(config, true); + }} + /> )}
-
-
- {config.name} - {isAuto && ( - - Recommended - - )} - {isSelected && } -
- - {isAuto ? "Auto load balancing" : config.model_name} - -
- {onEdit && ( - { - e.stopPropagation(); - setOpen(false); - onEdit(config, true); - }} - /> - )} -
- - ); - })} + + ); + })} )} @@ -290,51 +289,49 @@ export function ImageModelSelector({ className, onAddNew, onEdit }: ImageModelSe Your Image Models
- {filteredUser.map((config) => { - const isSelected = currentConfig?.id === config.id; - return ( - handleSelect(config.id)} - className={cn( - "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50", - isSelected && "bg-accent/80" - )} - > -
-
- -
-
-
- {config.name} - {isSelected && ( - - )} -
- - {config.model_name} - -
- {onEdit && ( - + {filteredUser.map((config) => { + const isSelected = currentConfig?.id === config.id; + return ( + handleSelect(config.id)} + className={cn( + "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50", + isSelected && "bg-accent/80" )} -
- - ); - })} + > +
+
+ +
+
+
+ {config.name} + {isSelected && } +
+ + {config.model_name} + +
+ {onEdit && ( + + )} +
+ + ); + })} )} diff --git a/surfsense_web/components/new-chat/model-selector.tsx b/surfsense_web/components/new-chat/model-selector.tsx index 148028df2..ec1143e04 100644 --- a/surfsense_web/components/new-chat/model-selector.tsx +++ b/surfsense_web/components/new-chat/model-selector.tsx @@ -392,8 +392,8 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp )} - {/* Add New Config Button */} -
+ {/* Add New Config Button */} +

- {plan.billingText ?? (isNaN(Number(plan.price)) ? "" : isMonthly ? "billed monthly" : "billed annually")} + {plan.billingText ?? + (isNaN(Number(plan.price)) + ? "" + : isMonthly + ? "billed monthly" + : "billed annually")}

    diff --git a/surfsense_web/components/settings/image-model-manager.tsx b/surfsense_web/components/settings/image-model-manager.tsx index 4e3f1840a..e87cc9a95 100644 --- a/surfsense_web/components/settings/image-model-manager.tsx +++ b/surfsense_web/components/settings/image-model-manager.tsx @@ -95,16 +95,29 @@ const item = { export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { // Image gen config atoms - const { mutateAsync: createConfig, isPending: isCreating, error: createError } = - useAtomValue(createImageGenConfigMutationAtom); - const { mutateAsync: updateConfig, isPending: isUpdating, error: updateError } = - useAtomValue(updateImageGenConfigMutationAtom); - const { mutateAsync: deleteConfig, isPending: isDeleting, error: deleteError } = - useAtomValue(deleteImageGenConfigMutationAtom); + const { + mutateAsync: createConfig, + isPending: isCreating, + error: createError, + } = useAtomValue(createImageGenConfigMutationAtom); + const { + mutateAsync: updateConfig, + isPending: isUpdating, + error: updateError, + } = useAtomValue(updateImageGenConfigMutationAtom); + const { + mutateAsync: deleteConfig, + isPending: isDeleting, + error: deleteError, + } = useAtomValue(deleteImageGenConfigMutationAtom); const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom); - const { data: userConfigs, isFetching: configsLoading, error: fetchError, refetch: refreshConfigs } = - useAtomValue(imageGenConfigsAtom); + const { + data: userConfigs, + isFetching: configsLoading, + error: fetchError, + refetch: refreshConfigs, + } = useAtomValue(imageGenConfigsAtom); const { data: globalConfigs = [], isFetching: globalLoading } = useAtomValue(globalImageGenConfigsAtom); const { data: preferences = {}, isFetching: prefsLoading } = useAtomValue(llmPreferencesAtom); @@ -249,7 +262,9 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { data: { image_generation_config_id: typeof selectedPrefId === "string" - ? selectedPrefId ? parseInt(selectedPrefId) : undefined + ? selectedPrefId + ? parseInt(selectedPrefId) + : undefined : selectedPrefId, }, }); @@ -289,7 +304,12 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { {/* Errors */} {errors.map((err) => ( - + {err?.message} @@ -304,7 +324,8 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { - {globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length} global image model(s) + {globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length} global + image model(s) {" "} available from your administrator. @@ -342,18 +363,27 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { {globalConfigs.length > 0 && ( <> -
    Global
    +
    + Global +
    {globalConfigs.map((c) => { const isAuto = "is_auto_mode" in c && c.is_auto_mode; return (
    {isAuto ? ( - - AUTO + + + AUTO ) : ( - + {c.provider} )} @@ -366,11 +396,15 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { )} {(userConfigs?.length ?? 0) > 0 && ( <> -
    Your Models
    +
    + Your Models +
    {userConfigs?.map((c) => (
    - {c.provider} + + {c.provider} + {c.name} ({c.model_name})
    @@ -382,10 +416,23 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { {hasPrefChanges && (
    - -
    @@ -409,7 +456,10 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) {

    Your Image Models

    - @@ -435,7 +485,12 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { {userConfigs?.map((config) => ( - +
    @@ -448,8 +503,13 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) {
    -

    {config.name}

    - +

    + {config.name} +

    + {config.provider}
    @@ -457,7 +517,9 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { {config.model_name} {config.description && ( -

    {config.description}

    +

    + {config.description} +

    )}
    @@ -469,7 +531,12 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { - @@ -479,7 +546,12 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { - @@ -501,15 +573,30 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { )} {/* Create/Edit Dialog */} - { if (!open) { setIsDialogOpen(false); setEditingConfig(null); resetForm(); } }}> + { + if (!open) { + setIsDialogOpen(false); + setEditingConfig(null); + resetForm(); + } + }} + > - {editingConfig ? : } + {editingConfig ? ( + + ) : ( + + )} {editingConfig ? "Edit Image Model" : "Add Image Model"} - {editingConfig ? "Update your image generation model" : "Configure a new image generation model (DALL-E 3, GPT Image 1, etc.)"} + {editingConfig + ? "Update your image generation model" + : "Configure a new image generation model (DALL-E 3, GPT Image 1, etc.)"} @@ -541,7 +628,9 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { handleRoleAssignment(`${key}_llm_id`, value)} - > - - - - - - Unassigned - +
    + + -
    + {/* Custom Configurations */} + {newLLMConfigs.length > 0 && ( + <> +
    + Your Configurations +
    + {newLLMConfigs + .filter( + (config) => config.id && config.id.toString().trim() !== "" + ) + .map((config) => ( + +
    + + {config.provider} + + {config.name} + + ({config.model_name}) + +
    +     + ))} + + )} +     + +
    {assignedConfig && (
    {isGenerated && } diff --git a/surfsense_web/contracts/types/inbox.types.ts b/surfsense_web/contracts/types/inbox.types.ts index ebf1889a1..bd04eefd1 100644 --- a/surfsense_web/contracts/types/inbox.types.ts +++ b/surfsense_web/contracts/types/inbox.types.ts @@ -205,6 +205,7 @@ export const getNotificationsRequest = z.object({ search_space_id: z.number().optional(), type: inboxItemTypeEnum.optional(), before_date: z.string().optional(), + search: z.string().optional(), limit: z.number().min(1).max(100).optional(), offset: z.number().min(0).optional(), }), diff --git a/surfsense_web/contracts/types/new-llm-config.types.ts b/surfsense_web/contracts/types/new-llm-config.types.ts index 3f0d39e5a..b99df1022 100644 --- a/surfsense_web/contracts/types/new-llm-config.types.ts +++ b/surfsense_web/contracts/types/new-llm-config.types.ts @@ -213,9 +213,7 @@ export const getImageGenConfigsResponse = z.array(imageGenerationConfig); export const updateImageGenConfigRequest = z.object({ id: z.number(), - data: imageGenerationConfig - .omit({ id: true, created_at: true, search_space_id: true }) - .partial(), + data: imageGenerationConfig.omit({ id: true, created_at: true, search_space_id: true }).partial(), }); export const updateImageGenConfigResponse = imageGenerationConfig; diff --git a/surfsense_web/hooks/use-documents.ts b/surfsense_web/hooks/use-documents.ts index 369cc7b41..6060b9572 100644 --- a/surfsense_web/hooks/use-documents.ts +++ b/surfsense_web/hooks/use-documents.ts @@ -71,8 +71,14 @@ function isValidDocument(doc: DocumentElectric): boolean { * 3. Use syncHandle.isUpToDate to determine if deletions can be trusted * 4. Handles bulk deletions correctly by checking sync state * + * Filtering strategy: + * - Internal state always stores ALL documents (unfiltered) + * - typeFilter is applied client-side when returning documents + * - typeCounts always reflect the full dataset so the filter sidebar stays complete + * - Changing filters is instant (no API re-fetch or Electric re-sync) + * * @param searchSpaceId - The search space ID to filter documents - * @param typeFilter - Optional document types to filter by + * @param typeFilter - Optional document types to filter by (applied client-side) */ export function useDocuments( searchSpaceId: number | null, @@ -80,7 +86,8 @@ export function useDocuments( ) { const electricClient = useElectricClient(); - const [documents, setDocuments] = useState([]); + // Internal state: ALL documents (unfiltered) + const [allDocuments, setAllDocuments] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); @@ -94,14 +101,21 @@ export function useDocuments( const syncHandleRef = useRef(null); const liveQueryRef = useRef<{ unsubscribe?: () => void } | null>(null); - // Real-time type counts + // Type counts from ALL documents (unfiltered) — keeps filter sidebar complete const typeCounts = useMemo(() => { const counts: Record = {}; - for (const doc of documents) { + for (const doc of allDocuments) { counts[doc.document_type] = (counts[doc.document_type] || 0) + 1; } return counts; - }, [documents]); + }, [allDocuments]); + + // Client-side filtered documents for display + const documents = useMemo(() => { + if (typeFilter.length === 0) return allDocuments; + const filterSet = new Set(typeFilter); + return allDocuments.filter((doc) => filterSet.has(doc.document_type)); + }, [allDocuments, typeFilter]); // Populate user cache from API response const populateUserCache = useCallback( @@ -151,7 +165,8 @@ export function useDocuments( [] ); - // EFFECT 1: Load from API (PRIMARY source of truth) + // EFFECT 1: Load ALL documents from API (PRIMARY source of truth) + // No type filter — always fetches everything so typeCounts stay complete useEffect(() => { if (!searchSpaceId) { setLoading(false); @@ -160,7 +175,6 @@ export function useDocuments( // Capture validated value for async closure const spaceId = searchSpaceId; - const currentTypeFilter = typeFilter; let mounted = true; apiLoadedRef.current = false; @@ -174,8 +188,7 @@ export function useDocuments( queryParams: { search_space_id: spaceId, page: 0, - page_size: -1, // Fetch all documents - ...(currentTypeFilter.length > 0 && { document_types: currentTypeFilter }), + page_size: -1, // Fetch all documents (unfiltered) }, }); @@ -183,7 +196,7 @@ export function useDocuments( populateUserCache(response.items); const docs = response.items.map(apiToDisplayDoc); - setDocuments(docs); + setAllDocuments(docs); apiLoadedRef.current = true; setError(null); console.log("[useDocuments] API loaded", docs.length, "documents"); @@ -201,16 +214,16 @@ export function useDocuments( return () => { mounted = false; }; - }, [searchSpaceId, typeFilter, populateUserCache, apiToDisplayDoc]); + }, [searchSpaceId, populateUserCache, apiToDisplayDoc]); // EFFECT 2: Start Electric sync + live query for real-time updates + // No type filter — syncs and queries ALL documents; filtering is client-side useEffect(() => { if (!searchSpaceId || !electricClient) return; // Capture validated values for async closure const spaceId = searchSpaceId; const client = electricClient; - const currentTypeFilter = typeFilter; let mounted = true; @@ -228,7 +241,7 @@ export function useDocuments( try { console.log("[useDocuments] Starting Electric sync for real-time updates"); - // Start Electric sync + // Start Electric sync (all documents for this search space) const handle = await client.syncShape({ table: "documents", where: `search_space_id = ${spaceId}`, @@ -263,7 +276,7 @@ export function useDocuments( if (!mounted) return; - // Set up live query + // Set up live query (unfiltered — type filtering is done client-side) const db = client.db as { live?: { query: ( @@ -281,21 +294,12 @@ export function useDocuments( return; } - let query = `SELECT id, document_type, search_space_id, title, created_by_id, created_at, status + const query = `SELECT id, document_type, search_space_id, title, created_by_id, created_at, status FROM documents - WHERE search_space_id = $1`; + WHERE search_space_id = $1 + ORDER BY created_at DESC`; - const params: (number | string)[] = [spaceId]; - - if (currentTypeFilter.length > 0) { - const placeholders = currentTypeFilter.map((_, i) => `$${i + 2}`).join(", "); - query += ` AND document_type IN (${placeholders})`; - params.push(...currentTypeFilter); - } - - query += ` ORDER BY created_at DESC`; - - const liveQuery = await db.live.query(query, params); + const liveQuery = await db.live.query(query, [spaceId]); if (!mounted) { liveQuery.unsubscribe?.(); @@ -333,7 +337,7 @@ export function useDocuments( .then((response) => { populateUserCache(response.items); if (mounted) { - setDocuments((prev) => + setAllDocuments((prev) => prev.map((doc) => ({ ...doc, created_by_name: doc.created_by_id @@ -347,7 +351,7 @@ export function useDocuments( } // Smart update logic based on sync state - setDocuments((prev) => { + setAllDocuments((prev) => { // Don't process if API hasn't loaded yet if (!apiLoadedRef.current) { console.log("[useDocuments] Waiting for API load, skipping live update"); @@ -424,7 +428,7 @@ export function useDocuments( liveQueryRef.current = null; } }; - }, [searchSpaceId, electricClient, typeFilter, electricToDisplayDoc, populateUserCache]); + }, [searchSpaceId, electricClient, electricToDisplayDoc, populateUserCache]); // Track previous searchSpaceId to detect actual changes const prevSearchSpaceIdRef = useRef(null); @@ -432,7 +436,7 @@ export function useDocuments( // Reset on search space change (not on initial mount) useEffect(() => { if (prevSearchSpaceIdRef.current !== null && prevSearchSpaceIdRef.current !== searchSpaceId) { - setDocuments([]); + setAllDocuments([]); apiLoadedRef.current = false; userCacheRef.current.clear(); } diff --git a/surfsense_web/lib/apis/image-gen-config-api.service.ts b/surfsense_web/lib/apis/image-gen-config-api.service.ts index 84aeed3d8..379edfa53 100644 --- a/surfsense_web/lib/apis/image-gen-config-api.service.ts +++ b/surfsense_web/lib/apis/image-gen-config-api.service.ts @@ -32,11 +32,9 @@ class ImageGenConfigApiService { const msg = parsed.error.issues.map((i) => i.message).join(", "); throw new ValidationError(`Invalid request: ${msg}`); } - return baseApiService.post( - `/api/v1/image-generation-configs`, - createImageGenConfigResponse, - { body: parsed.data } - ); + return baseApiService.post(`/api/v1/image-generation-configs`, createImageGenConfigResponse, { + body: parsed.data, + }); }; /** diff --git a/surfsense_web/lib/apis/notifications-api.service.ts b/surfsense_web/lib/apis/notifications-api.service.ts index bd590dcd2..086633d81 100644 --- a/surfsense_web/lib/apis/notifications-api.service.ts +++ b/surfsense_web/lib/apis/notifications-api.service.ts @@ -51,6 +51,9 @@ class NotificationsApiService { if (queryParams.offset !== undefined) { params.append("offset", String(queryParams.offset)); } + if (queryParams.search) { + params.append("search", queryParams.search); + } const queryString = params.toString(); diff --git a/surfsense_web/lib/electric/client.ts b/surfsense_web/lib/electric/client.ts index 9d596a261..c8bbd60fa 100644 --- a/surfsense_web/lib/electric/client.ts +++ b/surfsense_web/lib/electric/client.ts @@ -67,14 +67,10 @@ const pendingSyncs = new Map>(); // v2: user-specific database architecture // v3: consistent cutoff date for sync+queries, visibility refresh support // v4: heartbeat-based stale notification detection with updated_at tracking -// v5: fixed duplicate key errors (root cause: unstable cutoff dates in use-inbox.ts) -// - added onMustRefetch handler for server-side refetch scenarios -// - fixed getSyncCutoffDate to use stable midnight UTC timestamps -// v6: real-time documents table - added title and created_by_id columns for live document display -// v7: removed use-documents-electric.ts - consolidated to single documents sync to prevent conflicts -// v8: added status column for real-time document processing status (ready/processing/failed) -// v9: added pending state for accurate document queue visibility -const SYNC_VERSION = 11; +// v5: fixed duplicate key errors, stable cutoff dates, onMustRefetch handler, +// real-time documents table with title/created_by_id/status columns, +// consolidated single documents sync, pending state for document queue visibility +const SYNC_VERSION = 5; // Database name prefix for identifying SurfSense databases const DB_PREFIX = "surfsense-"; diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts index c6981b28a..7dc6c54b6 100644 --- a/surfsense_web/lib/query-client/cache-keys.ts +++ b/surfsense_web/lib/query-client/cache-keys.ts @@ -92,4 +92,8 @@ export const cacheKeys = { bySearchSpace: (searchSpaceId: number) => ["public-chat-snapshots", "search-space", searchSpaceId] as const, }, + notifications: { + search: (searchSpaceId: number | null, search: string, tab: string) => + ["notifications", "search", searchSpaceId, search, tab] as const, + }, }; diff --git a/surfsense_web/messages/en.json b/surfsense_web/messages/en.json index fae4c7265..20e665586 100644 --- a/surfsense_web/messages/en.json +++ b/surfsense_web/messages/en.json @@ -376,7 +376,7 @@ "upload_documents": { "title": "Upload Documents", "subtitle": "Upload your files to make them searchable and accessible through AI-powered conversations.", - "file_size_limit": "Maximum file size: 50MB per file. Supported formats vary based on your ETL service configuration.", + "file_size_limit": "Maximum file size: 50MB per file.", "upload_limits": "Upload limit: {maxFiles} files, {maxSizeMB}MB total.", "drop_files": "Drop files here", "drag_drop": "Drag & drop files here", diff --git a/surfsense_web/messages/zh.json b/surfsense_web/messages/zh.json index 2667a06d1..4c194a5dc 100644 --- a/surfsense_web/messages/zh.json +++ b/surfsense_web/messages/zh.json @@ -360,7 +360,7 @@ "upload_documents": { "title": "上传文档", "subtitle": "上传您的文件，使其可通过 AI 对话进行搜索和访问。", - "file_size_limit": "最大文件大小：每个文件 50MB。支持的格式因您的 ETL 服务配置而异。", + "file_size_limit": "最大文件大小：每个文件 50MB。", "upload_limits": "上传限制：最多 {maxFiles} 个文件，总大小不超过 {maxSizeMB}MB。", "drop_files": "放下文件到这里", "drag_drop": "拖放文件到这里",