diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index c4facc84d..9aa4b0b34 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -9,6 +9,8 @@ CELERY_TASK_DEFAULT_QUEUE=surfsense # Redis for app-level features (heartbeats, podcast markers) # Defaults to CELERY_BROKER_URL when not set REDIS_APP_URL=redis://localhost:6379/0 +# Optional: TTL in seconds for connector indexing lock key +# CONNECTOR_INDEXING_LOCK_TTL_SECONDS=28800 #Electric(for migrations only) ELECTRIC_DB_USER=electric diff --git a/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py b/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py index a11e4ac38..fefd80f04 100644 --- a/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py +++ b/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py @@ -8,6 +8,7 @@ This module provides: - Tool factory for creating search_knowledge_base tools """ +import asyncio import json from datetime import datetime from typing import Any @@ -16,6 +17,7 @@ from langchain_core.tools import StructuredTool from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession +from app.db import async_session_maker from app.services.connector_service import ConnectorService # ============================================================================= @@ -333,7 +335,7 @@ async def search_knowledge_base_async( Returns: Formatted string with search results """ - all_documents = [] + all_documents: list[dict[str, Any]] = [] # Resolve date range (default last 2 years) from app.agents.new_chat.utils import resolve_date_range @@ -345,323 +347,131 @@ async def search_knowledge_base_async( connectors = _normalize_connectors(connectors_to_search, available_connectors) - for connector in connectors: + connector_specs: dict[str, tuple[str, bool, bool, dict[str, Any]]] = { + "YOUTUBE_VIDEO": ("search_youtube", True, True, {}), + "EXTENSION": ("search_extension", True, True, {}), + "CRAWLED_URL": ("search_crawled_urls", True, True, {}), + "FILE": ("search_files", True, True, {}), + "SLACK_CONNECTOR": ("search_slack", True, True, {}), + "TEAMS_CONNECTOR": ("search_teams", True, True, {}), + "NOTION_CONNECTOR": ("search_notion", True, True, {}), + "GITHUB_CONNECTOR": ("search_github", True, True, {}), + "LINEAR_CONNECTOR": ("search_linear", True, True, {}), + "TAVILY_API": ("search_tavily", False, True, {}), + "SEARXNG_API": ("search_searxng", False, True, {}), + "LINKUP_API": ("search_linkup", False, False, {"mode": "standard"}), + "BAIDU_SEARCH_API": ("search_baidu", False, True, {}), + "DISCORD_CONNECTOR": ("search_discord", True, True, {}), + "JIRA_CONNECTOR": ("search_jira", True, True, {}), + "GOOGLE_CALENDAR_CONNECTOR": ("search_google_calendar", True, True, {}), + "AIRTABLE_CONNECTOR": ("search_airtable", True, True, {}), + "GOOGLE_GMAIL_CONNECTOR": ("search_google_gmail", True, True, {}), + "GOOGLE_DRIVE_FILE": ("search_google_drive", True, True, {}), + "CONFLUENCE_CONNECTOR": ("search_confluence", True, True, {}), + "CLICKUP_CONNECTOR": ("search_clickup", True, True, {}), + "LUMA_CONNECTOR": ("search_luma", True, True, {}), + "ELASTICSEARCH_CONNECTOR": ("search_elasticsearch", True, True, {}), + "NOTE": ("search_notes", True, True, {}), + "BOOKSTACK_CONNECTOR": ("search_bookstack", True, True, {}), + "CIRCLEBACK": ("search_circleback", True, True, {}), + "OBSIDIAN_CONNECTOR": ("search_obsidian", True, True, {}), + # Composio connectors + "COMPOSIO_GOOGLE_DRIVE_CONNECTOR": ( + "search_composio_google_drive", + True, + True, + {}, + ), + "COMPOSIO_GMAIL_CONNECTOR": ("search_composio_gmail", True, True, {}), + "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": ( + "search_composio_google_calendar", + True, + True, + {}, + ), + } + + # Keep a conservative cap to avoid overloading DB/external services. + max_parallel_searches = 4 + semaphore = asyncio.Semaphore(max_parallel_searches) + + async def _search_one_connector(connector: str) -> list[dict[str, Any]]: + spec = connector_specs.get(connector) + if spec is None: + return [] + + method_name, includes_date_range, includes_top_k, extra_kwargs = spec + kwargs: dict[str, Any] = { + "user_query": query, + "search_space_id": search_space_id, + **extra_kwargs, + } + if includes_top_k: + kwargs["top_k"] = top_k + if includes_date_range: + kwargs["start_date"] = resolved_start_date + kwargs["end_date"] = resolved_end_date + try: - if connector == "YOUTUBE_VIDEO": - _, chunks = await connector_service.search_youtube( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, + # Use isolated session per connector. Shared AsyncSession cannot safely + # run concurrent DB operations. + async with semaphore, async_session_maker() as isolated_session: + isolated_connector_service = ConnectorService( + isolated_session, search_space_id ) - all_documents.extend(chunks) - - elif connector == "EXTENSION": - _, chunks = await connector_service.search_extension( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "CRAWLED_URL": - _, chunks = await connector_service.search_crawled_urls( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "FILE": - _, chunks = await connector_service.search_files( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "SLACK_CONNECTOR": - _, chunks = await connector_service.search_slack( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "TEAMS_CONNECTOR": - _, chunks = await connector_service.search_teams( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "NOTION_CONNECTOR": - _, chunks = await connector_service.search_notion( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "GITHUB_CONNECTOR": - _, chunks = await connector_service.search_github( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "LINEAR_CONNECTOR": - _, chunks = await connector_service.search_linear( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "TAVILY_API": - _, chunks = await connector_service.search_tavily( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - ) - all_documents.extend(chunks) - - elif connector == "SEARXNG_API": - _, chunks = await connector_service.search_searxng( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - ) - all_documents.extend(chunks) - - elif connector == "LINKUP_API": - # Keep behavior aligned with researcher: default "standard" - _, chunks = await connector_service.search_linkup( - user_query=query, - search_space_id=search_space_id, - mode="standard", - ) - all_documents.extend(chunks) - - elif connector == "BAIDU_SEARCH_API": - _, chunks = await connector_service.search_baidu( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - ) - all_documents.extend(chunks) - - elif connector == "DISCORD_CONNECTOR": - _, chunks = await connector_service.search_discord( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "JIRA_CONNECTOR": - _, chunks = await connector_service.search_jira( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "GOOGLE_CALENDAR_CONNECTOR": - _, chunks = await connector_service.search_google_calendar( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "AIRTABLE_CONNECTOR": - _, chunks = await connector_service.search_airtable( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "GOOGLE_GMAIL_CONNECTOR": - _, chunks = await connector_service.search_google_gmail( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "GOOGLE_DRIVE_FILE": - _, chunks = await connector_service.search_google_drive( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "CONFLUENCE_CONNECTOR": - _, chunks = await connector_service.search_confluence( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "CLICKUP_CONNECTOR": - _, chunks = await connector_service.search_clickup( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "LUMA_CONNECTOR": - _, chunks = await connector_service.search_luma( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "ELASTICSEARCH_CONNECTOR": - _, chunks = await connector_service.search_elasticsearch( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "NOTE": - _, chunks = await connector_service.search_notes( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "BOOKSTACK_CONNECTOR": - _, chunks = await connector_service.search_bookstack( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "CIRCLEBACK": - _, chunks = await connector_service.search_circleback( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "OBSIDIAN_CONNECTOR": - _, chunks = await connector_service.search_obsidian( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - # ========================================================= - # Composio Connectors - # ========================================================= - elif connector == "COMPOSIO_GOOGLE_DRIVE_CONNECTOR": - _, chunks = await connector_service.search_composio_google_drive( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "COMPOSIO_GMAIL_CONNECTOR": - _, chunks = await connector_service.search_composio_gmail( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - - elif connector == "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": - _, chunks = await connector_service.search_composio_google_calendar( - user_query=query, - search_space_id=search_space_id, - top_k=top_k, - start_date=resolved_start_date, - end_date=resolved_end_date, - ) - all_documents.extend(chunks) - + connector_method = getattr(isolated_connector_service, method_name) + _, chunks = await connector_method(**kwargs) + return chunks except Exception as e: print(f"Error searching connector {connector}: {e}") - continue + return [] - # Deduplicate by content hash + connector_results = await asyncio.gather( + *[_search_one_connector(connector) for connector in connectors] + ) + for chunks in connector_results: + all_documents.extend(chunks) + + # Deduplicate primarily by document ID. Only fall back to content hashing + # when a document has no ID. seen_doc_ids: set[Any] = set() - seen_hashes: set[int] = set() + seen_content_hashes: set[int] = set() deduplicated: list[dict[str, Any]] = [] + + def _content_fingerprint(document: dict[str, Any]) -> int | None: + chunks = document.get("chunks") + if isinstance(chunks, list): + chunk_texts = [] + for chunk in chunks: + if not isinstance(chunk, dict): + continue + chunk_content = (chunk.get("content") or "").strip() + if chunk_content: + chunk_texts.append(chunk_content) + if chunk_texts: + return hash("||".join(chunk_texts)) + + flat_content = (document.get("content") or "").strip() + if flat_content: + return hash(flat_content) + return None + for doc in all_documents: doc_id = (doc.get("document", {}) or {}).get("id") - content = (doc.get("content", "") or "").strip() - content_hash = hash(content) - if (doc_id and doc_id in seen_doc_ids) or content_hash in seen_hashes: + if doc_id is not None: + if doc_id in seen_doc_ids: + continue + seen_doc_ids.add(doc_id) + deduplicated.append(doc) continue - if doc_id: - seen_doc_ids.add(doc_id) - seen_hashes.add(content_hash) + content_hash = _content_fingerprint(doc) + if content_hash is not None: + if content_hash in seen_content_hashes: + continue + seen_content_hashes.add(content_hash) + deduplicated.append(doc) return format_documents_for_context(deduplicated) diff --git a/surfsense_backend/app/agents/new_chat/tools/podcast.py b/surfsense_backend/app/agents/new_chat/tools/podcast.py index e6412f4f2..8ac537f9a 100644 --- a/surfsense_backend/app/agents/new_chat/tools/podcast.py +++ b/surfsense_backend/app/agents/new_chat/tools/podcast.py @@ -11,21 +11,18 @@ Duplicate request prevention: - Returns a friendly message if a podcast is already being generated """ -import os from typing import Any import redis from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession +from app.config import config from app.db import Podcast, PodcastStatus # Redis connection for tracking active podcast tasks # Defaults to the Celery broker when REDIS_APP_URL is not set -REDIS_URL = os.getenv( - "REDIS_APP_URL", - os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0"), -) +REDIS_URL = config.REDIS_APP_URL _redis_client: redis.Redis | None = None diff --git a/surfsense_backend/app/agents/new_chat/tools/scrape_webpage.py b/surfsense_backend/app/agents/new_chat/tools/scrape_webpage.py index e3c58c857..014126927 100644 --- a/surfsense_backend/app/agents/new_chat/tools/scrape_webpage.py +++ b/surfsense_backend/app/agents/new_chat/tools/scrape_webpage.py @@ -110,7 +110,17 @@ async def _scrape_youtube_video( if residential_proxies: http_client.proxies.update(residential_proxies) ytt_api = YouTubeTranscriptApi(http_client=http_client) - captions = ytt_api.fetch(video_id) + + # List all available transcripts and pick the first one + # (the video's primary language) instead of defaulting to English + transcript_list = ytt_api.list(video_id) + transcript = next(iter(transcript_list)) + captions = transcript.fetch() + + logger.info( + f"[scrape_webpage] Fetched transcript for {video_id} " + f"in {transcript.language} ({transcript.language_code})" + ) transcript_segments = [] for line in captions: diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index e102c414d..68c65a818 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -213,6 +213,17 @@ class Config: # Database DATABASE_URL = os.getenv("DATABASE_URL") + # Celery / Redis + CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0") + CELERY_RESULT_BACKEND = os.getenv( + "CELERY_RESULT_BACKEND", "redis://localhost:6379/0" + ) + CELERY_TASK_DEFAULT_QUEUE = os.getenv("CELERY_TASK_DEFAULT_QUEUE", "surfsense") + REDIS_APP_URL = os.getenv("REDIS_APP_URL", CELERY_BROKER_URL) + CONNECTOR_INDEXING_LOCK_TTL_SECONDS = int( + os.getenv("CONNECTOR_INDEXING_LOCK_TTL_SECONDS", str(8 * 60 * 60)) + ) + NEXT_FRONTEND_URL = os.getenv("NEXT_FRONTEND_URL") # Backend URL to override the http to https in the OAuth redirect URI BACKEND_URL = os.getenv("BACKEND_URL") diff --git a/surfsense_backend/app/connectors/notion_history.py b/surfsense_backend/app/connectors/notion_history.py index ff8478905..525b0b4c3 100644 --- a/surfsense_backend/app/connectors/notion_history.py +++ b/surfsense_backend/app/connectors/notion_history.py @@ -27,6 +27,12 @@ T = TypeVar("T") MAX_RETRIES = 5 BASE_RETRY_DELAY = 1.0 # seconds MAX_RETRY_DELAY = 60.0 # seconds (Notion's max request timeout) +MAX_RATE_LIMIT_WAIT_SECONDS = float( + getattr(config, "NOTION_MAX_RETRY_AFTER_SECONDS", 30.0) +) +MAX_TOTAL_RETRY_WAIT_SECONDS = float( + getattr(config, "NOTION_MAX_TOTAL_RETRY_WAIT_SECONDS", 120.0) +) # Type alias for retry callback function # Signature: async callback(retry_reason, attempt, max_attempts, wait_seconds) -> None @@ -292,6 +298,7 @@ class NotionHistoryConnector: """ last_exception: APIResponseError | None = None retry_delay = BASE_RETRY_DELAY + total_wait_time = 0.0 for attempt in range(MAX_RETRIES): try: @@ -325,6 +332,15 @@ class NotionHistoryConnector: wait_time = retry_delay else: wait_time = retry_delay + + # Avoid very long worker sleeps from external Retry-After values. + if wait_time > MAX_RATE_LIMIT_WAIT_SECONDS: + logger.warning( + f"Notion Retry-After ({wait_time}s) exceeds cap " + f"({MAX_RATE_LIMIT_WAIT_SECONDS}s). Clamping wait time." + ) + wait_time = MAX_RATE_LIMIT_WAIT_SECONDS + logger.warning( f"Notion API rate limited (429). " f"Waiting {wait_time}s. Attempt {attempt + 1}/{MAX_RETRIES}" @@ -348,6 +364,14 @@ class NotionHistoryConnector: # Notify about retry via callback (for user notifications) # Call before sleeping so user sees the message while we wait + if total_wait_time + wait_time > MAX_TOTAL_RETRY_WAIT_SECONDS: + logger.error( + "Notion API retry budget exceeded " + f"({total_wait_time + wait_time:.1f}s > " + f"{MAX_TOTAL_RETRY_WAIT_SECONDS:.1f}s). Failing fast." + ) + raise + if on_retry: try: await on_retry( @@ -362,6 +386,7 @@ class NotionHistoryConnector: # Wait before retrying await asyncio.sleep(wait_time) + total_wait_time += wait_time # Exponential backoff for next attempt retry_delay = min(retry_delay * 2, MAX_RETRY_DELAY) diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index b20f8cd9c..226d511cc 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -178,9 +178,26 @@ async def create_documents_file_upload( session, unique_identifier_hash ) if existing: - # Clean up temp file for duplicates - os.unlink(temp_path) - skipped_duplicates += 1 + if DocumentStatus.is_state(existing.status, DocumentStatus.READY): + # True duplicate — content already indexed, skip + os.unlink(temp_path) + skipped_duplicates += 1 + continue + + # Existing document is stuck (failed/pending/processing) + # Reset it to pending and re-dispatch for processing + existing.status = DocumentStatus.pending() + existing.content = "Processing..." + existing.document_metadata = { + **(existing.document_metadata or {}), + "file_size": file_size, + "upload_time": datetime.now().isoformat(), + } + existing.updated_at = get_current_timestamp() + created_documents.append(existing) + files_to_process.append( + (existing, temp_path, file.filename or "unknown") + ) continue # Create pending document (visible immediately in UI via ElectricSQL) diff --git a/surfsense_backend/app/routes/notifications_routes.py b/surfsense_backend/app/routes/notifications_routes.py index e8e89e6c4..4f80c6529 100644 --- a/surfsense_backend/app/routes/notifications_routes.py +++ b/surfsense_backend/app/routes/notifications_routes.py @@ -144,6 +144,9 @@ async def list_notifications( before_date: str | None = Query( None, description="Get notifications before this ISO date (for pagination)" ), + search: str | None = Query( + None, description="Search notifications by title or message (case-insensitive)" + ), limit: int = Query(50, ge=1, le=100, description="Number of items to return"), offset: int = Query(0, ge=0, description="Number of items to skip"), user: User = Depends(current_active_user), @@ -191,6 +194,15 @@ async def list_notifications( detail="Invalid date format. Use ISO format (e.g., 2024-01-15T00:00:00Z)", ) from None + # Filter by search query (case-insensitive title/message search) + if search: + search_term = f"%{search}%" + search_filter = Notification.title.ilike( + search_term + ) | Notification.message.ilike(search_term) + query = query.where(search_filter) + count_query = count_query.where(search_filter) + # Get total count total_result = await session.execute(count_query) total = total_result.scalar() or 0 diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 747e02834..ba6877376 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -19,7 +19,7 @@ Non-OAuth connectors (BookStack, GitHub, etc.) are limited to one per search spa """ import logging -import os +from contextlib import suppress from datetime import UTC, datetime, timedelta from typing import Any @@ -32,6 +32,7 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select +from app.config import config from app.connectors.github_connector import GitHubConnector from app.db import ( Permission, @@ -70,6 +71,10 @@ from app.tasks.connector_indexers import ( index_slack_messages, ) from app.users import current_active_user +from app.utils.indexing_locks import ( + acquire_connector_indexing_lock, + release_connector_indexing_lock, +) from app.utils.periodic_scheduler import ( create_periodic_schedule, delete_periodic_schedule, @@ -91,11 +96,9 @@ def get_heartbeat_redis_client() -> redis.Redis: """Get or create Redis client for heartbeat tracking.""" global _heartbeat_redis_client if _heartbeat_redis_client is None: - redis_url = os.getenv( - "REDIS_APP_URL", - os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0"), + _heartbeat_redis_client = redis.from_url( + config.REDIS_APP_URL, decode_responses=True ) - _heartbeat_redis_client = redis.from_url(redis_url, decode_responses=True) return _heartbeat_redis_client @@ -1229,10 +1232,19 @@ async def _run_indexing_with_notifications( from celery.exceptions import SoftTimeLimitExceeded notification = None + connector_lock_acquired = False # Track indexed count for retry notifications and heartbeat current_indexed_count = 0 try: + connector_lock_acquired = acquire_connector_indexing_lock(connector_id) + if not connector_lock_acquired: + logger.info( + f"Skipping indexing for connector {connector_id} " + "(another worker already holds Redis connector lock)" + ) + return + # Get connector info for notification connector_result = await session.execute( select(SearchSourceConnector).where( @@ -1558,6 +1570,9 @@ async def _run_indexing_with_notifications( get_heartbeat_redis_client().delete(heartbeat_key) except Exception: pass # Ignore cleanup errors - key will expire anyway + if connector_lock_acquired: + with suppress(Exception): + release_connector_indexing_lock(connector_id) async def run_notion_indexing_with_new_session( diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index 6dfcbff46..3e60b5819 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -1,6 +1,8 @@ """Celery tasks for document processing.""" +import asyncio import logging +import os from uuid import UUID from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine @@ -17,6 +19,77 @@ from app.tasks.document_processors import ( logger = logging.getLogger(__name__) +# ===== Redis heartbeat for document processing tasks ===== +# Same mechanism as connector indexing heartbeats (search_source_connectors_routes.py). +# A background coroutine refreshes a Redis key every 60s with a 2-min TTL. +# If the Celery worker crashes, the coroutine dies, the key expires, and the +# stale_notification_cleanup_task detects the missing key and marks the +# notification + document as failed. +_doc_heartbeat_redis = None +HEARTBEAT_TTL_SECONDS = 120 # 2 minutes — same as connector indexing +HEARTBEAT_REFRESH_INTERVAL = 60 # Refresh every 60 seconds + + +def _get_doc_heartbeat_redis(): + """Get Redis client for document processing heartbeat.""" + import redis + + global _doc_heartbeat_redis + if _doc_heartbeat_redis is None: + _doc_heartbeat_redis = redis.from_url( + config.REDIS_APP_URL, decode_responses=True + ) + return _doc_heartbeat_redis + + +def _get_heartbeat_key(notification_id: int) -> str: + """Generate Redis key for document processing heartbeat. + + Uses same key pattern as connector indexing: indexing:heartbeat:{notification_id} + """ + return f"indexing:heartbeat:{notification_id}" + + +def _start_heartbeat(notification_id: int) -> None: + """Set initial Redis heartbeat key for a document processing task.""" + try: + key = _get_heartbeat_key(notification_id) + _get_doc_heartbeat_redis().setex(key, HEARTBEAT_TTL_SECONDS, "started") + except Exception as e: + logger.warning( + f"Failed to set initial heartbeat for notification {notification_id}: {e}" + ) + + +def _stop_heartbeat(notification_id: int) -> None: + """Delete Redis heartbeat key when task completes (success or failure).""" + try: + key = _get_heartbeat_key(notification_id) + _get_doc_heartbeat_redis().delete(key) + except Exception: + pass # Key will expire on its own + + +async def _run_heartbeat_loop(notification_id: int): + """Background coroutine that refreshes Redis heartbeat every 60 seconds. + + This keeps the heartbeat alive while the task is running. + When the task finishes, this coroutine is cancelled via heartbeat_task.cancel(). + When the worker crashes, this coroutine dies with it and the key expires. + """ + key = _get_heartbeat_key(notification_id) + try: + while True: + await asyncio.sleep(HEARTBEAT_REFRESH_INTERVAL) + try: + _get_doc_heartbeat_redis().setex(key, HEARTBEAT_TTL_SECONDS, "alive") + except Exception as e: + logger.warning( + f"Failed to refresh heartbeat for notification {notification_id}: {e}" + ) + except asyncio.CancelledError: + pass # Normal cancellation when task completes + def get_celery_session_maker(): """ @@ -44,8 +117,6 @@ def process_extension_document_task( search_space_id: ID of the search space user_id: ID of the user """ - import asyncio - # Create a new event loop for this task loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -196,8 +267,6 @@ def process_youtube_video_task(self, url: str, search_space_id: int, user_id: st search_space_id: ID of the search space user_id: ID of the user """ - import asyncio - loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -226,6 +295,10 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str): ) ) + # Start Redis heartbeat for stale task detection + _start_heartbeat(notification.id) + heartbeat_task = asyncio.create_task(_run_heartbeat_loop(notification.id)) + log_entry = await task_logger.log_task_start( task_name="process_youtube_video", source="document_processor", @@ -243,7 +316,7 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str): ) result = await add_youtube_video_document( - session, url, search_space_id, user_id + session, url, search_space_id, user_id, notification=notification ) if result: @@ -307,6 +380,10 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str): logger.error(f"Error processing YouTube video: {e!s}") raise + finally: + # Stop heartbeat — key deleted on success, expires on crash + heartbeat_task.cancel() + _stop_heartbeat(notification.id) @celery_app.task(name="process_file_upload", bind=True) @@ -322,8 +399,6 @@ def process_file_upload_task( search_space_id: ID of the search space user_id: ID of the user """ - import asyncio - import os import traceback logger.info( @@ -336,7 +411,7 @@ def process_file_upload_task( if not os.path.exists(file_path): logger.error( f"[process_file_upload] File does not exist: {file_path}. " - "The temp file may have been cleaned up before the task ran." + "File may have been removed before syncing could start." ) return @@ -370,8 +445,6 @@ async def _process_file_upload( file_path: str, filename: str, search_space_id: int, user_id: str ): """Process file upload with new session.""" - import os - from app.tasks.document_processors.file_processors import process_file_in_background logger.info(f"[_process_file_upload] Starting async processing for: {filename}") @@ -404,6 +477,10 @@ async def _process_file_upload( f"[_process_file_upload] Notification created with ID: {notification.id if notification else 'None'}" ) + # Start Redis heartbeat for stale task detection + _start_heartbeat(notification.id) + heartbeat_task = asyncio.create_task(_run_heartbeat_loop(notification.id)) + log_entry = await task_logger.log_task_start( task_name="process_file_upload", source="document_processor", @@ -535,6 +612,10 @@ async def _process_file_upload( ) logger.error(error_message) raise + finally: + # Stop heartbeat — key deleted on success, expires on crash + heartbeat_task.cancel() + _stop_heartbeat(notification.id) @celery_app.task(name="process_file_upload_with_document", bind=True) @@ -560,8 +641,6 @@ def process_file_upload_with_document_task( search_space_id: ID of the search space user_id: ID of the user """ - import asyncio - import os import traceback logger.info( @@ -573,7 +652,7 @@ def process_file_upload_with_document_task( if not os.path.exists(temp_path): logger.error( f"[process_file_upload_with_document] File does not exist: {temp_path}. " - "The temp file may have been cleaned up before the task ran." + "File may have been removed before syncing could start." ) # Mark document as failed since file is missing loop = asyncio.new_event_loop() @@ -582,7 +661,7 @@ def process_file_upload_with_document_task( loop.run_until_complete( _mark_document_failed( document_id, - "File not found - temp file may have been cleaned up", + "File not found. Please re-upload the file.", ) ) finally: @@ -640,8 +719,6 @@ async def _process_file_with_document( - Processes the file (parsing, embedding, chunking) - Updates document to 'ready' on success or 'failed' on error """ - import os - from app.db import Document, DocumentStatus from app.tasks.document_processors.base import get_current_timestamp from app.tasks.document_processors.file_processors import ( @@ -689,6 +766,19 @@ async def _process_file_with_document( ) ) + # Store document_id in notification metadata so cleanup task can find the document + if notification and notification.notification_metadata is not None: + notification.notification_metadata["document_id"] = document_id + from sqlalchemy.orm.attributes import flag_modified + + flag_modified(notification, "notification_metadata") + await session.commit() + await session.refresh(notification) + + # Start Redis heartbeat for stale task detection + _start_heartbeat(notification.id) + heartbeat_task = asyncio.create_task(_run_heartbeat_loop(notification.id)) + log_entry = await task_logger.log_task_start( task_name="process_file_upload_with_document", source="document_processor", @@ -822,6 +912,10 @@ async def _process_file_with_document( raise finally: + # Stop heartbeat — key deleted on success, expires on crash + heartbeat_task.cancel() + _stop_heartbeat(notification.id) + # Clean up temp file if os.path.exists(temp_path): try: @@ -856,8 +950,6 @@ def process_circleback_meeting_task( search_space_id: ID of the search space connector_id: ID of the Circleback connector (for deletion support) """ - import asyncio - loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -897,6 +989,7 @@ async def _process_circleback_meeting( # Create notification if user_id is available notification = None + heartbeat_task = None if user_id: notification = ( await NotificationService.document_processing.notify_processing_started( @@ -908,6 +1001,10 @@ async def _process_circleback_meeting( ) ) + # Start Redis heartbeat for stale task detection + _start_heartbeat(notification.id) + heartbeat_task = asyncio.create_task(_run_heartbeat_loop(notification.id)) + log_entry = await task_logger.log_task_start( task_name="process_circleback_meeting", source="circleback_webhook", @@ -1000,3 +1097,9 @@ async def _process_circleback_meeting( logger.error(f"Error processing Circleback meeting: {e!s}") raise + finally: + # Stop heartbeat — key deleted on success, expires on crash + if heartbeat_task: + heartbeat_task.cancel() + if notification: + _stop_heartbeat(notification.id) diff --git a/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py b/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py index 14df83508..973e7e750 100644 --- a/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py @@ -46,16 +46,10 @@ def get_celery_session_maker(): def _clear_generating_podcast(search_space_id: int) -> None: """Clear the generating podcast marker from Redis when task completes.""" - import os - import redis try: - redis_url = os.getenv( - "REDIS_APP_URL", - os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0"), - ) - client = redis.from_url(redis_url, decode_responses=True) + client = redis.from_url(config.REDIS_APP_URL, decode_responses=True) key = f"podcast:generating:{search_space_id}" client.delete(key) logger.info( diff --git a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py index b33e25170..80d271aaa 100644 --- a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py +++ b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py @@ -9,7 +9,8 @@ from sqlalchemy.pool import NullPool from app.celery_app import celery_app from app.config import config -from app.db import SearchSourceConnector, SearchSourceConnectorType +from app.db import Notification, SearchSourceConnector, SearchSourceConnectorType +from app.utils.indexing_locks import is_connector_indexing_locked logger = logging.getLogger(__name__) @@ -107,6 +108,32 @@ async def _check_and_trigger_schedules(): # Trigger indexing for each due connector for connector in due_connectors: + # Primary guard: Redis lock indicates a task is currently running. + if is_connector_indexing_locked(connector.id): + logger.info( + f"Skipping periodic indexing for connector {connector.id} " + "(Redis lock indicates indexing is already in progress)" + ) + continue + + # Skip scheduling if a sync for this connector is already in progress. + # This prevents duplicate tasks from piling up under slow/rate-limited providers. + in_progress_result = await session.execute( + select(Notification.id).where( + Notification.type == "connector_indexing", + Notification.notification_metadata["connector_id"].astext + == str(connector.id), + Notification.notification_metadata["status"].astext + == "in_progress", + ) + ) + if in_progress_result.first(): + logger.info( + f"Skipping periodic indexing for connector {connector.id} " + "(already has in-progress indexing notification)" + ) + continue + task = task_map.get(connector.connector_type) if task: logger.info( diff --git a/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py b/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py index ef3a30e43..c2c82dd2c 100644 --- a/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py +++ b/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py @@ -1,24 +1,30 @@ -"""Celery task to detect and mark stale connector indexing notifications as failed. +"""Celery task to detect and mark stale notifications as failed. This task runs periodically (every 5 minutes by default) to find notifications that are stuck in "in_progress" status but don't have an active Redis heartbeat key. -These are marked as "failed" to prevent the frontend from showing a perpetual "syncing" state. +These are marked as "failed" to prevent the frontend from showing a perpetual +"syncing" or "processing" state. -Additionally, it cleans up documents stuck in pending/processing state that belong -to connectors with stale notifications. +It handles two notification types: +1. **connector_indexing** — connector sync tasks (Google Calendar, etc.) +2. **document_processing** — manual file uploads, YouTube videos, etc. + +Additionally, it cleans up documents stuck in pending/processing state: +- For connectors: by connector_id +- For non-connector documents (FILE uploads, YouTube): by document_id from notification metadata Detection mechanism: -- Active indexing tasks set a Redis key with TTL (2 minutes) as a heartbeat -- If the task crashes, the Redis key expires automatically +- Active tasks set a Redis key with TTL (2 minutes) as a heartbeat +- A background coroutine refreshes the key every 60 seconds +- If the task/worker crashes, the Redis key expires automatically - This cleanup task checks for in-progress notifications without a Redis heartbeat key - Such notifications are marked as failed with O(1) batch UPDATE -- Documents with pending/processing status for those connectors are also marked as failed +- Associated documents are also marked as failed """ import contextlib import json import logging -import os from datetime import UTC, datetime import redis @@ -36,19 +42,16 @@ logger = logging.getLogger(__name__) # Redis client for checking heartbeats _redis_client: redis.Redis | None = None -# Error message shown to users when sync is interrupted +# Error messages shown to users when tasks are interrupted STALE_SYNC_ERROR_MESSAGE = "Sync was interrupted unexpectedly. Please retry." +STALE_PROCESSING_ERROR_MESSAGE = "Syncing was interrupted unexpectedly. Please retry." def get_redis_client() -> redis.Redis: """Get or create Redis client for heartbeat checking.""" global _redis_client if _redis_client is None: - redis_url = os.getenv( - "REDIS_APP_URL", - os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0"), - ) - _redis_client = redis.from_url(redis_url, decode_responses=True) + _redis_client = redis.from_url(config.REDIS_APP_URL, decode_responses=True) return _redis_client @@ -70,14 +73,13 @@ def get_celery_session_maker(): @celery_app.task(name="cleanup_stale_indexing_notifications") def cleanup_stale_indexing_notifications_task(): """ - Check for stale connector indexing notifications and mark them as failed. + Check for stale notifications and mark them as failed. - This task finds notifications that: - - Have type = 'connector_indexing' - - Have metadata.status = 'in_progress' - - Do NOT have a corresponding Redis heartbeat key (meaning task crashed) + Handles two notification types: + 1. connector_indexing — connector sync tasks + 2. document_processing — manual file uploads, YouTube videos, etc. - And marks them as failed with O(1) batch UPDATE. + Detection: Redis heartbeat key with 2-min TTL. Missing key = stale task. Also marks associated pending/processing documents as failed. """ import asyncio @@ -87,6 +89,7 @@ def cleanup_stale_indexing_notifications_task(): try: loop.run_until_complete(_cleanup_stale_notifications()) + loop.run_until_complete(_cleanup_stale_document_processing_notifications()) finally: loop.close() @@ -269,3 +272,186 @@ async def _cleanup_stuck_documents(session, connector_ids: list[int]): exc_info=True, ) # Don't raise - let the notification cleanup continue even if document cleanup fails + + +# ===== Document Processing Cleanup (FILE uploads, YouTube, etc.) ===== + + +async def _cleanup_stale_document_processing_notifications(): + """Find and mark stale document processing notifications as failed. + + Same Redis heartbeat mechanism as connector indexing cleanup, but for + document_processing type notifications (FILE uploads, YouTube videos, etc.). + + For each stale notification that contains a document_id in its metadata, + the associated document is also marked as failed. + """ + async with get_celery_session_maker()() as session: + try: + # Find all in-progress document processing notifications + result = await session.execute( + select( + Notification.id, + Notification.notification_metadata, + ).where( + and_( + Notification.type == "document_processing", + Notification.notification_metadata["status"].astext + == "in_progress", + ) + ) + ) + in_progress_rows = result.fetchall() + + if not in_progress_rows: + logger.debug("No in-progress document processing notifications found") + return + + # Check which ones are missing heartbeat keys in Redis + redis_client = get_redis_client() + stale_notification_ids = [] + stale_document_ids = [] + + for row in in_progress_rows: + notification_id = row[0] + metadata = row[1] # Full metadata dict + heartbeat_key = _get_heartbeat_key(notification_id) + if not redis_client.exists(heartbeat_key): + stale_notification_ids.append(notification_id) + # Extract document_id from metadata for document cleanup + if metadata and isinstance(metadata, dict): + doc_id = metadata.get("document_id") + if doc_id is not None: + with contextlib.suppress(ValueError, TypeError): + stale_document_ids.append(int(doc_id)) + + if not stale_notification_ids: + logger.debug( + f"All {len(in_progress_rows)} in-progress document processing " + "notifications have active Redis heartbeats" + ) + return + + logger.warning( + f"Found {len(stale_notification_ids)} stale document processing " + f"notifications (no Redis heartbeat): {stale_notification_ids}" + ) + + # O(1) Batch UPDATE: Mark stale notifications as failed + update_data = { + "status": "failed", + "completed_at": datetime.now(UTC).isoformat(), + "error_message": STALE_PROCESSING_ERROR_MESSAGE, + "processing_stage": "failed", + } + + await session.execute( + text(""" + UPDATE notifications + SET metadata = metadata || CAST(:update_json AS jsonb), + title = 'Failed: ' || COALESCE(metadata->>'document_name', 'Document'), + message = :display_message + WHERE id = ANY(:ids) + """), + { + "update_json": json.dumps(update_data), + "display_message": STALE_PROCESSING_ERROR_MESSAGE, + "ids": stale_notification_ids, + }, + ) + + logger.info( + f"Successfully marked {len(stale_notification_ids)} stale document " + "processing notifications as failed" + ) + + # Clean up stuck documents by document_id from notification metadata + if stale_document_ids: + await _cleanup_stuck_non_connector_documents( + session, stale_document_ids + ) + + await session.commit() + + except Exception as e: + logger.error( + f"Error cleaning up stale document processing notifications: {e!s}", + exc_info=True, + ) + await session.rollback() + + +async def _cleanup_stuck_non_connector_documents(session, document_ids: list[int]): + """ + Mark specific non-connector documents stuck in pending/processing as failed. + + These are documents (FILE uploads, YouTube, etc.) identified from stale + notification metadata. Only documents that are still in pending/processing + state are updated — already-completed documents are left untouched. + + Args: + session: Database session + document_ids: List of document IDs to check and potentially mark as failed + """ + if not document_ids: + return + + try: + # Find which of these documents are actually stuck + count_result = await session.execute( + select(Document.id).where( + and_( + Document.id.in_(document_ids), + or_( + Document.status["state"].astext == DocumentStatus.PENDING, + Document.status["state"].astext == DocumentStatus.PROCESSING, + ), + ) + ) + ) + stuck_doc_ids = [row[0] for row in count_result.fetchall()] + + if not stuck_doc_ids: + logger.debug( + f"No stuck non-connector documents found for IDs: {document_ids}" + ) + return + + logger.warning( + f"Found {len(stuck_doc_ids)} stuck non-connector documents " + f"(pending/processing): {stuck_doc_ids}" + ) + + failed_status = DocumentStatus.failed(STALE_PROCESSING_ERROR_MESSAGE) + + await session.execute( + text(""" + UPDATE documents + SET status = CAST(:failed_status AS jsonb), + updated_at = :now + WHERE id = ANY(:doc_ids) + AND ( + status->>'state' = :pending_state + OR status->>'state' = :processing_state + ) + """), + { + "failed_status": json.dumps(failed_status), + "now": datetime.now(UTC), + "doc_ids": stuck_doc_ids, + "pending_state": DocumentStatus.PENDING, + "processing_state": DocumentStatus.PROCESSING, + }, + ) + + logger.info( + f"Successfully marked {len(stuck_doc_ids)} stuck non-connector " + "documents as failed" + ) + + except Exception as e: + logger.error( + f"Error cleaning up stuck non-connector documents {document_ids}: {e!s}", + exc_info=True, + ) + # Don't raise — let the rest of the cleanup continue diff --git a/surfsense_backend/app/tasks/connector_indexers/base.py b/surfsense_backend/app/tasks/connector_indexers/base.py index 6a1226230..139aed1d3 100644 --- a/surfsense_backend/app/tasks/connector_indexers/base.py +++ b/surfsense_backend/app/tasks/connector_indexers/base.py @@ -52,10 +52,22 @@ def safe_set_chunks(document: Document, chunks: list) -> None: # Instead of: document.chunks = chunks (DANGEROUS!) safe_set_chunks(document, chunks) # Always safe """ + from sqlalchemy.orm import object_session from sqlalchemy.orm.attributes import set_committed_value + # Keep relationship assignment lazy-load-safe. set_committed_value(document, "chunks", chunks) + # Ensure chunk rows are actually persisted. + # set_committed_value bypasses normal unit-of-work tracking, so we need to + # explicitly attach chunk objects to the current session. + session = object_session(document) + if session is not None: + if document.id is not None: + for chunk in chunks: + chunk.document_id = document.id + session.add_all(chunks) + def parse_date_flexible(date_str: str) -> datetime: """ diff --git a/surfsense_backend/app/tasks/document_processors/base.py b/surfsense_backend/app/tasks/document_processors/base.py index 2047ec63d..2edc48e91 100644 --- a/surfsense_backend/app/tasks/document_processors/base.py +++ b/surfsense_backend/app/tasks/document_processors/base.py @@ -38,10 +38,22 @@ def safe_set_chunks(document: Document, chunks: list) -> None: # Instead of: document.chunks = chunks (DANGEROUS!) safe_set_chunks(document, chunks) # Always safe """ + from sqlalchemy.orm import object_session from sqlalchemy.orm.attributes import set_committed_value + # Keep relationship assignment lazy-load-safe. set_committed_value(document, "chunks", chunks) + # Ensure chunk rows are actually persisted. + # set_committed_value bypasses normal unit-of-work tracking, so we need to + # explicitly attach chunk objects to the current session. + session = object_session(document) + if session is not None: + if document.id is not None: + for chunk in chunks: + chunk.document_id = document.id + session.add_all(chunks) + def get_current_timestamp() -> datetime: """ diff --git a/surfsense_backend/app/tasks/document_processors/youtube_processor.py b/surfsense_backend/app/tasks/document_processors/youtube_processor.py index 9dac6d554..80cdaae4d 100644 --- a/surfsense_backend/app/tasks/document_processors/youtube_processor.py +++ b/surfsense_backend/app/tasks/document_processors/youtube_processor.py @@ -61,7 +61,11 @@ def get_youtube_video_id(url: str) -> str | None: async def add_youtube_video_document( - session: AsyncSession, url: str, search_space_id: int, user_id: str + session: AsyncSession, + url: str, + search_space_id: int, + user_id: str, + notification=None, ) -> Document: """ Process a YouTube video URL, extract transcripts, and store as a document. @@ -75,6 +79,9 @@ async def add_youtube_video_document( url: YouTube video URL (supports standard, shortened, and embed formats) search_space_id: ID of the search space to add the document to user_id: ID of the user + notification: Optional notification object — if provided, the document_id + is stored in its metadata right after document creation so the stale + cleanup task can identify stuck documents. Returns: Document: The created document object @@ -182,6 +189,15 @@ async def add_youtube_video_document( await session.commit() # Document visible in UI now with pending status! is_new_document = True + # Store document_id in notification metadata so stale cleanup task + # can identify this document if the worker crashes. + if notification and notification.notification_metadata is not None: + from sqlalchemy.orm.attributes import flag_modified + + notification.notification_metadata["document_id"] = document.id + flag_modified(notification, "notification_metadata") + await session.commit() + logging.info(f"Created pending document for YouTube video {video_id}") # ======================================================================= @@ -244,7 +260,13 @@ async def add_youtube_video_document( if residential_proxies: http_client.proxies.update(residential_proxies) ytt_api = YouTubeTranscriptApi(http_client=http_client) - captions = ytt_api.fetch(video_id) + + # List all available transcripts and pick the first one + # (the video's primary language) instead of defaulting to English + transcript_list = ytt_api.list(video_id) + transcript = next(iter(transcript_list)) + captions = transcript.fetch() + # Include complete caption information with timestamps transcript_segments = [] for line in captions: @@ -257,11 +279,14 @@ async def add_youtube_video_document( await task_logger.log_task_progress( log_entry, - f"Transcript fetched successfully: {len(captions)} segments", + f"Transcript fetched successfully: {len(captions)} segments ({transcript.language})", { "stage": "transcript_fetched", "segments_count": len(captions), "transcript_length": len(transcript_text), + "language": transcript.language, + "language_code": transcript.language_code, + "is_generated": transcript.is_generated, }, ) except Exception as e: diff --git a/surfsense_backend/app/utils/indexing_locks.py b/surfsense_backend/app/utils/indexing_locks.py new file mode 100644 index 000000000..7790bcc11 --- /dev/null +++ b/surfsense_backend/app/utils/indexing_locks.py @@ -0,0 +1,46 @@ +"""Redis-based connector indexing locks to prevent duplicate sync tasks.""" + +import redis + +from app.config import config + +_redis_client: redis.Redis | None = None +LOCK_TTL_SECONDS = config.CONNECTOR_INDEXING_LOCK_TTL_SECONDS + + +def get_indexing_lock_redis_client() -> redis.Redis: + """Get or create Redis client for connector indexing locks.""" + global _redis_client + if _redis_client is None: + _redis_client = redis.from_url(config.REDIS_APP_URL, decode_responses=True) + return _redis_client + + +def _get_connector_lock_key(connector_id: int) -> str: + """Generate Redis key for a connector indexing lock.""" + return f"indexing:connector_lock:{connector_id}" + + +def acquire_connector_indexing_lock(connector_id: int) -> bool: + """Acquire lock for connector indexing. Returns True if acquired.""" + key = _get_connector_lock_key(connector_id) + return bool( + get_indexing_lock_redis_client().set( + key, + "1", + nx=True, + ex=LOCK_TTL_SECONDS, + ) + ) + + +def release_connector_indexing_lock(connector_id: int) -> None: + """Release lock for connector indexing.""" + key = _get_connector_lock_key(connector_id) + get_indexing_lock_redis_client().delete(key) + + +def is_connector_indexing_locked(connector_id: int) -> bool: + """Check if connector indexing lock exists.""" + key = _get_connector_lock_key(connector_id) + return bool(get_indexing_lock_redis_client().exists(key)) diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx index b214c96be..92ddb0057 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx @@ -35,9 +35,9 @@ export function DocumentTypeChip({ type, className }: { type: string; className? const chip = ( - {icon} + {icon} {fullLabel} diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx index ebdf431e4..c954474c6 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx @@ -178,7 +178,7 @@ export function DocumentsFilters({
setTypeSearchQuery(e.target.value)} className="h-6 pl-6 text-sm bg-transparent border-0 focus-visible:ring-0" diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx index d5ee00dfb..a44295ec0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx @@ -3,6 +3,7 @@ import { formatDistanceToNow } from "date-fns"; import { AlertCircle, + BadgeInfo, Calendar, CheckCircle2, ChevronDown, @@ -372,7 +373,7 @@ export function DocumentsTableShell({ )} {columnVisibility.status && ( - + )} @@ -414,7 +415,7 @@ export function DocumentsTableShell({ )} {columnVisibility.status && ( - + )} @@ -466,7 +467,7 @@ export function DocumentsTableShell({ className="flex flex-col items-center gap-4 max-w-md px-4 text-center" >
- +

{t("no_documents")}

@@ -544,8 +545,11 @@ export function DocumentsTableShell({ )} {columnVisibility.status && ( - - Status + + + + Status + )} @@ -644,7 +648,7 @@ export function DocumentsTableShell({ )} {columnVisibility.status && ( - + )} diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx index ec355f576..eb44d114a 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx @@ -9,6 +9,7 @@ import { AlertDialogAction, AlertDialogCancel, AlertDialogContent, + AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, AlertDialogTitle, @@ -205,7 +206,10 @@ export function RowActions({ - Are you sure? + Delete document? + + This action cannot be undone. This will permanently delete this document from your search space. + Cancel diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index 8cf2fe8da..585a09c43 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -140,6 +140,9 @@ export default function DocumentsTable() { } }); setPageIndex(0); + // Clear selections when filter changes — selected IDs from the previous + // filter view are no longer visible and would cause misleading bulk actions + setSelectedIds(new Set()); }; const onBulkDelete = async () => { diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 1a535539d..22085e064 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -844,7 +844,12 @@ export default function NewChatPage() { }); // Invalidate thread detail for breadcrumb update queryClient.invalidateQueries({ - queryKey: ["threads", String(searchSpaceId), "detail", String(titleData.threadId)], + queryKey: [ + "threads", + String(searchSpaceId), + "detail", + String(titleData.threadId), + ], }); } break; @@ -1403,7 +1408,7 @@ export default function NewChatPage() { // Show loading state only when loading an existing thread if (isInitializing) { return ( -
+
{/* User message */}
@@ -1444,7 +1449,7 @@ export default function NewChatPage() { // For new chats (urlChatId === 0), threadId being null is expected (lazy creation) if (!threadId && urlChatId > 0) { return ( -
+
Failed to load chat
); + const menuItems = ( + <> + {onSettings && ( + + + {tCommon("settings")} + + )} + {onSettings && onDelete && } + {onDelete && isOwner && ( + + + {tCommon("delete")} + + )} + {onDelete && !isOwner && ( + + + {t("leave")} + + )} + + ); + // If delete or settings handlers are provided, wrap with context menu if (onDelete || onSettings) { + // Mobile: use long-press triggered DropdownMenu + if (disableTooltip) { + return ( + + +
+ {avatarButton} +
+ + {menuItems} + + ); + } + + // Desktop: use right-click ContextMenu + Tooltip return ( @@ -150,6 +234,10 @@ export function SearchSpaceAvatar({ } // No context menu needed + if (disableTooltip) { + return avatarButton; + } + return ( {avatarButton} diff --git a/surfsense_web/components/layout/ui/sidebar/AllPrivateChatsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/AllPrivateChatsSidebar.tsx index d39d88d61..1d4f590bd 100644 --- a/surfsense_web/components/layout/ui/sidebar/AllPrivateChatsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/AllPrivateChatsSidebar.tsx @@ -27,10 +27,12 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { Input } from "@/components/ui/input"; +import { Skeleton } from "@/components/ui/skeleton"; import { Spinner } from "@/components/ui/spinner"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; +import { useIsMobile } from "@/hooks/use-mobile"; import { deleteThread, fetchThreads, @@ -56,6 +58,7 @@ export function AllPrivateChatsSidebar({ const router = useRouter(); const params = useParams(); const queryClient = useQueryClient(); + const isMobile = useIsMobile(); const currentChatId = Array.isArray(params.chat_id) ? Number(params.chat_id[0]) @@ -303,8 +306,16 @@ export function AllPrivateChatsSidebar({
{isLoading ? ( -
- +
+ {[75, 90, 55, 80, 65, 85].map((titleWidth, i) => ( +
+ + +
+ ))}
) : error ? (
@@ -329,25 +340,37 @@ export function AllPrivateChatsSidebar({ isBusy && "opacity-50 pointer-events-none" )} > - - - - - -

- {t("updated") || "Updated"}:{" "} - {format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")} -

- - + {isMobile ? ( + + ) : ( + + + + + +

+ {t("updated") || "Updated"}:{" "} + {format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")} +

+ + + )} {isLoading ? ( -
- +
+ {[75, 90, 55, 80, 65, 85].map((titleWidth, i) => ( +
+ + +
+ ))}
) : error ? (
@@ -329,25 +340,37 @@ export function AllSharedChatsSidebar({ isBusy && "opacity-50 pointer-events-none" )} > - - - - - -

- {t("updated") || "Updated"}:{" "} - {format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")} -

- - + {isMobile ? ( + + ) : ( + + + + + +

+ {t("updated") || "Updated"}:{" "} + {format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")} +

+ + + )} {t("more_options")} - - {onRename && ( - { - e.stopPropagation(); - onRename(); - }} - > - - {t("rename") || "Rename"} - - )} - {onArchive && ( - { - e.stopPropagation(); - onArchive(); - }} - > - {archived ? ( + + {onRename && ( + { + e.stopPropagation(); + onRename(); + }} + > + + {t("rename") || "Rename"} + + )} + {onArchive && ( + { + e.stopPropagation(); + onArchive(); + }} + > + {archived ? ( <> {t("unarchive") || "Restore"} diff --git a/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx index f313dd6f9..b6caed330 100644 --- a/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx @@ -1,5 +1,6 @@ "use client"; +import { useQuery } from "@tanstack/react-query"; import { useAtom } from "jotai"; import { AlertCircle, @@ -19,7 +20,7 @@ import { X, } from "lucide-react"; import { AnimatePresence, motion } from "motion/react"; -import { useRouter } from "next/navigation"; +import { useParams, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { setCommentsCollapsedAtom, setTargetCommentIdAtom } from "@/atoms/chat/current-thread.atom"; @@ -41,6 +42,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { Input } from "@/components/ui/input"; +import { Skeleton } from "@/components/ui/skeleton"; import { Spinner } from "@/components/ui/spinner"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; @@ -52,7 +54,10 @@ import { isPageLimitExceededMetadata, } from "@/contracts/types/inbox.types"; import type { InboxItem } from "@/hooks/use-inbox"; +import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useMediaQuery } from "@/hooks/use-media-query"; +import { notificationsApiService } from "@/lib/apis/notifications-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; import { cn } from "@/lib/utils"; import { useSidebarContextSafe } from "../../hooks"; @@ -179,7 +184,9 @@ export function InboxSidebar({ }: InboxSidebarProps) { const t = useTranslations("sidebar"); const router = useRouter(); + const params = useParams(); const isMobile = !useMediaQuery("(min-width: 640px)"); + const searchSpaceId = params?.search_space_id ? Number(params.search_space_id) : null; // Comments collapsed state (desktop only, when docked) const [, setCommentsCollapsed] = useAtom(setCommentsCollapsedAtom); @@ -187,12 +194,22 @@ export function InboxSidebar({ const [, setTargetCommentId] = useAtom(setTargetCommentIdAtom); const [searchQuery, setSearchQuery] = useState(""); + const debouncedSearch = useDebouncedValue(searchQuery, 300); + const isSearchMode = !!debouncedSearch.trim(); const [activeTab, setActiveTab] = useState("comments"); const [activeFilter, setActiveFilter] = useState("all"); const [selectedConnector, setSelectedConnector] = useState(null); const [mounted, setMounted] = useState(false); // Dropdown state for filter menu (desktop only) const [openDropdown, setOpenDropdown] = useState<"filter" | null>(null); + // Scroll shadow state for connector list + const [connectorScrollPos, setConnectorScrollPos] = useState<"top" | "middle" | "bottom">("top"); + const handleConnectorScroll = useCallback((e: React.UIEvent) => { + const el = e.currentTarget; + const atTop = el.scrollTop <= 2; + const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight <= 2; + setConnectorScrollPos(atTop ? "top" : atBottom ? "bottom" : "middle"); + }, []); // Drawer state for filter menu (mobile only) const [filterDrawerOpen, setFilterDrawerOpen] = useState(false); const [markingAsReadId, setMarkingAsReadId] = useState(null); @@ -200,6 +217,24 @@ export function InboxSidebar({ // Prefetch trigger ref - placed on item near the end const prefetchTriggerRef = useRef(null); + // Server-side search query (enabled only when user is typing a search) + // Determines which notification types to search based on active tab + const searchTypeFilter = activeTab === "comments" ? ("new_mention" as const) : undefined; + const { data: searchResponse, isLoading: isSearchLoading } = useQuery({ + queryKey: cacheKeys.notifications.search(searchSpaceId, debouncedSearch.trim(), activeTab), + queryFn: () => + notificationsApiService.getNotifications({ + queryParams: { + search_space_id: searchSpaceId ?? undefined, + type: searchTypeFilter, + search: debouncedSearch.trim(), + limit: 50, + }, + }), + staleTime: 30 * 1000, // 30 seconds (search results don't need to be super fresh) + enabled: isSearchMode && open, + }); + useEffect(() => { setMounted(true); }, []); @@ -234,17 +269,11 @@ export function InboxSidebar({ } }, [activeTab]); - // Both tabs now derive items from status (all types), so use status for pagination - const { loading, loadingMore = false, hasMore = false, loadMore } = status; + // Each tab uses its own data source for independent pagination + // Comments tab: uses mentions data source (fetches only mention/reply types from server) + const commentsItems = mentions.items; - // Comments tab: mentions and comment replies - const commentsItems = useMemo( - () => - status.items.filter((item) => item.type === "new_mention" || item.type === "comment_reply"), - [status.items] - ); - - // Status tab: connector indexing, document processing, page limit exceeded, connector deletion + // Status tab: filters status data source (fetches all types) to status-specific types const statusItems = useMemo( () => status.items.filter( @@ -257,6 +286,14 @@ export function InboxSidebar({ [status.items] ); + // Pagination switches based on active tab + const loading = activeTab === "comments" ? mentions.loading : status.loading; + const loadingMore = + activeTab === "comments" ? (mentions.loadingMore ?? false) : (status.loadingMore ?? false); + const hasMore = + activeTab === "comments" ? (mentions.hasMore ?? false) : (status.hasMore ?? false); + const loadMore = activeTab === "comments" ? mentions.loadMore : status.loadMore; + // Get unique connector types from status items for filtering const uniqueConnectorTypes = useMemo(() => { const connectorTypes = new Set(); @@ -279,9 +316,23 @@ export function InboxSidebar({ // Get items for current tab const displayItems = activeTab === "comments" ? commentsItems : statusItems; - // Filter items based on filter type, connector filter, and search query + // Filter items based on filter type, connector filter, and search mode + // When searching: use server-side API results (searches ALL notifications) + // When not searching: use Electric real-time items (fast, local) const filteredItems = useMemo(() => { - let items = displayItems; + // In search mode, use API results + let items: InboxItem[] = isSearchMode ? (searchResponse?.items ?? []) : displayItems; + + // For status tab search results, filter to status-specific types + if (isSearchMode && activeTab === "status") { + items = items.filter( + (item) => + item.type === "connector_indexing" || + item.type === "document_processing" || + item.type === "page_limit_exceeded" || + item.type === "connector_deletion" + ); + } // Apply read/unread filter if (activeFilter === "unread") { @@ -302,22 +353,14 @@ export function InboxSidebar({ }); } - // Apply search query - if (searchQuery.trim()) { - const query = searchQuery.toLowerCase(); - items = items.filter( - (item) => - item.title.toLowerCase().includes(query) || item.message.toLowerCase().includes(query) - ); - } - return items; - }, [displayItems, activeFilter, activeTab, selectedConnector, searchQuery]); + }, [displayItems, searchResponse, isSearchMode, activeFilter, activeTab, selectedConnector]); // Intersection Observer for infinite scroll with prefetching - // Only active when not searching (search results are client-side filtered) + // Re-runs when active tab changes so each tab gets its own pagination + // Disabled during server-side search (search results are not paginated via infinite scroll) useEffect(() => { - if (!loadMore || !hasMore || loadingMore || !open || searchQuery.trim()) return; + if (!loadMore || !hasMore || loadingMore || !open || isSearchMode) return; const observer = new IntersectionObserver( (entries) => { @@ -338,17 +381,11 @@ export function InboxSidebar({ } return () => observer.disconnect(); - }, [loadMore, hasMore, loadingMore, open, searchQuery]); + }, [loadMore, hasMore, loadingMore, open, isSearchMode, activeTab]); - // Unread counts derived from filtered items - const unreadCommentsCount = useMemo( - () => commentsItems.filter((item) => !item.read).length, - [commentsItems] - ); - const unreadStatusCount = useMemo( - () => statusItems.filter((item) => !item.read).length, - [statusItems] - ); + // Unread counts from server-side accurate totals (passed via props) + const unreadCommentsCount = mentions.unreadCount; + const unreadStatusCount = status.unreadCount; const handleItemClick = useCallback( async (item: InboxItem) => { @@ -539,26 +576,33 @@ export function InboxSidebar({
+ {/* Back button - mobile only */} + {isMobile && ( + + )}

{t("inbox") || "Inbox"}

{/* Mobile: Button that opens bottom drawer */} {isMobile ? ( <> - - - - - {t("filter") || "Filter"} - + {t("connectors") || "Connectors"} - setSelectedConnector(null)} - className="flex items-center justify-between" +
- - - {t("all_connectors") || "All connectors"} - - {selectedConnector === null && } - - {uniqueConnectorTypes.map((connector) => ( setSelectedConnector(connector.type)} + onClick={() => setSelectedConnector(null)} className="flex items-center justify-between" > - {getConnectorIcon(connector.type, "h-4 w-4")} - {connector.displayName} + + {t("all_connectors") || "All connectors"} - {selectedConnector === connector.type && } + {selectedConnector === null && } - ))} + {uniqueConnectorTypes.map((connector) => ( + setSelectedConnector(connector.type)} + className="flex items-center justify-between" + > + + {getConnectorIcon(connector.type, "h-4 w-4")} + {connector.displayName} + + {selectedConnector === connector.type && } + + ))} +
)} )} - - - - - - {t("mark_all_read") || "Mark all as read"} - - - {/* Close button - mobile only */} - {isMobile && ( + {isMobile ? ( + + ) : ( - {t("close") || "Close"} + + {t("mark_all_read") || "Mark all as read"} + )} {/* Dock/Undock button - desktop only */} @@ -881,17 +930,48 @@ export function InboxSidebar({
- {loading ? ( -
- + {(isSearchMode ? isSearchLoading : loading) ? ( +
+ {activeTab === "comments" + ? /* Comments skeleton: avatar + two-line text + time */ + [85, 60, 90, 70, 50, 75].map((titleWidth, i) => ( +
+ +
+ + +
+ +
+ )) + : /* Status skeleton: status icon circle + two-line text + time */ + [75, 90, 55, 80, 65, 85].map((titleWidth, i) => ( +
+ +
+ + +
+
+ + +
+
+ ))}
) : filteredItems.length > 0 ? (
{filteredItems.map((item, index) => { const isMarkingAsRead = markingAsReadId === item.id; - // Place prefetch trigger on 5th item from end (only if not searching) + // Place prefetch trigger on 5th item from end (only when not searching) const isPrefetchTrigger = - !searchQuery && hasMore && index === filteredItems.length - 5; + !isSearchMode && hasMore && index === filteredItems.length - 5; return (
- - - - - -

{item.title}

-

- {convertRenderedToDisplay(item.message)} -

- - + {isMobile ? ( + + ) : ( + + + + + +

{item.title}

+

+ {convertRenderedToDisplay(item.message)} +

+ + + )} {/* Time and unread dot - fixed width to prevent content shift */}
@@ -947,11 +1051,43 @@ export function InboxSidebar({ ); })} {/* Fallback trigger at the very end if less than 5 items and not searching */} - {!searchQuery && filteredItems.length < 5 && hasMore && ( + {!isSearchMode && filteredItems.length < 5 && hasMore && (
)} + {/* Loading more skeletons at the bottom during infinite scroll */} + {loadingMore && + (activeTab === "comments" + ? [80, 60, 90].map((titleWidth, i) => ( +
+ +
+ + +
+ +
+ )) + : [70, 85, 55].map((titleWidth, i) => ( +
+ +
+ + +
+
+ + +
+
+ )))}
- ) : searchQuery ? ( + ) : isSearchMode ? (

diff --git a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx index 71d85f600..377bf65f5 100644 --- a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx @@ -1,7 +1,8 @@ "use client"; -import { Menu, Plus } from "lucide-react"; +import { PanelRightClose, Plus } from "lucide-react"; import { Button } from "@/components/ui/button"; +import { ScrollArea } from "@/components/ui/scroll-area"; import { Sheet, SheetContent, SheetTitle } from "@/components/ui/sheet"; import type { ChatItem, NavItem, PageUsage, SearchSpace, User } from "../../types/layout.types"; import { SearchSpaceAvatar } from "../icon-rail/SearchSpaceAvatar"; @@ -43,7 +44,7 @@ interface MobileSidebarProps { export function MobileSidebarTrigger({ onClick }: { onClick: () => void }) { return ( ); @@ -97,15 +98,16 @@ export function MobileSidebar({ return ( - + Navigation - {/* Horizontal Search Spaces Rail */} -

-
- {searchSpaces.map((space) => ( -
+ {/* Vertical Search Spaces Rail - left side */} +
+ +
+ {searchSpaces.map((space) => ( 1} @@ -116,26 +118,28 @@ export function MobileSidebar({ onSearchSpaceSettings ? () => onSearchSpaceSettings(space) : undefined } size="md" + disableTooltip /> -
- ))} - -
+ ))} + +
+
- {/* Sidebar Content */} -
+ {/* Sidebar Content - right side */} +
onOpenChange(false)} navItems={navItems} onNavItemClick={handleNavItemClick} chats={chats} @@ -149,8 +153,22 @@ export function MobileSidebar({ onChatRename={onChatRename} onChatDelete={onChatDelete} onChatArchive={onChatArchive} - onViewAllSharedChats={onViewAllSharedChats} - onViewAllPrivateChats={onViewAllPrivateChats} + onViewAllSharedChats={ + onViewAllSharedChats + ? () => { + onOpenChange(false); + onViewAllSharedChats(); + } + : undefined + } + onViewAllPrivateChats={ + onViewAllPrivateChats + ? () => { + onOpenChange(false); + onViewAllPrivateChats(); + } + : undefined + } user={user} onSettings={onSettings} onManageMembers={onManageMembers} @@ -161,6 +179,7 @@ export function MobileSidebar({ setTheme={setTheme} className="w-full border-none" isLoadingChats={isLoadingChats} + disableTooltips />
diff --git a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx index 7b53fdc6a..883fa5890 100644 --- a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx @@ -50,6 +50,7 @@ interface SidebarProps { setTheme?: (theme: "light" | "dark" | "system") => void; className?: string; isLoadingChats?: boolean; + disableTooltips?: boolean; } export function Sidebar({ @@ -78,6 +79,7 @@ export function Sidebar({ setTheme, className, isLoadingChats = false, + disableTooltips = false, }: SidebarProps) { const t = useTranslations("sidebar"); @@ -95,20 +97,22 @@ export function Sidebar({ {})} + disableTooltip={disableTooltips} />
) : ( -
+
-
+
{})} + disableTooltip={disableTooltips} />
@@ -138,7 +142,7 @@ export function Sidebar({ {isCollapsed ? (
) : ( -
+
{/* Shared Chats Section - takes only space needed, max 50% */} - - - - - {t("view_all_shared_chats") || "View all shared chats"} - - + disableTooltips ? ( + + ) : ( + + + + + + {t("view_all_shared_chats") || "View all shared chats"} + + + ) ) : undefined } > @@ -208,21 +223,32 @@ export function Sidebar({ fillHeight={true} action={ onViewAllPrivateChats ? ( - - - - - - {t("view_all_private_chats") || "View all private chats"} - - + disableTooltips ? ( + + ) : ( + + + + + + {t("view_all_private_chats") || "View all private chats"} + + + ) ) : undefined } > diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarCollapseButton.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarCollapseButton.tsx index 3eaa87070..44f05249c 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarCollapseButton.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarCollapseButton.tsx @@ -8,21 +8,30 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip interface SidebarCollapseButtonProps { isCollapsed: boolean; onToggle: () => void; + disableTooltip?: boolean; } -export function SidebarCollapseButton({ isCollapsed, onToggle }: SidebarCollapseButtonProps) { +export function SidebarCollapseButton({ + isCollapsed, + onToggle, + disableTooltip = false, +}: SidebarCollapseButtonProps) { const t = useTranslations("sidebar"); + const button = ( + + ); + + if (disableTooltip) { + return button; + } + return ( - - - + {button} {isCollapsed ? `${t("expand_sidebar")} (⌘B)` : `${t("collapse_sidebar")} (⌘B)`} diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx index 28c359e64..cdcf2384e 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx @@ -1,6 +1,6 @@ "use client"; -import { ChevronsUpDown, Logs, Settings, Users } from "lucide-react"; +import { ChevronsUpDown, Settings, Users } from "lucide-react"; import { useParams, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { Button } from "@/components/ui/button"; @@ -35,14 +35,14 @@ export function SidebarHeader({ const searchSpaceId = params.search_space_id as string; return ( -
+
-
@@ -379,7 +410,9 @@ export function ImageConfigSidebar({ setFormData((p) => ({ ...p, description: e.target.value }))} + onChange={(e) => + setFormData((p) => ({ ...p, description: e.target.value })) + } />
@@ -390,7 +423,9 @@ export function ImageConfigSidebar({ setFormData((p) => ({ ...p, model_name: e.target.value }))} + onChange={(e) => + setFormData((p) => ({ ...p, model_name: e.target.value })) + } /> )}
@@ -489,14 +543,20 @@ export function ImageConfigSidebar({ setFormData((p) => ({ ...p, api_version: e.target.value }))} + onChange={(e) => + setFormData((p) => ({ ...p, api_version: e.target.value })) + } />
)} {/* Actions */}
-
- {filteredGlobal.map((config) => { - const isSelected = currentConfig?.id === config.id; - const isAuto = "is_auto_mode" in config && config.is_auto_mode; - return ( - handleSelect(config.id)} - className={cn( - "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50", - isSelected && "bg-accent/80", - isAuto && "border border-violet-200 dark:border-violet-800/50" - )} - > -
-
- {isAuto ? ( - - ) : ( - + {filteredGlobal.map((config) => { + const isSelected = currentConfig?.id === config.id; + const isAuto = "is_auto_mode" in config && config.is_auto_mode; + return ( + handleSelect(config.id)} + className={cn( + "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50", + isSelected && "bg-accent/80", + isAuto && "border border-violet-200 dark:border-violet-800/50" + )} + > +
+
+ {isAuto ? ( + + ) : ( + + )} +
+
+
+ {config.name} + {isAuto && ( + + Recommended + + )} + {isSelected && } +
+ + {isAuto ? "Auto load balancing" : config.model_name} + +
+ {onEdit && ( + { + e.stopPropagation(); + setOpen(false); + onEdit(config, true); + }} + /> )}
-
-
- {config.name} - {isAuto && ( - - Recommended - - )} - {isSelected && } -
- - {isAuto ? "Auto load balancing" : config.model_name} - -
- {onEdit && ( - { - e.stopPropagation(); - setOpen(false); - onEdit(config, true); - }} - /> - )} -
- - ); - })} + + ); + })} )} @@ -290,51 +289,49 @@ export function ImageModelSelector({ className, onAddNew, onEdit }: ImageModelSe Your Image Models
- {filteredUser.map((config) => { - const isSelected = currentConfig?.id === config.id; - return ( - handleSelect(config.id)} - className={cn( - "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50", - isSelected && "bg-accent/80" - )} - > -
-
- -
-
-
- {config.name} - {isSelected && ( - - )} -
- - {config.model_name} - -
- {onEdit && ( - + {filteredUser.map((config) => { + const isSelected = currentConfig?.id === config.id; + return ( + handleSelect(config.id)} + className={cn( + "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50", + isSelected && "bg-accent/80" )} -
- - ); - })} + > +
+
+ +
+
+
+ {config.name} + {isSelected && } +
+ + {config.model_name} + +
+ {onEdit && ( + + )} +
+ + ); + })} )} diff --git a/surfsense_web/components/new-chat/model-selector.tsx b/surfsense_web/components/new-chat/model-selector.tsx index 148028df2..ec1143e04 100644 --- a/surfsense_web/components/new-chat/model-selector.tsx +++ b/surfsense_web/components/new-chat/model-selector.tsx @@ -392,8 +392,8 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp )} - {/* Add New Config Button */} -
+ {/* Add New Config Button */} +

- {plan.billingText ?? (isNaN(Number(plan.price)) ? "" : isMonthly ? "billed monthly" : "billed annually")} + {plan.billingText ?? + (isNaN(Number(plan.price)) + ? "" + : isMonthly + ? "billed monthly" + : "billed annually")}

    diff --git a/surfsense_web/components/settings/image-model-manager.tsx b/surfsense_web/components/settings/image-model-manager.tsx index 4e3f1840a..e87cc9a95 100644 --- a/surfsense_web/components/settings/image-model-manager.tsx +++ b/surfsense_web/components/settings/image-model-manager.tsx @@ -95,16 +95,29 @@ const item = { export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { // Image gen config atoms - const { mutateAsync: createConfig, isPending: isCreating, error: createError } = - useAtomValue(createImageGenConfigMutationAtom); - const { mutateAsync: updateConfig, isPending: isUpdating, error: updateError } = - useAtomValue(updateImageGenConfigMutationAtom); - const { mutateAsync: deleteConfig, isPending: isDeleting, error: deleteError } = - useAtomValue(deleteImageGenConfigMutationAtom); + const { + mutateAsync: createConfig, + isPending: isCreating, + error: createError, + } = useAtomValue(createImageGenConfigMutationAtom); + const { + mutateAsync: updateConfig, + isPending: isUpdating, + error: updateError, + } = useAtomValue(updateImageGenConfigMutationAtom); + const { + mutateAsync: deleteConfig, + isPending: isDeleting, + error: deleteError, + } = useAtomValue(deleteImageGenConfigMutationAtom); const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom); - const { data: userConfigs, isFetching: configsLoading, error: fetchError, refetch: refreshConfigs } = - useAtomValue(imageGenConfigsAtom); + const { + data: userConfigs, + isFetching: configsLoading, + error: fetchError, + refetch: refreshConfigs, + } = useAtomValue(imageGenConfigsAtom); const { data: globalConfigs = [], isFetching: globalLoading } = useAtomValue(globalImageGenConfigsAtom); const { data: preferences = {}, isFetching: prefsLoading } = useAtomValue(llmPreferencesAtom); @@ -249,7 +262,9 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { data: { image_generation_config_id: typeof selectedPrefId === "string" - ? selectedPrefId ? parseInt(selectedPrefId) : undefined + ? selectedPrefId + ? parseInt(selectedPrefId) + : undefined : selectedPrefId, }, }); @@ -289,7 +304,12 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { {/* Errors */} {errors.map((err) => ( - + {err?.message} @@ -304,7 +324,8 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { - {globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length} global image model(s) + {globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length} global + image model(s) {" "} available from your administrator. @@ -342,18 +363,27 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { {globalConfigs.length > 0 && ( <> -
    Global
    +
    + Global +
    {globalConfigs.map((c) => { const isAuto = "is_auto_mode" in c && c.is_auto_mode; return (
    {isAuto ? ( - - AUTO + + + AUTO ) : ( - + {c.provider} )} @@ -366,11 +396,15 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { )} {(userConfigs?.length ?? 0) > 0 && ( <> -
    Your Models
    +
    + Your Models +
    {userConfigs?.map((c) => (
    - {c.provider} + + {c.provider} + {c.name} ({c.model_name})
    @@ -382,10 +416,23 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { {hasPrefChanges && (
    - -
    @@ -409,7 +456,10 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) {

    Your Image Models

    - @@ -435,7 +485,12 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { {userConfigs?.map((config) => ( - +
    @@ -448,8 +503,13 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) {
    -

    {config.name}

    - +

    + {config.name} +

    + {config.provider}
    @@ -457,7 +517,9 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { {config.model_name} {config.description && ( -

    {config.description}

    +

    + {config.description} +

    )}
    @@ -469,7 +531,12 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { - @@ -479,7 +546,12 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { - @@ -501,15 +573,30 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { )} {/* Create/Edit Dialog */} - { if (!open) { setIsDialogOpen(false); setEditingConfig(null); resetForm(); } }}> + { + if (!open) { + setIsDialogOpen(false); + setEditingConfig(null); + resetForm(); + } + }} + > - {editingConfig ? : } + {editingConfig ? ( + + ) : ( + + )} {editingConfig ? "Edit Image Model" : "Add Image Model"} - {editingConfig ? "Update your image generation model" : "Configure a new image generation model (DALL-E 3, GPT Image 1, etc.)"} + {editingConfig + ? "Update your image generation model" + : "Configure a new image generation model (DALL-E 3, GPT Image 1, etc.)"} @@ -541,7 +628,9 @@ export function ImageModelManager({ searchSpaceId }: ImageModelManagerProps) { handleRoleAssignment(`${key}_llm_id`, value)} - > - - - - - - Unassigned - +
    + + -
    + {/* Custom Configurations */} + {newLLMConfigs.length > 0 && ( + <> +
    + Your Configurations +
    + {newLLMConfigs + .filter( + (config) => config.id && config.id.toString().trim() !== "" + ) + .map((config) => ( + +
    + + {config.provider} + + {config.name} + + ({config.model_name}) + +
    +     + ))} + + )} +     + +
    {assignedConfig && (
    {isGenerated && } diff --git a/surfsense_web/contracts/types/inbox.types.ts b/surfsense_web/contracts/types/inbox.types.ts index ebf1889a1..bd04eefd1 100644 --- a/surfsense_web/contracts/types/inbox.types.ts +++ b/surfsense_web/contracts/types/inbox.types.ts @@ -205,6 +205,7 @@ export const getNotificationsRequest = z.object({ search_space_id: z.number().optional(), type: inboxItemTypeEnum.optional(), before_date: z.string().optional(), + search: z.string().optional(), limit: z.number().min(1).max(100).optional(), offset: z.number().min(0).optional(), }), diff --git a/surfsense_web/contracts/types/new-llm-config.types.ts b/surfsense_web/contracts/types/new-llm-config.types.ts index 3f0d39e5a..b99df1022 100644 --- a/surfsense_web/contracts/types/new-llm-config.types.ts +++ b/surfsense_web/contracts/types/new-llm-config.types.ts @@ -213,9 +213,7 @@ export const getImageGenConfigsResponse = z.array(imageGenerationConfig); export const updateImageGenConfigRequest = z.object({ id: z.number(), - data: imageGenerationConfig - .omit({ id: true, created_at: true, search_space_id: true }) - .partial(), + data: imageGenerationConfig.omit({ id: true, created_at: true, search_space_id: true }).partial(), }); export const updateImageGenConfigResponse = imageGenerationConfig; diff --git a/surfsense_web/hooks/use-documents.ts b/surfsense_web/hooks/use-documents.ts index 369cc7b41..6060b9572 100644 --- a/surfsense_web/hooks/use-documents.ts +++ b/surfsense_web/hooks/use-documents.ts @@ -71,8 +71,14 @@ function isValidDocument(doc: DocumentElectric): boolean { * 3. Use syncHandle.isUpToDate to determine if deletions can be trusted * 4. Handles bulk deletions correctly by checking sync state * + * Filtering strategy: + * - Internal state always stores ALL documents (unfiltered) + * - typeFilter is applied client-side when returning documents + * - typeCounts always reflect the full dataset so the filter sidebar stays complete + * - Changing filters is instant (no API re-fetch or Electric re-sync) + * * @param searchSpaceId - The search space ID to filter documents - * @param typeFilter - Optional document types to filter by + * @param typeFilter - Optional document types to filter by (applied client-side) */ export function useDocuments( searchSpaceId: number | null, @@ -80,7 +86,8 @@ export function useDocuments( ) { const electricClient = useElectricClient(); - const [documents, setDocuments] = useState([]); + // Internal state: ALL documents (unfiltered) + const [allDocuments, setAllDocuments] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); @@ -94,14 +101,21 @@ export function useDocuments( const syncHandleRef = useRef(null); const liveQueryRef = useRef<{ unsubscribe?: () => void } | null>(null); - // Real-time type counts + // Type counts from ALL documents (unfiltered) — keeps filter sidebar complete const typeCounts = useMemo(() => { const counts: Record = {}; - for (const doc of documents) { + for (const doc of allDocuments) { counts[doc.document_type] = (counts[doc.document_type] || 0) + 1; } return counts; - }, [documents]); + }, [allDocuments]); + + // Client-side filtered documents for display + const documents = useMemo(() => { + if (typeFilter.length === 0) return allDocuments; + const filterSet = new Set(typeFilter); + return allDocuments.filter((doc) => filterSet.has(doc.document_type)); + }, [allDocuments, typeFilter]); // Populate user cache from API response const populateUserCache = useCallback( @@ -151,7 +165,8 @@ export function useDocuments( [] ); - // EFFECT 1: Load from API (PRIMARY source of truth) + // EFFECT 1: Load ALL documents from API (PRIMARY source of truth) + // No type filter — always fetches everything so typeCounts stay complete useEffect(() => { if (!searchSpaceId) { setLoading(false); @@ -160,7 +175,6 @@ export function useDocuments( // Capture validated value for async closure const spaceId = searchSpaceId; - const currentTypeFilter = typeFilter; let mounted = true; apiLoadedRef.current = false; @@ -174,8 +188,7 @@ export function useDocuments( queryParams: { search_space_id: spaceId, page: 0, - page_size: -1, // Fetch all documents - ...(currentTypeFilter.length > 0 && { document_types: currentTypeFilter }), + page_size: -1, // Fetch all documents (unfiltered) }, }); @@ -183,7 +196,7 @@ export function useDocuments( populateUserCache(response.items); const docs = response.items.map(apiToDisplayDoc); - setDocuments(docs); + setAllDocuments(docs); apiLoadedRef.current = true; setError(null); console.log("[useDocuments] API loaded", docs.length, "documents"); @@ -201,16 +214,16 @@ export function useDocuments( return () => { mounted = false; }; - }, [searchSpaceId, typeFilter, populateUserCache, apiToDisplayDoc]); + }, [searchSpaceId, populateUserCache, apiToDisplayDoc]); // EFFECT 2: Start Electric sync + live query for real-time updates + // No type filter — syncs and queries ALL documents; filtering is client-side useEffect(() => { if (!searchSpaceId || !electricClient) return; // Capture validated values for async closure const spaceId = searchSpaceId; const client = electricClient; - const currentTypeFilter = typeFilter; let mounted = true; @@ -228,7 +241,7 @@ export function useDocuments( try { console.log("[useDocuments] Starting Electric sync for real-time updates"); - // Start Electric sync + // Start Electric sync (all documents for this search space) const handle = await client.syncShape({ table: "documents", where: `search_space_id = ${spaceId}`, @@ -263,7 +276,7 @@ export function useDocuments( if (!mounted) return; - // Set up live query + // Set up live query (unfiltered — type filtering is done client-side) const db = client.db as { live?: { query: ( @@ -281,21 +294,12 @@ export function useDocuments( return; } - let query = `SELECT id, document_type, search_space_id, title, created_by_id, created_at, status + const query = `SELECT id, document_type, search_space_id, title, created_by_id, created_at, status FROM documents - WHERE search_space_id = $1`; + WHERE search_space_id = $1 + ORDER BY created_at DESC`; - const params: (number | string)[] = [spaceId]; - - if (currentTypeFilter.length > 0) { - const placeholders = currentTypeFilter.map((_, i) => `$${i + 2}`).join(", "); - query += ` AND document_type IN (${placeholders})`; - params.push(...currentTypeFilter); - } - - query += ` ORDER BY created_at DESC`; - - const liveQuery = await db.live.query(query, params); + const liveQuery = await db.live.query(query, [spaceId]); if (!mounted) { liveQuery.unsubscribe?.(); @@ -333,7 +337,7 @@ export function useDocuments( .then((response) => { populateUserCache(response.items); if (mounted) { - setDocuments((prev) => + setAllDocuments((prev) => prev.map((doc) => ({ ...doc, created_by_name: doc.created_by_id @@ -347,7 +351,7 @@ export function useDocuments( } // Smart update logic based on sync state - setDocuments((prev) => { + setAllDocuments((prev) => { // Don't process if API hasn't loaded yet if (!apiLoadedRef.current) { console.log("[useDocuments] Waiting for API load, skipping live update"); @@ -424,7 +428,7 @@ export function useDocuments( liveQueryRef.current = null; } }; - }, [searchSpaceId, electricClient, typeFilter, electricToDisplayDoc, populateUserCache]); + }, [searchSpaceId, electricClient, electricToDisplayDoc, populateUserCache]); // Track previous searchSpaceId to detect actual changes const prevSearchSpaceIdRef = useRef(null); @@ -432,7 +436,7 @@ export function useDocuments( // Reset on search space change (not on initial mount) useEffect(() => { if (prevSearchSpaceIdRef.current !== null && prevSearchSpaceIdRef.current !== searchSpaceId) { - setDocuments([]); + setAllDocuments([]); apiLoadedRef.current = false; userCacheRef.current.clear(); } diff --git a/surfsense_web/lib/apis/image-gen-config-api.service.ts b/surfsense_web/lib/apis/image-gen-config-api.service.ts index 84aeed3d8..379edfa53 100644 --- a/surfsense_web/lib/apis/image-gen-config-api.service.ts +++ b/surfsense_web/lib/apis/image-gen-config-api.service.ts @@ -32,11 +32,9 @@ class ImageGenConfigApiService { const msg = parsed.error.issues.map((i) => i.message).join(", "); throw new ValidationError(`Invalid request: ${msg}`); } - return baseApiService.post( - `/api/v1/image-generation-configs`, - createImageGenConfigResponse, - { body: parsed.data } - ); + return baseApiService.post(`/api/v1/image-generation-configs`, createImageGenConfigResponse, { + body: parsed.data, + }); }; /** diff --git a/surfsense_web/lib/apis/notifications-api.service.ts b/surfsense_web/lib/apis/notifications-api.service.ts index bd590dcd2..086633d81 100644 --- a/surfsense_web/lib/apis/notifications-api.service.ts +++ b/surfsense_web/lib/apis/notifications-api.service.ts @@ -51,6 +51,9 @@ class NotificationsApiService { if (queryParams.offset !== undefined) { params.append("offset", String(queryParams.offset)); } + if (queryParams.search) { + params.append("search", queryParams.search); + } const queryString = params.toString(); diff --git a/surfsense_web/lib/electric/client.ts b/surfsense_web/lib/electric/client.ts index 9d596a261..c8bbd60fa 100644 --- a/surfsense_web/lib/electric/client.ts +++ b/surfsense_web/lib/electric/client.ts @@ -67,14 +67,10 @@ const pendingSyncs = new Map>(); // v2: user-specific database architecture // v3: consistent cutoff date for sync+queries, visibility refresh support // v4: heartbeat-based stale notification detection with updated_at tracking -// v5: fixed duplicate key errors (root cause: unstable cutoff dates in use-inbox.ts) -// - added onMustRefetch handler for server-side refetch scenarios -// - fixed getSyncCutoffDate to use stable midnight UTC timestamps -// v6: real-time documents table - added title and created_by_id columns for live document display -// v7: removed use-documents-electric.ts - consolidated to single documents sync to prevent conflicts -// v8: added status column for real-time document processing status (ready/processing/failed) -// v9: added pending state for accurate document queue visibility -const SYNC_VERSION = 11; +// v5: fixed duplicate key errors, stable cutoff dates, onMustRefetch handler, +// real-time documents table with title/created_by_id/status columns, +// consolidated single documents sync, pending state for document queue visibility +const SYNC_VERSION = 5; // Database name prefix for identifying SurfSense databases const DB_PREFIX = "surfsense-"; diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts index c6981b28a..7dc6c54b6 100644 --- a/surfsense_web/lib/query-client/cache-keys.ts +++ b/surfsense_web/lib/query-client/cache-keys.ts @@ -92,4 +92,8 @@ export const cacheKeys = { bySearchSpace: (searchSpaceId: number) => ["public-chat-snapshots", "search-space", searchSpaceId] as const, }, + notifications: { + search: (searchSpaceId: number | null, search: string, tab: string) => + ["notifications", "search", searchSpaceId, search, tab] as const, + }, }; diff --git a/surfsense_web/messages/en.json b/surfsense_web/messages/en.json index fae4c7265..20e665586 100644 --- a/surfsense_web/messages/en.json +++ b/surfsense_web/messages/en.json @@ -376,7 +376,7 @@ "upload_documents": { "title": "Upload Documents", "subtitle": "Upload your files to make them searchable and accessible through AI-powered conversations.", - "file_size_limit": "Maximum file size: 50MB per file. Supported formats vary based on your ETL service configuration.", + "file_size_limit": "Maximum file size: 50MB per file.", "upload_limits": "Upload limit: {maxFiles} files, {maxSizeMB}MB total.", "drop_files": "Drop files here", "drag_drop": "Drag & drop files here", diff --git a/surfsense_web/messages/zh.json b/surfsense_web/messages/zh.json index 2667a06d1..4c194a5dc 100644 --- a/surfsense_web/messages/zh.json +++ b/surfsense_web/messages/zh.json @@ -360,7 +360,7 @@ "upload_documents": { "title": "上传文档", "subtitle": "上传您的文件，使其可通过 AI 对话进行搜索和访问。", - "file_size_limit": "最大文件大小：每个文件 50MB。支持的格式因您的 ETL 服务配置而异。", + "file_size_limit": "最大文件大小：每个文件 50MB。", "upload_limits": "上传限制：最多 {maxFiles} 个文件，总大小不超过 {maxSizeMB}MB。", "drop_files": "放下文件到这里", "drag_drop": "拖放文件到这里",