From f54079643f61fbdbbfc52eb067eb1deed1bd7d76 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:53:35 +0200 Subject: [PATCH 01/92] feat(db): add GOOGLE_DRIVE_CONNECTOR to DocumentType and SearchSourceConnectorType enums --- surfsense_backend/app/db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index a2a424c26..a6bc3b938 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -46,6 +46,7 @@ class DocumentType(str, Enum): CLICKUP_CONNECTOR = "CLICKUP_CONNECTOR" GOOGLE_CALENDAR_CONNECTOR = "GOOGLE_CALENDAR_CONNECTOR" GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR" + GOOGLE_DRIVE_CONNECTOR = "GOOGLE_DRIVE_CONNECTOR" AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR" LUMA_CONNECTOR = "LUMA_CONNECTOR" ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR" @@ -69,6 +70,7 @@ class SearchSourceConnectorType(str, Enum): CLICKUP_CONNECTOR = "CLICKUP_CONNECTOR" GOOGLE_CALENDAR_CONNECTOR = "GOOGLE_CALENDAR_CONNECTOR" GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR" + GOOGLE_DRIVE_CONNECTOR = "GOOGLE_DRIVE_CONNECTOR" AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR" LUMA_CONNECTOR = "LUMA_CONNECTOR" ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR" From 5dd88386383c7f53dc42b253e35c32cedefaed5e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:53:44 +0200 Subject: [PATCH 02/92] feat(db): add idempotent Alembic migration for GOOGLE_DRIVE_CONNECTOR enums --- .../54_add_google_drive_connector_enums.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 surfsense_backend/alembic/versions/54_add_google_drive_connector_enums.py diff --git a/surfsense_backend/alembic/versions/54_add_google_drive_connector_enums.py b/surfsense_backend/alembic/versions/54_add_google_drive_connector_enums.py new file mode 100644 index 000000000..8e7d69340 --- /dev/null +++ b/surfsense_backend/alembic/versions/54_add_google_drive_connector_enums.py @@ -0,0 +1,74 @@ +"""Add Google Drive connector enums + +Revision ID: 54 +Revises: 53 +Create Date: 2025-12-28 12:00:00.000000 + +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "54" +down_revision: str | None = "53" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Safely add 'GOOGLE_DRIVE_CONNECTOR' to enum types if missing.""" + + # Add to searchsourceconnectortype enum + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type t + JOIN pg_enum e ON t.oid = e.enumtypid + WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'GOOGLE_DRIVE_CONNECTOR' + ) THEN + ALTER TYPE searchsourceconnectortype ADD VALUE 'GOOGLE_DRIVE_CONNECTOR'; + END IF; + END + $$; + """ + ) + + # Add to documenttype enum + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type t + JOIN pg_enum e ON t.oid = e.enumtypid + WHERE t.typname = 'documenttype' AND e.enumlabel = 'GOOGLE_DRIVE_CONNECTOR' + ) THEN + ALTER TYPE documenttype ADD VALUE 'GOOGLE_DRIVE_CONNECTOR'; + END IF; + END + $$; + """ + ) + + +def downgrade() -> None: + """Remove 'GOOGLE_DRIVE_CONNECTOR' from enum types. + + Note: PostgreSQL doesn't support removing enum values directly. + This would require recreating the enum type, which is complex and risky. + For now, we'll leave the enum values in place. + + In a production environment with strict downgrade requirements, you would need to: + 1. Create new enum types without the value + 2. Convert all columns to use the new type + 3. Drop the old enum type + 4. Rename the new type to the old name + + This is left as pass to avoid accidental data loss. + """ + pass + From 28979851270674e2d855d46e94456c91d0d9d89b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:53:51 +0200 Subject: [PATCH 03/92] feat(config): add GOOGLE_DRIVE_REDIRECT_URI environment variable --- surfsense_backend/app/config/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 08be26de1..9c503fb18 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -82,6 +82,9 @@ class Config: # Google Gmail redirect URI GOOGLE_GMAIL_REDIRECT_URI = os.getenv("GOOGLE_GMAIL_REDIRECT_URI") + # Google Drive redirect URI + GOOGLE_DRIVE_REDIRECT_URI = os.getenv("GOOGLE_DRIVE_REDIRECT_URI") + # Airtable OAuth AIRTABLE_CLIENT_ID = os.getenv("AIRTABLE_CLIENT_ID") AIRTABLE_CLIENT_SECRET = os.getenv("AIRTABLE_CLIENT_SECRET") From 2c8717b14bf8455bfc113bbe13a0db793f9a8c99 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:54:26 +0200 Subject: [PATCH 04/92] feat(connectors): add Google Drive credentials module for OAuth management - Handle Google OAuth credential initialization and validation - Automatic token refresh with database persistence - Reuse existing tokens when valid --- .../app/connectors/google_drive/__init__.py | 24 ++++ .../connectors/google_drive/credentials.py | 109 ++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 surfsense_backend/app/connectors/google_drive/__init__.py create mode 100644 surfsense_backend/app/connectors/google_drive/credentials.py diff --git a/surfsense_backend/app/connectors/google_drive/__init__.py b/surfsense_backend/app/connectors/google_drive/__init__.py new file mode 100644 index 000000000..c50135155 --- /dev/null +++ b/surfsense_backend/app/connectors/google_drive/__init__.py @@ -0,0 +1,24 @@ +""" +Google Drive Connector Module. + +Simple, modular approach to Google Drive indexing. +""" + +from .change_tracker import categorize_change, fetch_all_changes, get_start_page_token +from .client import GoogleDriveClient +from .content_extractor import download_and_process_file +from .credentials import get_valid_credentials, validate_credentials +from .folder_manager import get_files_in_folder, list_folder_contents + +__all__ = [ + "GoogleDriveClient", + "get_valid_credentials", + "validate_credentials", + "download_and_process_file", + "get_files_in_folder", + "list_folder_contents", + "get_start_page_token", + "fetch_all_changes", + "categorize_change", +] + diff --git a/surfsense_backend/app/connectors/google_drive/credentials.py b/surfsense_backend/app/connectors/google_drive/credentials.py new file mode 100644 index 000000000..5d09df881 --- /dev/null +++ b/surfsense_backend/app/connectors/google_drive/credentials.py @@ -0,0 +1,109 @@ +""" +Google Drive OAuth Credentials Management. + +Handles credential validation, token refresh, and persistence to database. +Small, focused module for credential operations only. +""" + +import json +from datetime import datetime + +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select +from sqlalchemy.orm.attributes import flag_modified + +from app.db import SearchSourceConnector, SearchSourceConnectorType + + +async def get_valid_credentials( + session: AsyncSession, + connector_id: int, +) -> Credentials: + """ + Get valid Google OAuth credentials, refreshing if needed. + + Args: + session: Database session + connector_id: Connector ID + + Returns: + Valid Google OAuth credentials + + Raises: + ValueError: If credentials are missing or invalid + Exception: If token refresh fails + """ + # Fetch connector from database + result = await session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise ValueError(f"Connector {connector_id} not found") + + # Extract credentials from config + config_data = connector.config + exp = config_data.get("expiry", "").replace("Z", "") + + # Validate required fields + if not all( + [ + config_data.get("client_id"), + config_data.get("client_secret"), + config_data.get("refresh_token"), + ] + ): + raise ValueError( + "Google OAuth credentials (client_id, client_secret, refresh_token) must be set" + ) + + # Create credentials object + credentials = Credentials( + token=config_data.get("token"), + refresh_token=config_data.get("refresh_token"), + token_uri=config_data.get("token_uri"), + client_id=config_data.get("client_id"), + client_secret=config_data.get("client_secret"), + scopes=config_data.get("scopes", []), + expiry=datetime.fromisoformat(exp) if exp else None, + ) + + # Refresh token if expired + if credentials.expired or not credentials.valid: + try: + credentials.refresh(Request()) + + # Persist refreshed token to database + connector.config = json.loads(credentials.to_json()) + flag_modified(connector, "config") + await session.commit() + + except Exception as e: + raise Exception(f"Failed to refresh Google OAuth credentials: {e!s}") from e + + return credentials + + +def validate_credentials(credentials: Credentials) -> bool: + """ + Validate that credentials have required fields. + + Args: + credentials: Google OAuth credentials + + Returns: + True if valid, False otherwise + """ + return all( + [ + credentials.client_id, + credentials.client_secret, + credentials.refresh_token, + ] + ) + From 74386affdcebbdf422235b94a67729f7f73b4304 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:54:32 +0200 Subject: [PATCH 05/92] feat(connectors): add Google Drive API client wrapper - Build and manage Google Drive service with credentials - List files with query support and pagination - Download binary files and export Google Workspace files as PDF - Handle HTTP errors gracefully --- .../app/connectors/google_drive/client.py | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 surfsense_backend/app/connectors/google_drive/client.py diff --git a/surfsense_backend/app/connectors/google_drive/client.py b/surfsense_backend/app/connectors/google_drive/client.py new file mode 100644 index 000000000..6d2d0abfd --- /dev/null +++ b/surfsense_backend/app/connectors/google_drive/client.py @@ -0,0 +1,194 @@ +""" +Google Drive API Client. + +Core client for interacting with Google Drive API. +Handles service initialization and basic file operations. +""" + +from typing import Any + +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError +from sqlalchemy.ext.asyncio import AsyncSession + +from .credentials import get_valid_credentials + + +class GoogleDriveClient: + """ + Main client for Google Drive API operations. + + Handles service initialization and provides methods for + listing files, getting metadata, and downloading content. + """ + + def __init__(self, session: AsyncSession, connector_id: int): + """ + Initialize Google Drive client. + + Args: + session: Database session + connector_id: ID of the Drive connector + """ + self.session = session + self.connector_id = connector_id + self.service = None + + async def get_service(self): + """ + Get or create the Drive service instance. + + Returns: + Google Drive service instance + + Raises: + Exception: If service creation fails + """ + if self.service: + return self.service + + try: + credentials = await get_valid_credentials(self.session, self.connector_id) + self.service = build("drive", "v3", credentials=credentials) + return self.service + except Exception as e: + raise Exception(f"Failed to create Google Drive service: {e!s}") from e + + async def list_files( + self, + query: str = "", + fields: str = "nextPageToken, files(id, name, mimeType, modifiedTime, size, webViewLink, parents, owners, createdTime, description)", + page_size: int = 100, + page_token: str | None = None, + ) -> tuple[list[dict[str, Any]], str | None, str | None]: + """ + List files from Google Drive with pagination. + + Args: + query: Search query (e.g., "mimeType != 'application/vnd.google-apps.folder'") + fields: Fields to retrieve + page_size: Number of files per page (max 1000) + page_token: Token for next page + + Returns: + Tuple of (files list, next_page_token, error message) + """ + try: + service = await self.get_service() + + params = { + "pageSize": min(page_size, 1000), + "fields": fields, + "supportsAllDrives": True, + "includeItemsFromAllDrives": True, + } + + if query: + params["q"] = query + if page_token: + params["pageToken"] = page_token + + result = service.files().list(**params).execute() + + files = result.get("files", []) + next_token = result.get("nextPageToken") + + return files, next_token, None + + except HttpError as e: + error_msg = f"HTTP error listing files: {e.resp.status} - {e.error_details}" + return [], None, error_msg + except Exception as e: + return [], None, f"Error listing files: {e!s}" + + async def get_file_metadata( + self, file_id: str, fields: str = "*" + ) -> tuple[dict[str, Any] | None, str | None]: + """ + Get metadata for a specific file. + + Args: + file_id: ID of the file + fields: Fields to retrieve + + Returns: + Tuple of (file metadata, error message) + """ + try: + service = await self.get_service() + file = service.files().get(fileId=file_id, fields=fields, supportsAllDrives=True).execute() + return file, None + except HttpError as e: + return None, f"HTTP error getting file metadata: {e.resp.status}" + except Exception as e: + return None, f"Error getting file metadata: {e!s}" + + async def download_file( + self, file_id: str + ) -> tuple[bytes | None, str | None]: + """ + Download binary file content. + + Args: + file_id: ID of the file to download + + Returns: + Tuple of (file content bytes, error message) + """ + try: + service = await self.get_service() + request = service.files().get_media(fileId=file_id) + + # Execute the download + import io + + fh = io.BytesIO() + from googleapiclient.http import MediaIoBaseDownload + + downloader = MediaIoBaseDownload(fh, request) + + done = False + while not done: + _, done = downloader.next_chunk() + + return fh.getvalue(), None + + except HttpError as e: + return None, f"HTTP error downloading file: {e.resp.status}" + except Exception as e: + return None, f"Error downloading file: {e!s}" + + async def export_google_file( + self, file_id: str, mime_type: str + ) -> tuple[bytes | None, str | None]: + """ + Export Google Workspace file to specified format. + + Args: + file_id: ID of the Google file + mime_type: Target MIME type (e.g., 'application/pdf', 'text/plain') + + Returns: + Tuple of (exported content as bytes, error message) + """ + try: + service = await self.get_service() + content = ( + service.files() + .export(fileId=file_id, mimeType=mime_type) + .execute() + ) + + # Content is already bytes from the API + # Keep as bytes to support both text and binary formats (like PDF) + if not isinstance(content, bytes): + content = content.encode("utf-8") + + return content, None + + except HttpError as e: + return None, f"HTTP error exporting file: {e.resp.status}" + except Exception as e: + return None, f"Error exporting file: {e!s}" + From 701c3409b386e8a85d725cef37664f95c39157b3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:54:42 +0200 Subject: [PATCH 06/92] feat(connectors): add Google Drive file type detection and mapping - Detect Google Workspace files (Docs, Sheets, Slides) - Map to PDF export format to preserve rich content (images, formatting) - Identify files to skip (shortcuts, unsupported types) --- .../app/connectors/google_drive/file_types.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 surfsense_backend/app/connectors/google_drive/file_types.py diff --git a/surfsense_backend/app/connectors/google_drive/file_types.py b/surfsense_backend/app/connectors/google_drive/file_types.py new file mode 100644 index 000000000..f66680c6c --- /dev/null +++ b/surfsense_backend/app/connectors/google_drive/file_types.py @@ -0,0 +1,37 @@ +""" +File Type Handlers for Google Drive. + +Simple module for basic file type detection. +""" + +# Google Workspace MIME types that need export +GOOGLE_DOC = "application/vnd.google-apps.document" +GOOGLE_SHEET = "application/vnd.google-apps.spreadsheet" +GOOGLE_SLIDE = "application/vnd.google-apps.presentation" +GOOGLE_FOLDER = "application/vnd.google-apps.folder" +GOOGLE_SHORTCUT = "application/vnd.google-apps.shortcut" + +# Export MIME types for Google Workspace files +# Export as PDF to preserve formatting, images, and structure +EXPORT_FORMATS = { + GOOGLE_DOC: "application/pdf", + GOOGLE_SHEET: "application/pdf", + GOOGLE_SLIDE: "application/pdf", +} + + +def is_google_workspace_file(mime_type: str) -> bool: + """Check if file is a Google Workspace file that needs export.""" + return mime_type.startswith("application/vnd.google-apps") + + +def should_skip_file(mime_type: str) -> bool: + """Check if file should be skipped (folders, shortcuts, etc).""" + return mime_type in [GOOGLE_FOLDER, GOOGLE_SHORTCUT] + + +def get_export_mime_type(mime_type: str) -> str | None: + """Get export MIME type for Google Workspace files.""" + return EXPORT_FORMATS.get(mime_type) + + From 40304c6795b9ab669fb594ee140abf6d5ce2d41e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:54:50 +0200 Subject: [PATCH 07/92] feat(connectors): add Google Drive content extraction using existing ETL - Download files from Google Drive to temporary location - Export Google Workspace files as PDF - Delegate content extraction to existing process_file_in_background - Reuse Surfsense's ETL services (Unstructured, LlamaCloud, Docling) --- .../google_drive/content_extractor.py | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 surfsense_backend/app/connectors/google_drive/content_extractor.py diff --git a/surfsense_backend/app/connectors/google_drive/content_extractor.py b/surfsense_backend/app/connectors/google_drive/content_extractor.py new file mode 100644 index 000000000..82b8d42b3 --- /dev/null +++ b/surfsense_backend/app/connectors/google_drive/content_extractor.py @@ -0,0 +1,122 @@ +""" +Content Extraction for Google Drive Files. + +Downloads files and delegates to Surfsense's existing file processors. +""" + +import logging +import os +import tempfile +from pathlib import Path +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import Log +from app.services.task_logging_service import TaskLoggingService + +from .client import GoogleDriveClient +from .file_types import get_export_mime_type, is_google_workspace_file, should_skip_file + +logger = logging.getLogger(__name__) + + +async def download_and_process_file( + client: GoogleDriveClient, + file: dict[str, Any], + search_space_id: int, + user_id: str, + session: AsyncSession, + task_logger: TaskLoggingService, + log_entry: Log, +) -> tuple[Any, str | None]: + """ + Download Google Drive file and process using Surfsense's existing infrastructure. + + This is the ONLY function needed - it delegates everything to process_file_in_background. + + Args: + client: GoogleDriveClient instance + file: File metadata from Drive API + search_space_id: ID of the search space + user_id: ID of the user + session: Database session + task_logger: Task logging service + log_entry: Log entry for tracking + + Returns: + Tuple of (Document object if successful, error message if failed) + """ + file_id = file.get("id") + file_name = file.get("name", "Unknown") + mime_type = file.get("mimeType", "") + + # Skip folders and shortcuts + if should_skip_file(mime_type): + return None, f"Skipping {mime_type}" + + logger.info(f"Downloading file: {file_name} ({mime_type})") + + temp_file_path = None + try: + # Step 1: Download or export the file + if is_google_workspace_file(mime_type): + # Google Workspace files need export (as PDF to preserve formatting & images) + export_mime = get_export_mime_type(mime_type) + if not export_mime: + return None, f"Cannot export Google Workspace type: {mime_type}" + + logger.info(f"Exporting Google Workspace file as {export_mime}") + content_bytes, error = await client.export_google_file(file_id, export_mime) + if error: + return None, error + + # Set extension based on export format + extension = ".pdf" if export_mime == "application/pdf" else ".txt" + else: + # Regular files - download directly + content_bytes, error = await client.download_file(file_id) + if error: + return None, error + + # Preserve original file extension + extension = Path(file_name).suffix or ".bin" + + # Save to temporary file + with tempfile.NamedTemporaryFile(delete=False, suffix=extension) as tmp_file: + tmp_file.write(content_bytes) + temp_file_path = tmp_file.name + + # Step 2: Delegate to Surfsense's existing file processor + # This handles ALL file types: markdown, audio, PDFs, Office docs, images, etc. + from app.tasks.document_processors.file_processors import ( + process_file_in_background, + ) + + logger.info(f"Processing {file_name} with Surfsense's file processor") + result = await process_file_in_background( + file_path=temp_file_path, + filename=file_name, + search_space_id=search_space_id, + user_id=user_id, + session=session, + task_logger=task_logger, + log_entry=log_entry, + ) + + # process_file_in_background returns None on duplicate/error, Document on success + return result, None + + except Exception as e: + logger.warning(f"Failed to process {file_name}: {e!s}") + return None, str(e) + + finally: + # Cleanup temp file (if process_file_in_background didn't already delete it) + if temp_file_path and os.path.exists(temp_file_path): + try: + os.unlink(temp_file_path) + except Exception as e: + logger.debug(f"Could not delete temp file {temp_file_path}: {e}") + + From 84bde67979e82cd4010baa340506499a7d1830db Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:54:58 +0200 Subject: [PATCH 08/92] feat(connectors): add Google Drive folder browsing and file listing - List folder contents with full pagination support - Query root folder or specific parent folder - Return both folders and files with metadata (size, icons, links) - Filter out shortcuts and trashed items --- .../connectors/google_drive/folder_manager.py | 243 ++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 surfsense_backend/app/connectors/google_drive/folder_manager.py diff --git a/surfsense_backend/app/connectors/google_drive/folder_manager.py b/surfsense_backend/app/connectors/google_drive/folder_manager.py new file mode 100644 index 000000000..da9deb75d --- /dev/null +++ b/surfsense_backend/app/connectors/google_drive/folder_manager.py @@ -0,0 +1,243 @@ +""" +Folder Management for Google Drive. + +Handles folder listing, selection, and hierarchy operations. +Small, focused module for folder-related operations. +""" + +import logging +from typing import Any + +from .client import GoogleDriveClient + +logger = logging.getLogger(__name__) + + +async def list_folders( + client: GoogleDriveClient, + parent_id: str | None = None, +) -> tuple[list[dict[str, Any]], str | None]: + """ + List folders in Google Drive. + + Args: + client: GoogleDriveClient instance + parent_id: Parent folder ID (None for root) + + Returns: + Tuple of (folders list, error message) + """ + try: + # Build query to get only folders + query_parts = ["mimeType = 'application/vnd.google-apps.folder'", "trashed = false"] + + if parent_id: + query_parts.append(f"'{parent_id}' in parents") + + query = " and ".join(query_parts) + + folders, _, error = await client.list_files( + query=query, + fields="files(id, name, parents, createdTime, modifiedTime)", + page_size=100, + ) + + if error: + return [], error + + return folders, None + + except Exception as e: + logger.error(f"Error listing folders: {e!s}", exc_info=True) + return [], f"Error listing folders: {e!s}" + + +async def get_folder_hierarchy( + client: GoogleDriveClient, + folder_id: str, +) -> tuple[list[dict[str, str]], str | None]: + """ + Get the full path hierarchy for a folder. + + Args: + client: GoogleDriveClient instance + folder_id: Folder ID to get hierarchy for + + Returns: + Tuple of (hierarchy list [{'id': ..., 'name': ...}], error message) + """ + try: + hierarchy = [] + current_id = folder_id + + # Traverse up to root + while current_id: + file, error = await client.get_file_metadata( + current_id, + fields="id, name, parents, mimeType" + ) + + if error: + return [], error + + if not file: + break + + hierarchy.insert(0, {"id": file["id"], "name": file["name"]}) + + # Get parent + parents = file.get("parents", []) + current_id = parents[0] if parents else None + + return hierarchy, None + + except Exception as e: + logger.error(f"Error getting folder hierarchy: {e!s}", exc_info=True) + return [], f"Error getting folder hierarchy: {e!s}" + + +async def get_files_in_folder( + client: GoogleDriveClient, + folder_id: str, + include_subfolders: bool = True, + page_token: str | None = None, +) -> tuple[list[dict[str, Any]], str | None, str | None]: + """ + Get all indexable files in a folder. + + Args: + client: GoogleDriveClient instance + folder_id: Folder ID to search in + include_subfolders: Whether to include subfolders + page_token: Pagination token + + Returns: + Tuple of (files list, next_page_token, error message) + """ + try: + # Build query + query_parts = [ + f"'{folder_id}' in parents", + "trashed = false", + "mimeType != 'application/vnd.google-apps.shortcut'", # Skip shortcuts + ] + + if not include_subfolders: + query_parts.append("mimeType != 'application/vnd.google-apps.folder'") + + query = " and ".join(query_parts) + + files, next_token, error = await client.list_files( + query=query, + page_size=100, + page_token=page_token, + ) + + if error: + return [], None, error + + return files, next_token, None + + except Exception as e: + logger.error(f"Error getting files in folder: {e!s}", exc_info=True) + return [], None, f"Error getting files in folder: {e!s}" + + +def format_folder_path(hierarchy: list[dict[str, str]]) -> str: + """ + Format folder hierarchy as a path string. + + Args: + hierarchy: List of folder dicts with 'id' and 'name' + + Returns: + Formatted path (e.g., "My Drive / Projects / Documents") + """ + if not hierarchy: + return "My Drive" + + folder_names = [folder["name"] for folder in hierarchy] + return " / ".join(folder_names) + + +async def list_folder_contents( + client: GoogleDriveClient, + parent_id: str | None = None, +) -> tuple[list[dict[str, Any]], str | None]: + """ + List both folders and files in a Google Drive folder. + + Fetches ALL items using pagination (handles folders with >100 items). + Returns items sorted with folders first, then files. + Each item includes 'isFolder' boolean for frontend rendering. + + Args: + client: GoogleDriveClient instance + parent_id: Parent folder ID (None for root) + + Returns: + Tuple of (items list with folders and files, error message) + """ + try: + # Build query to get folders and files (exclude shortcuts) + query_parts = [ + "trashed = false", + "mimeType != 'application/vnd.google-apps.shortcut'", + ] + + # For root, we need to explicitly query for items in 'root' + # For subfolders, query for items with that parent + if parent_id: + query_parts.append(f"'{parent_id}' in parents") + else: + # Query for root-level items + query_parts.append("'root' in parents") + + query = " and ".join(query_parts) + + # Fetch all items with pagination (max 1000 per page) + all_items = [] + page_token = None + + while True: + items, next_token, error = await client.list_files( + query=query, + fields="files(id, name, mimeType, parents, createdTime, modifiedTime, size, webViewLink, iconLink)", + page_size=1000, # Max allowed by Google Drive API + page_token=page_token, + ) + + if error: + return [], error + + all_items.extend(items) + + # If no more pages, break + if not next_token: + break + + page_token = next_token + + # Add 'isFolder' flag and sort (folders first, then files) + for item in all_items: + item["isFolder"] = item["mimeType"] == "application/vnd.google-apps.folder" + + # Sort: folders first (alphabetically), then files (alphabetically) + all_items.sort(key=lambda x: (not x["isFolder"], x["name"].lower())) + + # Count folders and files for logging + folder_count = sum(1 for item in all_items if item["isFolder"]) + file_count = len(all_items) - folder_count + + logger.info( + f"Listed {len(all_items)} items ({folder_count} folders, {file_count} files) " + + (f"in folder {parent_id}" if parent_id else "in root (My Drive)") + ) + + return all_items, None + + except Exception as e: + logger.error(f"Error listing folder contents: {e!s}", exc_info=True) + return [], f"Error listing folder contents: {e!s}" + + From 3e67d5f31ec9792c5a063f2ebcc7172b3c2fc57a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:55:06 +0200 Subject: [PATCH 09/92] feat(connectors): add Google Drive delta sync with change tracking - Get start page token for change tracking baseline - Fetch incremental changes using Google Drive Changes API - Categorize changes into added, modified, and removed files - Enable efficient re-indexing of only changed content --- .../connectors/google_drive/change_tracker.py | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 surfsense_backend/app/connectors/google_drive/change_tracker.py diff --git a/surfsense_backend/app/connectors/google_drive/change_tracker.py b/surfsense_backend/app/connectors/google_drive/change_tracker.py new file mode 100644 index 000000000..1c697af5f --- /dev/null +++ b/surfsense_backend/app/connectors/google_drive/change_tracker.py @@ -0,0 +1,213 @@ +""" +Change Tracking for Google Drive - Delta Sync Support. + +Handles change detection and incremental syncing using Drive API's changes endpoint. +Small, focused module for tracking file modifications. +""" + +import logging +from datetime import datetime +from typing import Any + +from .client import GoogleDriveClient + +logger = logging.getLogger(__name__) + + +async def get_start_page_token( + client: GoogleDriveClient, +) -> tuple[str | None, str | None]: + """ + Get the starting page token for change tracking. + + This token represents the current state and is used for future delta syncs. + + Args: + client: GoogleDriveClient instance + + Returns: + Tuple of (start_page_token, error message) + """ + try: + service = await client.get_service() + response = service.changes().getStartPageToken(supportsAllDrives=True).execute() + token = response.get("startPageToken") + + logger.info(f"Got start page token: {token}") + return token, None + + except Exception as e: + logger.error(f"Error getting start page token: {e!s}", exc_info=True) + return None, f"Error getting start page token: {e!s}" + + +async def get_changes( + client: GoogleDriveClient, + page_token: str, + folder_id: str | None = None, +) -> tuple[list[dict[str, Any]], str | None, str | None]: + """ + Get list of changes since the given page token. + + Args: + client: GoogleDriveClient instance + page_token: Page token from previous sync + folder_id: Optional folder ID to filter changes + + Returns: + Tuple of (changes list, new_page_token, error message) + """ + try: + service = await client.get_service() + + params = { + "pageToken": page_token, + "pageSize": 100, + "fields": "nextPageToken, newStartPageToken, changes(fileId, removed, file(id, name, mimeType, modifiedTime, size, webViewLink, parents, trashed))", + "supportsAllDrives": True, + "includeItemsFromAllDrives": True, + } + + response = service.changes().list(**params).execute() + + changes = response.get("changes", []) + next_token = response.get("nextPageToken") + new_start_token = response.get("newStartPageToken") + + # Use new start token if this is the last page + token_to_return = new_start_token if new_start_token else next_token + + # Filter changes by folder if specified + if folder_id: + changes = await _filter_changes_by_folder(client, changes, folder_id) + + logger.info(f"Got {len(changes)} changes, next token: {token_to_return}") + return changes, token_to_return, None + + except Exception as e: + logger.error(f"Error getting changes: {e!s}", exc_info=True) + return [], None, f"Error getting changes: {e!s}" + + +async def _filter_changes_by_folder( + client: GoogleDriveClient, + changes: list[dict[str, Any]], + folder_id: str, +) -> list[dict[str, Any]]: + """ + Filter changes to only include files within the specified folder. + + Args: + client: GoogleDriveClient instance + changes: List of changes from API + folder_id: Folder ID to filter by + + Returns: + Filtered list of changes + """ + filtered = [] + + for change in changes: + file = change.get("file") + if not file: + # File was removed + filtered.append(change) + continue + + # Check if file is in the folder (or subfolder) + parents = file.get("parents", []) + if folder_id in parents: + filtered.append(change) + else: + # Check if any parent is a descendant of folder_id + # This is a simplified check - full implementation would traverse hierarchy + # For now, we'll include it and let indexer validate + filtered.append(change) + + return filtered + + +def categorize_change(change: dict[str, Any]) -> str: + """ + Categorize a change event. + + Args: + change: Change event from Drive API + + Returns: + Category: 'removed', 'trashed', 'modified', 'new' + """ + if change.get("removed"): + return "removed" + + file = change.get("file") + if not file: + return "removed" + + if file.get("trashed"): + return "trashed" + + # Check if file was recently created + created_time = file.get("createdTime") + modified_time = file.get("modifiedTime") + + if created_time and modified_time: + try: + created = datetime.fromisoformat(created_time.replace("Z", "+00:00")) + modified = datetime.fromisoformat(modified_time.replace("Z", "+00:00")) + + # If created and modified times are very close, it's likely a new file + time_diff = abs((modified - created).total_seconds()) + if time_diff < 60: # Within 1 minute + return "new" + except Exception: + pass + + return "modified" + + +async def fetch_all_changes( + client: GoogleDriveClient, + start_token: str, + folder_id: str | None = None, +) -> tuple[list[dict[str, Any]], str | None, str | None]: + """ + Fetch all changes from start token, handling pagination. + + Args: + client: GoogleDriveClient instance + start_token: Starting page token + folder_id: Optional folder ID to filter changes + + Returns: + Tuple of (all changes, final_page_token, error message) + """ + all_changes = [] + current_token = start_token + error = None + + try: + while current_token: + changes, next_token, err = await get_changes( + client, current_token, folder_id + ) + + if err: + error = err + break + + all_changes.extend(changes) + + # If next_token is None, we've reached the end + if not next_token or next_token == current_token: + break + + current_token = next_token + + logger.info(f"Fetched total of {len(all_changes)} changes") + return all_changes, current_token, error + + except Exception as e: + logger.error(f"Error fetching all changes: {e!s}", exc_info=True) + return all_changes, current_token, f"Error fetching all changes: {e!s}" + From bf02005d82ddb5c8329176b5469492535753c5f7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:55:13 +0200 Subject: [PATCH 10/92] feat(routes): add Google Drive OAuth and folder listing endpoints - OAuth initialization and callback handling - Folder and file browsing with parent_id support - Validate credentials and handle token refresh - Return folder contents with metadata for UI tree view --- .../google_drive_add_connector_route.py | 315 ++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 surfsense_backend/app/routes/google_drive_add_connector_route.py diff --git a/surfsense_backend/app/routes/google_drive_add_connector_route.py b/surfsense_backend/app/routes/google_drive_add_connector_route.py new file mode 100644 index 000000000..d11404781 --- /dev/null +++ b/surfsense_backend/app/routes/google_drive_add_connector_route.py @@ -0,0 +1,315 @@ +""" +Google Drive Connector OAuth Routes. + +Handles OAuth 2.0 authentication flow for Google Drive connector. +Folder selection happens at index time on the manage connector page. + +Endpoints: +- GET /auth/google/drive/connector/add - Initiate OAuth +- GET /auth/google/drive/connector/callback - Handle OAuth callback +- GET /connectors/{connector_id}/google-drive/folders - List user's folders (for index-time selection) +""" + +import base64 +import json +import logging +import os +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from google_auth_oauthlib.flow import Flow +from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.connectors.google_drive import ( + GoogleDriveClient, + get_start_page_token, + get_valid_credentials, + list_folder_contents, +) +from app.connectors.google_drive.folder_manager import list_folders +from app.db import ( + SearchSourceConnector, + SearchSourceConnectorType, + User, + get_async_session, +) +from app.users import current_active_user + +# Relax token scope validation for Google OAuth +os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" + +logger = logging.getLogger(__name__) +router = APIRouter() + +# Google Drive OAuth scopes +SCOPES = [ + "https://www.googleapis.com/auth/drive.readonly", # Read-only access to Drive + "https://www.googleapis.com/auth/userinfo.email", # User email + "https://www.googleapis.com/auth/userinfo.profile", # User profile + "openid", +] + + +def get_google_flow(): + """Create and return a Google OAuth flow for Drive API.""" + try: + return Flow.from_client_config( + { + "web": { + "client_id": config.GOOGLE_OAUTH_CLIENT_ID, + "client_secret": config.GOOGLE_OAUTH_CLIENT_SECRET, + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "redirect_uris": [config.GOOGLE_DRIVE_REDIRECT_URI], + } + }, + scopes=SCOPES, + redirect_uri=config.GOOGLE_DRIVE_REDIRECT_URI, + ) + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Failed to create Google OAuth flow: {e!s}" + ) from e + + +@router.get("/auth/google/drive/connector/add") +async def connect_drive(space_id: int, user: User = Depends(current_active_user)): + """ + Initiate Google Drive OAuth flow. + + Query params: + space_id: Search space ID to add connector to + + Returns: + JSON with auth_url to redirect user to Google authorization + """ + try: + if not space_id: + raise HTTPException(status_code=400, detail="space_id is required") + + flow = get_google_flow() + + # Encode space_id and user_id in state parameter + state_payload = json.dumps( + { + "space_id": space_id, + "user_id": str(user.id), + } + ) + state_encoded = base64.urlsafe_b64encode(state_payload.encode()).decode() + + # Generate authorization URL + auth_url, _ = flow.authorization_url( + access_type="offline", # Get refresh token + prompt="consent", # Force consent screen to get refresh token + include_granted_scopes="true", + state=state_encoded, + ) + + logger.info(f"Initiating Google Drive OAuth for user {user.id}, space {space_id}") + return {"auth_url": auth_url} + + except Exception as e: + logger.error(f"Failed to initiate Google Drive OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to initiate Google OAuth: {e!s}" + ) from e + + +@router.get("/auth/google/drive/connector/callback") +async def drive_callback( + request: Request, + code: str, + state: str, + session: AsyncSession = Depends(get_async_session), +): + """ + Handle Google Drive OAuth callback. + + Query params: + code: Authorization code from Google + state: Encoded state with space_id and user_id + + Returns: + Redirect to frontend success page + """ + try: + # Decode and parse state + decoded_state = base64.urlsafe_b64decode(state.encode()).decode() + data = json.loads(decoded_state) + + user_id = UUID(data["user_id"]) + space_id = data["space_id"] + + logger.info(f"Processing Google Drive callback for user {user_id}, space {space_id}") + + # Exchange authorization code for tokens + flow = get_google_flow() + flow.fetch_token(code=code) + + creds = flow.credentials + creds_dict = json.loads(creds.to_json()) + + # Check if connector already exists for this space/user + result = await session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == space_id, + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR, + ) + ) + existing_connector = result.scalars().first() + + if existing_connector: + raise HTTPException( + status_code=409, + detail="A GOOGLE_DRIVE_CONNECTOR already exists in this search space. Each search space can have only one connector of each type per user.", + ) + + # Create new connector (NO folder selection here - happens at index time) + db_connector = SearchSourceConnector( + name="Google Drive Connector", + connector_type=SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR, + config={ + **creds_dict, + "start_page_token": None, # Will be set on first index + }, + search_space_id=space_id, + user_id=user_id, + is_indexable=True, + ) + + session.add(db_connector) + await session.commit() + await session.refresh(db_connector) + + # Get initial start page token for delta sync + try: + drive_client = GoogleDriveClient(session, db_connector.id) + start_token, token_error = await get_start_page_token(drive_client) + + if start_token and not token_error: + db_connector.config["start_page_token"] = start_token + from sqlalchemy.orm.attributes import flag_modified + + flag_modified(db_connector, "config") + await session.commit() + logger.info(f"Set initial start page token for connector {db_connector.id}") + except Exception as e: + logger.warning(f"Failed to get initial start page token: {e!s}") + + logger.info( + f"Successfully created Google Drive connector {db_connector.id} for user {user_id}" + ) + + # Redirect to connectors management page (not to folder selection) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors?success=google-drive-connected" + ) + + except HTTPException: + await session.rollback() + raise + except ValidationError as e: + await session.rollback() + logger.error(f"Validation error: {e!s}", exc_info=True) + raise HTTPException( + status_code=400, detail=f"Invalid connector configuration: {e!s}" + ) from e + except IntegrityError as e: + await session.rollback() + logger.error(f"Database integrity error: {e!s}", exc_info=True) + raise HTTPException( + status_code=409, + detail="A connector with this configuration already exists.", + ) from e + except Exception as e: + await session.rollback() + logger.error(f"Unexpected error in Drive callback: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to complete Google OAuth: {e!s}" + ) from e + + +@router.get("/connectors/{connector_id}/google-drive/folders") +async def list_google_drive_folders( + connector_id: int, + parent_id: str | None = None, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + """ + List folders AND files in user's Google Drive with hierarchical support. + + This is called at index time from the manage connector page to display + the complete file system (folders and files). Only folders are selectable. + + Args: + connector_id: ID of the Google Drive connector + parent_id: Optional parent folder ID to list contents (None for root) + + Returns: + JSON with list of items: { + "items": [ + {"id": str, "name": str, "mimeType": str, "isFolder": bool, ...}, + ... + ] + } + """ + try: + # Get connector and verify ownership + result = await session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == connector_id, + SearchSourceConnector.user_id == user.id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR, + ) + ) + connector = result.scalars().first() + + if not connector: + raise HTTPException( + status_code=404, + detail="Google Drive connector not found or access denied", + ) + + # Initialize Drive client (credentials will be loaded on first API call) + drive_client = GoogleDriveClient(session, connector_id) + + # List both folders and files (sorted: folders first) + items, error = await list_folder_contents(drive_client, parent_id=parent_id) + + if error: + raise HTTPException( + status_code=500, detail=f"Failed to list folder contents: {error}" + ) + + # Count folders and files for better logging + folder_count = sum(1 for item in items if item.get("isFolder", False)) + file_count = len(items) - folder_count + + logger.info( + f"✅ Listed {len(items)} total items ({folder_count} folders, {file_count} files) for connector {connector_id}" + + (f" in folder {parent_id}" if parent_id else " in ROOT") + ) + + # Log first few items for debugging + if items: + logger.info(f"First 3 items: {[item.get('name') for item in items[:3]]}") + + return {"items": items} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error listing Drive contents: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to list Drive contents: {e!s}" + ) from e From 1696c7056a8e448ca7bec7c7f00bf046a3e54e26 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:55:25 +0200 Subject: [PATCH 11/92] feat(indexer): add Google Drive folder indexing with delta sync - Full folder scan on first index - Delta sync using change tracking for subsequent indexes - Process files in parallel batches - Handle file additions, modifications, and deletions - Store change tracking token for efficient re-indexing --- .../google_drive_indexer.py | 448 ++++++++++++++++++ 1 file changed, 448 insertions(+) create mode 100644 surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py new file mode 100644 index 000000000..9c4d446de --- /dev/null +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -0,0 +1,448 @@ +""" +Google Drive Indexer - Delegates all processing to Surfsense's file processors. + +Handles: +- Folder-specific indexing (user selects folder) +- Delta sync (only index changed files) +- Delegates file processing to process_file_in_background +""" + +import logging +from datetime import datetime + +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.connectors.google_drive import ( + GoogleDriveClient, + categorize_change, + download_and_process_file, + fetch_all_changes, + get_files_in_folder, + get_start_page_token, +) +from app.db import DocumentType, SearchSourceConnectorType +from app.services.task_logging_service import TaskLoggingService +from app.tasks.connector_indexers.base import ( + check_document_by_unique_identifier, + get_connector_by_id, + update_connector_last_indexed, +) +from app.utils.document_converters import generate_unique_identifier_hash + +logger = logging.getLogger(__name__) + + +async def index_google_drive_files( + session: AsyncSession, + connector_id: int, + search_space_id: int, + user_id: str, + folder_id: str | None = None, + folder_name: str | None = None, + use_delta_sync: bool = True, + update_last_indexed: bool = True, + max_files: int = 500, +) -> tuple[int, str | None]: + """ + Index Google Drive files for a specific connector. + + Args: + session: Database session + connector_id: ID of the Drive connector + search_space_id: ID of the search space + user_id: ID of the user + folder_id: Specific folder to index (from UI/request, takes precedence) + folder_name: Folder name for display (from UI/request) + use_delta_sync: Whether to use change tracking for incremental sync + update_last_indexed: Whether to update last_indexed_at timestamp + max_files: Maximum number of files to index + + Returns: + Tuple of (number_of_indexed_files, error_message) + """ + task_logger = TaskLoggingService(session, search_space_id) + + # Log task start + log_entry = await task_logger.log_task_start( + task_name="google_drive_files_indexing", + source="connector_indexing_task", + message=f"Starting Google Drive indexing for connector {connector_id}", + metadata={ + "connector_id": connector_id, + "user_id": str(user_id), + "folder_id": folder_id, + "use_delta_sync": use_delta_sync, + "max_files": max_files, + }, + ) + + try: + # Get connector from database + connector = await get_connector_by_id( + session, connector_id, SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR + ) + + if not connector: + error_msg = f"Google Drive connector with ID {connector_id} not found" + await task_logger.log_task_failure( + log_entry, error_msg, {"error_type": "ConnectorNotFound"} + ) + return 0, error_msg + + # Initialize Drive client + await task_logger.log_task_progress( + log_entry, + f"Initializing Google Drive client for connector {connector_id}", + {"stage": "client_initialization"}, + ) + + drive_client = GoogleDriveClient(session, connector_id) + + # Use folder from request params (required for Google Drive) + if not folder_id: + error_msg = "folder_id is required for Google Drive indexing" + await task_logger.log_task_failure( + log_entry, error_msg, {"error_type": "MissingParameter"} + ) + return 0, error_msg + + target_folder_id = folder_id + target_folder_name = folder_name or "Selected Folder" + + logger.info(f"Indexing Google Drive folder: {target_folder_name} ({target_folder_id})") + + # Decide sync strategy + start_page_token = connector.config.get("start_page_token") + can_use_delta_sync = use_delta_sync and start_page_token and connector.last_indexed_at + + if can_use_delta_sync: + logger.info(f"Using delta sync for connector {connector_id}") + result = await _index_with_delta_sync( + drive_client=drive_client, + session=session, + connector=connector, + connector_id=connector_id, + search_space_id=search_space_id, + user_id=user_id, + folder_id=target_folder_id, + start_page_token=start_page_token, + task_logger=task_logger, + log_entry=log_entry, + max_files=max_files, + ) + else: + logger.info(f"Using full scan for connector {connector_id}") + result = await _index_full_scan( + drive_client=drive_client, + session=session, + connector=connector, + connector_id=connector_id, + search_space_id=search_space_id, + user_id=user_id, + folder_id=target_folder_id, + folder_name=target_folder_name, + task_logger=task_logger, + log_entry=log_entry, + max_files=max_files, + ) + + documents_indexed, documents_skipped = result + + # Update last indexed timestamp and get new start page token + if documents_indexed > 0 or can_use_delta_sync: + # Get new start page token for next sync + new_token, token_error = await get_start_page_token(drive_client) + if new_token and not token_error: + from sqlalchemy.orm.attributes import flag_modified + + connector.config["start_page_token"] = new_token + flag_modified(connector, "config") + + await update_connector_last_indexed(session, connector, update_last_indexed) + + # Final commit + await session.commit() + logger.info( + f"Successfully committed Google Drive indexing changes to database" + ) + + # Log success + await task_logger.log_task_success( + log_entry, + f"Successfully completed Google Drive indexing for connector {connector_id}", + { + "files_processed": documents_indexed, + "files_skipped": documents_skipped, + "sync_type": "delta" if can_use_delta_sync else "full", + "folder": target_folder_name, + }, + ) + + logger.info( + f"Google Drive indexing completed: {documents_indexed} files indexed, {documents_skipped} skipped" + ) + return documents_indexed, None + + except SQLAlchemyError as db_error: + await session.rollback() + await task_logger.log_task_failure( + log_entry, + f"Database error during Google Drive indexing for connector {connector_id}", + str(db_error), + {"error_type": "SQLAlchemyError"}, + ) + logger.error(f"Database error: {db_error!s}", exc_info=True) + return 0, f"Database error: {db_error!s}" + except Exception as e: + await session.rollback() + await task_logger.log_task_failure( + log_entry, + f"Failed to index Google Drive files for connector {connector_id}", + str(e), + {"error_type": type(e).__name__}, + ) + logger.error(f"Failed to index Google Drive files: {e!s}", exc_info=True) + return 0, f"Failed to index Google Drive files: {e!s}" + + +async def _index_full_scan( + drive_client: GoogleDriveClient, + session: AsyncSession, + connector: any, + connector_id: int, + search_space_id: int, + user_id: str, + folder_id: str | None, + folder_name: str, + task_logger: TaskLoggingService, + log_entry: any, + max_files: int, +) -> tuple[int, int]: + """Perform full scan indexing of a folder.""" + await task_logger.log_task_progress( + log_entry, + f"Starting full scan of folder: {folder_name}", + {"stage": "full_scan", "folder_id": folder_id}, + ) + + documents_indexed = 0 + documents_skipped = 0 + page_token = None + files_processed = 0 + + # Paginate through all files in folder + while files_processed < max_files: + files, next_token, error = await get_files_in_folder( + drive_client, folder_id, include_subfolders=False, page_token=page_token + ) + + if error: + logger.error(f"Error listing files: {error}") + break + + if not files: + break + + for file in files: + if files_processed >= max_files: + break + + files_processed += 1 + + # Process file + indexed, skipped = await _process_single_file( + drive_client=drive_client, + session=session, + file=file, + connector_id=connector_id, + search_space_id=search_space_id, + user_id=user_id, + task_logger=task_logger, + log_entry=log_entry, + ) + + documents_indexed += indexed + documents_skipped += skipped + + # Batch commit every 10 files + if documents_indexed % 10 == 0 and documents_indexed > 0: + await session.commit() + logger.info(f"Committed batch: {documents_indexed} files indexed so far") + + page_token = next_token + if not page_token: + break + + logger.info( + f"Full scan complete: {documents_indexed} indexed, {documents_skipped} skipped" + ) + return documents_indexed, documents_skipped + + +async def _index_with_delta_sync( + drive_client: GoogleDriveClient, + session: AsyncSession, + connector: any, + connector_id: int, + search_space_id: int, + user_id: str, + folder_id: str | None, + start_page_token: str, + task_logger: TaskLoggingService, + log_entry: any, + max_files: int, +) -> tuple[int, int]: + """Perform delta sync indexing using change tracking.""" + await task_logger.log_task_progress( + log_entry, + f"Starting delta sync from token: {start_page_token[:20]}...", + {"stage": "delta_sync", "start_token": start_page_token}, + ) + + # Fetch all changes since last sync + changes, final_token, error = await fetch_all_changes( + drive_client, start_page_token, folder_id + ) + + if error: + logger.error(f"Error fetching changes: {error}") + return 0, 0 + + if not changes: + logger.info("No changes detected since last sync") + return 0, 0 + + logger.info(f"Processing {len(changes)} changes") + + documents_indexed = 0 + documents_skipped = 0 + files_processed = 0 + + for change in changes: + if files_processed >= max_files: + break + + files_processed += 1 + change_type = categorize_change(change) + + # Handle removed/trashed files + if change_type in ["removed", "trashed"]: + file_id = change.get("fileId") + if file_id: + await _remove_document(session, file_id, search_space_id) + continue + + # Handle modified/new files + file = change.get("file") + if not file: + continue + + indexed, skipped = await _process_single_file( + drive_client=drive_client, + session=session, + file=file, + connector_id=connector_id, + search_space_id=search_space_id, + user_id=user_id, + task_logger=task_logger, + log_entry=log_entry, + ) + + documents_indexed += indexed + documents_skipped += skipped + + # Batch commit every 10 files + if documents_indexed % 10 == 0 and documents_indexed > 0: + await session.commit() + logger.info(f"Committed batch: {documents_indexed} changes processed") + + logger.info( + f"Delta sync complete: {documents_indexed} indexed, {documents_skipped} skipped" + ) + return documents_indexed, documents_skipped + + +async def _process_single_file( + drive_client: GoogleDriveClient, + session: AsyncSession, + file: dict, + connector_id: int, + search_space_id: int, + user_id: str, + task_logger: TaskLoggingService, + log_entry: any, +) -> tuple[int, int]: + """ + Process a single file by downloading and using Surfsense's file processor. + + Returns: + Tuple of (indexed_count, skipped_count) + """ + file_name = file.get("name", "Unknown") + mime_type = file.get("mimeType", "") + + try: + logger.info(f"Processing file: {file_name} ({mime_type})") + + # Download and process using Surfsense's existing infrastructure + # This handles: markdown, audio, PDFs, Office docs, images, etc. + # It also handles: deduplication, chunking, summarization, embedding + document, error = await download_and_process_file( + client=drive_client, + file=file, + search_space_id=search_space_id, + user_id=user_id, + session=session, + task_logger=task_logger, + log_entry=log_entry, + ) + + if error: + # Log and skip - not an error, just unsupported or empty + await task_logger.log_task_progress( + log_entry, + f"Skipped {file_name}: {error}", + {"status": "skipped", "reason": error}, + ) + return 0, 1 + + if document: + # Successfully indexed + await task_logger.log_task_progress( + log_entry, + f"Successfully indexed: {file_name}", + { + "status": "indexed", + "document_id": document.id, + "file_name": file_name, + }, + ) + return 1, 0 + else: + # Likely a duplicate or unsupported type + logger.info(f"No document created for {file_name} (duplicate or unsupported)") + return 0, 1 + + except Exception as e: + logger.error(f"Error processing file {file_name}: {e!s}", exc_info=True) + return 0, 1 + + +async def _remove_document( + session: AsyncSession, file_id: str, search_space_id: int +): + """Remove a document that was deleted in Drive.""" + unique_identifier_hash = generate_unique_identifier_hash( + DocumentType.GOOGLE_DRIVE_CONNECTOR, file_id, search_space_id + ) + + existing_document = await check_document_by_unique_identifier( + session, unique_identifier_hash + ) + + if existing_document: + await session.delete(existing_document) + logger.info(f"Removed deleted file document: {file_id}") + + From 501d08f2f4b52d939a6adede37b7f6bb96ce1326 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:55:38 +0200 Subject: [PATCH 12/92] feat(routes): register Google Drive OAuth router --- surfsense_backend/app/routes/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index a055bf549..24751e596 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -11,6 +11,9 @@ from .google_calendar_add_connector_route import ( from .google_gmail_add_connector_route import ( router as google_gmail_add_connector_router, ) +from .google_drive_add_connector_route import ( + router as google_drive_add_connector_router, +) from .logs_routes import router as logs_router from .luma_add_connector_route import router as luma_add_connector_router from .new_chat_routes import router as new_chat_router @@ -33,6 +36,7 @@ router.include_router(podcasts_router) # Podcast task status and audio router.include_router(search_source_connectors_router) router.include_router(google_calendar_add_connector_router) router.include_router(google_gmail_add_connector_router) +router.include_router(google_drive_add_connector_router) router.include_router(airtable_add_connector_router) router.include_router(luma_add_connector_router) router.include_router(new_llm_config_router) # LLM configs with prompt configuration From 7b8900d51f119c9c0549eec37f6a8756aeda8221 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:55:46 +0200 Subject: [PATCH 13/92] feat(indexer): export Google Drive indexer function --- surfsense_backend/app/tasks/connector_indexers/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_backend/app/tasks/connector_indexers/__init__.py b/surfsense_backend/app/tasks/connector_indexers/__init__.py index dcfca33c3..80a9eaf19 100644 --- a/surfsense_backend/app/tasks/connector_indexers/__init__.py +++ b/surfsense_backend/app/tasks/connector_indexers/__init__.py @@ -35,6 +35,7 @@ from .elasticsearch_indexer import index_elasticsearch_documents from .github_indexer import index_github_repos from .google_calendar_indexer import index_google_calendar_events from .google_gmail_indexer import index_google_gmail_messages +from .google_drive_indexer import index_google_drive_files from .jira_indexer import index_jira_issues # Issue tracking and project management @@ -57,6 +58,7 @@ __all__ = [ # noqa: RUF022 "index_github_repos", # Calendar and scheduling "index_google_calendar_events", + "index_google_drive_files", "index_luma_events", "index_jira_issues", # Issue tracking and project management From 358abdf02f4124d99c280e5ee019874f582bf62b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:55:57 +0200 Subject: [PATCH 14/92] feat(routes): add Google Drive indexing support with folder selection - Accept folder_id and folder_name as indexing parameters - Hide date range for Google Drive connectors - Create wrapper function to avoid circular imports - Trigger Google Drive indexing Celery task --- .../routes/search_source_connectors_routes.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 5a7db7f37..d530163f4 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -45,6 +45,7 @@ from app.tasks.connector_indexers import ( index_github_repos, index_google_calendar_events, index_google_gmail_messages, + index_google_drive_files, index_jira_issues, index_linear_issues, index_luma_events, @@ -542,6 +543,14 @@ async def index_connector_content( None, description="End date for indexing (YYYY-MM-DD format). If not provided, uses today's date", ), + folder_id: str = Query( + None, + description="[Google Drive only] Folder ID to index. If not provided, uses the connector's saved selected_folder_id", + ), + folder_name: str = Query( + None, + description="[Google Drive only] Folder name for display purposes", + ), session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): @@ -747,6 +756,25 @@ async def index_connector_content( ) response_message = "Google Gmail indexing started in the background." + elif ( + connector.connector_type == SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR + ): + from app.tasks.celery_tasks.connector_tasks import ( + index_google_drive_files_task, + ) + + logger.info( + f"Triggering Google Drive indexing for connector {connector_id} into search space {search_space_id}, folder: {folder_name or 'default'}" + ) + index_google_drive_files_task.delay( + connector_id, + search_space_id, + str(user.id), + folder_id, + folder_name, + ) + response_message = "Google Drive indexing started in the background." + elif connector.connector_type == SearchSourceConnectorType.DISCORD_CONNECTOR: from app.tasks.celery_tasks.connector_tasks import ( index_discord_messages_task, @@ -1515,6 +1543,50 @@ async def run_google_gmail_indexing( # Optionally update status in DB to indicate failure +async def run_google_drive_indexing( + session: AsyncSession, + connector_id: int, + search_space_id: int, + user_id: str, + folder_id: str, + folder_name: str, +): + """Runs the Google Drive indexing task and updates the timestamp.""" + try: + from app.tasks.connector_indexers.google_drive_indexer import ( + index_google_drive_files, + ) + + indexed_count, error_message = await index_google_drive_files( + session, + connector_id, + search_space_id, + user_id, + folder_id, + folder_name, + use_delta_sync=True, + update_last_indexed=False, + ) + if error_message: + logger.error( + f"Google Drive indexing failed for connector {connector_id}: {error_message}" + ) + # Optionally update status in DB to indicate failure + else: + logger.info( + f"Google Drive indexing successful for connector {connector_id}. Indexed {indexed_count} documents." + ) + # Update the last indexed timestamp only on success + await update_connector_last_indexed(session, connector_id) + await session.commit() # Commit timestamp update + except Exception as e: + logger.error( + f"Critical error in run_google_drive_indexing for connector {connector_id}: {e}", + exc_info=True, + ) + # Optionally update status in DB to indicate failure + + # Add new helper functions for luma indexing async def run_luma_indexing_with_new_session( connector_id: int, From 1c83327fc7dc6c3272c27503e61269cbf543d463 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:56:11 +0200 Subject: [PATCH 15/92] feat(celery): add Google Drive indexing Celery task - Create async task for Google Drive folder indexing - Accept folder_id and folder_name parameters - Call indexing wrapper to avoid circular imports --- .../app/tasks/celery_tasks/connector_tasks.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py index 6cd557dc4..8e507915f 100644 --- a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py @@ -473,6 +473,58 @@ async def _index_google_gmail_messages( ) +@celery_app.task(name="index_google_drive_files", bind=True) +def index_google_drive_files_task( + self, + connector_id: int, + search_space_id: int, + user_id: str, + folder_id: str, + folder_name: str, +): + """Celery task to index Google Drive files.""" + import asyncio + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + try: + loop.run_until_complete( + _index_google_drive_files( + connector_id, + search_space_id, + user_id, + folder_id, + folder_name, + ) + ) + finally: + loop.close() + + +async def _index_google_drive_files( + connector_id: int, + search_space_id: int, + user_id: str, + folder_id: str, + folder_name: str, +): + """Index Google Drive files with new session.""" + from app.routes.search_source_connectors_routes import ( + run_google_drive_indexing, + ) + + async with get_celery_session_maker()() as session: + await run_google_drive_indexing( + session, + connector_id, + search_space_id, + user_id, + folder_id, + folder_name, + ) + + @celery_app.task(name="index_discord_messages", bind=True) def index_discord_messages_task( self, From 2d24f9ac7921d4c8cc1f3296e43c27b303ca1e3d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:56:30 +0200 Subject: [PATCH 16/92] feat(types): add GOOGLE_DRIVE_CONNECTOR to frontend enum --- surfsense_web/contracts/enums/connector.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_web/contracts/enums/connector.ts b/surfsense_web/contracts/enums/connector.ts index 6cdbc5656..eb2cf7ad8 100644 --- a/surfsense_web/contracts/enums/connector.ts +++ b/surfsense_web/contracts/enums/connector.ts @@ -14,6 +14,7 @@ export enum EnumConnectorName { CLICKUP_CONNECTOR = "CLICKUP_CONNECTOR", GOOGLE_CALENDAR_CONNECTOR = "GOOGLE_CALENDAR_CONNECTOR", GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR", + GOOGLE_DRIVE_CONNECTOR = "GOOGLE_DRIVE_CONNECTOR", AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR", LUMA_CONNECTOR = "LUMA_CONNECTOR", ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR", From 11d94e0ea6ed8a5146001c2c228674aa2071b30d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:56:36 +0200 Subject: [PATCH 17/92] feat(ui): add Google Drive icon to connector icons mapping --- surfsense_web/contracts/enums/connectorIcons.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/surfsense_web/contracts/enums/connectorIcons.tsx b/surfsense_web/contracts/enums/connectorIcons.tsx index 87840d7e4..661be5253 100644 --- a/surfsense_web/contracts/enums/connectorIcons.tsx +++ b/surfsense_web/contracts/enums/connectorIcons.tsx @@ -26,6 +26,7 @@ import { Sparkles, Telescope, Webhook, + HardDrive, } from "lucide-react"; import { EnumConnectorName } from "./connector"; @@ -57,6 +58,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas return ; case EnumConnectorName.GOOGLE_GMAIL_CONNECTOR: return ; + case EnumConnectorName.GOOGLE_DRIVE_CONNECTOR: + return ; case EnumConnectorName.AIRTABLE_CONNECTOR: return ; case EnumConnectorName.CONFLUENCE_CONNECTOR: From bfbd813f4297605522b665cb532731739447dee0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:56:42 +0200 Subject: [PATCH 18/92] feat(i18n): add Google Drive connector translation keys --- surfsense_web/messages/en.json | 1 + surfsense_web/messages/zh.json | 1 + 2 files changed, 2 insertions(+) diff --git a/surfsense_web/messages/en.json b/surfsense_web/messages/en.json index eac362b9c..f70c854e0 100644 --- a/surfsense_web/messages/en.json +++ b/surfsense_web/messages/en.json @@ -303,6 +303,7 @@ "luma_desc": "Connect to Luma to search events, meetups and gatherings.", "calendar_desc": "Connect to Google Calendar to search events, meetings and schedules.", "gmail_desc": "Connect to your Gmail account to search through your emails.", + "google_drive_desc": "Connect to Google Drive to search and index your files and documents.", "zoom_desc": "Connect to Zoom to access meeting recordings and transcripts.", "webcrawler_desc": "Crawl and index content from any public web pages." }, diff --git a/surfsense_web/messages/zh.json b/surfsense_web/messages/zh.json index b943a3c2c..483a10a10 100644 --- a/surfsense_web/messages/zh.json +++ b/surfsense_web/messages/zh.json @@ -303,6 +303,7 @@ "luma_desc": "连接到 Luma 以搜索活动、聚会和集会。", "calendar_desc": "连接到 Google 日历以搜索活动、会议和日程。", "gmail_desc": "连接到您的 Gmail 账户以搜索您的电子邮件。", + "google_drive_desc": "连接到 Google 云端硬盘以搜索和索引您的文件和文档。", "zoom_desc": "连接到 Zoom 以访问会议录制和转录。", "webcrawler_desc": "爬取和索引任何公开网页的内容。" }, From 48112f66df4096b6b44f898e11ec01d18f175e7c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:56:52 +0200 Subject: [PATCH 19/92] feat(ui): add Google Drive connector card to Productivity category --- surfsense_web/components/sources/connector-data.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/surfsense_web/components/sources/connector-data.tsx b/surfsense_web/components/sources/connector-data.tsx index 338c3ae20..7fca3e6b9 100644 --- a/surfsense_web/components/sources/connector-data.tsx +++ b/surfsense_web/components/sources/connector-data.tsx @@ -183,6 +183,13 @@ export const connectorCategories: ConnectorCategory[] = [ icon: getConnectorIcon(EnumConnectorName.GOOGLE_GMAIL_CONNECTOR, "h-6 w-6"), status: "available", }, + { + id: "google-drive-connector", + title: "Google Drive", + description: "google_drive_desc", + icon: getConnectorIcon(EnumConnectorName.GOOGLE_DRIVE_CONNECTOR, "h-6 w-6"), + status: "available", + }, { id: "luma-connector", title: "Luma", From 90b3474b47d9d34e8182b0adda2251faee8feaed Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:57:02 +0200 Subject: [PATCH 20/92] feat(hooks): add folder parameters to indexConnector function - Accept folderId and folderName for Google Drive indexing - Pass folder parameters to backend API --- surfsense_web/hooks/use-search-source-connectors.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/surfsense_web/hooks/use-search-source-connectors.ts b/surfsense_web/hooks/use-search-source-connectors.ts index 2f77d7d82..ee8ce5518 100644 --- a/surfsense_web/hooks/use-search-source-connectors.ts +++ b/surfsense_web/hooks/use-search-source-connectors.ts @@ -267,7 +267,9 @@ export const useSearchSourceConnectors = (lazy: boolean = false, searchSpaceId?: connectorId: number, searchSpaceId: string | number, startDate?: string, - endDate?: string + endDate?: string, + folderId?: string, + folderName?: string ) => { try { // Build query parameters @@ -280,6 +282,12 @@ export const useSearchSourceConnectors = (lazy: boolean = false, searchSpaceId?: if (endDate) { params.append("end_date", endDate); } + if (folderId) { + params.append("folder_id", folderId); + } + if (folderName) { + params.append("folder_name", folderName); + } const response = await authenticatedFetch( `${ From ad4d424d3815b35335c703975dedd561ceb7aadb Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:57:10 +0200 Subject: [PATCH 21/92] feat(ui): add Google Drive OAuth connection page - Handle OAuth flow similar to Gmail/Calendar - Show connection status and redirect to manage page - Display connector features and file type support - No folder selection at connection time (done at index time) --- .../add/google-drive-connector/page.tsx | 218 ++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-drive-connector/page.tsx diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-drive-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-drive-connector/page.tsx new file mode 100644 index 000000000..b9fb8d953 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-drive-connector/page.tsx @@ -0,0 +1,218 @@ +"use client"; + +import { ArrowLeft, Check, ExternalLink, Loader2 } from "lucide-react"; +import { motion } from "motion/react"; +import Link from "next/link"; +import { useParams, useRouter, useSearchParams } from "next/navigation"; +import { useEffect, useState } from "react"; +import { toast } from "sonner"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { EnumConnectorName } from "@/contracts/enums/connector"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { + type SearchSourceConnector, + useSearchSourceConnectors, +} from "@/hooks/use-search-source-connectors"; +import { authenticatedFetch } from "@/lib/auth-utils"; + +export default function GoogleDriveConnectorPage() { + const router = useRouter(); + const params = useParams(); + const searchParams = useSearchParams(); + const searchSpaceId = params.search_space_id as string; + + const [isConnecting, setIsConnecting] = useState(false); + const [doesConnectorExist, setDoesConnectorExist] = useState(false); + + const { fetchConnectors } = useSearchSourceConnectors(true, Number.parseInt(searchSpaceId)); + + // Check if connector exists and handle OAuth success + useEffect(() => { + const success = searchParams.get("success"); + + fetchConnectors(Number.parseInt(searchSpaceId)).then((data) => { + const driveConnector = data.find( + (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.GOOGLE_DRIVE_CONNECTOR + ); + + if (driveConnector) { + setDoesConnectorExist(true); + + // If just connected, show success and redirect + if (success === "true") { + toast.success("Google Drive connected successfully!"); + setTimeout(() => { + router.push(`/dashboard/${searchSpaceId}/connectors`); + }, 1500); + } + } + }); + }, [searchParams, fetchConnectors, searchSpaceId, router]); + + const handleConnectGoogle = async () => { + try { + setIsConnecting(true); + const response = await authenticatedFetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/auth/google/drive/connector/add/?space_id=${searchSpaceId}`, + { method: "GET" } + ); + + if (!response.ok) { + throw new Error("Failed to initiate Google OAuth"); + } + + const data = await response.json(); + window.location.href = data.auth_url; + } catch (error) { + console.error("Error connecting to Google:", error); + toast.error("Failed to connect to Google Drive"); + } finally { + setIsConnecting(false); + } + }; + + return ( +
+ + {/* Header */} +
+ + + Back to connectors + +
+
+ {getConnectorIcon(EnumConnectorName.GOOGLE_DRIVE_CONNECTOR, "h-6 w-6")} +
+
+

Connect Google Drive

+

+ Securely connect your Google Drive account +

+
+
+
+ + {/* Connection Card */} + {!doesConnectorExist ? ( + + + Connect Your Google Account + + Authorize read-only access to your Google Drive. You'll select which folder to + index when you start indexing. + + + +
+ + Read-only access to your Drive files +
+
+ + Index documents, spreadsheets, presentations, PDFs & more +
+
+ + Automatic updates with change tracking +
+
+ + Secure OAuth 2.0 authentication +
+ + + + + + + ) : ( + + + ✅ Already Connected + + Your Google Drive connector is already set up. Go to the connectors page to + start indexing. + + + + + + + )} + + {/* Information Card */} + + + How Google Drive Integration Works + + +
+

1️⃣ Connect Your Account

+

+ First, securely connect your Google Drive account using OAuth 2.0. We only + request read-only access. +

+
+
+

2️⃣ Select Folder to Index

+

+ When you're ready to index, go to the connectors page and click "Index". You'll + choose which folder to process. +

+
+
+

3️⃣ Automatic Change Detection

+

+ We use Google Drive's change tracking API to detect when files are modified, + added, or deleted. Only changed files are re-indexed. +

+
+
+

📄 Comprehensive File Support

+

+ Supports Google Workspace files (Docs, Sheets, Slides), Microsoft Office + documents, PDFs, text files, images (with OCR), and more. +

+
+ + + +
+ ); +} From 5df04c3caa54573723c0a0158cebf6e6a4d2647c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:57:18 +0200 Subject: [PATCH 22/92] feat(ui): add hierarchical Google Drive folder tree browser - Display folders and files with lazy loading - Show different icons for file types (docs, sheets, slides, etc) - Expandable folder tree with proper indentation - Selectable folders for indexing - Handle overflow with proper truncation - Full pagination support for large folder structures --- .../connectors/google-drive-folder-tree.tsx | 340 ++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 surfsense_web/components/connectors/google-drive-folder-tree.tsx diff --git a/surfsense_web/components/connectors/google-drive-folder-tree.tsx b/surfsense_web/components/connectors/google-drive-folder-tree.tsx new file mode 100644 index 000000000..22ef97556 --- /dev/null +++ b/surfsense_web/components/connectors/google-drive-folder-tree.tsx @@ -0,0 +1,340 @@ +"use client"; + +import { + ChevronDown, + ChevronRight, + File, + FileText, + Folder, + FolderOpen, + HardDrive, + Image, + Loader2, + Sheet, + Presentation, +} from "lucide-react"; +import { useState } from "react"; +import { Button } from "@/components/ui/button"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { cn } from "@/lib/utils"; +import { authenticatedFetch } from "@/lib/auth-utils"; + +interface DriveItem { + id: string; + name: string; + mimeType: string; + isFolder: boolean; + parents?: string[]; + size?: number; + iconLink?: string; +} + +interface ItemTreeNode { + item: DriveItem; + children: DriveItem[] | null; // null = not loaded, [] = loaded but empty + isExpanded: boolean; + isLoading: boolean; +} + +interface GoogleDriveFolderTreeProps { + connectorId: number; + selectedFolderId: string | null; + onSelectFolder: (folderId: string, folderName: string) => void; +} + +// Helper to get appropriate icon for file type +function getFileIcon(mimeType: string, className: string = "h-4 w-4") { + if (mimeType.includes("spreadsheet") || mimeType.includes("excel")) { + return ; + } + if (mimeType.includes("presentation") || mimeType.includes("powerpoint")) { + return ; + } + if (mimeType.includes("document") || mimeType.includes("word") || mimeType.includes("text")) { + return ; + } + if (mimeType.includes("image")) { + return ; + } + return ; +} + +// Helper to format file size +function formatFileSize(bytes: number | undefined): string { + if (!bytes) return ""; + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`; +} + +export function GoogleDriveFolderTree({ + connectorId, + selectedFolderId, + onSelectFolder, +}: GoogleDriveFolderTreeProps) { + const [rootItems, setRootItems] = useState([]); + const [itemStates, setItemStates] = useState>(new Map()); + const [isLoadingRoot, setIsLoadingRoot] = useState(false); + const [isInitialized, setIsInitialized] = useState(false); + + // Load root items (folders and files) on mount + const loadRootItems = async () => { + if (isInitialized) return; // Already loaded + + setIsLoadingRoot(true); + try { + const response = await authenticatedFetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/connectors/${connectorId}/google-drive/folders` + ); + if (!response.ok) throw new Error("Failed to load items"); + + const data = await response.json(); + setRootItems(data.items || []); + setIsInitialized(true); + } catch (error) { + console.error("Error loading root items:", error); + } finally { + setIsLoadingRoot(false); + } + }; + + // Helper function to find an item recursively through all loaded items + const findItem = (itemId: string): DriveItem | undefined => { + // First check if we have it in itemStates + const state = itemStates.get(itemId); + if (state?.item) return state.item; + + // Check root items + const rootItem = rootItems.find((item) => item.id === itemId); + if (rootItem) return rootItem; + + // Recursively search through all loaded children + for (const [, nodeState] of itemStates) { + if (nodeState.children) { + const found = nodeState.children.find((child) => child.id === itemId); + if (found) return found; + } + } + + return undefined; + }; + + // Load children (folders and files) for a specific folder + const loadFolderContents = async (folderId: string) => { + try { + // Set loading state + setItemStates((prev) => { + const newMap = new Map(prev); + const existing = newMap.get(folderId); + if (existing) { + newMap.set(folderId, { ...existing, isLoading: true }); + } else { + // First time loading this folder - create initial state + const item = findItem(folderId); + if (item) { + newMap.set(folderId, { + item, + children: null, + isExpanded: false, + isLoading: true, + }); + } + } + return newMap; + }); + + const response = await authenticatedFetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/connectors/${connectorId}/google-drive/folders?parent_id=${folderId}` + ); + if (!response.ok) throw new Error("Failed to load folder contents"); + + const data = await response.json(); + const items = data.items || []; + + // Check if folder only contains files (no subfolders) + const hasSubfolders = items.some((item: DriveItem) => item.isFolder); + + // Update item state with loaded children + setItemStates((prev) => { + const newMap = new Map(prev); + const existing = newMap.get(folderId); + const item = existing?.item || findItem(folderId); + + if (item) { + newMap.set(folderId, { + item, + children: items, + isExpanded: true, // Always expand after loading + isLoading: false, + }); + } else { + console.error(`Could not find item for folderId: ${folderId}`); + } + return newMap; + }); + } catch (error) { + console.error("Error loading folder contents:", error); + // Clear loading state on error + setItemStates((prev) => { + const newMap = new Map(prev); + const existing = newMap.get(folderId); + if (existing) { + newMap.set(folderId, { ...existing, isLoading: false }); + } + return newMap; + }); + } + }; + + // Toggle folder expansion + const toggleFolder = async (item: DriveItem) => { + if (!item.isFolder) return; // Only folders can be expanded + + const state = itemStates.get(item.id); + + if (!state || state.children === null) { + // First time expanding - load children + await loadFolderContents(item.id); + } else { + // Toggle expansion state + setItemStates((prev) => { + const newMap = new Map(prev); + newMap.set(item.id, { + ...state, + isExpanded: !state.isExpanded, + }); + return newMap; + }); + } + }; + + // Recursive render function for item tree + const renderItem = (item: DriveItem, level: number = 0) => { + const state = itemStates.get(item.id); + const isExpanded = state?.isExpanded || false; + const isLoading = state?.isLoading || false; + const children = state?.children; + const isSelected = selectedFolderId === item.id; + const isFolder = item.isFolder; + + // Separate folders and files for children + const childFolders = children?.filter((c) => c.isFolder) || []; + const childFiles = children?.filter((c) => !c.isFolder) || []; + + return ( +
+ + + {/* Render children if expanded (folders first, then files) */} + {isExpanded && isFolder && children && ( +
+ {/* Render folders first */} + {childFolders.map((child) => renderItem(child, level + 1))} + + {/* Render files */} + {childFiles.map((child) => renderItem(child, level + 1))} + + {/* Empty state */} + {children.length === 0 && ( +
+ Empty folder +
+ )} +
+ )} +
+ ); + }; + + // Initialize on first render + if (!isInitialized && !isLoadingRoot) { + loadRootItems(); + } + + return ( +
+ +
+ {/* My Drive Header (always visible, selectable) */} +
+ +
+ + {/* Loading indicator */} + {isLoadingRoot && ( +
+ +
+ )} + + {/* Root items (folders and files) - same level as Google Drive shows */} +
+ {!isLoadingRoot && rootItems.map((item) => renderItem(item, 0))} +
+ + {/* Empty state */} + {!isLoadingRoot && rootItems.length === 0 && ( +
+ No files or folders found in your Google Drive +
+ )} +
+ +
+ ); +} From c4a95ecc024ca9ef8b0f0705bb4200a7279d9aa4 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 15:57:26 +0200 Subject: [PATCH 23/92] feat(ui): integrate Google Drive folder selection into manage connectors page - Add folder selection dialog for Google Drive indexing - Hide date picker and quick index for Google Drive - Show folder tree browser in modal - Pass selected folder to indexing API - Adjust modal size to prevent overflow --- .../connectors/(manage)/page.tsx | 215 ++++++++++++++++-- 1 file changed, 190 insertions(+), 25 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index e2f219448..fd1f7da1d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -5,6 +5,8 @@ import { Calendar as CalendarIcon, Clock, Edit, + Folder, + HardDrive, Loader2, Plus, RefreshCw, @@ -61,6 +63,13 @@ import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; import { cn } from "@/lib/utils"; +import { authenticatedFetch } from "@/lib/auth-utils"; +import { GoogleDriveFolderTree } from "@/components/connectors/google-drive-folder-tree"; + +interface DriveFolder { + id: string; + name: string; +} export default function ConnectorsPage() { const t = useTranslations("connectors"); @@ -105,6 +114,13 @@ export default function ConnectorsPage() { const [customFrequency, setCustomFrequency] = useState(""); const [isSavingPeriodic, setIsSavingPeriodic] = useState(false); + // Google Drive folder selection state + const [driveFolderDialogOpen, setDriveFolderDialogOpen] = useState(false); + const [driveFolders, setDriveFolders] = useState([]); + const [selectedFolderId, setSelectedFolderId] = useState(""); + const [selectedFolderName, setSelectedFolderName] = useState(""); + const [isLoadingFolders, setIsLoadingFolders] = useState(false); + useEffect(() => { if (error) { toast.error(t("failed_load")); @@ -129,8 +145,78 @@ export default function ConnectorsPage() { // Handle opening date picker for indexing const handleOpenDatePicker = (connectorId: number) => { + // Check if this is a Google Drive connector + const connector = connectors.find((c) => c.id === connectorId); + if (connector?.connector_type === EnumConnectorName.GOOGLE_DRIVE_CONNECTOR) { + // Open folder selection dialog for Google Drive + handleOpenDriveFolderDialog(connectorId); + } else { + // Open date picker for other connectors + setSelectedConnectorForIndexing(connectorId); + setDatePickerOpen(true); + } + }; + + // Handle opening Google Drive folder selection dialog + const handleOpenDriveFolderDialog = async (connectorId: number) => { setSelectedConnectorForIndexing(connectorId); - setDatePickerOpen(true); + setDriveFolderDialogOpen(true); + setIsLoadingFolders(true); + + try { + const response = await authenticatedFetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/connectors/${connectorId}/google-drive/folders`, + { method: "GET" } + ); + + if (!response.ok) { + throw new Error("Failed to load folders"); + } + + const data = await response.json(); + setDriveFolders(data.folders || []); + } catch (error) { + console.error("Error loading folders:", error); + toast.error("Failed to load Google Drive folders"); + setDriveFolderDialogOpen(false); + } finally { + setIsLoadingFolders(false); + } + }; + + // Handle Google Drive folder indexing + const handleIndexDriveFolder = async () => { + if (selectedConnectorForIndexing === null || !selectedFolderId) { + toast.error("Please select a folder"); + return; + } + + setDriveFolderDialogOpen(false); + + try { + setIndexingConnectorId(selectedConnectorForIndexing); + const selectedFolder = driveFolders.find((f) => f.id === selectedFolderId); + const folderName = selectedFolder?.name || "Selected Folder"; + + // Call indexConnector with folder_id and folder_name as query params + await indexConnector( + selectedConnectorForIndexing, + searchSpaceId, + undefined, + undefined, + selectedFolderId, + folderName + ); + toast.success(t("indexing_started")); + } catch (error) { + console.error("Error indexing connector content:", error); + toast.error(error instanceof Error ? error.message : t("indexing_failed")); + } finally { + setIndexingConnectorId(null); + setSelectedConnectorForIndexing(null); + setSelectedFolderId(""); + setDriveFolders([]); + } }; // Handle connector indexing with dates @@ -361,39 +447,52 @@ export default function ConnectorsPage() { > {indexingConnectorId === connector.id ? ( + ) : connector.connector_type === EnumConnectorName.GOOGLE_DRIVE_CONNECTOR ? ( + ) : ( )} - {t("index_date_range")} + + {connector.connector_type === EnumConnectorName.GOOGLE_DRIVE_CONNECTOR + ? "Select folder to index" + : t("index_date_range")} + -

{t("index_date_range")}

- - - - - - - - - -

{t("quick_index_auto")}

+

+ {connector.connector_type === EnumConnectorName.GOOGLE_DRIVE_CONNECTOR + ? "Select folder to index" + : t("index_date_range")} +

+ {/* Hide quick index button for Google Drive (requires folder selection) */} + {connector.connector_type !== EnumConnectorName.GOOGLE_DRIVE_CONNECTOR && ( + + + + + + +

{t("quick_index_auto")}

+ + + + )} )} {connector.is_indexable && ( @@ -581,6 +680,72 @@ export default function ConnectorsPage() { + {/* Google Drive Folder Selection Dialog */} + + + + Select Google Drive Folder + + Browse and select a folder to index. Click folders to expand and see subfolders. + + +
+
+ + {selectedConnectorForIndexing && ( + { + setSelectedFolderId(folderId); + setSelectedFolderName(folderName); + }} + /> + )} +

+ Changes to files in this folder will be automatically detected and re-indexed. +

+
+ {selectedFolderId && selectedFolderName && ( +
+
+

Selected folder:

+

+ {selectedFolderName} +

+
+
+

What will be indexed:

+
    +
  • Google Docs, Sheets, Slides (as PDFs)
    • +
  • PDFs, Word, Excel, PowerPoint files
    • +
  • Text files, markdown, code files
    • +
  • Images (with OCR if enabled)
    • +
+
+
+ )} +
+ + + + + + + {/* Periodic Indexing Configuration Dialog */} From e0edfef5fcce0d40e09505dd871a4f44bf7dad4a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 16:48:34 +0200 Subject: [PATCH 24/92] feat(ui): add multiple folder selection with checkboxes to Google Drive tree - Replace single folder selection with multi-select checkboxes - Remove cascading auto-select for clearer UX - Each folder must be selected individually - Visual indicators for selected folders --- .../connectors/google-drive-folder-tree.tsx | 118 +++++++++++------- 1 file changed, 72 insertions(+), 46 deletions(-) diff --git a/surfsense_web/components/connectors/google-drive-folder-tree.tsx b/surfsense_web/components/connectors/google-drive-folder-tree.tsx index 22ef97556..793fdc750 100644 --- a/surfsense_web/components/connectors/google-drive-folder-tree.tsx +++ b/surfsense_web/components/connectors/google-drive-folder-tree.tsx @@ -15,6 +15,7 @@ import { } from "lucide-react"; import { useState } from "react"; import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; import { ScrollArea } from "@/components/ui/scroll-area"; import { cn } from "@/lib/utils"; import { authenticatedFetch } from "@/lib/auth-utils"; @@ -36,10 +37,15 @@ interface ItemTreeNode { isLoading: boolean; } +interface SelectedFolder { + id: string; + name: string; +} + interface GoogleDriveFolderTreeProps { connectorId: number; - selectedFolderId: string | null; - onSelectFolder: (folderId: string, folderName: string) => void; + selectedFolders: SelectedFolder[]; + onSelectFolders: (folders: SelectedFolder[]) => void; } // Helper to get appropriate icon for file type @@ -59,25 +65,32 @@ function getFileIcon(mimeType: string, className: string = "h-4 w-4") { return ; } -// Helper to format file size -function formatFileSize(bytes: number | undefined): string { - if (!bytes) return ""; - if (bytes < 1024) return `${bytes} B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; - return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`; -} - export function GoogleDriveFolderTree({ connectorId, - selectedFolderId, - onSelectFolder, + selectedFolders, + onSelectFolders, }: GoogleDriveFolderTreeProps) { const [rootItems, setRootItems] = useState([]); const [itemStates, setItemStates] = useState>(new Map()); const [isLoadingRoot, setIsLoadingRoot] = useState(false); const [isInitialized, setIsInitialized] = useState(false); + // Helper to check if a folder is selected + const isFolderSelected = (folderId: string): boolean => { + return selectedFolders.some((f) => f.id === folderId); + }; + + // Handle folder checkbox toggle + const toggleFolderSelection = (folderId: string, folderName: string) => { + if (isFolderSelected(folderId)) { + // Remove from selection + onSelectFolders(selectedFolders.filter((f) => f.id !== folderId)); + } else { + // Add to selection + onSelectFolders([...selectedFolders, { id: folderId, name: folderName }]); + } + }; + // Load root items (folders and files) on mount const loadRootItems = async () => { if (isInitialized) return; // Already loaded @@ -215,7 +228,7 @@ export function GoogleDriveFolderTree({ const isExpanded = state?.isExpanded || false; const isLoading = state?.isLoading || false; const children = state?.children; - const isSelected = selectedFolderId === item.id; + const isSelected = isFolderSelected(item.id); const isFolder = item.isFolder; // Separate folders and files for children @@ -224,15 +237,13 @@ export function GoogleDriveFolderTree({ return (
- + isFolder && toggleFolder(item)} + > + {item.name} + +
{/* Render children if expanded (folders first, then files) */} {isExpanded && isFolder && children && (
{/* Render folders first */} {childFolders.map((child) => renderItem(child, level + 1))} - + {/* Render files */} {childFiles.map((child) => renderItem(child, level + 1))} - + {/* Empty state */} {children.length === 0 && ( -
- Empty folder -
+
Empty folder
)}
)} @@ -302,17 +328,17 @@ export function GoogleDriveFolderTree({
{/* My Drive Header (always visible, selectable) */}
- + toggleFolderSelection("root", "My Drive")}> + My Drive + +
{/* Loading indicator */} From 27a4bcdfc20466f936c0e4a3cf608264aa89b0f4 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 16:48:56 +0200 Subject: [PATCH 25/92] feat(ui): support multiple folder selection in Google Drive indexing - Update manage page to handle array of selected folders - Add info icon with clear description about folder-level indexing - Display list of all selected folders before indexing - Remove unnecessary file type details section - Pass comma-separated folder IDs and names to backend --- .../connectors/(manage)/page.tsx | 128 +++++++++--------- 1 file changed, 61 insertions(+), 67 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index fd1f7da1d..bbbfd61e0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -7,6 +7,7 @@ import { Edit, Folder, HardDrive, + Info, Loader2, Plus, RefreshCw, @@ -117,8 +118,7 @@ export default function ConnectorsPage() { // Google Drive folder selection state const [driveFolderDialogOpen, setDriveFolderDialogOpen] = useState(false); const [driveFolders, setDriveFolders] = useState([]); - const [selectedFolderId, setSelectedFolderId] = useState(""); - const [selectedFolderName, setSelectedFolderName] = useState(""); + const [selectedFolders, setSelectedFolders] = useState>([]); const [isLoadingFolders, setIsLoadingFolders] = useState(false); useEffect(() => { @@ -186,8 +186,8 @@ export default function ConnectorsPage() { // Handle Google Drive folder indexing const handleIndexDriveFolder = async () => { - if (selectedConnectorForIndexing === null || !selectedFolderId) { - toast.error("Please select a folder"); + if (selectedConnectorForIndexing === null || selectedFolders.length === 0) { + toast.error("Please select at least one folder"); return; } @@ -195,28 +195,26 @@ export default function ConnectorsPage() { try { setIndexingConnectorId(selectedConnectorForIndexing); - const selectedFolder = driveFolders.find((f) => f.id === selectedFolderId); - const folderName = selectedFolder?.name || "Selected Folder"; - // Call indexConnector with folder_id and folder_name as query params + // Call indexConnector with folder_ids and folder_names as query params await indexConnector( selectedConnectorForIndexing, searchSpaceId, undefined, undefined, - selectedFolderId, - folderName + selectedFolders.map((f) => f.id).join(","), + selectedFolders.map((f) => f.name).join(", ") ); toast.success(t("indexing_started")); } catch (error) { console.error("Error indexing connector content:", error); toast.error(error instanceof Error ? error.message : t("indexing_failed")); } finally { - setIndexingConnectorId(null); - setSelectedConnectorForIndexing(null); - setSelectedFolderId(""); - setDriveFolders([]); - } + setIndexingConnectorId(null); + setSelectedConnectorForIndexing(null); + setSelectedFolders([]); + setDriveFolders([]); + } }; // Handle connector indexing with dates @@ -683,66 +681,62 @@ export default function ConnectorsPage() { {/* Google Drive Folder Selection Dialog */} - - Select Google Drive Folder - - Browse and select a folder to index. Click folders to expand and see subfolders. - - -
+ + Select Google Drive Folders + + + + Select folders to index. Only files directly in each folder will be + processed—subfolders must be selected separately. + + + +
- {selectedConnectorForIndexing && ( - { - setSelectedFolderId(folderId); - setSelectedFolderName(folderName); - }} - /> - )} -

- Changes to files in this folder will be automatically detected and re-indexed. -

-
- {selectedFolderId && selectedFolderName && ( -
-
-

Selected folder:

-

- {selectedFolderName} -

-
-
-

What will be indexed:

-
    -
  • Google Docs, Sheets, Slides (as PDFs)
    • -
  • PDFs, Word, Excel, PowerPoint files
    • -
  • Text files, markdown, code files
    • -
  • Images (with OCR if enabled)
    • -
-
-
+ {selectedConnectorForIndexing && ( + { + setSelectedFolders(folders); + }} + /> )}
+ {selectedFolders.length > 0 && ( +
+
+

+ Selected {selectedFolders.length} folder{selectedFolders.length > 1 ? "s" : ""}: +

+
+ {selectedFolders.map((folder) => ( +

+ • {folder.name} +

+ ))} +
+
+
+ )} +
- - + onClick={() => { + setDriveFolderDialogOpen(false); + setSelectedConnectorForIndexing(null); + setSelectedFolders([]); + setDriveFolders([]); + }} + > + {tCommon("cancel")} + + + From 634eeb887e35ebc173c2de43e255d0d3739021e1 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 16:49:20 +0200 Subject: [PATCH 26/92] feat(routes): support multiple Google Drive folder indexing - Accept comma-separated folder_ids and folder_names - Loop through each folder and index sequentially - Collect total indexed count and errors - Update timestamp only on full success --- .../routes/search_source_connectors_routes.py | 56 +++++++++++++------ 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index d530163f4..af1f18513 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -1548,35 +1548,55 @@ async def run_google_drive_indexing( connector_id: int, search_space_id: int, user_id: str, - folder_id: str, - folder_name: str, + folder_ids: str, # Comma-separated folder IDs + folder_names: str, # Comma-separated folder names ): - """Runs the Google Drive indexing task and updates the timestamp.""" + """Runs the Google Drive indexing task for multiple folders and updates the timestamp.""" try: from app.tasks.connector_indexers.google_drive_indexer import ( index_google_drive_files, ) - indexed_count, error_message = await index_google_drive_files( - session, - connector_id, - search_space_id, - user_id, - folder_id, - folder_name, - use_delta_sync=True, - update_last_indexed=False, - ) - if error_message: + # Split comma-separated IDs and names into lists + folder_id_list = [fid.strip() for fid in folder_ids.split(",")] + folder_name_list = [fname.strip() for fname in folder_names.split(",")] + + total_indexed = 0 + errors = [] + + # Index each folder + for folder_id, folder_name in zip(folder_id_list, folder_name_list): + try: + indexed_count, error_message = await index_google_drive_files( + session, + connector_id, + search_space_id, + user_id, + folder_id, + folder_name, + use_delta_sync=True, + update_last_indexed=False, + ) + if error_message: + errors.append(f"{folder_name}: {error_message}") + else: + total_indexed += indexed_count + except Exception as e: + errors.append(f"{folder_name}: {str(e)}") + logger.error( + f"Error indexing folder {folder_name} ({folder_id}): {e}", + exc_info=True, + ) + + if errors: logger.error( - f"Google Drive indexing failed for connector {connector_id}: {error_message}" + f"Google Drive indexing completed with errors for connector {connector_id}: {'; '.join(errors)}" ) - # Optionally update status in DB to indicate failure else: logger.info( - f"Google Drive indexing successful for connector {connector_id}. Indexed {indexed_count} documents." + f"Google Drive indexing successful for connector {connector_id}. Indexed {total_indexed} documents from {len(folder_id_list)} folder(s)." ) - # Update the last indexed timestamp only on success + # Update the last indexed timestamp only on full success await update_connector_last_indexed(session, connector_id) await session.commit() # Commit timestamp update except Exception as e: From c9815fd6fb78037629409dd25673807122514dc4 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 16:49:47 +0200 Subject: [PATCH 27/92] feat(celery): update Google Drive task for multiple folders - Accept comma-separated folder_ids and folder_names parameters - Pass through to indexing function for batch processing --- .../app/tasks/celery_tasks/connector_tasks.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py index 8e507915f..44f57d464 100644 --- a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py @@ -479,10 +479,10 @@ def index_google_drive_files_task( connector_id: int, search_space_id: int, user_id: str, - folder_id: str, - folder_name: str, + folder_ids: str, # Comma-separated folder IDs + folder_names: str, # Comma-separated folder names ): - """Celery task to index Google Drive files.""" + """Celery task to index Google Drive files from multiple folders.""" import asyncio loop = asyncio.new_event_loop() @@ -494,8 +494,8 @@ def index_google_drive_files_task( connector_id, search_space_id, user_id, - folder_id, - folder_name, + folder_ids, + folder_names, ) ) finally: @@ -506,10 +506,10 @@ async def _index_google_drive_files( connector_id: int, search_space_id: int, user_id: str, - folder_id: str, - folder_name: str, + folder_ids: str, # Comma-separated folder IDs + folder_names: str, # Comma-separated folder names ): - """Index Google Drive files with new session.""" + """Index Google Drive files from multiple folders with new session.""" from app.routes.search_source_connectors_routes import ( run_google_drive_indexing, ) @@ -520,8 +520,8 @@ async def _index_google_drive_files( connector_id, search_space_id, user_id, - folder_id, - folder_name, + folder_ids, + folder_names, ) From 9f1fd20944d46a9475ec68b826addcfb3ce61f6c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 16:55:14 +0200 Subject: [PATCH 28/92] feat(connectors): mark Google Drive documents with GOOGLE_DRIVE_CONNECTOR type - Change document_type from file type (PDF, DOCX) to GOOGLE_DRIVE_CONNECTOR - Store original file type in metadata for reference - Add Google Drive specific metadata (file_id, mime_type, source) - Include export format info for Google Workspace files - Enables proper source tracking and bulk management --- .../google_drive/content_extractor.py | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/connectors/google_drive/content_extractor.py b/surfsense_backend/app/connectors/google_drive/content_extractor.py index 82b8d42b3..88aca8f46 100644 --- a/surfsense_backend/app/connectors/google_drive/content_extractor.py +++ b/surfsense_backend/app/connectors/google_drive/content_extractor.py @@ -94,7 +94,7 @@ async def download_and_process_file( ) logger.info(f"Processing {file_name} with Surfsense's file processor") - result = await process_file_in_background( + document = await process_file_in_background( file_path=temp_file_path, filename=file_name, search_space_id=search_space_id, @@ -104,8 +104,38 @@ async def download_and_process_file( log_entry=log_entry, ) + # Step 3: Update document type to GOOGLE_DRIVE_CONNECTOR and add metadata + if document: + from app.db import DocumentType + + # Store original file type in metadata before changing document_type + original_type = document.document_type + + # Update document type to mark it as from Google Drive + document.document_type = DocumentType.GOOGLE_DRIVE_CONNECTOR + + # Add Google Drive specific metadata + if not document.metadata: + document.metadata = {} + + document.metadata.update({ + "google_drive_file_id": file_id, + "google_drive_file_name": file_name, + "google_drive_mime_type": mime_type, + "original_document_type": original_type, + "source_connector": "google_drive", + }) + + # If it was a Google Workspace file, note the export format + if is_google_workspace_file(mime_type): + document.metadata["exported_as"] = "pdf" + document.metadata["original_workspace_type"] = mime_type.split(".")[-1] # e.g., "document", "spreadsheet" + + await session.flush() # Persist the changes + logger.info(f"Updated document type to GOOGLE_DRIVE_CONNECTOR for {file_name}") + # process_file_in_background returns None on duplicate/error, Document on success - return result, None + return document, None except Exception as e: logger.warning(f"Failed to process {file_name}: {e!s}") From b2b891e4d746b0d2add1f7f3bf0fb6f341e9ee85 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 17:15:29 +0200 Subject: [PATCH 29/92] fix(connectors): properly commit Google Drive document type changes - Return file metadata from content_extractor for indexer to use - Update document type and metadata in indexer after processing - Explicitly commit changes to database - Ensures documents are properly marked as GOOGLE_DRIVE_CONNECTOR type --- .../google_drive/content_extractor.py | 55 +++++++------------ .../google_drive_indexer.py | 26 ++++++++- 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/surfsense_backend/app/connectors/google_drive/content_extractor.py b/surfsense_backend/app/connectors/google_drive/content_extractor.py index 88aca8f46..005e7b0ae 100644 --- a/surfsense_backend/app/connectors/google_drive/content_extractor.py +++ b/surfsense_backend/app/connectors/google_drive/content_extractor.py @@ -29,7 +29,7 @@ async def download_and_process_file( session: AsyncSession, task_logger: TaskLoggingService, log_entry: Log, -) -> tuple[Any, str | None]: +) -> tuple[Any, str | None, dict[str, Any] | None]: """ Download Google Drive file and process using Surfsense's existing infrastructure. @@ -45,7 +45,7 @@ async def download_and_process_file( log_entry: Log entry for tracking Returns: - Tuple of (Document object if successful, error message if failed) + Tuple of (Document object if successful, error message if failed, file metadata dict) """ file_id = file.get("id") file_name = file.get("name", "Unknown") @@ -53,7 +53,7 @@ async def download_and_process_file( # Skip folders and shortcuts if should_skip_file(mime_type): - return None, f"Skipping {mime_type}" + return None, f"Skipping {mime_type}", None logger.info(f"Downloading file: {file_name} ({mime_type})") @@ -104,42 +104,27 @@ async def download_and_process_file( log_entry=log_entry, ) - # Step 3: Update document type to GOOGLE_DRIVE_CONNECTOR and add metadata - if document: - from app.db import DocumentType - - # Store original file type in metadata before changing document_type - original_type = document.document_type - - # Update document type to mark it as from Google Drive - document.document_type = DocumentType.GOOGLE_DRIVE_CONNECTOR - - # Add Google Drive specific metadata - if not document.metadata: - document.metadata = {} - - document.metadata.update({ - "google_drive_file_id": file_id, - "google_drive_file_name": file_name, - "google_drive_mime_type": mime_type, - "original_document_type": original_type, - "source_connector": "google_drive", - }) - - # If it was a Google Workspace file, note the export format - if is_google_workspace_file(mime_type): - document.metadata["exported_as"] = "pdf" - document.metadata["original_workspace_type"] = mime_type.split(".")[-1] # e.g., "document", "spreadsheet" - - await session.flush() # Persist the changes - logger.info(f"Updated document type to GOOGLE_DRIVE_CONNECTOR for {file_name}") - + # Note: Document type update happens in the indexer after this returns + # to ensure proper session management and commit timing + + # Prepare file metadata for the indexer to use + file_metadata = { + "google_drive_file_id": file_id, + "google_drive_file_name": file_name, + "google_drive_mime_type": mime_type, + } + + # If it was a Google Workspace file, note the export format + if is_google_workspace_file(mime_type): + file_metadata["exported_as"] = "pdf" + file_metadata["original_workspace_type"] = mime_type.split(".")[-1] # e.g., "document", "spreadsheet" + # process_file_in_background returns None on duplicate/error, Document on success - return document, None + return document, None, file_metadata except Exception as e: logger.warning(f"Failed to process {file_name}: {e!s}") - return None, str(e) + return None, str(e), None finally: # Cleanup temp file (if process_file_in_background didn't already delete it) diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index 9c4d446de..9ed295424 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -388,7 +388,7 @@ async def _process_single_file( # Download and process using Surfsense's existing infrastructure # This handles: markdown, audio, PDFs, Office docs, images, etc. # It also handles: deduplication, chunking, summarization, embedding - document, error = await download_and_process_file( + document, error, file_metadata = await download_and_process_file( client=drive_client, file=file, search_space_id=search_space_id, @@ -407,7 +407,28 @@ async def _process_single_file( ) return 0, 1 - if document: + if document and file_metadata: + # Update document type to GOOGLE_DRIVE_CONNECTOR and add metadata + original_type = document.document_type + document.document_type = DocumentType.GOOGLE_DRIVE_CONNECTOR + + # Add Google Drive specific metadata + if not document.metadata: + document.metadata = {} + + document.metadata.update({ + **file_metadata, + "original_document_type": original_type, + "source_connector": "google_drive", + }) + + # Commit the document type and metadata changes + await session.commit() + + logger.info( + f"Updated document {document.id} to GOOGLE_DRIVE_CONNECTOR type with metadata" + ) + # Successfully indexed await task_logger.log_task_progress( log_entry, @@ -416,6 +437,7 @@ async def _process_single_file( "status": "indexed", "document_id": document.id, "file_name": file_name, + "document_type": DocumentType.GOOGLE_DRIVE_CONNECTOR, }, ) return 1, 0 From 8da58be9e01406161b99d73bc6521b0f45511f16 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 17:21:44 +0200 Subject: [PATCH 30/92] fix(connectors): refresh document from DB before updating type - Query document from database to ensure it's attached to session - Prevents detached instance errors after process_file_in_background commits - Properly updates document_type and metadata with session management --- .../connector_indexers/google_drive_indexer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index 9ed295424..190792f1a 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -408,6 +408,20 @@ async def _process_single_file( return 0, 1 if document and file_metadata: + # Refresh document from database to ensure it's attached to session + from app.db import Document + from sqlalchemy import select + + # Get fresh document from database + result = await session.execute( + select(Document).where(Document.id == document.id) + ) + document = result.scalar_one_or_none() + + if not document: + logger.error(f"Could not find document {document.id} in database") + return 0, 1 + # Update document type to GOOGLE_DRIVE_CONNECTOR and add metadata original_type = document.document_type document.document_type = DocumentType.GOOGLE_DRIVE_CONNECTOR From a5935bc6775d13e9c321e49a0ef6809012042f1a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 18:01:39 +0200 Subject: [PATCH 31/92] feat(connectors): add connector parameter to file processor for source tracking - Add optional 'connector' parameter with 'type' and 'metadata' fields - Create helper function _update_document_from_connector - Use document_metadata column (not metadata) for JSON field - Merge metadata with existing using dict spread operator - Google Drive documents now marked as GOOGLE_DRIVE_CONNECTOR - Backward compatible - no changes to existing logic - Simple and clean implementation --- .../google_drive/content_extractor.py | 39 +++++++------ .../google_drive_indexer.py | 58 ++----------------- .../document_processors/file_processors.py | 34 +++++++++++ 3 files changed, 60 insertions(+), 71 deletions(-) diff --git a/surfsense_backend/app/connectors/google_drive/content_extractor.py b/surfsense_backend/app/connectors/google_drive/content_extractor.py index 005e7b0ae..04c48f47f 100644 --- a/surfsense_backend/app/connectors/google_drive/content_extractor.py +++ b/surfsense_backend/app/connectors/google_drive/content_extractor.py @@ -92,9 +92,26 @@ async def download_and_process_file( from app.tasks.document_processors.file_processors import ( process_file_in_background, ) + from app.db import DocumentType + + # Prepare connector info + connector_info = { + "type": DocumentType.GOOGLE_DRIVE_CONNECTOR, + "metadata": { + "google_drive_file_id": file_id, + "google_drive_file_name": file_name, + "google_drive_mime_type": mime_type, + "source_connector": "google_drive", + }, + } + + # If it was a Google Workspace file, note the export format + if is_google_workspace_file(mime_type): + connector_info["metadata"]["exported_as"] = "pdf" + connector_info["metadata"]["original_workspace_type"] = mime_type.split(".")[-1] logger.info(f"Processing {file_name} with Surfsense's file processor") - document = await process_file_in_background( + await process_file_in_background( file_path=temp_file_path, filename=file_name, search_space_id=search_space_id, @@ -102,25 +119,11 @@ async def download_and_process_file( session=session, task_logger=task_logger, log_entry=log_entry, + connector=connector_info, # Pass connector info ) - # Note: Document type update happens in the indexer after this returns - # to ensure proper session management and commit timing - - # Prepare file metadata for the indexer to use - file_metadata = { - "google_drive_file_id": file_id, - "google_drive_file_name": file_name, - "google_drive_mime_type": mime_type, - } - - # If it was a Google Workspace file, note the export format - if is_google_workspace_file(mime_type): - file_metadata["exported_as"] = "pdf" - file_metadata["original_workspace_type"] = mime_type.split(".")[-1] # e.g., "document", "spreadsheet" - - # process_file_in_background returns None on duplicate/error, Document on success - return document, None, file_metadata + # process_file_in_background doesn't return the document + return None, None, connector_info["metadata"] except Exception as e: logger.warning(f"Failed to process {file_name}: {e!s}") diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index 190792f1a..a2899853e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -388,7 +388,8 @@ async def _process_single_file( # Download and process using Surfsense's existing infrastructure # This handles: markdown, audio, PDFs, Office docs, images, etc. # It also handles: deduplication, chunking, summarization, embedding - document, error, file_metadata = await download_and_process_file( + # Document type is set to GOOGLE_DRIVE_CONNECTOR during processing + _, error, _ = await download_and_process_file( client=drive_client, file=file, search_space_id=search_space_id, @@ -407,58 +408,9 @@ async def _process_single_file( ) return 0, 1 - if document and file_metadata: - # Refresh document from database to ensure it's attached to session - from app.db import Document - from sqlalchemy import select - - # Get fresh document from database - result = await session.execute( - select(Document).where(Document.id == document.id) - ) - document = result.scalar_one_or_none() - - if not document: - logger.error(f"Could not find document {document.id} in database") - return 0, 1 - - # Update document type to GOOGLE_DRIVE_CONNECTOR and add metadata - original_type = document.document_type - document.document_type = DocumentType.GOOGLE_DRIVE_CONNECTOR - - # Add Google Drive specific metadata - if not document.metadata: - document.metadata = {} - - document.metadata.update({ - **file_metadata, - "original_document_type": original_type, - "source_connector": "google_drive", - }) - - # Commit the document type and metadata changes - await session.commit() - - logger.info( - f"Updated document {document.id} to GOOGLE_DRIVE_CONNECTOR type with metadata" - ) - - # Successfully indexed - await task_logger.log_task_progress( - log_entry, - f"Successfully indexed: {file_name}", - { - "status": "indexed", - "document_id": document.id, - "file_name": file_name, - "document_type": DocumentType.GOOGLE_DRIVE_CONNECTOR, - }, - ) - return 1, 0 - else: - # Likely a duplicate or unsupported type - logger.info(f"No document created for {file_name} (duplicate or unsupported)") - return 0, 1 + # File was processed successfully (document type already set in processor) + logger.info(f"Successfully indexed Google Drive file: {file_name}") + return 1, 0 except Exception as e: logger.error(f"Error processing file {file_name}: {e!s}", exc_info=True) diff --git a/surfsense_backend/app/tasks/document_processors/file_processors.py b/surfsense_backend/app/tasks/document_processors/file_processors.py index a32e75a32..61f484ae1 100644 --- a/surfsense_backend/app/tasks/document_processors/file_processors.py +++ b/surfsense_backend/app/tasks/document_processors/file_processors.py @@ -447,6 +447,24 @@ async def add_received_file_document_using_docling( ) from e +async def _update_document_from_connector( + document: Document | None, connector: dict | None, session: AsyncSession +) -> None: + """Helper to update document type and metadata from connector info.""" + if document and connector: + if "type" in connector: + document.document_type = connector["type"] + if "metadata" in connector: + # Merge with existing document_metadata (the actual column name) + if not document.document_metadata: + document.document_metadata = connector["metadata"] + else: + # Expand existing metadata with connector metadata + merged = {**document.document_metadata, **connector["metadata"]} + document.document_metadata = merged + await session.commit() + + async def process_file_in_background( file_path: str, filename: str, @@ -455,6 +473,7 @@ async def process_file_in_background( session: AsyncSession, task_logger: TaskLoggingService, log_entry: Log, + connector: dict | None = None, # Optional: {"type": "GOOGLE_DRIVE_CONNECTOR", "metadata": {...}} ): try: # Check if the file is a markdown or text file @@ -492,6 +511,9 @@ async def process_file_in_background( session, filename, markdown_content, search_space_id, user_id ) + # Update from connector if provided + await _update_document_from_connector(result, connector, session) + if result: await task_logger.log_task_success( log_entry, @@ -608,6 +630,9 @@ async def process_file_in_background( session, filename, transcribed_text, search_space_id, user_id ) + # Update from connector if provided + await _update_document_from_connector(result, connector, session) + if result: await task_logger.log_task_success( log_entry, @@ -753,6 +778,9 @@ async def process_file_in_background( session, filename, docs, search_space_id, user_id ) + # Update from connector if provided + await _update_document_from_connector(result, connector, session) + if result: # Update page usage after successful processing # allow_exceed=True because document was already created after passing initial check @@ -897,6 +925,9 @@ async def process_file_in_background( user_id, final_page_count, allow_exceed=True ) + # Update from connector if provided + await _update_document_from_connector(last_created_doc, connector, session) + await task_logger.log_task_success( log_entry, f"Successfully processed file with LlamaCloud: {filename}", @@ -1021,6 +1052,9 @@ async def process_file_in_background( user_id, final_page_count, allow_exceed=True ) + # Update from connector if provided + await _update_document_from_connector(doc_result, connector, session) + await task_logger.log_task_success( log_entry, f"Successfully processed file with Docling: {filename}", From 506a9297a90c6fcf64a983a8b9d850c9398ad7dc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 18:32:59 +0200 Subject: [PATCH 32/92] fix(connectors): track delta sync tokens per folder for Google Drive - Store tokens in folder_tokens dict instead of single global token - Each folder now tracks its own sync state independently - Fixes issue where indexing folder 2 incorrectly used delta sync after folder 1 was indexed - First-time indexing now correctly uses full scan for each new folder --- .../tasks/connector_indexers/google_drive_indexer.py | 10 +++++++--- surfsense_web/contracts/types/document.types.ts | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index a2899853e..335c3b41d 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -112,8 +112,9 @@ async def index_google_drive_files( logger.info(f"Indexing Google Drive folder: {target_folder_name} ({target_folder_id})") - # Decide sync strategy - start_page_token = connector.config.get("start_page_token") + # Decide sync strategy - track tokens per folder + folder_tokens = connector.config.get("folder_tokens", {}) + start_page_token = folder_tokens.get(target_folder_id) can_use_delta_sync = use_delta_sync and start_page_token and connector.last_indexed_at if can_use_delta_sync: @@ -156,7 +157,10 @@ async def index_google_drive_files( if new_token and not token_error: from sqlalchemy.orm.attributes import flag_modified - connector.config["start_page_token"] = new_token + # Store token per folder + if "folder_tokens" not in connector.config: + connector.config["folder_tokens"] = {} + connector.config["folder_tokens"][target_folder_id] = new_token flag_modified(connector, "config") await update_connector_last_indexed(session, connector, update_last_indexed) diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts index 3ce5388dd..b2cdb79c3 100644 --- a/surfsense_web/contracts/types/document.types.ts +++ b/surfsense_web/contracts/types/document.types.ts @@ -15,6 +15,7 @@ export const documentTypeEnum = z.enum([ "CLICKUP_CONNECTOR", "GOOGLE_CALENDAR_CONNECTOR", "GOOGLE_GMAIL_CONNECTOR", + "GOOGLE_DRIVE_CONNECTOR", "AIRTABLE_CONNECTOR", "LUMA_CONNECTOR", "ELASTICSEARCH_CONNECTOR", From acf47e3b0cb6b4ba24defee4d38f07b10abad493 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 18:53:13 +0200 Subject: [PATCH 33/92] refactor(connectors): remove verbose docstrings and obvious comments - Simplify module docstrings (remove meta-commentary about 'small focused modules') - Remove redundant inline comments (e.g., 'Log task start', 'Get connector from database') - Trim verbose function docstrings to essential information only - Remove over-explanatory comments that restate what code does - Keep necessary documentation, remove noise for better readability --- .../app/connectors/google_drive/__init__.py | 6 +--- .../connectors/google_drive/change_tracker.py | 10 +----- .../app/connectors/google_drive/client.py | 15 ++------- .../google_drive/content_extractor.py | 20 ++---------- .../connectors/google_drive/credentials.py | 13 +------- .../app/connectors/google_drive/file_types.py | 9 +----- .../connectors/google_drive/folder_manager.py | 17 ++-------- .../google_drive_indexer.py | 32 +------------------ 8 files changed, 12 insertions(+), 110 deletions(-) diff --git a/surfsense_backend/app/connectors/google_drive/__init__.py b/surfsense_backend/app/connectors/google_drive/__init__.py index c50135155..6e0d25725 100644 --- a/surfsense_backend/app/connectors/google_drive/__init__.py +++ b/surfsense_backend/app/connectors/google_drive/__init__.py @@ -1,8 +1,4 @@ -""" -Google Drive Connector Module. - -Simple, modular approach to Google Drive indexing. -""" +"""Google Drive Connector Module.""" from .change_tracker import categorize_change, fetch_all_changes, get_start_page_token from .client import GoogleDriveClient diff --git a/surfsense_backend/app/connectors/google_drive/change_tracker.py b/surfsense_backend/app/connectors/google_drive/change_tracker.py index 1c697af5f..860e2dbef 100644 --- a/surfsense_backend/app/connectors/google_drive/change_tracker.py +++ b/surfsense_backend/app/connectors/google_drive/change_tracker.py @@ -1,9 +1,4 @@ -""" -Change Tracking for Google Drive - Delta Sync Support. - -Handles change detection and incremental syncing using Drive API's changes endpoint. -Small, focused module for tracking file modifications. -""" +"""Change tracking for Google Drive delta sync.""" import logging from datetime import datetime @@ -110,7 +105,6 @@ async def _filter_changes_by_folder( for change in changes: file = change.get("file") if not file: - # File was removed filtered.append(change) continue @@ -147,7 +141,6 @@ def categorize_change(change: dict[str, Any]) -> str: if file.get("trashed"): return "trashed" - # Check if file was recently created created_time = file.get("createdTime") modified_time = file.get("modifiedTime") @@ -198,7 +191,6 @@ async def fetch_all_changes( all_changes.extend(changes) - # If next_token is None, we've reached the end if not next_token or next_token == current_token: break diff --git a/surfsense_backend/app/connectors/google_drive/client.py b/surfsense_backend/app/connectors/google_drive/client.py index 6d2d0abfd..5053aa449 100644 --- a/surfsense_backend/app/connectors/google_drive/client.py +++ b/surfsense_backend/app/connectors/google_drive/client.py @@ -1,9 +1,4 @@ -""" -Google Drive API Client. - -Core client for interacting with Google Drive API. -Handles service initialization and basic file operations. -""" +"""Google Drive API client.""" from typing import Any @@ -16,12 +11,7 @@ from .credentials import get_valid_credentials class GoogleDriveClient: - """ - Main client for Google Drive API operations. - - Handles service initialization and provides methods for - listing files, getting metadata, and downloading content. - """ + """Client for Google Drive API operations.""" def __init__(self, session: AsyncSession, connector_id: int): """ @@ -140,7 +130,6 @@ class GoogleDriveClient: service = await self.get_service() request = service.files().get_media(fileId=file_id) - # Execute the download import io fh = io.BytesIO() diff --git a/surfsense_backend/app/connectors/google_drive/content_extractor.py b/surfsense_backend/app/connectors/google_drive/content_extractor.py index 04c48f47f..00211957a 100644 --- a/surfsense_backend/app/connectors/google_drive/content_extractor.py +++ b/surfsense_backend/app/connectors/google_drive/content_extractor.py @@ -1,8 +1,4 @@ -""" -Content Extraction for Google Drive Files. - -Downloads files and delegates to Surfsense's existing file processors. -""" +"""Content extraction for Google Drive files.""" import logging import os @@ -31,9 +27,7 @@ async def download_and_process_file( log_entry: Log, ) -> tuple[Any, str | None, dict[str, Any] | None]: """ - Download Google Drive file and process using Surfsense's existing infrastructure. - - This is the ONLY function needed - it delegates everything to process_file_in_background. + Download Google Drive file and process using Surfsense file processors. Args: client: GoogleDriveClient instance @@ -71,10 +65,8 @@ async def download_and_process_file( if error: return None, error - # Set extension based on export format extension = ".pdf" if export_mime == "application/pdf" else ".txt" else: - # Regular files - download directly content_bytes, error = await client.download_file(file_id) if error: return None, error @@ -82,19 +74,15 @@ async def download_and_process_file( # Preserve original file extension extension = Path(file_name).suffix or ".bin" - # Save to temporary file with tempfile.NamedTemporaryFile(delete=False, suffix=extension) as tmp_file: tmp_file.write(content_bytes) temp_file_path = tmp_file.name - # Step 2: Delegate to Surfsense's existing file processor - # This handles ALL file types: markdown, audio, PDFs, Office docs, images, etc. from app.tasks.document_processors.file_processors import ( process_file_in_background, ) from app.db import DocumentType - # Prepare connector info connector_info = { "type": DocumentType.GOOGLE_DRIVE_CONNECTOR, "metadata": { @@ -105,7 +93,6 @@ async def download_and_process_file( }, } - # If it was a Google Workspace file, note the export format if is_google_workspace_file(mime_type): connector_info["metadata"]["exported_as"] = "pdf" connector_info["metadata"]["original_workspace_type"] = mime_type.split(".")[-1] @@ -119,10 +106,9 @@ async def download_and_process_file( session=session, task_logger=task_logger, log_entry=log_entry, - connector=connector_info, # Pass connector info + connector=connector_info, ) - # process_file_in_background doesn't return the document return None, None, connector_info["metadata"] except Exception as e: diff --git a/surfsense_backend/app/connectors/google_drive/credentials.py b/surfsense_backend/app/connectors/google_drive/credentials.py index 5d09df881..4c1ef9c03 100644 --- a/surfsense_backend/app/connectors/google_drive/credentials.py +++ b/surfsense_backend/app/connectors/google_drive/credentials.py @@ -1,9 +1,4 @@ -""" -Google Drive OAuth Credentials Management. - -Handles credential validation, token refresh, and persistence to database. -Small, focused module for credential operations only. -""" +"""Google Drive OAuth credential management.""" import json from datetime import datetime @@ -35,7 +30,6 @@ async def get_valid_credentials( ValueError: If credentials are missing or invalid Exception: If token refresh fails """ - # Fetch connector from database result = await session.execute( select(SearchSourceConnector).filter( SearchSourceConnector.id == connector_id @@ -46,11 +40,9 @@ async def get_valid_credentials( if not connector: raise ValueError(f"Connector {connector_id} not found") - # Extract credentials from config config_data = connector.config exp = config_data.get("expiry", "").replace("Z", "") - # Validate required fields if not all( [ config_data.get("client_id"), @@ -62,7 +54,6 @@ async def get_valid_credentials( "Google OAuth credentials (client_id, client_secret, refresh_token) must be set" ) - # Create credentials object credentials = Credentials( token=config_data.get("token"), refresh_token=config_data.get("refresh_token"), @@ -73,12 +64,10 @@ async def get_valid_credentials( expiry=datetime.fromisoformat(exp) if exp else None, ) - # Refresh token if expired if credentials.expired or not credentials.valid: try: credentials.refresh(Request()) - # Persist refreshed token to database connector.config = json.loads(credentials.to_json()) flag_modified(connector, "config") await session.commit() diff --git a/surfsense_backend/app/connectors/google_drive/file_types.py b/surfsense_backend/app/connectors/google_drive/file_types.py index f66680c6c..cb2354585 100644 --- a/surfsense_backend/app/connectors/google_drive/file_types.py +++ b/surfsense_backend/app/connectors/google_drive/file_types.py @@ -1,18 +1,11 @@ -""" -File Type Handlers for Google Drive. +"""File type handlers for Google Drive.""" -Simple module for basic file type detection. -""" - -# Google Workspace MIME types that need export GOOGLE_DOC = "application/vnd.google-apps.document" GOOGLE_SHEET = "application/vnd.google-apps.spreadsheet" GOOGLE_SLIDE = "application/vnd.google-apps.presentation" GOOGLE_FOLDER = "application/vnd.google-apps.folder" GOOGLE_SHORTCUT = "application/vnd.google-apps.shortcut" -# Export MIME types for Google Workspace files -# Export as PDF to preserve formatting, images, and structure EXPORT_FORMATS = { GOOGLE_DOC: "application/pdf", GOOGLE_SHEET: "application/pdf", diff --git a/surfsense_backend/app/connectors/google_drive/folder_manager.py b/surfsense_backend/app/connectors/google_drive/folder_manager.py index da9deb75d..599475a46 100644 --- a/surfsense_backend/app/connectors/google_drive/folder_manager.py +++ b/surfsense_backend/app/connectors/google_drive/folder_manager.py @@ -1,9 +1,4 @@ -""" -Folder Management for Google Drive. - -Handles folder listing, selection, and hierarchy operations. -Small, focused module for folder-related operations. -""" +"""Folder management for Google Drive.""" import logging from typing import Any @@ -165,11 +160,7 @@ async def list_folder_contents( parent_id: str | None = None, ) -> tuple[list[dict[str, Any]], str | None]: """ - List both folders and files in a Google Drive folder. - - Fetches ALL items using pagination (handles folders with >100 items). - Returns items sorted with folders first, then files. - Each item includes 'isFolder' boolean for frontend rendering. + List folders and files in a Google Drive folder with pagination support. Args: client: GoogleDriveClient instance @@ -212,20 +203,16 @@ async def list_folder_contents( all_items.extend(items) - # If no more pages, break if not next_token: break page_token = next_token - # Add 'isFolder' flag and sort (folders first, then files) for item in all_items: item["isFolder"] = item["mimeType"] == "application/vnd.google-apps.folder" - # Sort: folders first (alphabetically), then files (alphabetically) all_items.sort(key=lambda x: (not x["isFolder"], x["name"].lower())) - # Count folders and files for logging folder_count = sum(1 for item in all_items if item["isFolder"]) file_count = len(all_items) - folder_count diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index 335c3b41d..cd862e372 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -1,11 +1,4 @@ -""" -Google Drive Indexer - Delegates all processing to Surfsense's file processors. - -Handles: -- Folder-specific indexing (user selects folder) -- Delta sync (only index changed files) -- Delegates file processing to process_file_in_background -""" +"""Google Drive indexer using Surfsense file processors.""" import logging from datetime import datetime @@ -63,7 +56,6 @@ async def index_google_drive_files( """ task_logger = TaskLoggingService(session, search_space_id) - # Log task start log_entry = await task_logger.log_task_start( task_name="google_drive_files_indexing", source="connector_indexing_task", @@ -78,7 +70,6 @@ async def index_google_drive_files( ) try: - # Get connector from database connector = await get_connector_by_id( session, connector_id, SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR ) @@ -90,7 +81,6 @@ async def index_google_drive_files( ) return 0, error_msg - # Initialize Drive client await task_logger.log_task_progress( log_entry, f"Initializing Google Drive client for connector {connector_id}", @@ -99,7 +89,6 @@ async def index_google_drive_files( drive_client = GoogleDriveClient(session, connector_id) - # Use folder from request params (required for Google Drive) if not folder_id: error_msg = "folder_id is required for Google Drive indexing" await task_logger.log_task_failure( @@ -112,7 +101,6 @@ async def index_google_drive_files( logger.info(f"Indexing Google Drive folder: {target_folder_name} ({target_folder_id})") - # Decide sync strategy - track tokens per folder folder_tokens = connector.config.get("folder_tokens", {}) start_page_token = folder_tokens.get(target_folder_id) can_use_delta_sync = use_delta_sync and start_page_token and connector.last_indexed_at @@ -150,14 +138,11 @@ async def index_google_drive_files( documents_indexed, documents_skipped = result - # Update last indexed timestamp and get new start page token if documents_indexed > 0 or can_use_delta_sync: - # Get new start page token for next sync new_token, token_error = await get_start_page_token(drive_client) if new_token and not token_error: from sqlalchemy.orm.attributes import flag_modified - # Store token per folder if "folder_tokens" not in connector.config: connector.config["folder_tokens"] = {} connector.config["folder_tokens"][target_folder_id] = new_token @@ -165,13 +150,11 @@ async def index_google_drive_files( await update_connector_last_indexed(session, connector, update_last_indexed) - # Final commit await session.commit() logger.info( f"Successfully committed Google Drive indexing changes to database" ) - # Log success await task_logger.log_task_success( log_entry, f"Successfully completed Google Drive indexing for connector {connector_id}", @@ -235,7 +218,6 @@ async def _index_full_scan( page_token = None files_processed = 0 - # Paginate through all files in folder while files_processed < max_files: files, next_token, error = await get_files_in_folder( drive_client, folder_id, include_subfolders=False, page_token=page_token @@ -254,7 +236,6 @@ async def _index_full_scan( files_processed += 1 - # Process file indexed, skipped = await _process_single_file( drive_client=drive_client, session=session, @@ -269,7 +250,6 @@ async def _index_full_scan( documents_indexed += indexed documents_skipped += skipped - # Batch commit every 10 files if documents_indexed % 10 == 0 and documents_indexed > 0: await session.commit() logger.info(f"Committed batch: {documents_indexed} files indexed so far") @@ -304,7 +284,6 @@ async def _index_with_delta_sync( {"stage": "delta_sync", "start_token": start_page_token}, ) - # Fetch all changes since last sync changes, final_token, error = await fetch_all_changes( drive_client, start_page_token, folder_id ) @@ -330,14 +309,12 @@ async def _index_with_delta_sync( files_processed += 1 change_type = categorize_change(change) - # Handle removed/trashed files if change_type in ["removed", "trashed"]: file_id = change.get("fileId") if file_id: await _remove_document(session, file_id, search_space_id) continue - # Handle modified/new files file = change.get("file") if not file: continue @@ -356,7 +333,6 @@ async def _index_with_delta_sync( documents_indexed += indexed documents_skipped += skipped - # Batch commit every 10 files if documents_indexed % 10 == 0 and documents_indexed > 0: await session.commit() logger.info(f"Committed batch: {documents_indexed} changes processed") @@ -389,10 +365,6 @@ async def _process_single_file( try: logger.info(f"Processing file: {file_name} ({mime_type})") - # Download and process using Surfsense's existing infrastructure - # This handles: markdown, audio, PDFs, Office docs, images, etc. - # It also handles: deduplication, chunking, summarization, embedding - # Document type is set to GOOGLE_DRIVE_CONNECTOR during processing _, error, _ = await download_and_process_file( client=drive_client, file=file, @@ -404,7 +376,6 @@ async def _process_single_file( ) if error: - # Log and skip - not an error, just unsupported or empty await task_logger.log_task_progress( log_entry, f"Skipped {file_name}: {error}", @@ -412,7 +383,6 @@ async def _process_single_file( ) return 0, 1 - # File was processed successfully (document type already set in processor) logger.info(f"Successfully indexed Google Drive file: {file_name}") return 1, 0 From 0b006de32dbfd0aba418920da65107acb2654db8 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 18:59:30 +0200 Subject: [PATCH 34/92] refactor(web): clean up Google Drive folder tree component - Replace inline comments with JSDoc multiline comments for main functions - Remove obvious/noisy inline comments from JSX - Simplify component documentation while keeping it clear - Improve readability by reducing comment clutter --- .../connectors/google-drive-folder-tree.tsx | 59 ++++++------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/surfsense_web/components/connectors/google-drive-folder-tree.tsx b/surfsense_web/components/connectors/google-drive-folder-tree.tsx index 793fdc750..05f4cc9e2 100644 --- a/surfsense_web/components/connectors/google-drive-folder-tree.tsx +++ b/surfsense_web/components/connectors/google-drive-folder-tree.tsx @@ -75,25 +75,23 @@ export function GoogleDriveFolderTree({ const [isLoadingRoot, setIsLoadingRoot] = useState(false); const [isInitialized, setIsInitialized] = useState(false); - // Helper to check if a folder is selected const isFolderSelected = (folderId: string): boolean => { return selectedFolders.some((f) => f.id === folderId); }; - // Handle folder checkbox toggle const toggleFolderSelection = (folderId: string, folderName: string) => { if (isFolderSelected(folderId)) { - // Remove from selection onSelectFolders(selectedFolders.filter((f) => f.id !== folderId)); } else { - // Add to selection onSelectFolders([...selectedFolders, { id: folderId, name: folderName }]); } }; - // Load root items (folders and files) on mount + /** + * Load root-level folders and files from Google Drive. + */ const loadRootItems = async () => { - if (isInitialized) return; // Already loaded + if (isInitialized) return; setIsLoadingRoot(true); try { @@ -112,17 +110,16 @@ export function GoogleDriveFolderTree({ } }; - // Helper function to find an item recursively through all loaded items + /** + * Find an item by ID across all loaded items (root and nested). + */ const findItem = (itemId: string): DriveItem | undefined => { - // First check if we have it in itemStates const state = itemStates.get(itemId); if (state?.item) return state.item; - // Check root items const rootItem = rootItems.find((item) => item.id === itemId); if (rootItem) return rootItem; - // Recursively search through all loaded children for (const [, nodeState] of itemStates) { if (nodeState.children) { const found = nodeState.children.find((child) => child.id === itemId); @@ -133,17 +130,17 @@ export function GoogleDriveFolderTree({ return undefined; }; - // Load children (folders and files) for a specific folder + /** + * Load and display contents of a specific folder. + */ const loadFolderContents = async (folderId: string) => { try { - // Set loading state setItemStates((prev) => { const newMap = new Map(prev); const existing = newMap.get(folderId); if (existing) { newMap.set(folderId, { ...existing, isLoading: true }); } else { - // First time loading this folder - create initial state const item = findItem(folderId); if (item) { newMap.set(folderId, { @@ -165,10 +162,6 @@ export function GoogleDriveFolderTree({ const data = await response.json(); const items = data.items || []; - // Check if folder only contains files (no subfolders) - const hasSubfolders = items.some((item: DriveItem) => item.isFolder); - - // Update item state with loaded children setItemStates((prev) => { const newMap = new Map(prev); const existing = newMap.get(folderId); @@ -178,7 +171,7 @@ export function GoogleDriveFolderTree({ newMap.set(folderId, { item, children: items, - isExpanded: true, // Always expand after loading + isExpanded: true, isLoading: false, }); } else { @@ -188,7 +181,6 @@ export function GoogleDriveFolderTree({ }); } catch (error) { console.error("Error loading folder contents:", error); - // Clear loading state on error setItemStates((prev) => { const newMap = new Map(prev); const existing = newMap.get(folderId); @@ -200,17 +192,17 @@ export function GoogleDriveFolderTree({ } }; - // Toggle folder expansion + /** + * Toggle folder expand/collapse state. + */ const toggleFolder = async (item: DriveItem) => { - if (!item.isFolder) return; // Only folders can be expanded + if (!item.isFolder) return; const state = itemStates.get(item.id); if (!state || state.children === null) { - // First time expanding - load children await loadFolderContents(item.id); } else { - // Toggle expansion state setItemStates((prev) => { const newMap = new Map(prev); newMap.set(item.id, { @@ -222,7 +214,9 @@ export function GoogleDriveFolderTree({ } }; - // Recursive render function for item tree + /** + * Render a single item (folder or file) with its children. + */ const renderItem = (item: DriveItem, level: number = 0) => { const state = itemStates.get(item.id); const isExpanded = state?.isExpanded || false; @@ -231,7 +225,6 @@ export function GoogleDriveFolderTree({ const isSelected = isFolderSelected(item.id); const isFolder = item.isFolder; - // Separate folders and files for children const childFolders = children?.filter((c) => c.isFolder) || []; const childFiles = children?.filter((c) => !c.isFolder) || []; @@ -245,7 +238,6 @@ export function GoogleDriveFolderTree({ isSelected && isFolder && "bg-accent/50" )} > - {/* Expand/Collapse Icon (only for folders) */} {isFolder ? ( ) : ( - // Empty space for alignment + )} - {/* Checkbox (only for folders) */} {isFolder && ( )} - {/* Icon */}
{isFolder ? ( isExpanded ? ( @@ -289,7 +279,6 @@ export function GoogleDriveFolderTree({ )}
- {/* Item Name */} isFolder && toggleFolder(item)} @@ -298,16 +287,11 @@ export function GoogleDriveFolderTree({ - {/* Render children if expanded (folders first, then files) */} {isExpanded && isFolder && children && (
- {/* Render folders first */} {childFolders.map((child) => renderItem(child, level + 1))} - - {/* Render files */} {childFiles.map((child) => renderItem(child, level + 1))} - {/* Empty state */} {children.length === 0 && (
Empty folder
)} @@ -317,7 +301,6 @@ export function GoogleDriveFolderTree({ ); }; - // Initialize on first render if (!isInitialized && !isLoadingRoot) { loadRootItems(); } @@ -326,7 +309,6 @@ export function GoogleDriveFolderTree({
- {/* My Drive Header (always visible, selectable) */}
- {/* Loading indicator */} {isLoadingRoot && (
)} - {/* Root items (folders and files) - same level as Google Drive shows */}
{!isLoadingRoot && rootItems.map((item) => renderItem(item, 0))}
- {/* Empty state */} {!isLoadingRoot && rootItems.length === 0 && (
No files or folders found in your Google Drive From 10c98745cdc3a2e7231d27dc8b05d1c9b6b609b8 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 28 Dec 2025 19:17:37 +0200 Subject: [PATCH 35/92] refactor(web): use React Query for Google Drive folder operations - Fix errors in connectors-api.service (use .issues instead of .errors) - Create useGoogleDriveFolders hook with proper React Query integration - Add Google Drive folders cache keys with proper query invalidation - Refactor GoogleDriveFolderTree to use React Query hook for root data - Remove manual state management (isInitialized, setRootItems, loadRootItems) - Remove unused state (driveFolders, isLoadingFolders) from manage page - Simplify handleOpenDriveFolderDialog function - Automatic loading, caching, error handling, and refetching via React Query - Better performance with proper caching and state management --- .../connectors/(manage)/page.tsx | 68 ++++++------------- .../connectors/google-drive-folder-tree.tsx | 49 ++++--------- .../contracts/types/connector.types.ts | 32 +++++++++ .../hooks/use-google-drive-folders.ts | 29 ++++++++ .../lib/apis/connectors-api.service.ts | 40 +++++++++-- surfsense_web/lib/query-client/cache-keys.ts | 4 ++ 6 files changed, 129 insertions(+), 93 deletions(-) create mode 100644 surfsense_web/hooks/use-google-drive-folders.ts diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index 5854cb706..1e0e76ca9 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -70,14 +70,8 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { cn } from "@/lib/utils"; -import { authenticatedFetch } from "@/lib/auth-utils"; import { GoogleDriveFolderTree } from "@/components/connectors/google-drive-folder-tree"; -interface DriveFolder { - id: string; - name: string; -} - export default function ConnectorsPage() { const t = useTranslations("connectors"); const tCommon = useTranslations("common"); @@ -127,9 +121,7 @@ export default function ConnectorsPage() { // Google Drive folder selection state const [driveFolderDialogOpen, setDriveFolderDialogOpen] = useState(false); - const [driveFolders, setDriveFolders] = useState([]); const [selectedFolders, setSelectedFolders] = useState>([]); - const [isLoadingFolders, setIsLoadingFolders] = useState(false); useEffect(() => { if (error) { @@ -165,31 +157,9 @@ export default function ConnectorsPage() { } }; - // Handle opening Google Drive folder selection dialog - const handleOpenDriveFolderDialog = async (connectorId: number) => { + const handleOpenDriveFolderDialog = (connectorId: number) => { setSelectedConnectorForIndexing(connectorId); setDriveFolderDialogOpen(true); - setIsLoadingFolders(true); - - try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/connectors/${connectorId}/google-drive/folders`, - { method: "GET" } - ); - - if (!response.ok) { - throw new Error("Failed to load folders"); - } - - const data = await response.json(); - setDriveFolders(data.folders || []); - } catch (error) { - console.error("Error loading folders:", error); - toast.error("Failed to load Google Drive folders"); - setDriveFolderDialogOpen(false); - } finally { - setIsLoadingFolders(false); - } }; // Handle Google Drive folder indexing @@ -204,15 +174,17 @@ export default function ConnectorsPage() { try { setIndexingConnectorId(selectedConnectorForIndexing); - // Call indexConnector with folder_ids and folder_names as query params - await indexConnector( - selectedConnectorForIndexing, - searchSpaceId, - undefined, - undefined, - selectedFolders.map((f) => f.id).join(","), - selectedFolders.map((f) => f.name).join(", ") - ); + const folderIds = selectedFolders.map((f) => f.id).join(","); + const folderNames = selectedFolders.map((f) => f.name).join(", "); + + await indexConnector({ + connector_id: selectedConnectorForIndexing, + queryParams: { + search_space_id: searchSpaceId, + folder_ids: folderIds, + folder_names: folderNames, + }, + }); toast.success(t("indexing_started")); } catch (error) { console.error("Error indexing connector content:", error); @@ -221,7 +193,6 @@ export default function ConnectorsPage() { setIndexingConnectorId(null); setSelectedConnectorForIndexing(null); setSelectedFolders([]); - setDriveFolders([]); } }; @@ -747,14 +718,13 @@ export default function ConnectorsPage() { - {tooltip} + + {tooltip} + ); } diff --git a/surfsense_web/components/ui/tooltip.tsx b/surfsense_web/components/ui/tooltip.tsx index 98420c858..e1aa458e6 100644 --- a/surfsense_web/components/ui/tooltip.tsx +++ b/surfsense_web/components/ui/tooltip.tsx @@ -32,7 +32,7 @@ function TooltipTrigger({ ...props }: React.ComponentProps) { @@ -42,13 +42,12 @@ function TooltipContent({ data-slot="tooltip-content" sideOffset={sideOffset} className={cn( - "bg-popover text-popover-foreground border shadow-md animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-fit rounded-md px-3 py-1.5 text-xs text-balance", + "bg-popover text-popover-foreground fill-popover shadow-md animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-fit rounded-md px-3 py-1.5 text-xs text-balance", className )} {...props} > {children} - ); From 9f19bea28496e43d7ea0d1fe66ada8b9481cd0df Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 30 Dec 2025 04:03:34 +0530 Subject: [PATCH 43/92] feat: Extract connector indicator UI from thread into a new dedicated component. --- .../assistant-ui/connector-popup.tsx | 163 ++++++++++++++++++ .../components/assistant-ui/thread.tsx | 150 +--------------- 2 files changed, 164 insertions(+), 149 deletions(-) create mode 100644 surfsense_web/components/assistant-ui/connector-popup.tsx diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx new file mode 100644 index 000000000..fbe137287 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -0,0 +1,163 @@ +import { useAtomValue } from "jotai"; +import { + ChevronRightIcon, + Loader2, + Plug2, + Plus, +} from "lucide-react"; +import Link from "next/link"; +import { + type FC, + useCallback, + useRef, + useState, +} from "react"; +import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; +import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; +import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; +import { cn } from "@/lib/utils"; + +export const ConnectorIndicator: FC = () => { + const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); + const { connectors, isLoading: connectorsLoading } = useSearchSourceConnectors( + false, + searchSpaceId ? Number(searchSpaceId) : undefined + ); + const { data: documentTypeCounts, isLoading: documentTypesLoading } = + useAtomValue(documentTypeCountsAtom); + const [isOpen, setIsOpen] = useState(false); + const closeTimeoutRef = useRef(null); + + const isLoading = connectorsLoading || documentTypesLoading; + + // Get document types that have documents in the search space + const activeDocumentTypes = documentTypeCounts + ? Object.entries(documentTypeCounts).filter(([_, count]) => count > 0) + : []; + + const hasConnectors = connectors.length > 0; + const hasSources = hasConnectors || activeDocumentTypes.length > 0; + const totalSourceCount = connectors.length + activeDocumentTypes.length; + + const handleMouseEnter = useCallback(() => { + // Clear any pending close timeout + if (closeTimeoutRef.current) { + clearTimeout(closeTimeoutRef.current); + closeTimeoutRef.current = null; + } + setIsOpen(true); + }, []); + + const handleMouseLeave = useCallback(() => { + // Delay closing by 150ms for better UX + closeTimeoutRef.current = setTimeout(() => { + setIsOpen(false); + }, 150); + }, []); + + if (!searchSpaceId) return null; + + return ( + + + + + + {hasSources ? ( +
+
+

Connected Sources

+ + {totalSourceCount} + +
+
+ {/* Document types from the search space */} + {activeDocumentTypes.map(([docType]) => ( +
+ {getConnectorIcon(docType, "size-3.5")} + {getDocumentTypeLabel(docType)} +
+ ))} + {/* Search source connectors */} + {connectors.map((connector) => ( +
+ {getConnectorIcon(connector.connector_type, "size-3.5")} + {connector.name} +
+ ))} +
+
+ + + Add more sources + + +
+
+ ) : ( +
+

No sources yet

+

+ Add documents or connect data sources to enhance search results. +

+ + + Add Connector + +
+ )} + + + ); +}; + diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index cb01e7605..1023c4e18 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -23,8 +23,6 @@ import { FileText, Loader2, PencilIcon, - Plug2, - Plus, RefreshCwIcon, SquareIcon, } from "lucide-react"; @@ -41,25 +39,23 @@ import { useState, } from "react"; import { createPortal } from "react-dom"; -import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; import { mentionedDocumentIdsAtom, mentionedDocumentsAtom, messageDocumentsMapAtom, } from "@/atoms/chat/mentioned-documents.atom"; -import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; import { globalNewLLMConfigsAtom, llmPreferencesAtom, newLLMConfigsAtom, } from "@/atoms/new-llm-config/new-llm-config-query.atoms"; -import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { ComposerAddAttachment, ComposerAttachments, UserMessageAttachments, } from "@/components/assistant-ui/attachment"; +import { ConnectorIndicator } from "@/components/assistant-ui/connector-popup"; import { InlineMentionEditor, type InlineMentionEditorRef, @@ -75,10 +71,7 @@ import { ChainOfThoughtItem } from "@/components/prompt-kit/chain-of-thought"; import { TextShimmerLoader } from "@/components/prompt-kit/loader"; import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; import { Button } from "@/components/ui/button"; -import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; -import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { Document } from "@/contracts/types/document.types"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; import { cn } from "@/lib/utils"; /** @@ -625,147 +618,6 @@ const Composer: FC = () => { ); }; -const ConnectorIndicator: FC = () => { - const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); - const { connectors, isLoading: connectorsLoading } = useSearchSourceConnectors( - false, - searchSpaceId ? Number(searchSpaceId) : undefined - ); - const { data: documentTypeCounts, isLoading: documentTypesLoading } = - useAtomValue(documentTypeCountsAtom); - const [isOpen, setIsOpen] = useState(false); - const closeTimeoutRef = useRef(null); - - const isLoading = connectorsLoading || documentTypesLoading; - - // Get document types that have documents in the search space - const activeDocumentTypes = documentTypeCounts - ? Object.entries(documentTypeCounts).filter(([_, count]) => count > 0) - : []; - - const hasConnectors = connectors.length > 0; - const hasSources = hasConnectors || activeDocumentTypes.length > 0; - const totalSourceCount = connectors.length + activeDocumentTypes.length; - - const handleMouseEnter = useCallback(() => { - // Clear any pending close timeout - if (closeTimeoutRef.current) { - clearTimeout(closeTimeoutRef.current); - closeTimeoutRef.current = null; - } - setIsOpen(true); - }, []); - - const handleMouseLeave = useCallback(() => { - // Delay closing by 150ms for better UX - closeTimeoutRef.current = setTimeout(() => { - setIsOpen(false); - }, 150); - }, []); - - if (!searchSpaceId) return null; - - return ( - - - - - - {hasSources ? ( -
-
-

Connected Sources

- - {totalSourceCount} - -
-
- {/* Document types from the search space */} - {activeDocumentTypes.map(([docType]) => ( -
- {getConnectorIcon(docType, "size-3.5")} - {getDocumentTypeLabel(docType)} -
- ))} - {/* Search source connectors */} - {connectors.map((connector) => ( -
- {getConnectorIcon(connector.connector_type, "size-3.5")} - {connector.name} -
- ))} -
-
- - - Add more sources - - -
-
- ) : ( -
-

No sources yet

-

- Add documents or connect data sources to enhance search results. -

- - - Add Connector - -
- )} - - - ); -}; - const ComposerAction: FC = () => { // Check if any attachments are still being processed (running AND progress < 100) // When progress is 100, processing is done but waiting for send() From 577ebdb3e7b0c640c70f3cbdfb71eed7d304b4b0 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 30 Dec 2025 15:51:10 +0530 Subject: [PATCH 44/92] feat: Refactor connector selection UI to a dialog with tabs, search, and connection logic, and add new dialog sub-components. --- .../assistant-ui/connector-popup.tsx | 618 ++++++++++++++---- surfsense_web/components/ui/dialog.tsx | 2 +- 2 files changed, 500 insertions(+), 120 deletions(-) diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index fbe137287..12fd7144f 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -1,26 +1,170 @@ +"use client"; + import { useAtomValue } from "jotai"; import { - ChevronRightIcon, + ChevronRight, Loader2, Plug2, - Plus, + Search, } from "lucide-react"; import Link from "next/link"; -import { - type FC, - useCallback, - useRef, - useState, -} from "react"; +import { usePathname, useRouter } from "next/navigation"; +import { type FC, useCallback, useEffect, useState } from "react"; +import { toast } from "sonner"; import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; -import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; +import { authenticatedFetch } from "@/lib/auth-utils"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { cn } from "@/lib/utils"; +// OAuth Connectors (Quick Connect) +const OAUTH_CONNECTORS = [ + { + id: "google-gmail-connector", + title: "Gmail", + description: "Search through your emails", + connectorType: EnumConnectorName.GOOGLE_GMAIL_CONNECTOR, + authEndpoint: "/api/v1/auth/google/gmail/connector/add/", + }, + { + id: "google-calendar-connector", + title: "Google Calendar", + description: "Search through your events", + connectorType: EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR, + authEndpoint: "/api/v1/auth/google/calendar/connector/add/", + }, + { + id: "airtable-connector", + title: "Airtable", + description: "Search your Airtable bases", + connectorType: EnumConnectorName.AIRTABLE_CONNECTOR, + authEndpoint: "/api/v1/auth/airtable/connector/add/", + }, +]; + +// Non-OAuth Connectors +const OTHER_CONNECTORS = [ + { + id: "slack-connector", + title: "Slack", + description: "Search Slack messages", + connectorType: EnumConnectorName.SLACK_CONNECTOR, + }, + { + id: "discord-connector", + title: "Discord", + description: "Search Discord messages", + connectorType: EnumConnectorName.DISCORD_CONNECTOR, + }, + { + id: "notion-connector", + title: "Notion", + description: "Search Notion pages", + connectorType: EnumConnectorName.NOTION_CONNECTOR, + }, + { + id: "confluence-connector", + title: "Confluence", + description: "Search documentation", + connectorType: EnumConnectorName.CONFLUENCE_CONNECTOR, + }, + { + id: "bookstack-connector", + title: "BookStack", + description: "Search BookStack docs", + connectorType: EnumConnectorName.BOOKSTACK_CONNECTOR, + }, + { + id: "github-connector", + title: "GitHub", + description: "Search repositories", + connectorType: EnumConnectorName.GITHUB_CONNECTOR, + }, + { + id: "linear-connector", + title: "Linear", + description: "Search issues & projects", + connectorType: EnumConnectorName.LINEAR_CONNECTOR, + }, + { + id: "jira-connector", + title: "Jira", + description: "Search Jira issues", + connectorType: EnumConnectorName.JIRA_CONNECTOR, + }, + { + id: "clickup-connector", + title: "ClickUp", + description: "Search ClickUp tasks", + connectorType: EnumConnectorName.CLICKUP_CONNECTOR, + }, + { + id: "luma-connector", + title: "Luma", + description: "Search Luma events", + connectorType: EnumConnectorName.LUMA_CONNECTOR, + }, + { + id: "elasticsearch-connector", + title: "Elasticsearch", + description: "Search ES indexes", + connectorType: EnumConnectorName.ELASTICSEARCH_CONNECTOR, + }, + { + id: "webcrawler-connector", + title: "Web Pages", + description: "Crawl web content", + connectorType: EnumConnectorName.WEBCRAWLER_CONNECTOR, + }, + { + id: "tavily-api", + title: "Tavily AI", + description: "Search with Tavily", + connectorType: EnumConnectorName.TAVILY_API, + }, + { + id: "searxng", + title: "SearxNG", + description: "Search with SearxNG", + connectorType: EnumConnectorName.SEARXNG_API, + }, + { + id: "linkup-api", + title: "Linkup API", + description: "Search with Linkup", + connectorType: EnumConnectorName.LINKUP_API, + }, + { + id: "baidu-search-api", + title: "Baidu Search", + description: "Search with Baidu", + connectorType: EnumConnectorName.BAIDU_SEARCH_API, + }, +]; + +import { + Tabs, + TabsContent, + TabsList, + TabsTrigger, +} from "@/components/ui/tabs"; + export const ConnectorIndicator: FC = () => { + const router = useRouter(); const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); const { connectors, isLoading: connectorsLoading } = useSearchSourceConnectors( false, @@ -28,11 +172,34 @@ export const ConnectorIndicator: FC = () => { ); const { data: documentTypeCounts, isLoading: documentTypesLoading } = useAtomValue(documentTypeCountsAtom); + const { data: allConnectors } = useAtomValue(connectorsAtom); + const pathname = usePathname(); const [isOpen, setIsOpen] = useState(false); - const closeTimeoutRef = useRef(null); + const [activeTab, setActiveTab] = useState("all"); + const [connectingId, setConnectingId] = useState(null); + const [isScrolled, setIsScrolled] = useState(false); + const [searchQuery, setSearchQuery] = useState(""); const isLoading = connectorsLoading || documentTypesLoading; + // Synchronize state with URL path + useEffect(() => { + const pathParts = window.location.pathname.split("/"); + const connectorsIdx = pathParts.indexOf("connectors"); + + if (connectorsIdx !== -1) { + if (!isOpen) setIsOpen(true); + + // Detect tab from URL: .../connectors/active or .../connectors/all + const tabFromUrl = pathParts[connectorsIdx + 1]; + if (tabFromUrl === "active" || tabFromUrl === "all") { + if (activeTab !== tabFromUrl) setActiveTab(tabFromUrl); + } + } else { + if (isOpen) setIsOpen(false); + } + }, [pathname, isOpen, activeTab]); + // Get document types that have documents in the search space const activeDocumentTypes = documentTypeCounts ? Object.entries(documentTypeCounts).filter(([_, count]) => count > 0) @@ -42,122 +209,335 @@ export const ConnectorIndicator: FC = () => { const hasSources = hasConnectors || activeDocumentTypes.length > 0; const totalSourceCount = connectors.length + activeDocumentTypes.length; - const handleMouseEnter = useCallback(() => { - // Clear any pending close timeout - if (closeTimeoutRef.current) { - clearTimeout(closeTimeoutRef.current); - closeTimeoutRef.current = null; - } - setIsOpen(true); - }, []); + // Check which connectors are already connected + const connectedTypes = new Set( + (allConnectors || []).map((c: SearchSourceConnector) => c.connector_type) + ); - const handleMouseLeave = useCallback(() => { - // Delay closing by 150ms for better UX - closeTimeoutRef.current = setTimeout(() => { - setIsOpen(false); - }, 150); + // Filter connectors based on search + const filteredOAuth = OAUTH_CONNECTORS.filter(c => + c.title.toLowerCase().includes(searchQuery.toLowerCase()) || + c.description.toLowerCase().includes(searchQuery.toLowerCase()) + ); + + const filteredOther = OTHER_CONNECTORS.filter(c => + c.title.toLowerCase().includes(searchQuery.toLowerCase()) || + c.description.toLowerCase().includes(searchQuery.toLowerCase()) + ); + + // Handle OAuth connection + const handleConnectOAuth = useCallback( + async (connector: (typeof OAUTH_CONNECTORS)[0]) => { + if (!searchSpaceId || !connector.authEndpoint) return; + + try { + setConnectingId(connector.id); + const response = await authenticatedFetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}${connector.authEndpoint}?space_id=${searchSpaceId}`, + { method: "GET" } + ); + + if (!response.ok) { + throw new Error(`Failed to initiate ${connector.title} OAuth`); + } + + const data = await response.json(); + window.location.href = data.auth_url; + } catch (error) { + console.error(`Error connecting to ${connector.title}:`, error); + toast.error(`Failed to connect to ${connector.title}`); + } finally { + setConnectingId(null); + } + }, + [searchSpaceId] + ); + + const handleOpenChange = useCallback( + (open: boolean) => { + setIsOpen(open); + + const currentPath = window.location.pathname; + const basePath = currentPath.split("/connectors")[0].replace(/\/$/, ""); + + if (open) { + // Base state is /connectors/all + const newUrl = `${basePath}/connectors/${activeTab}`; + window.history.pushState({ modal: true }, "", newUrl); + } else { + // Return to base chat path + window.history.pushState({ modal: false }, "", basePath || "/"); + setIsScrolled(false); + setSearchQuery(""); + } + }, + [activeTab] + ); + + const handleTabChange = useCallback( + (value: string) => { + setActiveTab(value); + const currentPath = window.location.pathname; + const basePath = currentPath.split("/connectors")[0].replace(/\/$/, ""); + + // Update URL to reflect the new tab state + const newUrl = `${basePath}/connectors/${value}`; + window.history.replaceState({ modal: true }, "", newUrl); + }, + [] + ); + + const handleScroll = useCallback((e: React.UIEvent) => { + setIsScrolled(e.currentTarget.scrollTop > 0); }, []); if (!searchSpaceId) return null; return ( - - - - - + - {hasSources ? ( -
-
-

Connected Sources

- - {totalSourceCount} - -
-
- {/* Document types from the search space */} - {activeDocumentTypes.map(([docType]) => ( -
- {getConnectorIcon(docType, "size-3.5")} - {getDocumentTypeLabel(docType)} -
- ))} - {/* Search source connectors */} - {connectors.map((connector) => ( -
- {getConnectorIcon(connector.connector_type, "size-3.5")} - {connector.name} -
- ))} -
-
- - - Add more sources - - -
-
- ) : ( -
-

No sources yet

-

- Add documents or connect data sources to enhance search results. -

- - - Add Connector - -
+ className={cn( + "size-[34px] rounded-full p-1 flex items-center justify-center transition-colors relative", + "hover:bg-muted-foreground/15 dark:hover:bg-muted-foreground/30", + "outline-none focus:outline-none focus-visible:outline-none font-semibold text-xs", + "border-0 ring-0 focus:ring-0 shadow-none focus:shadow-none" )} - - + aria-label={ + hasSources ? `View ${totalSourceCount} connected sources` : "Add your first connector" + } + onClick={() => handleOpenChange(true)} + > + {isLoading ? ( + + ) : ( + <> + + {totalSourceCount > 0 && ( + + {totalSourceCount > 99 ? "99+" : totalSourceCount} + + )} + + )} + + + + + {/* Header */} +
+ + Connectors + + Search across all your apps and data in one place. + + + +
+ + + All Connectors + + + Active + {totalSourceCount > 0 && ( + + {totalSourceCount} + + )} + + +
+ + setSearchQuery(e.target.value)} + /> +
+
+
+ + {/* Content */} +
+
+ + {/* Quick Connect */} + {filteredOAuth.length > 0 && ( +
+
+

Quick Connect

+
+
+ {filteredOAuth.map((connector) => { + const isConnected = connectedTypes.has(connector.connectorType); + const isConnecting = connectingId === connector.id; + + return ( +
+
+ {getConnectorIcon(connector.connectorType, "size-6")} +
+
+
+ {connector.title} + {isConnected && ( + + )} +
+

+ {isConnected ? "Connected" : connector.description} +

+
+ +
+ ); + })} +
+
+ )} + + {/* More Integrations */} + {filteredOther.length > 0 && ( +
+
+

More Integrations

+
+
+ {filteredOther.map((connector) => { + const isConnected = connectedTypes.has(connector.connectorType); + + return ( + +
+ {getConnectorIcon(connector.connectorType, "size-6")} +
+
+
+ {connector.title} + {isConnected && ( + + )} +
+

+ {connector.description} +

+
+ + + ); + })} +
+
+ )} + + + + {hasSources ? ( +
+
+

Currently Active

+
+
+ {activeDocumentTypes.map(([docType, count]) => ( +
+
+ {getConnectorIcon(docType, "size-6")} +
+
+

+ {getDocumentTypeLabel(docType)} +

+

+ {count as number} documents indexed +

+
+
+ ))} + {connectors.map((connector) => ( +
+
+ {getConnectorIcon(connector.connector_type, "size-6")} +
+
+

+ {connector.name} +

+

Status: Active

+
+ +
+ ))} +
+
+ ) : ( +
+
+ +
+

No active sources

+

+ Connect your first service to start searching across all your data. +

+ + Browse available connectors + +
+ )} + +
+
+ + +
); }; - diff --git a/surfsense_web/components/ui/dialog.tsx b/surfsense_web/components/ui/dialog.tsx index 47fcfeece..d04d76520 100644 --- a/surfsense_web/components/ui/dialog.tsx +++ b/surfsense_web/components/ui/dialog.tsx @@ -44,7 +44,7 @@ const DialogContent = React.forwardRef< {...props} > {children} - + Close From 733ec5df1348b32436084eafa4fd3b17eb76d47b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 30 Dec 2025 16:05:38 +0530 Subject: [PATCH 45/92] feat: enhance connector popup responsiveness, update connector icon, and add a scroll fade effect. --- .../assistant-ui/connector-popup.tsx | 55 +++++++++++-------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index 12fd7144f..bfcb68847 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -2,9 +2,9 @@ import { useAtomValue } from "jotai"; import { + Cable, ChevronRight, Loader2, - Plug2, Search, } from "lucide-react"; import Link from "next/link"; @@ -313,7 +313,7 @@ export const ConnectorIndicator: FC = () => { ) : ( <> - + {totalSourceCount > 0 && ( {totalSourceCount > 99 ? "99+" : totalSourceCount} @@ -323,24 +323,24 @@ export const ConnectorIndicator: FC = () => { )} - + {/* Header */}
- Connectors - + Connectors + Search across all your apps and data in one place. -
- +
+ { )} -
- - setSearchQuery(e.target.value)} - /> + +
+
+ + setSearchQuery(e.target.value)} + /> +
{/* Content */} -
-
+
+
+
{/* Quick Connect */} {filteredOAuth.length > 0 && ( @@ -382,7 +386,7 @@ export const ConnectorIndicator: FC = () => {

Quick Connect

-
+
{filteredOAuth.map((connector) => { const isConnected = connectedTypes.has(connector.connectorType); const isConnecting = connectingId === connector.id; @@ -440,7 +444,7 @@ export const ConnectorIndicator: FC = () => {

More Integrations

-
+
{filteredOther.map((connector) => { const isConnected = connectedTypes.has(connector.connectorType); @@ -479,7 +483,7 @@ export const ConnectorIndicator: FC = () => {

Currently Active

-
+
{activeDocumentTypes.map(([docType, count]) => (
{ ) : (
- +

No active sources

@@ -533,8 +537,11 @@ export const ConnectorIndicator: FC = () => {

)} - + +
+ {/* Bottom fade shadow */} +
From 03559ddc01525e0269962eaf0759b71204645639 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 30 Dec 2025 17:28:15 +0530 Subject: [PATCH 46/92] style: Update connector popup styles for improved UI consistency, including border adjustments, background colors, and hover effects. --- .../assistant-ui/connector-popup.tsx | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index bfcb68847..9043ef657 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -323,7 +323,7 @@ export const ConnectorIndicator: FC = () => { )} - + {/* Header */}
{ -
+
All Connectors - Active + + Active + + {totalSourceCount > 0 && ( {totalSourceCount} @@ -360,13 +363,13 @@ export const ConnectorIndicator: FC = () => { -
+
setSearchQuery(e.target.value)} /> @@ -394,9 +397,9 @@ export const ConnectorIndicator: FC = () => { return (
-
+
{getConnectorIcon(connector.connectorType, "size-6")}
@@ -452,9 +455,9 @@ export const ConnectorIndicator: FC = () => { -
+
{getConnectorIcon(connector.connectorType, "size-6")}
@@ -487,9 +490,9 @@ export const ConnectorIndicator: FC = () => { {activeDocumentTypes.map(([docType, count]) => (
-
+
{getConnectorIcon(docType, "size-6")}
@@ -505,9 +508,9 @@ export const ConnectorIndicator: FC = () => { {connectors.map((connector) => (
-
+
{getConnectorIcon(connector.connector_type, "size-6")}
From 2898192ac4666d971c29f250357592cdfd36ca92 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 30 Dec 2025 19:49:28 +0530 Subject: [PATCH 47/92] feat: Update redirect URLs in connector routes to include success parameters and improve indexing configuration handling in the connector popup. --- .../routes/airtable_add_connector_route.py | 5 +- .../google_calendar_add_connector_route.py | 4 +- .../google_gmail_add_connector_route.py | 5 +- .../assistant-ui/connector-popup.tsx | 599 ++++++++++++++++-- 4 files changed, 550 insertions(+), 63 deletions(-) diff --git a/surfsense_backend/app/routes/airtable_add_connector_route.py b/surfsense_backend/app/routes/airtable_add_connector_route.py index fa124f1c2..3bcbe4dc0 100644 --- a/surfsense_backend/app/routes/airtable_add_connector_route.py +++ b/surfsense_backend/app/routes/airtable_add_connector_route.py @@ -255,9 +255,10 @@ async def airtable_callback( await session.commit() logger.info(f"Successfully saved Airtable connector for user {user_id}") - # Redirect to the frontend success page + # Redirect to the frontend with success params for indexing config + # Using query params to auto-open the popup with config view on new-chat page return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/add/airtable-connector?success=true" + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=airtable-connector" ) except ValidationError as e: diff --git a/surfsense_backend/app/routes/google_calendar_add_connector_route.py b/surfsense_backend/app/routes/google_calendar_add_connector_route.py index fa4ef5466..8bb685450 100644 --- a/surfsense_backend/app/routes/google_calendar_add_connector_route.py +++ b/surfsense_backend/app/routes/google_calendar_add_connector_route.py @@ -131,8 +131,10 @@ async def calendar_callback( session.add(db_connector) await session.commit() await session.refresh(db_connector) + # Redirect to the frontend with success params for indexing config + # Using query params to auto-open the popup with config view on new-chat page return RedirectResponse( - f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/add/google-calendar-connector?success=true" + f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-calendar-connector" ) except ValidationError as e: await session.rollback() diff --git a/surfsense_backend/app/routes/google_gmail_add_connector_route.py b/surfsense_backend/app/routes/google_gmail_add_connector_route.py index 6d37da244..21fcf2c38 100644 --- a/surfsense_backend/app/routes/google_gmail_add_connector_route.py +++ b/surfsense_backend/app/routes/google_gmail_add_connector_route.py @@ -135,9 +135,10 @@ async def gmail_callback( f"Successfully created Gmail connector for user {user_id} with ID {db_connector.id}" ) - # Redirect to the frontend success page + # Redirect to the frontend with success params for indexing config + # Using query params to auto-open the popup with config view on new-chat page return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/add/google-gmail-connector?success=true" + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-gmail-connector" ) except IntegrityError as e: diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index 9043ef657..e55c13448 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -1,17 +1,22 @@ "use client"; +import { format, subDays, subYears } from "date-fns"; import { useAtomValue } from "jotai"; import { + ArrowLeft, Cable, + Calendar as CalendarIcon, + Check, ChevronRight, Loader2, Search, } from "lucide-react"; import Link from "next/link"; -import { usePathname, useRouter } from "next/navigation"; -import { type FC, useCallback, useEffect, useState } from "react"; +import { useRouter, useSearchParams } from "next/navigation"; +import { type FC, useCallback, useEffect, useMemo, useState } from "react"; import { toast } from "sonner"; import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; +import { indexConnectorMutationAtom, updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; @@ -23,14 +28,35 @@ import { DialogTitle, } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; +import { Calendar } from "@/components/ui/calendar"; +import { Label } from "@/components/ui/label"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Switch } from "@/components/ui/switch"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import { useLogsSummary } from "@/hooks/use-logs"; import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; import { authenticatedFetch } from "@/lib/auth-utils"; +import { queryClient } from "@/lib/query-client/client"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { cn } from "@/lib/utils"; +// Type for the indexing configuration state +interface IndexingConfigState { + connectorType: string; + connectorId: number; + connectorTitle: string; +} + // OAuth Connectors (Quick Connect) const OAUTH_CONNECTORS = [ { @@ -165,44 +191,243 @@ import { export const ConnectorIndicator: FC = () => { const router = useRouter(); + const searchParams = useSearchParams(); const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); - const { connectors, isLoading: connectorsLoading } = useSearchSourceConnectors( + const { connectors, isLoading: connectorsLoading, refreshConnectors } = useSearchSourceConnectors( false, searchSpaceId ? Number(searchSpaceId) : undefined ); const { data: documentTypeCounts, isLoading: documentTypesLoading } = useAtomValue(documentTypeCountsAtom); - const { data: allConnectors } = useAtomValue(connectorsAtom); - const pathname = usePathname(); + const { data: allConnectors, refetch: refetchAllConnectors } = useAtomValue(connectorsAtom); + const { mutateAsync: indexConnector } = useAtomValue(indexConnectorMutationAtom); + const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom); const [isOpen, setIsOpen] = useState(false); const [activeTab, setActiveTab] = useState("all"); const [connectingId, setConnectingId] = useState(null); const [isScrolled, setIsScrolled] = useState(false); const [searchQuery, setSearchQuery] = useState(""); + + // Indexing configuration state (shown after OAuth success) + const [indexingConfig, setIndexingConfig] = useState(null); + const [startDate, setStartDate] = useState(undefined); + const [endDate, setEndDate] = useState(undefined); + const [isStartingIndexing, setIsStartingIndexing] = useState(false); + + // Periodic indexing state + const [periodicEnabled, setPeriodicEnabled] = useState(false); + const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); // Default: daily + + // Helper function to get frequency label + const getFrequencyLabel = useCallback((minutes: string): string => { + switch (minutes) { + case "15": return "15 minutes"; + case "60": return "hour"; + case "360": return "6 hours"; + case "720": return "12 hours"; + case "1440": return "day"; + case "10080": return "week"; + default: return `${minutes} minutes`; + } + }, []); + + // Track active indexing tasks + const { summary: logsSummary } = useLogsSummary( + searchSpaceId ? Number(searchSpaceId) : 0, + 24, + { + enablePolling: true, + refetchInterval: 5000, + } + ); + + // Get connector IDs that are currently being indexed + const indexingConnectorIds = useMemo(() => { + if (!logsSummary?.active_tasks) return new Set(); + return new Set( + logsSummary.active_tasks + .filter((task) => task.source?.includes("connector_indexing")) + .map((task) => { + // Extract connector ID from task metadata or source + const match = task.source?.match(/connector[_-]?(\d+)/i); + return match ? parseInt(match[1], 10) : null; + }) + .filter((id): id is number => id !== null) + ); + }, [logsSummary?.active_tasks]); const isLoading = connectorsLoading || documentTypesLoading; - // Synchronize state with URL path + // Synchronize state with URL query params useEffect(() => { - const pathParts = window.location.pathname.split("/"); - const connectorsIdx = pathParts.indexOf("connectors"); + const modalParam = searchParams.get("modal"); + const tabParam = searchParams.get("tab"); + const viewParam = searchParams.get("view"); + const connectorParam = searchParams.get("connector"); - if (connectorsIdx !== -1) { + if (modalParam === "connectors") { if (!isOpen) setIsOpen(true); - // Detect tab from URL: .../connectors/active or .../connectors/all - const tabFromUrl = pathParts[connectorsIdx + 1]; - if (tabFromUrl === "active" || tabFromUrl === "all") { - if (activeTab !== tabFromUrl) setActiveTab(tabFromUrl); + // Detect tab from URL query param + if (tabParam === "active" || tabParam === "all") { + if (activeTab !== tabParam) setActiveTab(tabParam); + } + + // Restore indexing config view from URL if present (e.g., on page refresh) + if (viewParam === "configure" && connectorParam && !indexingConfig) { + const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === connectorParam); + if (oauthConnector && allConnectors) { + const existingConnector = allConnectors.find( + (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType + ); + if (existingConnector) { + setIndexingConfig({ + connectorType: oauthConnector.connectorType, + connectorId: existingConnector.id, + connectorTitle: oauthConnector.title, + }); + } + } } } else { if (isOpen) setIsOpen(false); } - }, [pathname, isOpen, activeTab]); + }, [searchParams, isOpen, activeTab, indexingConfig, allConnectors]); + + // Detect OAuth success and transition to config view + useEffect(() => { + const success = searchParams.get("success"); + const connectorParam = searchParams.get("connector"); + const modalParam = searchParams.get("modal"); + + if (success === "true" && connectorParam && searchSpaceId && modalParam === "connectors") { + // Find the OAuth connector info + const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === connectorParam); + if (oauthConnector) { + // Refetch connectors to get the newly created connector + refetchAllConnectors().then((result) => { + const newConnector = result.data?.find( + (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType + ); + if (newConnector) { + setIndexingConfig({ + connectorType: oauthConnector.connectorType, + connectorId: newConnector.id, + connectorTitle: oauthConnector.title, + }); + setIsOpen(true); + // Update URL to reflect config view (replace success with view=configure) + const url = new URL(window.location.href); + url.searchParams.delete("success"); + url.searchParams.set("view", "configure"); + // Keep connector param for URL restoration + window.history.replaceState({}, "", url.toString()); + } + }); + } + } + }, [searchParams, searchSpaceId, refetchAllConnectors]); + + // Handle starting indexing + const handleStartIndexing = useCallback(async () => { + if (!indexingConfig || !searchSpaceId) return; + + setIsStartingIndexing(true); + try { + const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined; + const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined; + + // Update periodic indexing settings if enabled + if (periodicEnabled) { + const frequency = parseInt(frequencyMinutes, 10); + await updateConnector({ + id: indexingConfig.connectorId, + data: { + periodic_indexing_enabled: true, + indexing_frequency_minutes: frequency, + }, + }); + } + + await indexConnector({ + connector_id: indexingConfig.connectorId, + queryParams: { + search_space_id: searchSpaceId, + start_date: startDateStr, + end_date: endDateStr, + }, + }); + + toast.success(`${indexingConfig.connectorTitle} indexing started`, { + description: periodicEnabled + ? `Periodic sync enabled every ${getFrequencyLabel(frequencyMinutes)}.` + : "You can continue working while we sync your data.", + }); + + // Close the config view and reset state + setIndexingConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + + // Clear config-related URL params and switch to active tab + const url = new URL(window.location.href); + url.searchParams.delete("view"); + url.searchParams.delete("connector"); + url.searchParams.set("tab", "active"); + window.history.replaceState({}, "", url.toString()); + setActiveTab("active"); + + // Refresh connectors list + refreshConnectors(); + queryClient.invalidateQueries({ + queryKey: cacheKeys.logs.summary(Number(searchSpaceId)), + }); + } catch (error) { + console.error("Error starting indexing:", error); + toast.error("Failed to start indexing"); + } finally { + setIsStartingIndexing(false); + } + }, [indexingConfig, searchSpaceId, startDate, endDate, indexConnector, updateConnector, periodicEnabled, frequencyMinutes, refreshConnectors, getFrequencyLabel]); + + // Handle skipping indexing for now + const handleSkipIndexing = useCallback(() => { + setIndexingConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + + // Clear config-related URL params + const url = new URL(window.location.href); + url.searchParams.delete("view"); + url.searchParams.delete("connector"); + window.history.replaceState({}, "", url.toString()); + }, []); + + // Quick date range handlers + const handleLast30Days = useCallback(() => { + const today = new Date(); + setStartDate(subDays(today, 30)); + setEndDate(today); + }, []); + + const handleLastYear = useCallback(() => { + const today = new Date(); + setStartDate(subYears(today, 1)); + setEndDate(today); + }, []); + + const handleClearDates = useCallback(() => { + setStartDate(undefined); + setEndDate(undefined); + }, []); // Get document types that have documents in the search space const activeDocumentTypes = documentTypeCounts - ? Object.entries(documentTypeCounts).filter(([_, count]) => count > 0) + ? Object.entries(documentTypeCounts).filter(([, count]) => count > 0) : []; const hasConnectors = connectors.length > 0; @@ -257,32 +482,43 @@ export const ConnectorIndicator: FC = () => { (open: boolean) => { setIsOpen(open); - const currentPath = window.location.pathname; - const basePath = currentPath.split("/connectors")[0].replace(/\/$/, ""); - if (open) { - // Base state is /connectors/all - const newUrl = `${basePath}/connectors/${activeTab}`; - window.history.pushState({ modal: true }, "", newUrl); + // Add modal query params to current URL + const url = new URL(window.location.href); + url.searchParams.set("modal", "connectors"); + url.searchParams.set("tab", activeTab); + window.history.pushState({ modal: true }, "", url.toString()); } else { - // Return to base chat path - window.history.pushState({ modal: false }, "", basePath || "/"); + // Remove modal query params from URL + const url = new URL(window.location.href); + url.searchParams.delete("modal"); + url.searchParams.delete("tab"); + url.searchParams.delete("success"); + url.searchParams.delete("connector"); + url.searchParams.delete("view"); + window.history.pushState({ modal: false }, "", url.toString()); setIsScrolled(false); setSearchQuery(""); + // Reset indexing config when closing + if (!isStartingIndexing) { + setIndexingConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + } } }, - [activeTab] + [activeTab, isStartingIndexing] ); const handleTabChange = useCallback( (value: string) => { setActiveTab(value); - const currentPath = window.location.pathname; - const basePath = currentPath.split("/connectors")[0].replace(/\/$/, ""); - - // Update URL to reflect the new tab state - const newUrl = `${basePath}/connectors/${value}`; - window.history.replaceState({ modal: true }, "", newUrl); + // Update tab query param + const url = new URL(window.location.href); + url.searchParams.set("tab", value); + window.history.replaceState({ modal: true }, "", url.toString()); }, [] ); @@ -323,8 +559,215 @@ export const ConnectorIndicator: FC = () => { )} - - + + {/* Indexing Configuration View - shown after OAuth success */} + {indexingConfig ? ( +
+ {/* Fixed Header */} +
+ {/* Back button */} + + + {/* Success header */} +
+
+ +
+
+

+ {indexingConfig.connectorTitle} Connected! +

+

+ Configure when to start syncing your data +

+
+
+
+ + {/* Scrollable Content */} +
+
+
+

Select Date Range

+

+ Choose how far back you want to sync your data. You can always re-index later with different dates. +

+ +
+ {/* Start Date */} +
+ + + + + + + date > new Date()} + /> + + +
+ + {/* End Date */} +
+ + + + + + + date > new Date() || (startDate ? date < startDate : false)} + /> + + +
+
+ + {/* Quick date range buttons */} +
+ + + +
+
+ + {/* Periodic Indexing Configuration */} +
+
+
+

Enable Periodic Sync

+

+ Automatically re-index at regular intervals +

+
+ +
+ + {periodicEnabled && ( +
+
+ + +
+
+ )} +
+ + {/* Info box */} +
+
+ {getConnectorIcon(indexingConfig.connectorType, "size-4")} +
+
+

Indexing runs in the background

+

+ You can continue using SurfSense while we sync your data. Check the Active tab to see progress. +

+
+
+
+
+ + {/* Fixed Footer - Action buttons */} +
+ + +
+
+ ) : ( + {/* Header */}
{ -
+
{ setSearchQuery(e.target.value)} /> @@ -397,9 +840,9 @@ export const ConnectorIndicator: FC = () => { return (
-
+
{getConnectorIcon(connector.connectorType, "size-6")}
@@ -455,9 +898,9 @@ export const ConnectorIndicator: FC = () => { -
+
{getConnectorIcon(connector.connectorType, "size-6")}
@@ -490,9 +933,9 @@ export const ConnectorIndicator: FC = () => { {activeDocumentTypes.map(([docType, count]) => (
-
+
{getConnectorIcon(docType, "size-6")}
@@ -505,25 +948,64 @@ export const ConnectorIndicator: FC = () => {
))} - {connectors.map((connector) => ( -
-
- {getConnectorIcon(connector.connector_type, "size-6")} + {connectors.map((connector) => { + const isIndexing = indexingConnectorIds.has(connector.id); + const activeTask = logsSummary?.active_tasks?.find( + (task) => task.source?.includes(`connector_${connector.id}`) || task.source?.includes(`connector-${connector.id}`) + ); + + return ( +
+
+ {getConnectorIcon(connector.connector_type, "size-6")} +
+
+

+ {connector.name} +

+ {isIndexing ? ( +

+ + Indexing... + {activeTask?.message && ( + + • {activeTask.message} + + )} +

+ ) : ( +

+ {connector.last_indexed_at + ? `Last indexed: ${format(new Date(connector.last_indexed_at), "MMM d, yyyy")}` + : "Never indexed"} +

+ )} +
+
-
-

- {connector.name} -

-

Status: Active

-
- -
- ))} + ); + })}
) : ( @@ -547,6 +1029,7 @@ export const ConnectorIndicator: FC = () => {
+ )} ); From 32b4a09c0e803e1a2f66bbac8150da7108bb811b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 30 Dec 2025 20:07:15 +0530 Subject: [PATCH 48/92] feat: Refactor connector popup to include new components for active and all connectors, enhance indexing configuration view, and improve date range selection functionality. --- .../assistant-ui/connector-popup.tsx | 1022 ++--------------- .../connector-popup/active-connectors-tab.tsx | 152 +++ .../connector-popup/all-connectors-tab.tsx | 125 ++ .../connector-popup/connector-card.tsx | 66 ++ .../connector-popup/connector-constants.ts | 134 +++ .../connector-dialog-header.tsx | 89 ++ .../connector-popup/date-range-selector.tsx | 140 +++ .../assistant-ui/connector-popup/index.ts | 19 + .../indexing-configuration-view.tsx | 121 ++ .../connector-popup/periodic-sync-config.tsx | 65 ++ .../connector-popup/use-connector-dialog.ts | 295 +++++ 11 files changed, 1298 insertions(+), 930 deletions(-) create mode 100644 surfsense_web/components/assistant-ui/connector-popup/active-connectors-tab.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/all-connectors-tab.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-card.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-constants.ts create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-dialog-header.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/date-range-selector.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/index.ts create mode 100644 surfsense_web/components/assistant-ui/connector-popup/indexing-configuration-view.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/periodic-sync-config.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index e55c13448..1e9e09869 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -1,197 +1,30 @@ "use client"; -import { format, subDays, subYears } from "date-fns"; import { useAtomValue } from "jotai"; -import { - ArrowLeft, - Cable, - Calendar as CalendarIcon, - Check, - ChevronRight, - Loader2, - Search, -} from "lucide-react"; -import Link from "next/link"; -import { useRouter, useSearchParams } from "next/navigation"; -import { type FC, useCallback, useEffect, useMemo, useState } from "react"; -import { toast } from "sonner"; -import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; -import { indexConnectorMutationAtom, updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; -import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; +import { Cable, Loader2 } from "lucide-react"; +import { type FC, useMemo } from "react"; import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; +import { useLogsSummary } from "@/hooks/use-logs"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; import { Dialog, DialogContent, - DialogDescription, - DialogHeader, - DialogTitle, } from "@/components/ui/dialog"; -import { Button } from "@/components/ui/button"; -import { Calendar } from "@/components/ui/calendar"; -import { Label } from "@/components/ui/label"; -import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; -import { Switch } from "@/components/ui/switch"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { EnumConnectorName } from "@/contracts/enums/connector"; -import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import type { SearchSourceConnector } from "@/contracts/types/connector.types"; -import { useLogsSummary } from "@/hooks/use-logs"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; -import { authenticatedFetch } from "@/lib/auth-utils"; -import { queryClient } from "@/lib/query-client/client"; -import { cacheKeys } from "@/lib/query-client/cache-keys"; -import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; -import { cn } from "@/lib/utils"; - -// Type for the indexing configuration state -interface IndexingConfigState { - connectorType: string; - connectorId: number; - connectorTitle: string; -} - -// OAuth Connectors (Quick Connect) -const OAUTH_CONNECTORS = [ - { - id: "google-gmail-connector", - title: "Gmail", - description: "Search through your emails", - connectorType: EnumConnectorName.GOOGLE_GMAIL_CONNECTOR, - authEndpoint: "/api/v1/auth/google/gmail/connector/add/", - }, - { - id: "google-calendar-connector", - title: "Google Calendar", - description: "Search through your events", - connectorType: EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR, - authEndpoint: "/api/v1/auth/google/calendar/connector/add/", - }, - { - id: "airtable-connector", - title: "Airtable", - description: "Search your Airtable bases", - connectorType: EnumConnectorName.AIRTABLE_CONNECTOR, - authEndpoint: "/api/v1/auth/airtable/connector/add/", - }, -]; - -// Non-OAuth Connectors -const OTHER_CONNECTORS = [ - { - id: "slack-connector", - title: "Slack", - description: "Search Slack messages", - connectorType: EnumConnectorName.SLACK_CONNECTOR, - }, - { - id: "discord-connector", - title: "Discord", - description: "Search Discord messages", - connectorType: EnumConnectorName.DISCORD_CONNECTOR, - }, - { - id: "notion-connector", - title: "Notion", - description: "Search Notion pages", - connectorType: EnumConnectorName.NOTION_CONNECTOR, - }, - { - id: "confluence-connector", - title: "Confluence", - description: "Search documentation", - connectorType: EnumConnectorName.CONFLUENCE_CONNECTOR, - }, - { - id: "bookstack-connector", - title: "BookStack", - description: "Search BookStack docs", - connectorType: EnumConnectorName.BOOKSTACK_CONNECTOR, - }, - { - id: "github-connector", - title: "GitHub", - description: "Search repositories", - connectorType: EnumConnectorName.GITHUB_CONNECTOR, - }, - { - id: "linear-connector", - title: "Linear", - description: "Search issues & projects", - connectorType: EnumConnectorName.LINEAR_CONNECTOR, - }, - { - id: "jira-connector", - title: "Jira", - description: "Search Jira issues", - connectorType: EnumConnectorName.JIRA_CONNECTOR, - }, - { - id: "clickup-connector", - title: "ClickUp", - description: "Search ClickUp tasks", - connectorType: EnumConnectorName.CLICKUP_CONNECTOR, - }, - { - id: "luma-connector", - title: "Luma", - description: "Search Luma events", - connectorType: EnumConnectorName.LUMA_CONNECTOR, - }, - { - id: "elasticsearch-connector", - title: "Elasticsearch", - description: "Search ES indexes", - connectorType: EnumConnectorName.ELASTICSEARCH_CONNECTOR, - }, - { - id: "webcrawler-connector", - title: "Web Pages", - description: "Crawl web content", - connectorType: EnumConnectorName.WEBCRAWLER_CONNECTOR, - }, - { - id: "tavily-api", - title: "Tavily AI", - description: "Search with Tavily", - connectorType: EnumConnectorName.TAVILY_API, - }, - { - id: "searxng", - title: "SearxNG", - description: "Search with SearxNG", - connectorType: EnumConnectorName.SEARXNG_API, - }, - { - id: "linkup-api", - title: "Linkup API", - description: "Search with Linkup", - connectorType: EnumConnectorName.LINKUP_API, - }, - { - id: "baidu-search-api", - title: "Baidu Search", - description: "Search with Baidu", - connectorType: EnumConnectorName.BAIDU_SEARCH_API, - }, -]; - import { Tabs, TabsContent, - TabsList, - TabsTrigger, } from "@/components/ui/tabs"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; +import { cn } from "@/lib/utils"; +import { AllConnectorsTab } from "./connector-popup/all-connectors-tab"; +import { ActiveConnectorsTab } from "./connector-popup/active-connectors-tab"; +import { ConnectorDialogHeader } from "./connector-popup/connector-dialog-header"; +import { IndexingConfigurationView } from "./connector-popup/indexing-configuration-view"; +import { useConnectorDialog } from "./connector-popup/use-connector-dialog"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; export const ConnectorIndicator: FC = () => { - const router = useRouter(); - const searchParams = useSearchParams(); const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); const { connectors, isLoading: connectorsLoading, refreshConnectors } = useSearchSourceConnectors( false, @@ -199,37 +32,6 @@ export const ConnectorIndicator: FC = () => { ); const { data: documentTypeCounts, isLoading: documentTypesLoading } = useAtomValue(documentTypeCountsAtom); - const { data: allConnectors, refetch: refetchAllConnectors } = useAtomValue(connectorsAtom); - const { mutateAsync: indexConnector } = useAtomValue(indexConnectorMutationAtom); - const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom); - const [isOpen, setIsOpen] = useState(false); - const [activeTab, setActiveTab] = useState("all"); - const [connectingId, setConnectingId] = useState(null); - const [isScrolled, setIsScrolled] = useState(false); - const [searchQuery, setSearchQuery] = useState(""); - - // Indexing configuration state (shown after OAuth success) - const [indexingConfig, setIndexingConfig] = useState(null); - const [startDate, setStartDate] = useState(undefined); - const [endDate, setEndDate] = useState(undefined); - const [isStartingIndexing, setIsStartingIndexing] = useState(false); - - // Periodic indexing state - const [periodicEnabled, setPeriodicEnabled] = useState(false); - const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); // Default: daily - - // Helper function to get frequency label - const getFrequencyLabel = useCallback((minutes: string): string => { - switch (minutes) { - case "15": return "15 minutes"; - case "60": return "hour"; - case "360": return "6 hours"; - case "720": return "12 hours"; - case "1440": return "day"; - case "10080": return "week"; - default: return `${minutes} minutes`; - } - }, []); // Track active indexing tasks const { summary: logsSummary } = useLogsSummary( @@ -248,7 +50,6 @@ export const ConnectorIndicator: FC = () => { logsSummary.active_tasks .filter((task) => task.source?.includes("connector_indexing")) .map((task) => { - // Extract connector ID from task metadata or source const match = task.source?.match(/connector[_-]?(\d+)/i); return match ? parseInt(match[1], 10) : null; }) @@ -258,172 +59,32 @@ export const ConnectorIndicator: FC = () => { const isLoading = connectorsLoading || documentTypesLoading; - // Synchronize state with URL query params - useEffect(() => { - const modalParam = searchParams.get("modal"); - const tabParam = searchParams.get("tab"); - const viewParam = searchParams.get("view"); - const connectorParam = searchParams.get("connector"); - - if (modalParam === "connectors") { - if (!isOpen) setIsOpen(true); - - // Detect tab from URL query param - if (tabParam === "active" || tabParam === "all") { - if (activeTab !== tabParam) setActiveTab(tabParam); - } - - // Restore indexing config view from URL if present (e.g., on page refresh) - if (viewParam === "configure" && connectorParam && !indexingConfig) { - const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === connectorParam); - if (oauthConnector && allConnectors) { - const existingConnector = allConnectors.find( - (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType - ); - if (existingConnector) { - setIndexingConfig({ - connectorType: oauthConnector.connectorType, - connectorId: existingConnector.id, - connectorTitle: oauthConnector.title, - }); - } - } - } - } else { - if (isOpen) setIsOpen(false); - } - }, [searchParams, isOpen, activeTab, indexingConfig, allConnectors]); - - // Detect OAuth success and transition to config view - useEffect(() => { - const success = searchParams.get("success"); - const connectorParam = searchParams.get("connector"); - const modalParam = searchParams.get("modal"); - - if (success === "true" && connectorParam && searchSpaceId && modalParam === "connectors") { - // Find the OAuth connector info - const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === connectorParam); - if (oauthConnector) { - // Refetch connectors to get the newly created connector - refetchAllConnectors().then((result) => { - const newConnector = result.data?.find( - (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType - ); - if (newConnector) { - setIndexingConfig({ - connectorType: oauthConnector.connectorType, - connectorId: newConnector.id, - connectorTitle: oauthConnector.title, - }); - setIsOpen(true); - // Update URL to reflect config view (replace success with view=configure) - const url = new URL(window.location.href); - url.searchParams.delete("success"); - url.searchParams.set("view", "configure"); - // Keep connector param for URL restoration - window.history.replaceState({}, "", url.toString()); - } - }); - } - } - }, [searchParams, searchSpaceId, refetchAllConnectors]); - - // Handle starting indexing - const handleStartIndexing = useCallback(async () => { - if (!indexingConfig || !searchSpaceId) return; - - setIsStartingIndexing(true); - try { - const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined; - const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined; - - // Update periodic indexing settings if enabled - if (periodicEnabled) { - const frequency = parseInt(frequencyMinutes, 10); - await updateConnector({ - id: indexingConfig.connectorId, - data: { - periodic_indexing_enabled: true, - indexing_frequency_minutes: frequency, - }, - }); - } - - await indexConnector({ - connector_id: indexingConfig.connectorId, - queryParams: { - search_space_id: searchSpaceId, - start_date: startDateStr, - end_date: endDateStr, - }, - }); - - toast.success(`${indexingConfig.connectorTitle} indexing started`, { - description: periodicEnabled - ? `Periodic sync enabled every ${getFrequencyLabel(frequencyMinutes)}.` - : "You can continue working while we sync your data.", - }); - - // Close the config view and reset state - setIndexingConfig(null); - setStartDate(undefined); - setEndDate(undefined); - setPeriodicEnabled(false); - setFrequencyMinutes("1440"); - - // Clear config-related URL params and switch to active tab - const url = new URL(window.location.href); - url.searchParams.delete("view"); - url.searchParams.delete("connector"); - url.searchParams.set("tab", "active"); - window.history.replaceState({}, "", url.toString()); - setActiveTab("active"); - - // Refresh connectors list - refreshConnectors(); - queryClient.invalidateQueries({ - queryKey: cacheKeys.logs.summary(Number(searchSpaceId)), - }); - } catch (error) { - console.error("Error starting indexing:", error); - toast.error("Failed to start indexing"); - } finally { - setIsStartingIndexing(false); - } - }, [indexingConfig, searchSpaceId, startDate, endDate, indexConnector, updateConnector, periodicEnabled, frequencyMinutes, refreshConnectors, getFrequencyLabel]); - - // Handle skipping indexing for now - const handleSkipIndexing = useCallback(() => { - setIndexingConfig(null); - setStartDate(undefined); - setEndDate(undefined); - setPeriodicEnabled(false); - setFrequencyMinutes("1440"); - - // Clear config-related URL params - const url = new URL(window.location.href); - url.searchParams.delete("view"); - url.searchParams.delete("connector"); - window.history.replaceState({}, "", url.toString()); - }, []); - - // Quick date range handlers - const handleLast30Days = useCallback(() => { - const today = new Date(); - setStartDate(subDays(today, 30)); - setEndDate(today); - }, []); - - const handleLastYear = useCallback(() => { - const today = new Date(); - setStartDate(subYears(today, 1)); - setEndDate(today); - }, []); - - const handleClearDates = useCallback(() => { - setStartDate(undefined); - setEndDate(undefined); - }, []); + // Use the custom hook for dialog state management + const { + isOpen, + activeTab, + connectingId, + isScrolled, + searchQuery, + indexingConfig, + startDate, + endDate, + isStartingIndexing, + periodicEnabled, + frequencyMinutes, + allConnectors, + setSearchQuery, + setStartDate, + setEndDate, + setPeriodicEnabled, + setFrequencyMinutes, + handleOpenChange, + handleTabChange, + handleScroll, + handleConnectOAuth, + handleStartIndexing, + handleSkipIndexing, + } = useConnectorDialog(); // Get document types that have documents in the search space const activeDocumentTypes = documentTypeCounts @@ -439,94 +100,6 @@ export const ConnectorIndicator: FC = () => { (allConnectors || []).map((c: SearchSourceConnector) => c.connector_type) ); - // Filter connectors based on search - const filteredOAuth = OAUTH_CONNECTORS.filter(c => - c.title.toLowerCase().includes(searchQuery.toLowerCase()) || - c.description.toLowerCase().includes(searchQuery.toLowerCase()) - ); - - const filteredOther = OTHER_CONNECTORS.filter(c => - c.title.toLowerCase().includes(searchQuery.toLowerCase()) || - c.description.toLowerCase().includes(searchQuery.toLowerCase()) - ); - - // Handle OAuth connection - const handleConnectOAuth = useCallback( - async (connector: (typeof OAUTH_CONNECTORS)[0]) => { - if (!searchSpaceId || !connector.authEndpoint) return; - - try { - setConnectingId(connector.id); - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}${connector.authEndpoint}?space_id=${searchSpaceId}`, - { method: "GET" } - ); - - if (!response.ok) { - throw new Error(`Failed to initiate ${connector.title} OAuth`); - } - - const data = await response.json(); - window.location.href = data.auth_url; - } catch (error) { - console.error(`Error connecting to ${connector.title}:`, error); - toast.error(`Failed to connect to ${connector.title}`); - } finally { - setConnectingId(null); - } - }, - [searchSpaceId] - ); - - const handleOpenChange = useCallback( - (open: boolean) => { - setIsOpen(open); - - if (open) { - // Add modal query params to current URL - const url = new URL(window.location.href); - url.searchParams.set("modal", "connectors"); - url.searchParams.set("tab", activeTab); - window.history.pushState({ modal: true }, "", url.toString()); - } else { - // Remove modal query params from URL - const url = new URL(window.location.href); - url.searchParams.delete("modal"); - url.searchParams.delete("tab"); - url.searchParams.delete("success"); - url.searchParams.delete("connector"); - url.searchParams.delete("view"); - window.history.pushState({ modal: false }, "", url.toString()); - setIsScrolled(false); - setSearchQuery(""); - // Reset indexing config when closing - if (!isStartingIndexing) { - setIndexingConfig(null); - setStartDate(undefined); - setEndDate(undefined); - setPeriodicEnabled(false); - setFrequencyMinutes("1440"); - } - } - }, - [activeTab, isStartingIndexing] - ); - - const handleTabChange = useCallback( - (value: string) => { - setActiveTab(value); - // Update tab query param - const url = new URL(window.location.href); - url.searchParams.set("tab", value); - window.history.replaceState({ modal: true }, "", url.toString()); - }, - [] - ); - - const handleScroll = useCallback((e: React.UIEvent) => { - setIsScrolled(e.currentTarget.scrollTop > 0); - }, []); - if (!searchSpaceId) return null; return ( @@ -559,477 +132,66 @@ export const ConnectorIndicator: FC = () => { )} - - {/* Indexing Configuration View - shown after OAuth success */} - {indexingConfig ? ( -
- {/* Fixed Header */} -
- {/* Back button */} - + + {/* Indexing Configuration View - shown after OAuth success */} + {indexingConfig ? ( + handleStartIndexing(refreshConnectors)} + onSkip={handleSkipIndexing} + /> + ) : ( + + {/* Header */} + - {/* Success header */} -
-
- -
-
-

- {indexingConfig.connectorTitle} Connected! -

-

- Configure when to start syncing your data -

-
-
-
+ {/* Content */} +
+
+
+ + + - {/* Scrollable Content */} -
-
-
-

Select Date Range

-

- Choose how far back you want to sync your data. You can always re-index later with different dates. -

- -
- {/* Start Date */} -
- - - - - - - date > new Date()} - /> - - -
- - {/* End Date */} -
- - - - - - - date > new Date() || (startDate ? date < startDate : false)} - /> - - -
-
- - {/* Quick date range buttons */} -
- - - -
-
- - {/* Periodic Indexing Configuration */} -
-
-
-

Enable Periodic Sync

-

- Automatically re-index at regular intervals -

-
- -
- - {periodicEnabled && ( -
-
- - -
-
- )} -
- - {/* Info box */} -
-
- {getConnectorIcon(indexingConfig.connectorType, "size-4")} -
-
-

Indexing runs in the background

-

- You can continue using SurfSense while we sync your data. Check the Active tab to see progress. -

-
-
-
-
- - {/* Fixed Footer - Action buttons */} -
- - -
-
- ) : ( - - {/* Header */} -
- - Connectors - - Search across all your apps and data in one place. - - - -
- - - All Connectors - - - - Active - - - {totalSourceCount > 0 && ( - - {totalSourceCount} - - )} - - - -
-
- - setSearchQuery(e.target.value)} +
+ {/* Bottom fade shadow */} +
-
- - {/* Content */} -
-
-
- - {/* Quick Connect */} - {filteredOAuth.length > 0 && ( -
-
-

Quick Connect

-
-
- {filteredOAuth.map((connector) => { - const isConnected = connectedTypes.has(connector.connectorType); - const isConnecting = connectingId === connector.id; - - return ( -
-
- {getConnectorIcon(connector.connectorType, "size-6")} -
-
-
- {connector.title} - {isConnected && ( - - )} -
-

- {isConnected ? "Connected" : connector.description} -

-
- -
- ); - })} -
-
- )} - - {/* More Integrations */} - {filteredOther.length > 0 && ( -
-
-

More Integrations

-
-
- {filteredOther.map((connector) => { - const isConnected = connectedTypes.has(connector.connectorType); - - return ( - -
- {getConnectorIcon(connector.connectorType, "size-6")} -
-
-
- {connector.title} - {isConnected && ( - - )} -
-

- {connector.description} -

-
- - - ); - })} -
-
- )} - - - - {hasSources ? ( -
-
-

Currently Active

-
-
- {activeDocumentTypes.map(([docType, count]) => ( -
-
- {getConnectorIcon(docType, "size-6")} -
-
-

- {getDocumentTypeLabel(docType)} -

-

- {count as number} documents indexed -

-
-
- ))} - {connectors.map((connector) => { - const isIndexing = indexingConnectorIds.has(connector.id); - const activeTask = logsSummary?.active_tasks?.find( - (task) => task.source?.includes(`connector_${connector.id}`) || task.source?.includes(`connector-${connector.id}`) - ); - - return ( -
-
- {getConnectorIcon(connector.connector_type, "size-6")} -
-
-

- {connector.name} -

- {isIndexing ? ( -

- - Indexing... - {activeTask?.message && ( - - • {activeTask.message} - - )} -

- ) : ( -

- {connector.last_indexed_at - ? `Last indexed: ${format(new Date(connector.last_indexed_at), "MMM d, yyyy")}` - : "Never indexed"} -

- )} -
- -
- ); - })} -
-
- ) : ( -
-
- -
-

No active sources

-

- Connect your first service to start searching across all your data. -

- - Browse available connectors - -
- )} - -
-
- {/* Bottom fade shadow */} -
-
- - )} + + )} ); diff --git a/surfsense_web/components/assistant-ui/connector-popup/active-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/active-connectors-tab.tsx new file mode 100644 index 000000000..323fa34e7 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/active-connectors-tab.tsx @@ -0,0 +1,152 @@ +"use client"; + +import { format } from "date-fns"; +import { Cable, Loader2 } from "lucide-react"; +import { useRouter } from "next/navigation"; +import type { FC } from "react"; +import { Button } from "@/components/ui/button"; +import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import { cn } from "@/lib/utils"; +import { + TabsContent, + TabsTrigger, +} from "@/components/ui/tabs"; + +interface ActiveConnectorsTabProps { + hasSources: boolean; + totalSourceCount: number; + activeDocumentTypes: Array<[string, number]>; + connectors: SearchSourceConnector[]; + indexingConnectorIds: Set; + logsSummary: any; + searchSpaceId: string; + onTabChange: (value: string) => void; +} + +export const ActiveConnectorsTab: FC = ({ + hasSources, + activeDocumentTypes, + connectors, + indexingConnectorIds, + logsSummary, + searchSpaceId, + onTabChange, +}) => { + const router = useRouter(); + + return ( + + {hasSources ? ( +
+
+

+ Currently Active +

+
+
+ {activeDocumentTypes.map(([docType, count]) => ( +
+
+ {getConnectorIcon(docType, "size-6")} +
+
+

+ {getDocumentTypeLabel(docType)} +

+

+ {count as number} documents indexed +

+
+
+ ))} + {connectors.map((connector) => { + const isIndexing = indexingConnectorIds.has(connector.id); + const activeTask = logsSummary?.active_tasks?.find( + (task: any) => + task.source?.includes(`connector_${connector.id}`) || + task.source?.includes(`connector-${connector.id}`) + ); + + return ( +
+
+ {getConnectorIcon(connector.connector_type, "size-6")} +
+
+

+ {connector.name} +

+ {isIndexing ? ( +

+ + Indexing... + {activeTask?.message && ( + + • {activeTask.message} + + )} +

+ ) : ( +

+ {connector.last_indexed_at + ? `Last indexed: ${format(new Date(connector.last_indexed_at), "MMM d, yyyy")}` + : "Never indexed"} +

+ )} +
+ +
+ ); + })} +
+
+ ) : ( +
+
+ +
+

No active sources

+

+ Connect your first service to start searching across all your data. +

+ onTabChange("all")}> + Browse available connectors + +
+ )} + + ); +}; + diff --git a/surfsense_web/components/assistant-ui/connector-popup/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/all-connectors-tab.tsx new file mode 100644 index 000000000..e2f735b13 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/all-connectors-tab.tsx @@ -0,0 +1,125 @@ +"use client"; + +import { ChevronRight } from "lucide-react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { type FC } from "react"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "./connector-constants"; +import { ConnectorCard } from "./connector-card"; + +interface AllConnectorsTabProps { + searchQuery: string; + searchSpaceId: string; + connectedTypes: Set; + connectingId: string | null; + onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[0]) => void; +} + +export const AllConnectorsTab: FC = ({ + searchQuery, + searchSpaceId, + connectedTypes, + connectingId, + onConnectOAuth, +}) => { + const router = useRouter(); + + // Filter connectors based on search + const filteredOAuth = OAUTH_CONNECTORS.filter( + (c) => + c.title.toLowerCase().includes(searchQuery.toLowerCase()) || + c.description.toLowerCase().includes(searchQuery.toLowerCase()) + ); + + const filteredOther = OTHER_CONNECTORS.filter( + (c) => + c.title.toLowerCase().includes(searchQuery.toLowerCase()) || + c.description.toLowerCase().includes(searchQuery.toLowerCase()) + ); + + return ( +
+ {/* Quick Connect */} + {filteredOAuth.length > 0 && ( +
+
+

+ Quick Connect +

+
+
+ {filteredOAuth.map((connector) => { + const isConnected = connectedTypes.has(connector.connectorType); + const isConnecting = connectingId === connector.id; + + return ( + onConnectOAuth(connector)} + onManage={() => + router.push( + `/dashboard/${searchSpaceId}/connectors/add/${connector.id}` + ) + } + /> + ); + })} +
+
+ )} + + {/* More Integrations */} + {filteredOther.length > 0 && ( +
+
+

+ More Integrations +

+
+
+ {filteredOther.map((connector) => { + const isConnected = connectedTypes.has(connector.connectorType); + + return ( + +
+ {getConnectorIcon(connector.connectorType, "size-6")} +
+
+
+ + {connector.title} + + {isConnected && ( + + )} +
+

+ {connector.description} +

+
+ + + ); + })} +
+
+ )} +
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-card.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-card.tsx new file mode 100644 index 000000000..d1f79b16d --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-card.tsx @@ -0,0 +1,66 @@ +"use client"; + +import { Loader2 } from "lucide-react"; +import { type FC } from "react"; +import { Button } from "@/components/ui/button"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; + +interface ConnectorCardProps { + id: string; + title: string; + description: string; + connectorType: string; + isConnected?: boolean; + isConnecting?: boolean; + onConnect?: () => void; + onManage?: () => void; +} + +export const ConnectorCard: FC = ({ + id, + title, + description, + connectorType, + isConnected = false, + isConnecting = false, + onConnect, + onManage, +}) => { + return ( +
+
+ {getConnectorIcon(connectorType, "size-6")} +
+
+
+ {title} + {isConnected && ( + + )} +
+

+ {isConnected ? "Connected" : description} +

+
+ +
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-constants.ts b/surfsense_web/components/assistant-ui/connector-popup/connector-constants.ts new file mode 100644 index 000000000..65d5bd516 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-constants.ts @@ -0,0 +1,134 @@ +import { EnumConnectorName } from "@/contracts/enums/connector"; + +// OAuth Connectors (Quick Connect) +export const OAUTH_CONNECTORS = [ + { + id: "google-gmail-connector", + title: "Gmail", + description: "Search through your emails", + connectorType: EnumConnectorName.GOOGLE_GMAIL_CONNECTOR, + authEndpoint: "/api/v1/auth/google/gmail/connector/add/", + }, + { + id: "google-calendar-connector", + title: "Google Calendar", + description: "Search through your events", + connectorType: EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR, + authEndpoint: "/api/v1/auth/google/calendar/connector/add/", + }, + { + id: "airtable-connector", + title: "Airtable", + description: "Search your Airtable bases", + connectorType: EnumConnectorName.AIRTABLE_CONNECTOR, + authEndpoint: "/api/v1/auth/airtable/connector/add/", + }, +] as const; + +// Non-OAuth Connectors +export const OTHER_CONNECTORS = [ + { + id: "slack-connector", + title: "Slack", + description: "Search Slack messages", + connectorType: EnumConnectorName.SLACK_CONNECTOR, + }, + { + id: "discord-connector", + title: "Discord", + description: "Search Discord messages", + connectorType: EnumConnectorName.DISCORD_CONNECTOR, + }, + { + id: "notion-connector", + title: "Notion", + description: "Search Notion pages", + connectorType: EnumConnectorName.NOTION_CONNECTOR, + }, + { + id: "confluence-connector", + title: "Confluence", + description: "Search documentation", + connectorType: EnumConnectorName.CONFLUENCE_CONNECTOR, + }, + { + id: "bookstack-connector", + title: "BookStack", + description: "Search BookStack docs", + connectorType: EnumConnectorName.BOOKSTACK_CONNECTOR, + }, + { + id: "github-connector", + title: "GitHub", + description: "Search repositories", + connectorType: EnumConnectorName.GITHUB_CONNECTOR, + }, + { + id: "linear-connector", + title: "Linear", + description: "Search issues & projects", + connectorType: EnumConnectorName.LINEAR_CONNECTOR, + }, + { + id: "jira-connector", + title: "Jira", + description: "Search Jira issues", + connectorType: EnumConnectorName.JIRA_CONNECTOR, + }, + { + id: "clickup-connector", + title: "ClickUp", + description: "Search ClickUp tasks", + connectorType: EnumConnectorName.CLICKUP_CONNECTOR, + }, + { + id: "luma-connector", + title: "Luma", + description: "Search Luma events", + connectorType: EnumConnectorName.LUMA_CONNECTOR, + }, + { + id: "elasticsearch-connector", + title: "Elasticsearch", + description: "Search ES indexes", + connectorType: EnumConnectorName.ELASTICSEARCH_CONNECTOR, + }, + { + id: "webcrawler-connector", + title: "Web Pages", + description: "Crawl web content", + connectorType: EnumConnectorName.WEBCRAWLER_CONNECTOR, + }, + { + id: "tavily-api", + title: "Tavily AI", + description: "Search with Tavily", + connectorType: EnumConnectorName.TAVILY_API, + }, + { + id: "searxng", + title: "SearxNG", + description: "Search with SearxNG", + connectorType: EnumConnectorName.SEARXNG_API, + }, + { + id: "linkup-api", + title: "Linkup API", + description: "Search with Linkup", + connectorType: EnumConnectorName.LINKUP_API, + }, + { + id: "baidu-search-api", + title: "Baidu Search", + description: "Search with Baidu", + connectorType: EnumConnectorName.BAIDU_SEARCH_API, + }, +] as const; + +// Type for the indexing configuration state +export interface IndexingConfigState { + connectorType: string; + connectorId: number; + connectorTitle: string; +} + diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-dialog-header.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-dialog-header.tsx new file mode 100644 index 000000000..d4126f2ce --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-dialog-header.tsx @@ -0,0 +1,89 @@ +"use client"; + +import { Search } from "lucide-react"; +import { type FC } from "react"; +import { + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + TabsList, + TabsTrigger, +} from "@/components/ui/tabs"; +import { cn } from "@/lib/utils"; + +interface ConnectorDialogHeaderProps { + activeTab: string; + totalSourceCount: number; + searchQuery: string; + onTabChange: (value: string) => void; + onSearchChange: (query: string) => void; + isScrolled: boolean; +} + +export const ConnectorDialogHeader: FC = ({ + activeTab, + totalSourceCount, + searchQuery, + onTabChange, + onSearchChange, + isScrolled, +}) => { + return ( +
+ + + Connectors + + + Search across all your apps and data in one place. + + + +
+ + + All Connectors + + + + Active + + + {totalSourceCount > 0 && ( + + {totalSourceCount} + + )} + + + +
+
+ + onSearchChange(e.target.value)} + /> +
+
+
+
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/connector-popup/date-range-selector.tsx b/surfsense_web/components/assistant-ui/connector-popup/date-range-selector.tsx new file mode 100644 index 000000000..1112f3f36 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/date-range-selector.tsx @@ -0,0 +1,140 @@ +"use client"; + +import { format, subDays, subYears } from "date-fns"; +import { Calendar as CalendarIcon } from "lucide-react"; +import type { FC } from "react"; +import { Button } from "@/components/ui/button"; +import { Calendar } from "@/components/ui/calendar"; +import { Label } from "@/components/ui/label"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { cn } from "@/lib/utils"; + +interface DateRangeSelectorProps { + startDate: Date | undefined; + endDate: Date | undefined; + onStartDateChange: (date: Date | undefined) => void; + onEndDateChange: (date: Date | undefined) => void; +} + +export const DateRangeSelector: FC = ({ + startDate, + endDate, + onStartDateChange, + onEndDateChange, +}) => { + const handleLast30Days = () => { + const today = new Date(); + onStartDateChange(subDays(today, 30)); + onEndDateChange(today); + }; + + const handleLastYear = () => { + const today = new Date(); + onStartDateChange(subYears(today, 1)); + onEndDateChange(today); + }; + + const handleClearDates = () => { + onStartDateChange(undefined); + onEndDateChange(undefined); + }; + + return ( +
+

Select Date Range

+

+ Choose how far back you want to sync your data. You can always re-index later with different dates. +

+ +
+ {/* Start Date */} +
+ + + + + + + date > new Date()} + /> + + +
+ + {/* End Date */} +
+ + + + + + + date > new Date() || (startDate ? date < startDate : false)} + /> + + +
+
+ + {/* Quick date range buttons */} +
+ + + +
+
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/connector-popup/index.ts b/surfsense_web/components/assistant-ui/connector-popup/index.ts new file mode 100644 index 000000000..da1d4639e --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/index.ts @@ -0,0 +1,19 @@ +// Main component export +export { ConnectorIndicator } from "../connector-popup"; + +// Sub-components (if needed for external use) +export { ConnectorCard } from "./connector-card"; +export { DateRangeSelector } from "./date-range-selector"; +export { PeriodicSyncConfig } from "./periodic-sync-config"; +export { IndexingConfigurationView } from "./indexing-configuration-view"; +export { ConnectorDialogHeader } from "./connector-dialog-header"; +export { AllConnectorsTab } from "./all-connectors-tab"; +export { ActiveConnectorsTab } from "./active-connectors-tab"; + +// Constants and types +export { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "./connector-constants"; +export type { IndexingConfigState } from "./connector-constants"; + +// Hooks +export { useConnectorDialog } from "./use-connector-dialog"; + diff --git a/surfsense_web/components/assistant-ui/connector-popup/indexing-configuration-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/indexing-configuration-view.tsx new file mode 100644 index 000000000..cb9c3b66b --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/indexing-configuration-view.tsx @@ -0,0 +1,121 @@ +"use client"; + +import { ArrowLeft, Check, Loader2 } from "lucide-react"; +import { type FC } from "react"; +import { Button } from "@/components/ui/button"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import type { IndexingConfigState } from "./connector-constants"; +import { DateRangeSelector } from "./date-range-selector"; +import { PeriodicSyncConfig } from "./periodic-sync-config"; + +interface IndexingConfigurationViewProps { + config: IndexingConfigState; + startDate: Date | undefined; + endDate: Date | undefined; + periodicEnabled: boolean; + frequencyMinutes: string; + isStartingIndexing: boolean; + onStartDateChange: (date: Date | undefined) => void; + onEndDateChange: (date: Date | undefined) => void; + onPeriodicEnabledChange: (enabled: boolean) => void; + onFrequencyChange: (frequency: string) => void; + onStartIndexing: () => void; + onSkip: () => void; +} + +export const IndexingConfigurationView: FC = ({ + config, + startDate, + endDate, + periodicEnabled, + frequencyMinutes, + isStartingIndexing, + onStartDateChange, + onEndDateChange, + onPeriodicEnabledChange, + onFrequencyChange, + onStartIndexing, + onSkip, +}) => { + return ( +
+ {/* Fixed Header */} +
+ {/* Back button */} + + + {/* Success header */} +
+
+ +
+
+

+ {config.connectorTitle} Connected! +

+

+ Configure when to start syncing your data +

+
+
+
+ + {/* Scrollable Content */} +
+
+ + + + + {/* Info box */} +
+
+ {getConnectorIcon(config.connectorType, "size-4")} +
+
+

Indexing runs in the background

+

+ You can continue using SurfSense while we sync your data. Check the Active tab to see progress. +

+
+
+
+
+ + {/* Fixed Footer - Action buttons */} +
+ + +
+
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/connector-popup/periodic-sync-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/periodic-sync-config.tsx new file mode 100644 index 000000000..427a6ac86 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/periodic-sync-config.tsx @@ -0,0 +1,65 @@ +"use client"; + +import { type FC } from "react"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; + +interface PeriodicSyncConfigProps { + enabled: boolean; + frequencyMinutes: string; + onEnabledChange: (enabled: boolean) => void; + onFrequencyChange: (frequency: string) => void; +} + +export const PeriodicSyncConfig: FC = ({ + enabled, + frequencyMinutes, + onEnabledChange, + onFrequencyChange, +}) => { + return ( +
+
+
+

Enable Periodic Sync

+

+ Automatically re-index at regular intervals +

+
+ +
+ + {enabled && ( +
+
+ + +
+
+ )} +
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts new file mode 100644 index 000000000..5a309c2ce --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts @@ -0,0 +1,295 @@ +import { useAtomValue } from "jotai"; +import { useRouter, useSearchParams } from "next/navigation"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { toast } from "sonner"; +import { indexConnectorMutationAtom, updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; +import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; +import { authenticatedFetch } from "@/lib/auth-utils"; +import { queryClient } from "@/lib/query-client/client"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { format } from "date-fns"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import { OAUTH_CONNECTORS } from "./connector-constants"; +import type { IndexingConfigState } from "./connector-constants"; + +export const useConnectorDialog = () => { + const router = useRouter(); + const searchParams = useSearchParams(); + const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); + const { data: allConnectors, refetch: refetchAllConnectors } = useAtomValue(connectorsAtom); + const { mutateAsync: indexConnector } = useAtomValue(indexConnectorMutationAtom); + const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom); + + const [isOpen, setIsOpen] = useState(false); + const [activeTab, setActiveTab] = useState("all"); + const [connectingId, setConnectingId] = useState(null); + const [isScrolled, setIsScrolled] = useState(false); + const [searchQuery, setSearchQuery] = useState(""); + const [indexingConfig, setIndexingConfig] = useState(null); + const [startDate, setStartDate] = useState(undefined); + const [endDate, setEndDate] = useState(undefined); + const [isStartingIndexing, setIsStartingIndexing] = useState(false); + const [periodicEnabled, setPeriodicEnabled] = useState(false); + const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); + + // Helper function to get frequency label + const getFrequencyLabel = useCallback((minutes: string): string => { + switch (minutes) { + case "15": return "15 minutes"; + case "60": return "hour"; + case "360": return "6 hours"; + case "720": return "12 hours"; + case "1440": return "day"; + case "10080": return "week"; + default: return `${minutes} minutes`; + } + }, []); + + // Synchronize state with URL query params + useEffect(() => { + const modalParam = searchParams.get("modal"); + const tabParam = searchParams.get("tab"); + const viewParam = searchParams.get("view"); + const connectorParam = searchParams.get("connector"); + + if (modalParam === "connectors") { + if (!isOpen) setIsOpen(true); + + if (tabParam === "active" || tabParam === "all") { + if (activeTab !== tabParam) setActiveTab(tabParam); + } + + if (viewParam === "configure" && connectorParam && !indexingConfig) { + const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === connectorParam); + if (oauthConnector && allConnectors) { + const existingConnector = allConnectors.find( + (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType + ); + if (existingConnector) { + setIndexingConfig({ + connectorType: oauthConnector.connectorType, + connectorId: existingConnector.id, + connectorTitle: oauthConnector.title, + }); + } + } + } + } else { + if (isOpen) setIsOpen(false); + } + }, [searchParams, isOpen, activeTab, indexingConfig, allConnectors]); + + // Detect OAuth success and transition to config view + useEffect(() => { + const success = searchParams.get("success"); + const connectorParam = searchParams.get("connector"); + const modalParam = searchParams.get("modal"); + + if (success === "true" && connectorParam && searchSpaceId && modalParam === "connectors") { + const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === connectorParam); + if (oauthConnector) { + refetchAllConnectors().then((result) => { + const newConnector = result.data?.find( + (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType + ); + if (newConnector) { + setIndexingConfig({ + connectorType: oauthConnector.connectorType, + connectorId: newConnector.id, + connectorTitle: oauthConnector.title, + }); + setIsOpen(true); + const url = new URL(window.location.href); + url.searchParams.delete("success"); + url.searchParams.set("view", "configure"); + window.history.replaceState({}, "", url.toString()); + } + }); + } + } + }, [searchParams, searchSpaceId, refetchAllConnectors]); + + // Handle OAuth connection + const handleConnectOAuth = useCallback( + async (connector: (typeof OAUTH_CONNECTORS)[0]) => { + if (!searchSpaceId || !connector.authEndpoint) return; + + try { + setConnectingId(connector.id); + const response = await authenticatedFetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}${connector.authEndpoint}?space_id=${searchSpaceId}`, + { method: "GET" } + ); + + if (!response.ok) { + throw new Error(`Failed to initiate ${connector.title} OAuth`); + } + + const data = await response.json(); + window.location.href = data.auth_url; + } catch (error) { + console.error(`Error connecting to ${connector.title}:`, error); + toast.error(`Failed to connect to ${connector.title}`); + } finally { + setConnectingId(null); + } + }, + [searchSpaceId] + ); + + // Handle starting indexing + const handleStartIndexing = useCallback(async (refreshConnectors: () => void) => { + if (!indexingConfig || !searchSpaceId) return; + + setIsStartingIndexing(true); + try { + const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined; + const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined; + + if (periodicEnabled) { + const frequency = parseInt(frequencyMinutes, 10); + await updateConnector({ + id: indexingConfig.connectorId, + data: { + periodic_indexing_enabled: true, + indexing_frequency_minutes: frequency, + }, + }); + } + + await indexConnector({ + connector_id: indexingConfig.connectorId, + queryParams: { + search_space_id: searchSpaceId, + start_date: startDateStr, + end_date: endDateStr, + }, + }); + + toast.success(`${indexingConfig.connectorTitle} indexing started`, { + description: periodicEnabled + ? `Periodic sync enabled every ${getFrequencyLabel(frequencyMinutes)}.` + : "You can continue working while we sync your data.", + }); + + setIndexingConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + + const url = new URL(window.location.href); + url.searchParams.delete("view"); + url.searchParams.delete("connector"); + url.searchParams.set("tab", "active"); + window.history.replaceState({}, "", url.toString()); + setActiveTab("active"); + + refreshConnectors(); + queryClient.invalidateQueries({ + queryKey: cacheKeys.logs.summary(Number(searchSpaceId)), + }); + } catch (error) { + console.error("Error starting indexing:", error); + toast.error("Failed to start indexing"); + } finally { + setIsStartingIndexing(false); + } + }, [indexingConfig, searchSpaceId, startDate, endDate, indexConnector, updateConnector, periodicEnabled, frequencyMinutes, getFrequencyLabel]); + + // Handle skipping indexing + const handleSkipIndexing = useCallback(() => { + setIndexingConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + + const url = new URL(window.location.href); + url.searchParams.delete("view"); + url.searchParams.delete("connector"); + window.history.replaceState({}, "", url.toString()); + }, []); + + // Handle dialog open/close + const handleOpenChange = useCallback( + (open: boolean) => { + setIsOpen(open); + + if (open) { + const url = new URL(window.location.href); + url.searchParams.set("modal", "connectors"); + url.searchParams.set("tab", activeTab); + window.history.pushState({ modal: true }, "", url.toString()); + } else { + const url = new URL(window.location.href); + url.searchParams.delete("modal"); + url.searchParams.delete("tab"); + url.searchParams.delete("success"); + url.searchParams.delete("connector"); + url.searchParams.delete("view"); + window.history.pushState({ modal: false }, "", url.toString()); + setIsScrolled(false); + setSearchQuery(""); + if (!isStartingIndexing) { + setIndexingConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + } + } + }, + [activeTab, isStartingIndexing] + ); + + // Handle tab change + const handleTabChange = useCallback( + (value: string) => { + setActiveTab(value); + const url = new URL(window.location.href); + url.searchParams.set("tab", value); + window.history.replaceState({ modal: true }, "", url.toString()); + }, + [] + ); + + // Handle scroll + const handleScroll = useCallback((e: React.UIEvent) => { + setIsScrolled(e.currentTarget.scrollTop > 0); + }, []); + + return { + // State + isOpen, + activeTab, + connectingId, + isScrolled, + searchQuery, + indexingConfig, + startDate, + endDate, + isStartingIndexing, + periodicEnabled, + frequencyMinutes, + searchSpaceId, + allConnectors, + + // Setters + setSearchQuery, + setStartDate, + setEndDate, + setPeriodicEnabled, + setFrequencyMinutes, + + // Handlers + handleOpenChange, + handleTabChange, + handleScroll, + handleConnectOAuth, + handleStartIndexing, + handleSkipIndexing, + }; +}; + From 2b5377846d5e91f960576505b9c91e852b40152f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 30 Dec 2025 16:38:57 +0200 Subject: [PATCH 49/92] refactor: modularize thread.tsx into focused component modules - Split 1,089-line thread.tsx into 10 smaller, focused modules - Created dedicated files for thinking-steps, welcome, composer, messages, etc. - No breaking changes - all logic preserved exactly as before - Improved code organization and maintainability --- .../assistant-ui/assistant-message.tsx | 118 ++ .../components/assistant-ui/branch-picker.tsx | 33 + .../assistant-ui/composer-action.tsx | 269 +++++ .../components/assistant-ui/composer.tsx | 240 ++++ .../components/assistant-ui/edit-composer.tsx | 27 + .../assistant-ui/thinking-steps.tsx | 207 ++++ .../assistant-ui/thread-scroll-to-bottom.tsx | 19 + .../assistant-ui/thread-welcome.tsx | 72 ++ .../components/assistant-ui/thread.tsx | 1045 +---------------- .../components/assistant-ui/user-message.tsx | 73 ++ 10 files changed, 1067 insertions(+), 1036 deletions(-) create mode 100644 surfsense_web/components/assistant-ui/assistant-message.tsx create mode 100644 surfsense_web/components/assistant-ui/branch-picker.tsx create mode 100644 surfsense_web/components/assistant-ui/composer-action.tsx create mode 100644 surfsense_web/components/assistant-ui/composer.tsx create mode 100644 surfsense_web/components/assistant-ui/edit-composer.tsx create mode 100644 surfsense_web/components/assistant-ui/thinking-steps.tsx create mode 100644 surfsense_web/components/assistant-ui/thread-scroll-to-bottom.tsx create mode 100644 surfsense_web/components/assistant-ui/thread-welcome.tsx create mode 100644 surfsense_web/components/assistant-ui/user-message.tsx diff --git a/surfsense_web/components/assistant-ui/assistant-message.tsx b/surfsense_web/components/assistant-ui/assistant-message.tsx new file mode 100644 index 000000000..62fbe0dd4 --- /dev/null +++ b/surfsense_web/components/assistant-ui/assistant-message.tsx @@ -0,0 +1,118 @@ +import { + ActionBarPrimitive, + AssistantIf, + ErrorPrimitive, + MessagePrimitive, + useAssistantState, +} from "@assistant-ui/react"; +import { CheckIcon, CopyIcon, DownloadIcon, RefreshCwIcon } from "lucide-react"; +import type { FC } from "react"; +import { useContext } from "react"; +import { MarkdownText } from "@/components/assistant-ui/markdown-text"; +import { ThinkingStepsContext, ThinkingStepsDisplay } from "@/components/assistant-ui/thinking-steps"; +import { ToolFallback } from "@/components/assistant-ui/tool-fallback"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; +import { BranchPicker } from "@/components/assistant-ui/branch-picker"; + +export const MessageError: FC = () => { + return ( + + + + + + ); +}; + +/** + * Custom component to render thinking steps from Context + */ +const ThinkingStepsPart: FC = () => { + const thinkingStepsMap = useContext(ThinkingStepsContext); + + // Get the current message ID to look up thinking steps + const messageId = useAssistantState(({ message }) => message?.id); + const thinkingSteps = thinkingStepsMap.get(messageId) || []; + + // Check if this specific message is currently streaming + // A message is streaming if: thread is running AND this is the last assistant message + const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); + const isLastMessage = useAssistantState(({ message }) => message?.isLast ?? false); + const isMessageStreaming = isThreadRunning && isLastMessage; + + if (thinkingSteps.length === 0) return null; + + return ( +
+ +
+ ); +}; + +const AssistantMessageInner: FC = () => { + return ( + <> + {/* Render thinking steps from message content - this ensures proper scroll tracking */} + + +
+ + +
+ +
+ + +
+ + ); +}; + +export const AssistantMessage: FC = () => { + return ( + + + + ); +}; + +const AssistantActionBar: FC = () => { + return ( + + + + message.isCopied}> + + + !message.isCopied}> + + + + + + + + + + + + + + + + ); +}; + diff --git a/surfsense_web/components/assistant-ui/branch-picker.tsx b/surfsense_web/components/assistant-ui/branch-picker.tsx new file mode 100644 index 000000000..1d9041309 --- /dev/null +++ b/surfsense_web/components/assistant-ui/branch-picker.tsx @@ -0,0 +1,33 @@ +import { BranchPickerPrimitive } from "@assistant-ui/react"; +import { ChevronLeftIcon, ChevronRightIcon } from "lucide-react"; +import type { FC } from "react"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; +import { cn } from "@/lib/utils"; + +export const BranchPicker: FC = ({ className, ...rest }) => { + return ( + + + + + + + + / + + + + + + + + ); +}; + diff --git a/surfsense_web/components/assistant-ui/composer-action.tsx b/surfsense_web/components/assistant-ui/composer-action.tsx new file mode 100644 index 000000000..9c5a95d88 --- /dev/null +++ b/surfsense_web/components/assistant-ui/composer-action.tsx @@ -0,0 +1,269 @@ +import { AssistantIf, ComposerPrimitive, useAssistantState } from "@assistant-ui/react"; +import { useAtomValue } from "jotai"; +import { AlertCircle, ArrowUpIcon, Loader2, Plus, Plug2, SquareIcon } from "lucide-react"; +import Link from "next/link"; +import type { FC } from "react"; +import { useCallback, useMemo, useRef, useState } from "react"; +import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; +import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; +import { + globalNewLLMConfigsAtom, + llmPreferencesAtom, + newLLMConfigsAtom, +} from "@/atoms/new-llm-config/new-llm-config-query.atoms"; +import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; +import { ComposerAddAttachment } from "@/components/assistant-ui/attachment"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; +import { Button } from "@/components/ui/button"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; +import { cn } from "@/lib/utils"; +import { ChevronRightIcon } from "lucide-react"; + +const ConnectorIndicator: FC = () => { + const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); + const { connectors, isLoading: connectorsLoading } = useSearchSourceConnectors( + false, + searchSpaceId ? Number(searchSpaceId) : undefined + ); + const { data: documentTypeCounts, isLoading: documentTypesLoading } = + useAtomValue(documentTypeCountsAtom); + const [isOpen, setIsOpen] = useState(false); + const closeTimeoutRef = useRef(null); + + const isLoading = connectorsLoading || documentTypesLoading; + + // Get document types that have documents in the search space + const activeDocumentTypes = documentTypeCounts + ? Object.entries(documentTypeCounts).filter(([_, count]) => count > 0) + : []; + + const hasConnectors = connectors.length > 0; + const hasSources = hasConnectors || activeDocumentTypes.length > 0; + const totalSourceCount = connectors.length + activeDocumentTypes.length; + + const handleMouseEnter = useCallback(() => { + // Clear any pending close timeout + if (closeTimeoutRef.current) { + clearTimeout(closeTimeoutRef.current); + closeTimeoutRef.current = null; + } + setIsOpen(true); + }, []); + + const handleMouseLeave = useCallback(() => { + // Delay closing by 150ms for better UX + closeTimeoutRef.current = setTimeout(() => { + setIsOpen(false); + }, 150); + }, []); + + if (!searchSpaceId) return null; + + return ( + + + + + + {hasSources ? ( +
+
+

Connected Sources

+ + {totalSourceCount} + +
+
+ {/* Document types from the search space */} + {activeDocumentTypes.map(([docType]) => ( +
+ {getConnectorIcon(docType, "size-3.5")} + {getDocumentTypeLabel(docType)} +
+ ))} + {/* Search source connectors */} + {connectors.map((connector) => ( +
+ {getConnectorIcon(connector.connector_type, "size-3.5")} + {connector.name} +
+ ))} +
+
+ + + Add more sources + + +
+
+ ) : ( +
+

No sources yet

+

+ Add documents or connect data sources to enhance search results. +

+ + + Add Connector + +
+ )} + + + ); +}; + +export const ComposerAction: FC = () => { + // Check if any attachments are still being processed (running AND progress < 100) + // When progress is 100, processing is done but waiting for send() + const hasProcessingAttachments = useAssistantState(({ composer }) => + composer.attachments?.some((att) => { + const status = att.status; + if (status?.type !== "running") return false; + const progress = (status as { type: "running"; progress?: number }).progress; + return progress === undefined || progress < 100; + }) + ); + + // Check if composer text is empty + const isComposerEmpty = useAssistantState(({ composer }) => { + const text = composer.text?.trim() || ""; + return text.length === 0; + }); + + // Check if a model is configured + const { data: userConfigs } = useAtomValue(newLLMConfigsAtom); + const { data: globalConfigs } = useAtomValue(globalNewLLMConfigsAtom); + const { data: preferences } = useAtomValue(llmPreferencesAtom); + + const hasModelConfigured = useMemo(() => { + if (!preferences) return false; + const agentLlmId = preferences.agent_llm_id; + if (agentLlmId === null || agentLlmId === undefined) return false; + + // Check if the configured model actually exists + if (agentLlmId < 0) { + return globalConfigs?.some((c) => c.id === agentLlmId) ?? false; + } + return userConfigs?.some((c) => c.id === agentLlmId) ?? false; + }, [preferences, globalConfigs, userConfigs]); + + const isSendDisabled = hasProcessingAttachments || isComposerEmpty || !hasModelConfigured; + + return ( +
+
+ + +
+ + {/* Show processing indicator when attachments are being processed */} + {hasProcessingAttachments && ( +
+ + Processing... +
+ )} + + {/* Show warning when no model is configured */} + {!hasModelConfigured && !hasProcessingAttachments && ( +
+ + Select a model +
+ )} + + !thread.isRunning}> + + + + + + + + thread.isRunning}> + + + + +
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/composer.tsx b/surfsense_web/components/assistant-ui/composer.tsx new file mode 100644 index 000000000..1973726da --- /dev/null +++ b/surfsense_web/components/assistant-ui/composer.tsx @@ -0,0 +1,240 @@ +import { ComposerPrimitive, useAssistantState, useComposerRuntime } from "@assistant-ui/react"; +import { useAtom, useSetAtom } from "jotai"; +import { useParams } from "next/navigation"; +import type { FC } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; +import { + mentionedDocumentIdsAtom, + mentionedDocumentsAtom, +} from "@/atoms/chat/mentioned-documents.atom"; +import { + ComposerAddAttachment, + ComposerAttachments, +} from "@/components/assistant-ui/attachment"; +import { ComposerAction } from "@/components/assistant-ui/composer-action"; +import { + InlineMentionEditor, + type InlineMentionEditorRef, +} from "@/components/assistant-ui/inline-mention-editor"; +import { + DocumentMentionPicker, + type DocumentMentionPickerRef, +} from "@/components/new-chat/document-mention-picker"; +import type { Document } from "@/contracts/types/document.types"; + +export const Composer: FC = () => { + // ---- State for document mentions (using atoms to persist across remounts) ---- + const [mentionedDocuments, setMentionedDocuments] = useAtom(mentionedDocumentsAtom); + const [showDocumentPopover, setShowDocumentPopover] = useState(false); + const [mentionQuery, setMentionQuery] = useState(""); + const editorRef = useRef(null); + const editorContainerRef = useRef(null); + const documentPickerRef = useRef(null); + const { search_space_id } = useParams(); + const setMentionedDocumentIds = useSetAtom(mentionedDocumentIdsAtom); + const composerRuntime = useComposerRuntime(); + const hasAutoFocusedRef = useRef(false); + + // Check if thread is empty (new chat) + const isThreadEmpty = useAssistantState(({ thread }) => thread.isEmpty); + + // Check if thread is currently running (streaming response) + const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); + + // Auto-focus editor when on new chat page + useEffect(() => { + if (isThreadEmpty && !hasAutoFocusedRef.current && editorRef.current) { + // Small delay to ensure the editor is fully mounted + const timeoutId = setTimeout(() => { + editorRef.current?.focus(); + hasAutoFocusedRef.current = true; + }, 100); + return () => clearTimeout(timeoutId); + } + }, [isThreadEmpty]); + + // Sync mentioned document IDs to atom for use in chat request + useEffect(() => { + setMentionedDocumentIds(mentionedDocuments.map((doc) => doc.id)); + }, [mentionedDocuments, setMentionedDocumentIds]); + + // Handle text change from inline editor - sync with assistant-ui composer + const handleEditorChange = useCallback( + (text: string) => { + composerRuntime.setText(text); + }, + [composerRuntime] + ); + + // Handle @ mention trigger from inline editor + const handleMentionTrigger = useCallback((query: string) => { + setShowDocumentPopover(true); + setMentionQuery(query); + }, []); + + // Handle mention close + const handleMentionClose = useCallback(() => { + if (showDocumentPopover) { + setShowDocumentPopover(false); + setMentionQuery(""); + } + }, [showDocumentPopover]); + + // Handle keyboard navigation when popover is open + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (showDocumentPopover) { + if (e.key === "ArrowDown") { + e.preventDefault(); + documentPickerRef.current?.moveDown(); + return; + } + if (e.key === "ArrowUp") { + e.preventDefault(); + documentPickerRef.current?.moveUp(); + return; + } + if (e.key === "Enter") { + e.preventDefault(); + documentPickerRef.current?.selectHighlighted(); + return; + } + if (e.key === "Escape") { + e.preventDefault(); + setShowDocumentPopover(false); + setMentionQuery(""); + return; + } + } + }, + [showDocumentPopover] + ); + + // Handle submit from inline editor (Enter key) + const handleSubmit = useCallback(() => { + // Prevent sending while a response is still streaming + if (isThreadRunning) { + return; + } + if (!showDocumentPopover) { + composerRuntime.send(); + // Clear the editor after sending + editorRef.current?.clear(); + setMentionedDocuments([]); + setMentionedDocumentIds([]); + } + }, [ + showDocumentPopover, + isThreadRunning, + composerRuntime, + setMentionedDocuments, + setMentionedDocumentIds, + ]); + + // Handle document removal from inline editor + const handleDocumentRemove = useCallback( + (docId: number) => { + setMentionedDocuments((prev) => { + const updated = prev.filter((doc) => doc.id !== docId); + // Immediately sync document IDs to avoid race conditions + setMentionedDocumentIds(updated.map((doc) => doc.id)); + return updated; + }); + }, + [setMentionedDocuments, setMentionedDocumentIds] + ); + + // Handle document selection from picker + const handleDocumentsMention = useCallback( + (documents: Document[]) => { + // Insert chips into the inline editor for each new document + const existingIds = new Set(mentionedDocuments.map((d) => d.id)); + const newDocs = documents.filter((doc) => !existingIds.has(doc.id)); + + for (const doc of newDocs) { + editorRef.current?.insertDocumentChip(doc); + } + + // Update mentioned documents state + setMentionedDocuments((prev) => { + const existingIdSet = new Set(prev.map((d) => d.id)); + const uniqueNewDocs = documents.filter((doc) => !existingIdSet.has(doc.id)); + const updated = [...prev, ...uniqueNewDocs]; + // Immediately sync document IDs to avoid race conditions + setMentionedDocumentIds(updated.map((doc) => doc.id)); + return updated; + }); + + // Reset mention query but keep popover open for more selections + setMentionQuery(""); + }, + [mentionedDocuments, setMentionedDocuments, setMentionedDocumentIds] + ); + + return ( + + + + {/* -------- Inline Mention Editor -------- */} +
+ +
+ + {/* -------- Document mention popover (rendered via portal) -------- */} + {showDocumentPopover && + typeof document !== "undefined" && + createPortal( + <> + {/* Backdrop */} + + + + + +
+ + + ); +}; + diff --git a/surfsense_web/components/assistant-ui/thinking-steps.tsx b/surfsense_web/components/assistant-ui/thinking-steps.tsx new file mode 100644 index 000000000..f0cf4a7c1 --- /dev/null +++ b/surfsense_web/components/assistant-ui/thinking-steps.tsx @@ -0,0 +1,207 @@ +import { useAssistantState, useThreadViewport } from "@assistant-ui/react"; +import type { FC } from "react"; +import { createContext, useCallback, useContext, useEffect, useRef, useState } from "react"; +import { ChevronRightIcon } from "lucide-react"; +import { ChainOfThoughtItem } from "@/components/prompt-kit/chain-of-thought"; +import { TextShimmerLoader } from "@/components/prompt-kit/loader"; +import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; +import { cn } from "@/lib/utils"; + +// Context to pass thinking steps to AssistantMessage +export const ThinkingStepsContext = createContext>(new Map()); + +/** + * Chain of thought display component - single collapsible dropdown design + */ +export const ThinkingStepsDisplay: FC<{ steps: ThinkingStep[]; isThreadRunning?: boolean }> = ({ + steps, + isThreadRunning = true, +}) => { + const [isOpen, setIsOpen] = useState(true); + + // Derive effective status for each step + const getEffectiveStatus = useCallback( + (step: ThinkingStep): "pending" | "in_progress" | "completed" => { + if (step.status === "in_progress" && !isThreadRunning) { + return "completed"; + } + return step.status; + }, + [isThreadRunning] + ); + + // Calculate summary info + const completedSteps = steps.filter((s) => getEffectiveStatus(s) === "completed").length; + const inProgressStep = steps.find((s) => getEffectiveStatus(s) === "in_progress"); + const allCompleted = completedSteps === steps.length && steps.length > 0 && !isThreadRunning; + const isProcessing = isThreadRunning && !allCompleted; + + // Auto-collapse when all tasks are completed + useEffect(() => { + if (allCompleted) { + setIsOpen(false); + } + }, [allCompleted]); + + if (steps.length === 0) return null; + + // Generate header text + const getHeaderText = () => { + if (allCompleted) { + return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`; + } + if (inProgressStep) { + return inProgressStep.title; + } + if (isProcessing) { + return `Processing ${completedSteps}/${steps.length} steps`; + } + return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`; + }; + + return ( +
+
+ {/* Main collapsible header */} + + + {/* Collapsible content with CSS grid animation */} +
+
+
+ {steps.map((step, index) => { + const effectiveStatus = getEffectiveStatus(step); + const isLast = index === steps.length - 1; + + return ( +
+ {/* Dot and line column */} +
+ {/* Vertical connection line - extends to next dot */} + {!isLast && ( +
+ )} + {/* Step dot - on top of line */} +
+ {effectiveStatus === "in_progress" ? ( + + ) : ( + + )} +
+
+ + {/* Step content */} +
+ {/* Step title */} +
+ {effectiveStatus === "in_progress" ? ( + + ) : ( + step.title + )} +
+ + {/* Step items (sub-content) */} + {step.items && step.items.length > 0 && ( +
+ {step.items.map((item, idx) => ( + + {item} + + ))} +
+ )} +
+
+ ); + })} +
+
+
+
+
+ ); +}; + +/** + * Component that handles auto-scroll when thinking steps update. + * Uses useThreadViewport to scroll to bottom when thinking steps change, + * ensuring the user always sees the latest content during streaming. + */ +export const ThinkingStepsScrollHandler: FC = () => { + const thinkingStepsMap = useContext(ThinkingStepsContext); + const viewport = useThreadViewport(); + const isRunning = useAssistantState(({ thread }) => thread.isRunning); + // Track the serialized state to detect any changes + const prevStateRef = useRef(""); + + useEffect(() => { + // Only act during streaming + if (!isRunning) { + prevStateRef.current = ""; + return; + } + + // Serialize the thinking steps state to detect any changes + // This catches new steps, status changes, and item additions + let stateString = ""; + thinkingStepsMap.forEach((steps, msgId) => { + steps.forEach((step) => { + stateString += `${msgId}:${step.id}:${step.status}:${step.items?.length || 0};`; + }); + }); + + // If state changed at all during streaming, scroll + if (stateString !== prevStateRef.current && stateString !== "") { + prevStateRef.current = stateString; + + // Multiple attempts to ensure scroll happens after DOM updates + const scrollAttempt = () => { + try { + viewport.scrollToBottom(); + } catch { + // Ignore errors - viewport might not be ready + } + }; + + // Delayed attempts to handle async DOM updates + requestAnimationFrame(scrollAttempt); + setTimeout(scrollAttempt, 100); + } + }, [thinkingStepsMap, viewport, isRunning]); + + return null; // This component doesn't render anything +}; + diff --git a/surfsense_web/components/assistant-ui/thread-scroll-to-bottom.tsx b/surfsense_web/components/assistant-ui/thread-scroll-to-bottom.tsx new file mode 100644 index 000000000..6f641615e --- /dev/null +++ b/surfsense_web/components/assistant-ui/thread-scroll-to-bottom.tsx @@ -0,0 +1,19 @@ +import { ThreadPrimitive } from "@assistant-ui/react"; +import { ArrowDownIcon } from "lucide-react"; +import type { FC } from "react"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; + +export const ThreadScrollToBottom: FC = () => { + return ( + + + + + + ); +}; + diff --git a/surfsense_web/components/assistant-ui/thread-welcome.tsx b/surfsense_web/components/assistant-ui/thread-welcome.tsx new file mode 100644 index 000000000..b5e4bbac0 --- /dev/null +++ b/surfsense_web/components/assistant-ui/thread-welcome.tsx @@ -0,0 +1,72 @@ +import { useAtomValue } from "jotai"; +import type { FC } from "react"; +import { useMemo } from "react"; +import { currentUserAtom } from "@/atoms/user/user-query.atoms"; +import { Composer } from "@/components/assistant-ui/composer"; + +const getTimeBasedGreeting = (userEmail?: string): string => { + const hour = new Date().getHours(); + + // Extract first name from email if available + const firstName = userEmail + ? userEmail.split("@")[0].split(".")[0].charAt(0).toUpperCase() + + userEmail.split("@")[0].split(".")[0].slice(1) + : null; + + // Array of greeting variations for each time period + const morningGreetings = ["Good morning", "Fresh start today", "Morning", "Hey there"]; + + const afternoonGreetings = ["Good afternoon", "Afternoon", "Hey there", "Hi there"]; + + const eveningGreetings = ["Good evening", "Evening", "Hey there", "Hi there"]; + + const nightGreetings = ["Good night", "Evening", "Hey there", "Winding down"]; + + const lateNightGreetings = ["Still up", "Night owl mode", "Up past bedtime", "Hi there"]; + + // Select a random greeting based on time + let greeting: string; + if (hour < 5) { + // Late night: midnight to 5 AM + greeting = lateNightGreetings[Math.floor(Math.random() * lateNightGreetings.length)]; + } else if (hour < 12) { + greeting = morningGreetings[Math.floor(Math.random() * morningGreetings.length)]; + } else if (hour < 18) { + greeting = afternoonGreetings[Math.floor(Math.random() * afternoonGreetings.length)]; + } else if (hour < 22) { + greeting = eveningGreetings[Math.floor(Math.random() * eveningGreetings.length)]; + } else { + // Night: 10 PM to midnight + greeting = nightGreetings[Math.floor(Math.random() * nightGreetings.length)]; + } + + // Add personalization with first name if available + if (firstName) { + return `${greeting}, ${firstName}!`; + } + + return `${greeting}!`; +}; + +export const ThreadWelcome: FC = () => { + const { data: user } = useAtomValue(currentUserAtom); + + // Memoize greeting so it doesn't change on re-renders (only on user change) + const greeting = useMemo(() => getTimeBasedGreeting(user?.email), [user?.email]); + + return ( +
+ {/* Greeting positioned above the composer - fixed position */} +
+

+ {greeting} +

+
+ {/* Composer - top edge fixed, expands downward only */} +
+ +
+
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index cb01e7605..a354414f0 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -1,85 +1,13 @@ -import { - ActionBarPrimitive, - AssistantIf, - BranchPickerPrimitive, - ComposerPrimitive, - ErrorPrimitive, - MessagePrimitive, - ThreadPrimitive, - useAssistantState, - useComposerRuntime, - useThreadViewport, -} from "@assistant-ui/react"; -import { useAtom, useAtomValue, useSetAtom } from "jotai"; -import { - AlertCircle, - ArrowDownIcon, - ArrowUpIcon, - CheckIcon, - ChevronLeftIcon, - ChevronRightIcon, - CopyIcon, - DownloadIcon, - FileText, - Loader2, - PencilIcon, - Plug2, - Plus, - RefreshCwIcon, - SquareIcon, -} from "lucide-react"; -import Link from "next/link"; -import { useParams } from "next/navigation"; -import { - createContext, - type FC, - useCallback, - useContext, - useEffect, - useMemo, - useRef, - useState, -} from "react"; -import { createPortal } from "react-dom"; -import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; -import { - mentionedDocumentIdsAtom, - mentionedDocumentsAtom, - messageDocumentsMapAtom, -} from "@/atoms/chat/mentioned-documents.atom"; -import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; -import { - globalNewLLMConfigsAtom, - llmPreferencesAtom, - newLLMConfigsAtom, -} from "@/atoms/new-llm-config/new-llm-config-query.atoms"; -import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; -import { currentUserAtom } from "@/atoms/user/user-query.atoms"; -import { - ComposerAddAttachment, - ComposerAttachments, - UserMessageAttachments, -} from "@/components/assistant-ui/attachment"; -import { - InlineMentionEditor, - type InlineMentionEditorRef, -} from "@/components/assistant-ui/inline-mention-editor"; -import { MarkdownText } from "@/components/assistant-ui/markdown-text"; -import { ToolFallback } from "@/components/assistant-ui/tool-fallback"; -import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; -import { - DocumentMentionPicker, - type DocumentMentionPickerRef, -} from "@/components/new-chat/document-mention-picker"; -import { ChainOfThoughtItem } from "@/components/prompt-kit/chain-of-thought"; -import { TextShimmerLoader } from "@/components/prompt-kit/loader"; +import { AssistantIf, ThreadPrimitive } from "@assistant-ui/react"; +import type { FC } from "react"; import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; -import { Button } from "@/components/ui/button"; -import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; -import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import type { Document } from "@/contracts/types/document.types"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; -import { cn } from "@/lib/utils"; +import { ThinkingStepsContext } from "@/components/assistant-ui/thinking-steps"; +import { ThreadWelcome } from "@/components/assistant-ui/thread-welcome"; +import { Composer } from "@/components/assistant-ui/composer"; +import { ThreadScrollToBottom } from "@/components/assistant-ui/thread-scroll-to-bottom"; +import { AssistantMessage } from "@/components/assistant-ui/assistant-message"; +import { UserMessage } from "@/components/assistant-ui/user-message"; +import { EditComposer } from "@/components/assistant-ui/edit-composer"; /** * Props for the Thread component @@ -90,204 +18,6 @@ interface ThreadProps { header?: React.ReactNode; } -// Context to pass thinking steps to AssistantMessage -const ThinkingStepsContext = createContext>(new Map()); - -/** - * Chain of thought display component - single collapsible dropdown design - */ -const ThinkingStepsDisplay: FC<{ steps: ThinkingStep[]; isThreadRunning?: boolean }> = ({ - steps, - isThreadRunning = true, -}) => { - const [isOpen, setIsOpen] = useState(true); - - // Derive effective status for each step - const getEffectiveStatus = useCallback( - (step: ThinkingStep): "pending" | "in_progress" | "completed" => { - if (step.status === "in_progress" && !isThreadRunning) { - return "completed"; - } - return step.status; - }, - [isThreadRunning] - ); - - // Calculate summary info - const completedSteps = steps.filter((s) => getEffectiveStatus(s) === "completed").length; - const inProgressStep = steps.find((s) => getEffectiveStatus(s) === "in_progress"); - const allCompleted = completedSteps === steps.length && steps.length > 0 && !isThreadRunning; - const isProcessing = isThreadRunning && !allCompleted; - - // Auto-collapse when all tasks are completed - useEffect(() => { - if (allCompleted) { - setIsOpen(false); - } - }, [allCompleted]); - - if (steps.length === 0) return null; - - // Generate header text - const getHeaderText = () => { - if (allCompleted) { - return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`; - } - if (inProgressStep) { - return inProgressStep.title; - } - if (isProcessing) { - return `Processing ${completedSteps}/${steps.length} steps`; - } - return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`; - }; - - return ( -
-
- {/* Main collapsible header */} - - - {/* Collapsible content with CSS grid animation */} -
-
-
- {steps.map((step, index) => { - const effectiveStatus = getEffectiveStatus(step); - const isLast = index === steps.length - 1; - - return ( -
- {/* Dot and line column */} -
- {/* Vertical connection line - extends to next dot */} - {!isLast && ( -
- )} - {/* Step dot - on top of line */} -
- {effectiveStatus === "in_progress" ? ( - - ) : ( - - )} -
-
- - {/* Step content */} -
- {/* Step title */} -
- {effectiveStatus === "in_progress" ? ( - - ) : ( - step.title - )} -
- - {/* Step items (sub-content) */} - {step.items && step.items.length > 0 && ( -
- {step.items.map((item, idx) => ( - - {item} - - ))} -
- )} -
-
- ); - })} -
-
-
-
-
- ); -}; - -/** - * Component that handles auto-scroll when thinking steps update. - * Uses useThreadViewport to scroll to bottom when thinking steps change, - * ensuring the user always sees the latest content during streaming. - */ -const _ThinkingStepsScrollHandler: FC = () => { - const thinkingStepsMap = useContext(ThinkingStepsContext); - const viewport = useThreadViewport(); - const isRunning = useAssistantState(({ thread }) => thread.isRunning); - // Track the serialized state to detect any changes - const prevStateRef = useRef(""); - - useEffect(() => { - // Only act during streaming - if (!isRunning) { - prevStateRef.current = ""; - return; - } - - // Serialize the thinking steps state to detect any changes - // This catches new steps, status changes, and item additions - let stateString = ""; - thinkingStepsMap.forEach((steps, msgId) => { - steps.forEach((step) => { - stateString += `${msgId}:${step.id}:${step.status}:${step.items?.length || 0};`; - }); - }); - - // If state changed at all during streaming, scroll - if (stateString !== prevStateRef.current && stateString !== "") { - prevStateRef.current = stateString; - - // Multiple attempts to ensure scroll happens after DOM updates - const scrollAttempt = () => { - try { - viewport.scrollToBottom(); - } catch { - // Ignore errors - viewport might not be ready - } - }; - - // Delayed attempts to handle async DOM updates - requestAnimationFrame(scrollAttempt); - setTimeout(scrollAttempt, 100); - } - }, [thinkingStepsMap, viewport, isRunning]); - - return null; // This component doesn't render anything -}; - export const Thread: FC = ({ messageThinkingSteps = new Map(), header }) => { return ( @@ -329,760 +59,3 @@ export const Thread: FC = ({ messageThinkingSteps = new Map(), head ); }; - -const ThreadScrollToBottom: FC = () => { - return ( - - - - - - ); -}; - -const getTimeBasedGreeting = (userEmail?: string): string => { - const hour = new Date().getHours(); - - // Extract first name from email if available - const firstName = userEmail - ? userEmail.split("@")[0].split(".")[0].charAt(0).toUpperCase() + - userEmail.split("@")[0].split(".")[0].slice(1) - : null; - - // Array of greeting variations for each time period - const morningGreetings = ["Good morning", "Fresh start today", "Morning", "Hey there"]; - - const afternoonGreetings = ["Good afternoon", "Afternoon", "Hey there", "Hi there"]; - - const eveningGreetings = ["Good evening", "Evening", "Hey there", "Hi there"]; - - const nightGreetings = ["Good night", "Evening", "Hey there", "Winding down"]; - - const lateNightGreetings = ["Still up", "Night owl mode", "Up past bedtime", "Hi there"]; - - // Select a random greeting based on time - let greeting: string; - if (hour < 5) { - // Late night: midnight to 5 AM - greeting = lateNightGreetings[Math.floor(Math.random() * lateNightGreetings.length)]; - } else if (hour < 12) { - greeting = morningGreetings[Math.floor(Math.random() * morningGreetings.length)]; - } else if (hour < 18) { - greeting = afternoonGreetings[Math.floor(Math.random() * afternoonGreetings.length)]; - } else if (hour < 22) { - greeting = eveningGreetings[Math.floor(Math.random() * eveningGreetings.length)]; - } else { - // Night: 10 PM to midnight - greeting = nightGreetings[Math.floor(Math.random() * nightGreetings.length)]; - } - - // Add personalization with first name if available - if (firstName) { - return `${greeting}, ${firstName}!`; - } - - return `${greeting}!`; -}; - -const ThreadWelcome: FC = () => { - const { data: user } = useAtomValue(currentUserAtom); - - // Memoize greeting so it doesn't change on re-renders (only on user change) - const greeting = useMemo(() => getTimeBasedGreeting(user?.email), [user?.email]); - - return ( -
- {/* Greeting positioned above the composer - fixed position */} -
-

- {greeting} -

-
- {/* Composer - top edge fixed, expands downward only */} -
- -
-
- ); -}; - -const Composer: FC = () => { - // ---- State for document mentions (using atoms to persist across remounts) ---- - const [mentionedDocuments, setMentionedDocuments] = useAtom(mentionedDocumentsAtom); - const [showDocumentPopover, setShowDocumentPopover] = useState(false); - const [mentionQuery, setMentionQuery] = useState(""); - const editorRef = useRef(null); - const editorContainerRef = useRef(null); - const documentPickerRef = useRef(null); - const { search_space_id } = useParams(); - const setMentionedDocumentIds = useSetAtom(mentionedDocumentIdsAtom); - const composerRuntime = useComposerRuntime(); - const hasAutoFocusedRef = useRef(false); - - // Check if thread is empty (new chat) - const isThreadEmpty = useAssistantState(({ thread }) => thread.isEmpty); - - // Check if thread is currently running (streaming response) - const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); - - // Auto-focus editor when on new chat page - useEffect(() => { - if (isThreadEmpty && !hasAutoFocusedRef.current && editorRef.current) { - // Small delay to ensure the editor is fully mounted - const timeoutId = setTimeout(() => { - editorRef.current?.focus(); - hasAutoFocusedRef.current = true; - }, 100); - return () => clearTimeout(timeoutId); - } - }, [isThreadEmpty]); - - // Sync mentioned document IDs to atom for use in chat request - useEffect(() => { - setMentionedDocumentIds(mentionedDocuments.map((doc) => doc.id)); - }, [mentionedDocuments, setMentionedDocumentIds]); - - // Handle text change from inline editor - sync with assistant-ui composer - const handleEditorChange = useCallback( - (text: string) => { - composerRuntime.setText(text); - }, - [composerRuntime] - ); - - // Handle @ mention trigger from inline editor - const handleMentionTrigger = useCallback((query: string) => { - setShowDocumentPopover(true); - setMentionQuery(query); - }, []); - - // Handle mention close - const handleMentionClose = useCallback(() => { - if (showDocumentPopover) { - setShowDocumentPopover(false); - setMentionQuery(""); - } - }, [showDocumentPopover]); - - // Handle keyboard navigation when popover is open - const handleKeyDown = useCallback( - (e: React.KeyboardEvent) => { - if (showDocumentPopover) { - if (e.key === "ArrowDown") { - e.preventDefault(); - documentPickerRef.current?.moveDown(); - return; - } - if (e.key === "ArrowUp") { - e.preventDefault(); - documentPickerRef.current?.moveUp(); - return; - } - if (e.key === "Enter") { - e.preventDefault(); - documentPickerRef.current?.selectHighlighted(); - return; - } - if (e.key === "Escape") { - e.preventDefault(); - setShowDocumentPopover(false); - setMentionQuery(""); - return; - } - } - }, - [showDocumentPopover] - ); - - // Handle submit from inline editor (Enter key) - const handleSubmit = useCallback(() => { - // Prevent sending while a response is still streaming - if (isThreadRunning) { - return; - } - if (!showDocumentPopover) { - composerRuntime.send(); - // Clear the editor after sending - editorRef.current?.clear(); - setMentionedDocuments([]); - setMentionedDocumentIds([]); - } - }, [ - showDocumentPopover, - isThreadRunning, - composerRuntime, - setMentionedDocuments, - setMentionedDocumentIds, - ]); - - // Handle document removal from inline editor - const handleDocumentRemove = useCallback( - (docId: number) => { - setMentionedDocuments((prev) => { - const updated = prev.filter((doc) => doc.id !== docId); - // Immediately sync document IDs to avoid race conditions - setMentionedDocumentIds(updated.map((doc) => doc.id)); - return updated; - }); - }, - [setMentionedDocuments, setMentionedDocumentIds] - ); - - // Handle document selection from picker - const handleDocumentsMention = useCallback( - (documents: Document[]) => { - // Insert chips into the inline editor for each new document - const existingIds = new Set(mentionedDocuments.map((d) => d.id)); - const newDocs = documents.filter((doc) => !existingIds.has(doc.id)); - - for (const doc of newDocs) { - editorRef.current?.insertDocumentChip(doc); - } - - // Update mentioned documents state - setMentionedDocuments((prev) => { - const existingIdSet = new Set(prev.map((d) => d.id)); - const uniqueNewDocs = documents.filter((doc) => !existingIdSet.has(doc.id)); - const updated = [...prev, ...uniqueNewDocs]; - // Immediately sync document IDs to avoid race conditions - setMentionedDocumentIds(updated.map((doc) => doc.id)); - return updated; - }); - - // Reset mention query but keep popover open for more selections - setMentionQuery(""); - }, - [mentionedDocuments, setMentionedDocuments, setMentionedDocumentIds] - ); - - return ( - - - - {/* -------- Inline Mention Editor -------- */} -
- -
- - {/* -------- Document mention popover (rendered via portal) -------- */} - {showDocumentPopover && - typeof document !== "undefined" && - createPortal( - <> - {/* Backdrop */} - - - - {hasSources ? ( -
-
-

Connected Sources

- - {totalSourceCount} - -
-
- {/* Document types from the search space */} - {activeDocumentTypes.map(([docType]) => ( -
- {getConnectorIcon(docType, "size-3.5")} - {getDocumentTypeLabel(docType)} -
- ))} - {/* Search source connectors */} - {connectors.map((connector) => ( -
- {getConnectorIcon(connector.connector_type, "size-3.5")} - {connector.name} -
- ))} -
-
- - - Add more sources - - -
-
- ) : ( -
-

No sources yet

-

- Add documents or connect data sources to enhance search results. -

- - - Add Connector - -
- )} - - - ); -}; - -const ComposerAction: FC = () => { - // Check if any attachments are still being processed (running AND progress < 100) - // When progress is 100, processing is done but waiting for send() - const hasProcessingAttachments = useAssistantState(({ composer }) => - composer.attachments?.some((att) => { - const status = att.status; - if (status?.type !== "running") return false; - const progress = (status as { type: "running"; progress?: number }).progress; - return progress === undefined || progress < 100; - }) - ); - - // Check if composer text is empty - const isComposerEmpty = useAssistantState(({ composer }) => { - const text = composer.text?.trim() || ""; - return text.length === 0; - }); - - // Check if a model is configured - const { data: userConfigs } = useAtomValue(newLLMConfigsAtom); - const { data: globalConfigs } = useAtomValue(globalNewLLMConfigsAtom); - const { data: preferences } = useAtomValue(llmPreferencesAtom); - - const hasModelConfigured = useMemo(() => { - if (!preferences) return false; - const agentLlmId = preferences.agent_llm_id; - if (agentLlmId === null || agentLlmId === undefined) return false; - - // Check if the configured model actually exists - if (agentLlmId < 0) { - return globalConfigs?.some((c) => c.id === agentLlmId) ?? false; - } - return userConfigs?.some((c) => c.id === agentLlmId) ?? false; - }, [preferences, globalConfigs, userConfigs]); - - const isSendDisabled = hasProcessingAttachments || isComposerEmpty || !hasModelConfigured; - - return ( -
-
- - -
- - {/* Show processing indicator when attachments are being processed */} - {hasProcessingAttachments && ( -
- - Processing... -
- )} - - {/* Show warning when no model is configured */} - {!hasModelConfigured && !hasProcessingAttachments && ( -
- - Select a model -
- )} - - !thread.isRunning}> - - - - - - - - thread.isRunning}> - - - - -
- ); -}; - -const MessageError: FC = () => { - return ( - - - - - - ); -}; - -/** - * Custom component to render thinking steps from Context - */ -const ThinkingStepsPart: FC = () => { - const thinkingStepsMap = useContext(ThinkingStepsContext); - - // Get the current message ID to look up thinking steps - const messageId = useAssistantState(({ message }) => message?.id); - const thinkingSteps = thinkingStepsMap.get(messageId) || []; - - // Check if this specific message is currently streaming - // A message is streaming if: thread is running AND this is the last assistant message - const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); - const isLastMessage = useAssistantState(({ message }) => message?.isLast ?? false); - const isMessageStreaming = isThreadRunning && isLastMessage; - - if (thinkingSteps.length === 0) return null; - - return ( -
- -
- ); -}; - -const AssistantMessageInner: FC = () => { - return ( - <> - {/* Render thinking steps from message content - this ensures proper scroll tracking */} - - -
- - -
- -
- - -
- - ); -}; - -const AssistantMessage: FC = () => { - return ( - - - - ); -}; - -const AssistantActionBar: FC = () => { - return ( - - - - message.isCopied}> - - - !message.isCopied}> - - - - - - - - - - - - - - - - ); -}; - -const UserMessage: FC = () => { - const messageId = useAssistantState(({ message }) => message?.id); - const messageDocumentsMap = useAtomValue(messageDocumentsMapAtom); - const mentionedDocs = messageId ? messageDocumentsMap[messageId] : undefined; - const hasAttachments = useAssistantState( - ({ message }) => message?.attachments && message.attachments.length > 0 - ); - - return ( - -
- {/* Display attachments and mentioned documents */} - {(hasAttachments || (mentionedDocs && mentionedDocs.length > 0)) && ( -
- {/* Attachments (images show as thumbnails, documents as chips) */} - - {/* Mentioned documents as chips */} - {mentionedDocs?.map((doc) => ( - - - {doc.title} - - ))} -
- )} - {/* Message bubble with action bar positioned relative to it */} -
-
- -
-
- -
-
-
- - - - ); -}; - -const UserActionBar: FC = () => { - return ( - - - - - - - - ); -}; - -const EditComposer: FC = () => { - return ( - - - -
- - - - - - -
- - - ); -}; - -const BranchPicker: FC = ({ className, ...rest }) => { - return ( - - - - - - - - / - - - - - - - - ); -}; diff --git a/surfsense_web/components/assistant-ui/user-message.tsx b/surfsense_web/components/assistant-ui/user-message.tsx new file mode 100644 index 000000000..fbbcf42bf --- /dev/null +++ b/surfsense_web/components/assistant-ui/user-message.tsx @@ -0,0 +1,73 @@ +import { ActionBarPrimitive, MessagePrimitive, useAssistantState } from "@assistant-ui/react"; +import { useAtomValue } from "jotai"; +import { FileText, PencilIcon } from "lucide-react"; +import type { FC } from "react"; +import { messageDocumentsMapAtom } from "@/atoms/chat/mentioned-documents.atom"; +import { UserMessageAttachments } from "@/components/assistant-ui/attachment"; +import { BranchPicker } from "@/components/assistant-ui/branch-picker"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; + +export const UserMessage: FC = () => { + const messageId = useAssistantState(({ message }) => message?.id); + const messageDocumentsMap = useAtomValue(messageDocumentsMapAtom); + const mentionedDocs = messageId ? messageDocumentsMap[messageId] : undefined; + const hasAttachments = useAssistantState( + ({ message }) => message?.attachments && message.attachments.length > 0 + ); + + return ( + +
+ {/* Display attachments and mentioned documents */} + {(hasAttachments || (mentionedDocs && mentionedDocs.length > 0)) && ( +
+ {/* Attachments (images show as thumbnails, documents as chips) */} + + {/* Mentioned documents as chips */} + {mentionedDocs?.map((doc) => ( + + + {doc.title} + + ))} +
+ )} + {/* Message bubble with action bar positioned relative to it */} +
+
+ +
+
+ +
+
+
+ + + + ); +}; + +const UserActionBar: FC = () => { + return ( + + + + + + + + ); +}; + From 5b39b32ef6f5e8828e624db2c3782b1efb5291c0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 30 Dec 2025 17:37:22 +0200 Subject: [PATCH 50/92] fix: improve connector popover filtering and display - Show document count badges for each document type - Filter to only show non-indexable connectors - Only display document types with at least 1 document - Update total source count to reflect filtered connectors --- .../components/assistant-ui/composer-action.tsx | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/surfsense_web/components/assistant-ui/composer-action.tsx b/surfsense_web/components/assistant-ui/composer-action.tsx index 9c5a95d88..ba27f40c2 100644 --- a/surfsense_web/components/assistant-ui/composer-action.tsx +++ b/surfsense_web/components/assistant-ui/composer-action.tsx @@ -34,14 +34,15 @@ const ConnectorIndicator: FC = () => { const isLoading = connectorsLoading || documentTypesLoading; - // Get document types that have documents in the search space const activeDocumentTypes = documentTypeCounts ? Object.entries(documentTypeCounts).filter(([_, count]) => count > 0) : []; - const hasConnectors = connectors.length > 0; + const nonIndexableConnectors = connectors.filter((connector) => !connector.is_indexable); + + const hasConnectors = nonIndexableConnectors.length > 0; const hasSources = hasConnectors || activeDocumentTypes.length > 0; - const totalSourceCount = connectors.length + activeDocumentTypes.length; + const totalSourceCount = nonIndexableConnectors.length + activeDocumentTypes.length; const handleMouseEnter = useCallback(() => { // Clear any pending close timeout @@ -110,18 +111,19 @@ const ConnectorIndicator: FC = () => {
- {/* Document types from the search space */} - {activeDocumentTypes.map(([docType]) => ( + {activeDocumentTypes.map(([docType, count]) => (
{getConnectorIcon(docType, "size-3.5")} {getDocumentTypeLabel(docType)} + + {count > 999 ? "999+" : count} +
))} - {/* Search source connectors */} - {connectors.map((connector) => ( + {nonIndexableConnectors.map((connector) => (
Date: Tue, 30 Dec 2025 21:25:48 +0530 Subject: [PATCH 51/92] feat: Introduce new connector schemas and validation, enhance connector dialog with improved query parameter handling, and implement scroll detection in indexing configuration view. --- .../connector-popup/active-connectors-tab.tsx | 5 +- .../connector-popup/all-connectors-tab.tsx | 45 ++-- .../connector-popup/connector-card.tsx | 6 - .../connector-popup/connector-constants.ts | 15 +- .../connector-popup.schemas.ts | 108 +++++++++ .../assistant-ui/connector-popup/index.ts | 18 ++ .../indexing-configuration-view.tsx | 108 ++++++--- .../connector-popup/use-connector-dialog.ts | 214 ++++++++++++------ 8 files changed, 376 insertions(+), 143 deletions(-) create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-popup.schemas.ts diff --git a/surfsense_web/components/assistant-ui/connector-popup/active-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/active-connectors-tab.tsx index 323fa34e7..43deba278 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/active-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/active-connectors-tab.tsx @@ -8,6 +8,7 @@ import { Button } from "@/components/ui/button"; import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import type { LogSummary, LogActiveTask } from "@/contracts/types/log.types"; import { cn } from "@/lib/utils"; import { TabsContent, @@ -20,7 +21,7 @@ interface ActiveConnectorsTabProps { activeDocumentTypes: Array<[string, number]>; connectors: SearchSourceConnector[]; indexingConnectorIds: Set; - logsSummary: any; + logsSummary: LogSummary | undefined; searchSpaceId: string; onTabChange: (value: string) => void; } @@ -67,7 +68,7 @@ export const ActiveConnectorsTab: FC = ({ {connectors.map((connector) => { const isIndexing = indexingConnectorIds.has(connector.id); const activeTask = logsSummary?.active_tasks?.find( - (task: any) => + (task: LogActiveTask) => task.source?.includes(`connector_${connector.id}`) || task.source?.includes(`connector-${connector.id}`) ); diff --git a/surfsense_web/components/assistant-ui/connector-popup/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/all-connectors-tab.tsx index e2f735b13..4dd056c90 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/all-connectors-tab.tsx @@ -1,10 +1,7 @@ "use client"; -import { ChevronRight } from "lucide-react"; -import Link from "next/link"; import { useRouter } from "next/navigation"; import { type FC } from "react"; -import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "./connector-constants"; import { ConnectorCard } from "./connector-card"; @@ -88,32 +85,24 @@ export const AllConnectorsTab: FC = ({ const isConnected = connectedTypes.has(connector.connectorType); return ( - -
- {getConnectorIcon(connector.connectorType, "size-6")} -
-
-
- - {connector.title} - - {isConnected && ( - - )} -
-

- {connector.description} -

-
- - + id={connector.id} + title={connector.title} + description={connector.description} + connectorType={connector.connectorType} + isConnected={isConnected} + onConnect={() => + router.push( + `/dashboard/${searchSpaceId}/connectors/add/${connector.id}` + ) + } + onManage={() => + router.push( + `/dashboard/${searchSpaceId}/connectors/add/${connector.id}` + ) + } + /> ); })}
diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-card.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-card.tsx index d1f79b16d..1e5871579 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-card.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-card.tsx @@ -34,12 +34,6 @@ export const ConnectorCard: FC = ({
{title} - {isConnected && ( - - )}

{isConnected ? "Connected" : description} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-constants.ts b/surfsense_web/components/assistant-ui/connector-popup/connector-constants.ts index 65d5bd516..a2750e133 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-constants.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-constants.ts @@ -2,6 +2,13 @@ import { EnumConnectorName } from "@/contracts/enums/connector"; // OAuth Connectors (Quick Connect) export const OAUTH_CONNECTORS = [ + { + id: "google-drive-connector", + title: "Google Drive", + description: "Search your Drive files", + connectorType: EnumConnectorName.GOOGLE_DRIVE_CONNECTOR, + authEndpoint: "/api/v1/auth/google/drive/connector/add/", + }, { id: "google-gmail-connector", title: "Gmail", @@ -125,10 +132,6 @@ export const OTHER_CONNECTORS = [ }, ] as const; -// Type for the indexing configuration state -export interface IndexingConfigState { - connectorType: string; - connectorId: number; - connectorTitle: string; -} +// Re-export IndexingConfigState from schemas for backward compatibility +export type { IndexingConfigState } from "./connector-popup.schemas"; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-popup.schemas.ts b/surfsense_web/components/assistant-ui/connector-popup/connector-popup.schemas.ts new file mode 100644 index 000000000..118625e57 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-popup.schemas.ts @@ -0,0 +1,108 @@ +import { z } from "zod"; +import { searchSourceConnectorTypeEnum } from "@/contracts/types/connector.types"; + +/** + * Schema for URL query parameters used by the connector popup + */ +export const connectorPopupQueryParamsSchema = z.object({ + modal: z.enum(["connectors"]).optional(), + tab: z.enum(["all", "active"]).optional(), + view: z.enum(["configure"]).optional(), + connector: z.string().optional(), + success: z.enum(["true", "false"]).optional(), +}); + +export type ConnectorPopupQueryParams = z.infer; + +/** + * Schema for OAuth API response (auth_url) + */ +export const oauthAuthResponseSchema = z.object({ + auth_url: z.string().url("Invalid auth URL format"), +}); + +export type OAuthAuthResponse = z.infer; + +/** + * Schema for IndexingConfigState + */ +export const indexingConfigStateSchema = z.object({ + connectorType: searchSourceConnectorTypeEnum, + connectorId: z.number().int().positive("Connector ID must be a positive integer"), + connectorTitle: z.string().min(1, "Connector title is required"), +}); + +export type IndexingConfigState = z.infer; + +/** + * Schema for frequency minutes (must be one of the allowed values) + */ +export const frequencyMinutesSchema = z.enum(["15", "60", "360", "720", "1440", "10080"], { + errorMap: () => ({ message: "Invalid frequency value" }), +}); + +export type FrequencyMinutes = z.infer; + +/** + * Schema for date range validation + */ +export const dateRangeSchema = z + .object({ + startDate: z.date().optional(), + endDate: z.date().optional(), + }) + .refine( + (data) => { + if (data.startDate && data.endDate) { + return data.startDate <= data.endDate; + } + return true; + }, + { + message: "Start date must be before or equal to end date", + path: ["endDate"], + } + ); + +export type DateRange = z.infer; + +/** + * Schema for connector ID validation (used in URL params) + */ +export const connectorIdSchema = z.string().min(1, "Connector ID is required"); + +/** + * Helper function to safely parse query params + */ +export function parseConnectorPopupQueryParams( + params: URLSearchParams | Record +): ConnectorPopupQueryParams { + const obj: Record = {}; + + if (params instanceof URLSearchParams) { + params.forEach((value, key) => { + obj[key] = value || undefined; + }); + } else { + Object.entries(params).forEach(([key, value]) => { + obj[key] = value || undefined; + }); + } + + return connectorPopupQueryParamsSchema.parse(obj); +} + +/** + * Helper function to safely parse OAuth response + */ +export function parseOAuthAuthResponse(data: unknown): OAuthAuthResponse { + return oauthAuthResponseSchema.parse(data); +} + +/** + * Helper function to validate indexing config state + */ +export function validateIndexingConfigState(data: unknown): IndexingConfigState { + return indexingConfigStateSchema.parse(data); +} + diff --git a/surfsense_web/components/assistant-ui/connector-popup/index.ts b/surfsense_web/components/assistant-ui/connector-popup/index.ts index da1d4639e..1aea4de95 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/index.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/index.ts @@ -14,6 +14,24 @@ export { ActiveConnectorsTab } from "./active-connectors-tab"; export { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "./connector-constants"; export type { IndexingConfigState } from "./connector-constants"; +// Schemas and validation +export { + connectorPopupQueryParamsSchema, + oauthAuthResponseSchema, + indexingConfigStateSchema, + frequencyMinutesSchema, + dateRangeSchema, + parseConnectorPopupQueryParams, + parseOAuthAuthResponse, + validateIndexingConfigState, +} from "./connector-popup.schemas"; +export type { + ConnectorPopupQueryParams, + OAuthAuthResponse, + FrequencyMinutes, + DateRange, +} from "./connector-popup.schemas"; + // Hooks export { useConnectorDialog } from "./use-connector-dialog"; diff --git a/surfsense_web/components/assistant-ui/connector-popup/indexing-configuration-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/indexing-configuration-view.tsx index cb9c3b66b..3135c634b 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/indexing-configuration-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/indexing-configuration-view.tsx @@ -1,9 +1,10 @@ "use client"; import { ArrowLeft, Check, Loader2 } from "lucide-react"; -import { type FC } from "react"; +import { type FC, useState, useCallback, useRef, useEffect } from "react"; import { Button } from "@/components/ui/button"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { cn } from "@/lib/utils"; import type { IndexingConfigState } from "./connector-constants"; import { DateRangeSelector } from "./date-range-selector"; import { PeriodicSyncConfig } from "./periodic-sync-config"; @@ -37,10 +38,49 @@ export const IndexingConfigurationView: FC = ({ onStartIndexing, onSkip, }) => { + const [isScrolled, setIsScrolled] = useState(false); + const [hasMoreContent, setHasMoreContent] = useState(false); + const scrollContainerRef = useRef(null); + + const checkScrollState = useCallback(() => { + if (!scrollContainerRef.current) return; + + const target = scrollContainerRef.current; + const scrolled = target.scrollTop > 0; + const hasMore = target.scrollHeight > target.clientHeight && + target.scrollTop + target.clientHeight < target.scrollHeight - 10; + + setIsScrolled(scrolled); + setHasMoreContent(hasMore); + }, []); + + const handleScroll = useCallback(() => { + checkScrollState(); + }, [checkScrollState]); + + // Check initial scroll state and on resize + useEffect(() => { + checkScrollState(); + const resizeObserver = new ResizeObserver(() => { + checkScrollState(); + }); + + if (scrollContainerRef.current) { + resizeObserver.observe(scrollContainerRef.current); + } + + return () => { + resizeObserver.disconnect(); + }; + }, [checkScrollState]); + return (

{/* Fixed Header */} -
+
{/* Back button */}
{/* Scrollable Content */} -
-
- +
+
+
+ - + - {/* Info box */} -
-
- {getConnectorIcon(config.connectorType, "size-4")} -
-
-

Indexing runs in the background

-

- You can continue using SurfSense while we sync your data. Check the Active tab to see progress. -

+ {/* Info box */} +
+
+ {getConnectorIcon(config.connectorType, "size-4")} +
+
+

Indexing runs in the background

+

+ You can continue using SurfSense while we sync your data. Check the Active tab to see progress. +

+
+ {/* Top fade shadow - appears when scrolled */} + {isScrolled && ( +
+ )} + {/* Bottom fade shadow - appears when there's more content */} + {hasMoreContent && ( +
+ )}
{/* Fixed Footer - Action buttons */} -
+
diff --git a/surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts index 5a309c2ce..32a475300 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts @@ -10,8 +10,16 @@ import { queryClient } from "@/lib/query-client/client"; import { cacheKeys } from "@/lib/query-client/cache-keys"; import { format } from "date-fns"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import { searchSourceConnector } from "@/contracts/types/connector.types"; import { OAUTH_CONNECTORS } from "./connector-constants"; import type { IndexingConfigState } from "./connector-constants"; +import { + parseConnectorPopupQueryParams, + parseOAuthAuthResponse, + validateIndexingConfigState, + frequencyMinutesSchema, + dateRangeSchema, +} from "./connector-popup.schemas"; export const useConnectorDialog = () => { const router = useRouter(); @@ -48,65 +56,101 @@ export const useConnectorDialog = () => { // Synchronize state with URL query params useEffect(() => { - const modalParam = searchParams.get("modal"); - const tabParam = searchParams.get("tab"); - const viewParam = searchParams.get("view"); - const connectorParam = searchParams.get("connector"); - - if (modalParam === "connectors") { - if (!isOpen) setIsOpen(true); + try { + const params = parseConnectorPopupQueryParams(searchParams); - if (tabParam === "active" || tabParam === "all") { - if (activeTab !== tabParam) setActiveTab(tabParam); - } - - if (viewParam === "configure" && connectorParam && !indexingConfig) { - const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === connectorParam); - if (oauthConnector && allConnectors) { - const existingConnector = allConnectors.find( - (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType - ); - if (existingConnector) { - setIndexingConfig({ - connectorType: oauthConnector.connectorType, - connectorId: existingConnector.id, - connectorTitle: oauthConnector.title, - }); + if (params.modal === "connectors") { + setIsOpen(true); + + if (params.tab === "active" || params.tab === "all") { + setActiveTab(params.tab); + } + + // Clear indexing config if view is not "configure" anymore + if (params.view !== "configure" && indexingConfig) { + setIndexingConfig(null); + } + + if (params.view === "configure" && params.connector && !indexingConfig) { + const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === params.connector); + if (oauthConnector && allConnectors) { + const existingConnector = allConnectors.find( + (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType + ); + if (existingConnector) { + // Validate connector data before setting state + const connectorValidation = searchSourceConnector.safeParse(existingConnector); + if (connectorValidation.success) { + const config = validateIndexingConfigState({ + connectorType: oauthConnector.connectorType, + connectorId: existingConnector.id, + connectorTitle: oauthConnector.title, + }); + setIndexingConfig(config); + } + } } } + } else { + setIsOpen(false); + // Clear indexing config when modal is closed + if (indexingConfig) { + setIndexingConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + setIsScrolled(false); + setSearchQuery(""); + } } - } else { - if (isOpen) setIsOpen(false); + } catch (error) { + // Invalid query params - log but don't crash + console.warn("Invalid connector popup query params:", error); } - }, [searchParams, isOpen, activeTab, indexingConfig, allConnectors]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [searchParams, allConnectors]); // Detect OAuth success and transition to config view useEffect(() => { - const success = searchParams.get("success"); - const connectorParam = searchParams.get("connector"); - const modalParam = searchParams.get("modal"); - - if (success === "true" && connectorParam && searchSpaceId && modalParam === "connectors") { - const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === connectorParam); - if (oauthConnector) { - refetchAllConnectors().then((result) => { - const newConnector = result.data?.find( - (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType - ); - if (newConnector) { - setIndexingConfig({ - connectorType: oauthConnector.connectorType, - connectorId: newConnector.id, - connectorTitle: oauthConnector.title, - }); - setIsOpen(true); - const url = new URL(window.location.href); - url.searchParams.delete("success"); - url.searchParams.set("view", "configure"); - window.history.replaceState({}, "", url.toString()); - } - }); + try { + const params = parseConnectorPopupQueryParams(searchParams); + + if (params.success === "true" && params.connector && searchSpaceId && params.modal === "connectors") { + const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === params.connector); + if (oauthConnector) { + refetchAllConnectors().then((result) => { + if (!result.data) return; + + const newConnector = result.data.find( + (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType + ); + if (newConnector) { + // Validate connector data before setting state + const connectorValidation = searchSourceConnector.safeParse(newConnector); + if (connectorValidation.success) { + const config = validateIndexingConfigState({ + connectorType: oauthConnector.connectorType, + connectorId: newConnector.id, + connectorTitle: oauthConnector.title, + }); + setIndexingConfig(config); + setIsOpen(true); + const url = new URL(window.location.href); + url.searchParams.delete("success"); + url.searchParams.set("view", "configure"); + window.history.replaceState({}, "", url.toString()); + } else { + console.warn("Invalid connector data after OAuth:", connectorValidation.error); + toast.error("Failed to validate connector data"); + } + } + }); + } } + } catch (error) { + // Invalid query params - log but don't crash + console.warn("Invalid connector popup query params in OAuth success handler:", error); } }, [searchParams, searchSpaceId, refetchAllConnectors]); @@ -115,8 +159,10 @@ export const useConnectorDialog = () => { async (connector: (typeof OAUTH_CONNECTORS)[0]) => { if (!searchSpaceId || !connector.authEndpoint) return; + // Set connecting state immediately to disable button and show spinner + setConnectingId(connector.id); + try { - setConnectingId(connector.id); const response = await authenticatedFetch( `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}${connector.authEndpoint}?space_id=${searchSpaceId}`, { method: "GET" } @@ -127,11 +173,21 @@ export const useConnectorDialog = () => { } const data = await response.json(); - window.location.href = data.auth_url; + + // Validate OAuth response with Zod + const validatedData = parseOAuthAuthResponse(data); + + // Don't clear connectingId here - let the redirect happen with button still disabled + // The component will unmount on redirect anyway + window.location.href = validatedData.auth_url; } catch (error) { console.error(`Error connecting to ${connector.title}:`, error); - toast.error(`Failed to connect to ${connector.title}`); - } finally { + if (error instanceof Error && error.message.includes("Invalid auth URL")) { + toast.error(`Invalid response from ${connector.title} OAuth endpoint`); + } else { + toast.error(`Failed to connect to ${connector.title}`); + } + // Only clear connectingId on error so user can retry setConnectingId(null); } }, @@ -142,6 +198,22 @@ export const useConnectorDialog = () => { const handleStartIndexing = useCallback(async (refreshConnectors: () => void) => { if (!indexingConfig || !searchSpaceId) return; + // Validate date range + const dateRangeValidation = dateRangeSchema.safeParse({ startDate, endDate }); + if (!dateRangeValidation.success) { + toast.error(dateRangeValidation.error.errors[0]?.message || "Invalid date range"); + return; + } + + // Validate frequency minutes if periodic is enabled + if (periodicEnabled) { + const frequencyValidation = frequencyMinutesSchema.safeParse(frequencyMinutes); + if (!frequencyValidation.success) { + toast.error("Invalid frequency value"); + return; + } + } + setIsStartingIndexing(true); try { const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined; @@ -173,18 +245,14 @@ export const useConnectorDialog = () => { : "You can continue working while we sync your data.", }); - setIndexingConfig(null); - setStartDate(undefined); - setEndDate(undefined); - setPeriodicEnabled(false); - setFrequencyMinutes("1440"); - + // Update URL - the effect will handle closing the modal and clearing state const url = new URL(window.location.href); - url.searchParams.delete("view"); + url.searchParams.delete("modal"); + url.searchParams.delete("tab"); + url.searchParams.delete("success"); url.searchParams.delete("connector"); - url.searchParams.set("tab", "active"); - window.history.replaceState({}, "", url.toString()); - setActiveTab("active"); + url.searchParams.delete("view"); + router.replace(url.pathname + url.search, { scroll: false }); refreshConnectors(); queryClient.invalidateQueries({ @@ -196,21 +264,19 @@ export const useConnectorDialog = () => { } finally { setIsStartingIndexing(false); } - }, [indexingConfig, searchSpaceId, startDate, endDate, indexConnector, updateConnector, periodicEnabled, frequencyMinutes, getFrequencyLabel]); + }, [indexingConfig, searchSpaceId, startDate, endDate, indexConnector, updateConnector, periodicEnabled, frequencyMinutes, getFrequencyLabel, router]); // Handle skipping indexing const handleSkipIndexing = useCallback(() => { - setIndexingConfig(null); - setStartDate(undefined); - setEndDate(undefined); - setPeriodicEnabled(false); - setFrequencyMinutes("1440"); - + // Update URL - the effect will handle closing the modal and clearing state const url = new URL(window.location.href); - url.searchParams.delete("view"); + url.searchParams.delete("modal"); + url.searchParams.delete("tab"); + url.searchParams.delete("success"); url.searchParams.delete("connector"); - window.history.replaceState({}, "", url.toString()); - }, []); + url.searchParams.delete("view"); + router.replace(url.pathname + url.search, { scroll: false }); + }, [router]); // Handle dialog open/close const handleOpenChange = useCallback( From c19d300c9d098626220b14fa4311f996d234caec Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 30 Dec 2025 09:00:59 -0800 Subject: [PATCH 52/92] feat: added circleback connector --- .../54_add_google_drive_connector_enums.py | 7 +- ...5_rename_google_drive_connector_to_file.py | 17 +- .../56_add_circleback_connector_enums.py | 73 ++++ .../agents/new_chat/tools/knowledge_base.py | 1 + .../app/connectors/google_drive/__init__.py | 15 +- .../connectors/google_drive/change_tracker.py | 1 - .../app/connectors/google_drive/client.py | 16 +- .../google_drive/content_extractor.py | 12 +- .../connectors/google_drive/credentials.py | 7 +- .../app/connectors/google_drive/file_types.py | 2 - .../connectors/google_drive/folder_manager.py | 16 +- surfsense_backend/app/db.py | 2 + surfsense_backend/app/routes/__init__.py | 8 +- .../app/routes/circleback_webhook_route.py | 317 +++++++++++++++ .../google_drive_add_connector_route.py | 18 +- .../routes/search_source_connectors_routes.py | 7 +- .../app/tasks/celery_tasks/document_tasks.py | 102 +++++ .../app/tasks/connector_indexers/__init__.py | 2 +- .../google_drive_indexer.py | 23 +- .../circleback_processor.py | 183 +++++++++ .../document_processors/file_processors.py | 11 +- .../add/circleback-connector/page.tsx | 363 ++++++++++++++++++ .../components/sources/connector-data.tsx | 39 +- surfsense_web/contracts/enums/connector.ts | 1 + .../contracts/enums/connectorIcons.tsx | 5 + surfsense_web/messages/en.json | 1 + surfsense_web/messages/zh.json | 1 + 27 files changed, 1153 insertions(+), 97 deletions(-) create mode 100644 surfsense_backend/alembic/versions/56_add_circleback_connector_enums.py create mode 100644 surfsense_backend/app/routes/circleback_webhook_route.py create mode 100644 surfsense_backend/app/tasks/document_processors/circleback_processor.py create mode 100644 surfsense_web/app/dashboard/[search_space_id]/connectors/add/circleback-connector/page.tsx diff --git a/surfsense_backend/alembic/versions/54_add_google_drive_connector_enums.py b/surfsense_backend/alembic/versions/54_add_google_drive_connector_enums.py index 8e7d69340..d802deb0d 100644 --- a/surfsense_backend/alembic/versions/54_add_google_drive_connector_enums.py +++ b/surfsense_backend/alembic/versions/54_add_google_drive_connector_enums.py @@ -57,18 +57,17 @@ def upgrade() -> None: def downgrade() -> None: """Remove 'GOOGLE_DRIVE_CONNECTOR' from enum types. - + Note: PostgreSQL doesn't support removing enum values directly. This would require recreating the enum type, which is complex and risky. For now, we'll leave the enum values in place. - + In a production environment with strict downgrade requirements, you would need to: 1. Create new enum types without the value 2. Convert all columns to use the new type 3. Drop the old enum type 4. Rename the new type to the old name - + This is left as pass to avoid accidental data loss. """ pass - diff --git a/surfsense_backend/alembic/versions/55_rename_google_drive_connector_to_file.py b/surfsense_backend/alembic/versions/55_rename_google_drive_connector_to_file.py index 137274b16..9ce57d95f 100644 --- a/surfsense_backend/alembic/versions/55_rename_google_drive_connector_to_file.py +++ b/surfsense_backend/alembic/versions/55_rename_google_drive_connector_to_file.py @@ -19,9 +19,9 @@ depends_on: str | Sequence[str] | None = None def upgrade() -> None: from sqlalchemy import text - + connection = op.get_bind() - + connection.execute( text( """ @@ -39,9 +39,9 @@ def upgrade() -> None: """ ) ) - + connection.commit() - + connection.execute( text( """ @@ -51,15 +51,15 @@ def upgrade() -> None: """ ) ) - + connection.commit() def downgrade() -> None: from sqlalchemy import text - + connection = op.get_bind() - + connection.execute( text( """ @@ -69,6 +69,5 @@ def downgrade() -> None: """ ) ) - - connection.commit() + connection.commit() diff --git a/surfsense_backend/alembic/versions/56_add_circleback_connector_enums.py b/surfsense_backend/alembic/versions/56_add_circleback_connector_enums.py new file mode 100644 index 000000000..551b8180b --- /dev/null +++ b/surfsense_backend/alembic/versions/56_add_circleback_connector_enums.py @@ -0,0 +1,73 @@ +"""Add Circleback connector enums + +Revision ID: 56 +Revises: 55 +Create Date: 2025-12-30 12:00:00.000000 + +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "56" +down_revision: str | None = "55" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Safely add 'CIRCLEBACK' to documenttype and 'CIRCLEBACK_CONNECTOR' to searchsourceconnectortype enums if missing.""" + + # Add to documenttype enum + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type t + JOIN pg_enum e ON t.oid = e.enumtypid + WHERE t.typname = 'documenttype' AND e.enumlabel = 'CIRCLEBACK' + ) THEN + ALTER TYPE documenttype ADD VALUE 'CIRCLEBACK'; + END IF; + END + $$; + """ + ) + + # Add to searchsourceconnectortype enum + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type t + JOIN pg_enum e ON t.oid = e.enumtypid + WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'CIRCLEBACK_CONNECTOR' + ) THEN + ALTER TYPE searchsourceconnectortype ADD VALUE 'CIRCLEBACK_CONNECTOR'; + END IF; + END + $$; + """ + ) + + +def downgrade() -> None: + """Remove 'CIRCLEBACK' and 'CIRCLEBACK_CONNECTOR' from enum types. + + Note: PostgreSQL doesn't support removing enum values directly. + This would require recreating the enum type, which is complex and risky. + For now, we'll leave the enum values in place. + + In a production environment with strict downgrade requirements, you would need to: + 1. Create new enum types without the value + 2. Convert all columns to use the new type + 3. Drop the old enum type + 4. Rename the new type to the old name + + This is left as pass to avoid accidental data loss. + """ + pass diff --git a/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py b/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py index ecaff6f2f..2096ce2b9 100644 --- a/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py +++ b/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py @@ -47,6 +47,7 @@ _ALL_CONNECTORS: list[str] = [ "NOTE", "BOOKSTACK_CONNECTOR", "CRAWLED_URL", + "CIRCLEBACK", ] diff --git a/surfsense_backend/app/connectors/google_drive/__init__.py b/surfsense_backend/app/connectors/google_drive/__init__.py index 6e0d25725..561072661 100644 --- a/surfsense_backend/app/connectors/google_drive/__init__.py +++ b/surfsense_backend/app/connectors/google_drive/__init__.py @@ -8,13 +8,12 @@ from .folder_manager import get_files_in_folder, list_folder_contents __all__ = [ "GoogleDriveClient", - "get_valid_credentials", - "validate_credentials", - "download_and_process_file", - "get_files_in_folder", - "list_folder_contents", - "get_start_page_token", - "fetch_all_changes", "categorize_change", + "download_and_process_file", + "fetch_all_changes", + "get_files_in_folder", + "get_start_page_token", + "get_valid_credentials", + "list_folder_contents", + "validate_credentials", ] - diff --git a/surfsense_backend/app/connectors/google_drive/change_tracker.py b/surfsense_backend/app/connectors/google_drive/change_tracker.py index 860e2dbef..dee828219 100644 --- a/surfsense_backend/app/connectors/google_drive/change_tracker.py +++ b/surfsense_backend/app/connectors/google_drive/change_tracker.py @@ -202,4 +202,3 @@ async def fetch_all_changes( except Exception as e: logger.error(f"Error fetching all changes: {e!s}", exc_info=True) return all_changes, current_token, f"Error fetching all changes: {e!s}" - diff --git a/surfsense_backend/app/connectors/google_drive/client.py b/surfsense_backend/app/connectors/google_drive/client.py index 5053aa449..aec5704b8 100644 --- a/surfsense_backend/app/connectors/google_drive/client.py +++ b/surfsense_backend/app/connectors/google_drive/client.py @@ -2,7 +2,6 @@ from typing import Any -from google.oauth2.credentials import Credentials from googleapiclient.discovery import build from googleapiclient.errors import HttpError from sqlalchemy.ext.asyncio import AsyncSession @@ -107,16 +106,18 @@ class GoogleDriveClient: """ try: service = await self.get_service() - file = service.files().get(fileId=file_id, fields=fields, supportsAllDrives=True).execute() + file = ( + service.files() + .get(fileId=file_id, fields=fields, supportsAllDrives=True) + .execute() + ) return file, None except HttpError as e: return None, f"HTTP error getting file metadata: {e.resp.status}" except Exception as e: return None, f"Error getting file metadata: {e!s}" - async def download_file( - self, file_id: str - ) -> tuple[bytes | None, str | None]: + async def download_file(self, file_id: str) -> tuple[bytes | None, str | None]: """ Download binary file content. @@ -164,9 +165,7 @@ class GoogleDriveClient: try: service = await self.get_service() content = ( - service.files() - .export(fileId=file_id, mimeType=mime_type) - .execute() + service.files().export(fileId=file_id, mimeType=mime_type).execute() ) # Content is already bytes from the API @@ -180,4 +179,3 @@ class GoogleDriveClient: return None, f"HTTP error exporting file: {e.resp.status}" except Exception as e: return None, f"Error exporting file: {e!s}" - diff --git a/surfsense_backend/app/connectors/google_drive/content_extractor.py b/surfsense_backend/app/connectors/google_drive/content_extractor.py index 1246d9e43..28c14a757 100644 --- a/surfsense_backend/app/connectors/google_drive/content_extractor.py +++ b/surfsense_backend/app/connectors/google_drive/content_extractor.py @@ -78,10 +78,10 @@ async def download_and_process_file( tmp_file.write(content_bytes) temp_file_path = tmp_file.name + from app.db import DocumentType from app.tasks.document_processors.file_processors import ( process_file_in_background, ) - from app.db import DocumentType connector_info = { "type": DocumentType.GOOGLE_DRIVE_FILE, @@ -92,7 +92,7 @@ async def download_and_process_file( "source_connector": "google_drive", }, } - + # Add additional Drive metadata if available if "modifiedTime" in file: connector_info["metadata"]["modified_time"] = file["modifiedTime"] @@ -102,10 +102,12 @@ async def download_and_process_file( connector_info["metadata"]["file_size"] = file["size"] if "webViewLink" in file: connector_info["metadata"]["web_view_link"] = file["webViewLink"] - + if is_google_workspace_file(mime_type): connector_info["metadata"]["exported_as"] = "pdf" - connector_info["metadata"]["original_workspace_type"] = mime_type.split(".")[-1] + connector_info["metadata"]["original_workspace_type"] = mime_type.split( + "." + )[-1] logger.info(f"Processing {file_name} with Surfsense's file processor") await process_file_in_background( @@ -132,5 +134,3 @@ async def download_and_process_file( os.unlink(temp_file_path) except Exception as e: logger.debug(f"Could not delete temp file {temp_file_path}: {e}") - - diff --git a/surfsense_backend/app/connectors/google_drive/credentials.py b/surfsense_backend/app/connectors/google_drive/credentials.py index 4c1ef9c03..f88486468 100644 --- a/surfsense_backend/app/connectors/google_drive/credentials.py +++ b/surfsense_backend/app/connectors/google_drive/credentials.py @@ -9,7 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.orm.attributes import flag_modified -from app.db import SearchSourceConnector, SearchSourceConnectorType +from app.db import SearchSourceConnector async def get_valid_credentials( @@ -31,9 +31,7 @@ async def get_valid_credentials( Exception: If token refresh fails """ result = await session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == connector_id - ) + select(SearchSourceConnector).filter(SearchSourceConnector.id == connector_id) ) connector = result.scalars().first() @@ -95,4 +93,3 @@ def validate_credentials(credentials: Credentials) -> bool: credentials.refresh_token, ] ) - diff --git a/surfsense_backend/app/connectors/google_drive/file_types.py b/surfsense_backend/app/connectors/google_drive/file_types.py index cb2354585..a66463208 100644 --- a/surfsense_backend/app/connectors/google_drive/file_types.py +++ b/surfsense_backend/app/connectors/google_drive/file_types.py @@ -26,5 +26,3 @@ def should_skip_file(mime_type: str) -> bool: def get_export_mime_type(mime_type: str) -> str | None: """Get export MIME type for Google Workspace files.""" return EXPORT_FORMATS.get(mime_type) - - diff --git a/surfsense_backend/app/connectors/google_drive/folder_manager.py b/surfsense_backend/app/connectors/google_drive/folder_manager.py index 599475a46..b0ed425ef 100644 --- a/surfsense_backend/app/connectors/google_drive/folder_manager.py +++ b/surfsense_backend/app/connectors/google_drive/folder_manager.py @@ -24,7 +24,10 @@ async def list_folders( """ try: # Build query to get only folders - query_parts = ["mimeType = 'application/vnd.google-apps.folder'", "trashed = false"] + query_parts = [ + "mimeType = 'application/vnd.google-apps.folder'", + "trashed = false", + ] if parent_id: query_parts.append(f"'{parent_id}' in parents") @@ -68,8 +71,7 @@ async def get_folder_hierarchy( # Traverse up to root while current_id: file, error = await client.get_file_metadata( - current_id, - fields="id, name, parents, mimeType" + current_id, fields="id, name, parents, mimeType" ) if error: @@ -189,7 +191,7 @@ async def list_folder_contents( # Fetch all items with pagination (max 1000 per page) all_items = [] page_token = None - + while True: items, next_token, error = await client.list_files( query=query, @@ -202,10 +204,10 @@ async def list_folder_contents( return [], error all_items.extend(items) - + if not next_token: break - + page_token = next_token for item in all_items: @@ -226,5 +228,3 @@ async def list_folder_contents( except Exception as e: logger.error(f"Error listing folder contents: {e!s}", exc_info=True) return [], f"Error listing folder contents: {e!s}" - - diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index c761561b5..fbd53bd06 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -51,6 +51,7 @@ class DocumentType(str, Enum): LUMA_CONNECTOR = "LUMA_CONNECTOR" ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR" BOOKSTACK_CONNECTOR = "BOOKSTACK_CONNECTOR" + CIRCLEBACK = "CIRCLEBACK" NOTE = "NOTE" @@ -76,6 +77,7 @@ class SearchSourceConnectorType(str, Enum): ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR" WEBCRAWLER_CONNECTOR = "WEBCRAWLER_CONNECTOR" BOOKSTACK_CONNECTOR = "BOOKSTACK_CONNECTOR" + CIRCLEBACK_CONNECTOR = "CIRCLEBACK_CONNECTOR" class LiteLLMProvider(str, Enum): diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 24751e596..3c18650ae 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -3,17 +3,18 @@ from fastapi import APIRouter from .airtable_add_connector_route import ( router as airtable_add_connector_router, ) +from .circleback_webhook_route import router as circleback_webhook_router from .documents_routes import router as documents_router from .editor_routes import router as editor_router from .google_calendar_add_connector_route import ( router as google_calendar_add_connector_router, ) -from .google_gmail_add_connector_route import ( - router as google_gmail_add_connector_router, -) from .google_drive_add_connector_route import ( router as google_drive_add_connector_router, ) +from .google_gmail_add_connector_route import ( + router as google_gmail_add_connector_router, +) from .logs_routes import router as logs_router from .luma_add_connector_route import router as luma_add_connector_router from .new_chat_routes import router as new_chat_router @@ -41,3 +42,4 @@ router.include_router(airtable_add_connector_router) router.include_router(luma_add_connector_router) router.include_router(new_llm_config_router) # LLM configs with prompt configuration router.include_router(logs_router) +router.include_router(circleback_webhook_router) # Circleback meeting webhooks diff --git a/surfsense_backend/app/routes/circleback_webhook_route.py b/surfsense_backend/app/routes/circleback_webhook_route.py new file mode 100644 index 000000000..1285aadeb --- /dev/null +++ b/surfsense_backend/app/routes/circleback_webhook_route.py @@ -0,0 +1,317 @@ +""" +Circleback Webhook Route + +This module provides a webhook endpoint for receiving meeting data from Circleback. +It processes the incoming webhook payload and saves it as a document in the specified search space. +""" + +import logging +from datetime import datetime +from typing import Any + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +# Pydantic models for Circleback webhook payload +class CirclebackAttendee(BaseModel): + """Attendee model for Circleback meeting.""" + + name: str | None = None + email: str | None = None + + +class CirclebackActionItemAssignee(BaseModel): + """Assignee model for action items.""" + + name: str | None = None + email: str | None = None + + +class CirclebackActionItem(BaseModel): + """Action item model for Circleback meeting.""" + + id: int + title: str + description: str = "" + assignee: CirclebackActionItemAssignee | None = None + status: str = "PENDING" + + +class CirclebackTranscriptSegment(BaseModel): + """Transcript segment model for Circleback meeting.""" + + speaker: str + text: str + timestamp: float + + +class CirclebackInsightItem(BaseModel): + """Individual insight item.""" + + insight: str | dict[str, Any] + speaker: str | None = None + timestamp: float | None = None + + +class CirclebackWebhookPayload(BaseModel): + """ + Circleback webhook payload model. + + This model represents the data sent by Circleback when a meeting is processed. + """ + + model_config = {"populate_by_name": True} + + id: int = Field(..., description="Circleback meeting ID") + name: str = Field(..., description="Meeting name") + created_at: str = Field( + ..., alias="createdAt", description="Meeting creation date in ISO format" + ) + duration: float = Field(..., description="Meeting duration in seconds") + url: str | None = Field(None, description="URL of the virtual meeting") + recording_url: str | None = Field( + None, + alias="recordingUrl", + description="URL of the meeting recording (valid for 24 hours)", + ) + tags: list[str] = Field(default_factory=list, description="Meeting tags") + ical_uid: str | None = Field( + None, alias="icalUid", description="Unique identifier of the calendar event" + ) + attendees: list[CirclebackAttendee] = Field( + default_factory=list, description="Meeting attendees" + ) + notes: str = Field("", description="Meeting notes in Markdown format") + action_items: list[CirclebackActionItem] = Field( + default_factory=list, + alias="actionItems", + description="Action items from the meeting", + ) + transcript: list[CirclebackTranscriptSegment] = Field( + default_factory=list, description="Meeting transcript segments" + ) + insights: dict[str, list[CirclebackInsightItem]] = Field( + default_factory=dict, description="Custom insights from the meeting" + ) + + +def format_circleback_meeting_to_markdown(payload: CirclebackWebhookPayload) -> str: + """ + Convert Circleback webhook payload to a well-formatted Markdown document. + + Args: + payload: The Circleback webhook payload + + Returns: + Markdown string representation of the meeting + """ + lines = [] + + # Title + lines.append(f"# {payload.name}") + lines.append("") + + # Meeting metadata + lines.append("## Meeting Details") + lines.append("") + + # Parse and format date + try: + created_dt = datetime.fromisoformat(payload.created_at.replace("Z", "+00:00")) + formatted_date = created_dt.strftime("%Y-%m-%d %H:%M:%S UTC") + except (ValueError, AttributeError): + formatted_date = payload.created_at + + lines.append(f"- **Date:** {formatted_date}") + lines.append(f"- **Duration:** {int(payload.duration // 60)} minutes") + + if payload.url: + lines.append(f"- **Meeting URL:** {payload.url}") + + if payload.tags: + lines.append(f"- **Tags:** {', '.join(payload.tags)}") + + lines.append( + f"- **Circleback Link:** [View on Circleback](https://app.circleback.ai/meetings/{payload.id})" + ) + lines.append("") + + # Attendees + if payload.attendees: + lines.append("## Attendees") + lines.append("") + for attendee in payload.attendees: + name = attendee.name or "Unknown" + if attendee.email: + lines.append(f"- **{name}** ({attendee.email})") + else: + lines.append(f"- **{name}**") + lines.append("") + + # Notes (if provided) + if payload.notes: + lines.append("## Meeting Notes") + lines.append("") + lines.append(payload.notes) + lines.append("") + + # Action Items + if payload.action_items: + lines.append("## Action Items") + lines.append("") + for item in payload.action_items: + status_emoji = "✅" if item.status == "DONE" else "⬜" + assignee_text = "" + if item.assignee and item.assignee.name: + assignee_text = f" (Assigned to: {item.assignee.name})" + + lines.append(f"{status_emoji} **{item.title}**{assignee_text}") + if item.description: + lines.append(f" {item.description}") + lines.append("") + + # Insights + if payload.insights: + lines.append("## Insights") + lines.append("") + for insight_name, insight_items in payload.insights.items(): + lines.append(f"### {insight_name}") + lines.append("") + for insight_item in insight_items: + if isinstance(insight_item.insight, dict): + for key, value in insight_item.insight.items(): + lines.append(f"- **{key}:** {value}") + else: + speaker_info = ( + f" _{insight_item.speaker}_" if insight_item.speaker else "" + ) + lines.append(f"- {insight_item.insight}{speaker_info}") + lines.append("") + + # Transcript + if payload.transcript: + lines.append("## Transcript") + lines.append("") + for segment in payload.transcript: + # Format timestamp as MM:SS + minutes = int(segment.timestamp // 60) + seconds = int(segment.timestamp % 60) + timestamp_str = f"[{minutes:02d}:{seconds:02d}]" + lines.append(f"**{segment.speaker}** {timestamp_str}: {segment.text}") + lines.append("") + + return "\n".join(lines) + + +@router.post("/webhooks/circleback/{search_space_id}") +async def receive_circleback_webhook( + search_space_id: int, + payload: CirclebackWebhookPayload, +): + """ + Receive and process a Circleback webhook. + + This endpoint receives meeting data from Circleback and saves it as a document + in the specified search space. The meeting data is converted to Markdown format + and processed asynchronously. + + Args: + search_space_id: The ID of the search space to save the document to + payload: The Circleback webhook payload containing meeting data + + Returns: + Success message with document details + + Note: + This endpoint does not require authentication as it's designed to receive + webhooks from Circleback. Signature verification can be added later for security. + """ + try: + logger.info( + f"Received Circleback webhook for meeting {payload.id} in search space {search_space_id}" + ) + + # Convert to markdown + markdown_content = format_circleback_meeting_to_markdown(payload) + + # Trigger async document processing + from app.tasks.celery_tasks.document_tasks import ( + process_circleback_meeting_task, + ) + + # Prepare meeting metadata for the task + meeting_metadata = { + "circleback_meeting_id": payload.id, + "meeting_name": payload.name, + "meeting_date": payload.created_at, + "duration_seconds": payload.duration, + "meeting_url": payload.url, + "tags": payload.tags, + "attendees_count": len(payload.attendees), + "action_items_count": len(payload.action_items), + "has_transcript": len(payload.transcript) > 0, + } + + # Queue the processing task + process_circleback_meeting_task.delay( + meeting_id=payload.id, + meeting_name=payload.name, + markdown_content=markdown_content, + metadata=meeting_metadata, + search_space_id=search_space_id, + ) + + logger.info( + f"Queued Circleback meeting {payload.id} for processing in search space {search_space_id}" + ) + + return { + "status": "accepted", + "message": f"Meeting '{payload.name}' queued for processing", + "meeting_id": payload.id, + "search_space_id": search_space_id, + } + + except Exception as e: + logger.error(f"Error processing Circleback webhook: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, + detail=f"Failed to process Circleback webhook: {e!s}", + ) from e + + +@router.get("/webhooks/circleback/{search_space_id}/info") +async def get_circleback_webhook_info( + search_space_id: int, +): + """ + Get information about the Circleback webhook endpoint. + + This endpoint provides information about how to configure the Circleback + webhook integration. + + Args: + search_space_id: The ID of the search space + + Returns: + Webhook configuration information + """ + from app.config import config + + # Construct the webhook URL + base_url = getattr(config, "API_BASE_URL", "http://localhost:8000") + webhook_url = f"{base_url}/api/v1/webhooks/circleback/{search_space_id}" + + return { + "webhook_url": webhook_url, + "search_space_id": search_space_id, + "method": "POST", + "content_type": "application/json", + "description": "Use this URL in your Circleback automation to send meeting data to SurfSense", + "note": "Configure this URL in Circleback Settings → Automations → Create automation → Send webhook request", + } diff --git a/surfsense_backend/app/routes/google_drive_add_connector_route.py b/surfsense_backend/app/routes/google_drive_add_connector_route.py index d11404781..200441d33 100644 --- a/surfsense_backend/app/routes/google_drive_add_connector_route.py +++ b/surfsense_backend/app/routes/google_drive_add_connector_route.py @@ -28,10 +28,8 @@ from app.config import config from app.connectors.google_drive import ( GoogleDriveClient, get_start_page_token, - get_valid_credentials, list_folder_contents, ) -from app.connectors.google_drive.folder_manager import list_folders from app.db import ( SearchSourceConnector, SearchSourceConnectorType, @@ -111,7 +109,9 @@ async def connect_drive(space_id: int, user: User = Depends(current_active_user) state=state_encoded, ) - logger.info(f"Initiating Google Drive OAuth for user {user.id}, space {space_id}") + logger.info( + f"Initiating Google Drive OAuth for user {user.id}, space {space_id}" + ) return {"auth_url": auth_url} except Exception as e: @@ -146,7 +146,9 @@ async def drive_callback( user_id = UUID(data["user_id"]) space_id = data["space_id"] - logger.info(f"Processing Google Drive callback for user {user_id}, space {space_id}") + logger.info( + f"Processing Google Drive callback for user {user_id}, space {space_id}" + ) # Exchange authorization code for tokens flow = get_google_flow() @@ -200,7 +202,9 @@ async def drive_callback( flag_modified(db_connector, "config") await session.commit() - logger.info(f"Set initial start page token for connector {db_connector.id}") + logger.info( + f"Set initial start page token for connector {db_connector.id}" + ) except Exception as e: logger.warning(f"Failed to get initial start page token: {e!s}") @@ -246,7 +250,7 @@ async def list_google_drive_folders( ): """ List folders AND files in user's Google Drive with hierarchical support. - + This is called at index time from the manage connector page to display the complete file system (folders and files). Only folders are selectable. @@ -299,7 +303,7 @@ async def list_google_drive_folders( f"✅ Listed {len(items)} total items ({folder_count} folders, {file_count} files) for connector {connector_id}" + (f" in folder {parent_id}" if parent_id else " in ROOT") ) - + # Log first few items for debugging if items: logger.info(f"First 3 items: {[item.get('name') for item in items[:3]]}") diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 894be54c4..8efbbfa5f 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -45,7 +45,6 @@ from app.tasks.connector_indexers import ( index_github_repos, index_google_calendar_events, index_google_gmail_messages, - index_google_drive_files, index_jira_issues, index_linear_issues, index_luma_events, @@ -1572,7 +1571,9 @@ async def run_google_drive_indexing( errors = [] # Index each folder - for folder_id, folder_name in zip(folder_id_list, folder_name_list): + for folder_id, folder_name in zip( + folder_id_list, folder_name_list, strict=False + ): try: indexed_count, error_message = await index_google_drive_files( session, @@ -1589,7 +1590,7 @@ async def run_google_drive_indexing( else: total_indexed += indexed_count except Exception as e: - errors.append(f"{folder_name}: {str(e)}") + errors.append(f"{folder_name}: {e!s}") logger.error( f"Error indexing folder {folder_name} ({folder_id}): {e}", exc_info=True, diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index 5b7f9ce13..bb53fd042 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -268,3 +268,105 @@ async def _process_file_upload( ) logger.error(error_message) raise + + +@celery_app.task(name="process_circleback_meeting", bind=True) +def process_circleback_meeting_task( + self, + meeting_id: int, + meeting_name: str, + markdown_content: str, + metadata: dict, + search_space_id: int, +): + """ + Celery task to process Circleback meeting webhook data. + + Args: + meeting_id: Circleback meeting ID + meeting_name: Name of the meeting + markdown_content: Meeting content formatted as markdown + metadata: Meeting metadata dictionary + search_space_id: ID of the search space + """ + import asyncio + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + try: + loop.run_until_complete( + _process_circleback_meeting( + meeting_id, + meeting_name, + markdown_content, + metadata, + search_space_id, + ) + ) + finally: + loop.close() + + +async def _process_circleback_meeting( + meeting_id: int, + meeting_name: str, + markdown_content: str, + metadata: dict, + search_space_id: int, +): + """Process Circleback meeting with new session.""" + from app.tasks.document_processors.circleback_processor import ( + add_circleback_meeting_document, + ) + + async with get_celery_session_maker()() as session: + task_logger = TaskLoggingService(session, search_space_id) + + log_entry = await task_logger.log_task_start( + task_name="process_circleback_meeting", + source="circleback_webhook", + message=f"Starting Circleback meeting processing: {meeting_name}", + metadata={ + "document_type": "CIRCLEBACK", + "meeting_id": meeting_id, + "meeting_name": meeting_name, + **metadata, + }, + ) + + try: + result = await add_circleback_meeting_document( + session=session, + meeting_id=meeting_id, + meeting_name=meeting_name, + markdown_content=markdown_content, + metadata=metadata, + search_space_id=search_space_id, + ) + + if result: + await task_logger.log_task_success( + log_entry, + f"Successfully processed Circleback meeting: {meeting_name}", + { + "document_id": result.id, + "meeting_id": meeting_id, + "content_hash": result.content_hash, + }, + ) + else: + await task_logger.log_task_success( + log_entry, + f"Circleback meeting document already exists (duplicate): {meeting_name}", + {"duplicate_detected": True, "meeting_id": meeting_id}, + ) + except Exception as e: + await task_logger.log_task_failure( + log_entry, + f"Failed to process Circleback meeting: {meeting_name}", + str(e), + {"error_type": type(e).__name__, "meeting_id": meeting_id}, + ) + logger.error(f"Error processing Circleback meeting: {e!s}") + raise diff --git a/surfsense_backend/app/tasks/connector_indexers/__init__.py b/surfsense_backend/app/tasks/connector_indexers/__init__.py index 80a9eaf19..95e57ddf2 100644 --- a/surfsense_backend/app/tasks/connector_indexers/__init__.py +++ b/surfsense_backend/app/tasks/connector_indexers/__init__.py @@ -34,8 +34,8 @@ from .discord_indexer import index_discord_messages from .elasticsearch_indexer import index_elasticsearch_documents from .github_indexer import index_github_repos from .google_calendar_indexer import index_google_calendar_events -from .google_gmail_indexer import index_google_gmail_messages from .google_drive_indexer import index_google_drive_files +from .google_gmail_indexer import index_google_gmail_messages from .jira_indexer import index_jira_issues # Issue tracking and project management diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index 5695c084d..10f4b672c 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -1,7 +1,6 @@ """Google Drive indexer using Surfsense file processors.""" import logging -from datetime import datetime from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession @@ -99,11 +98,15 @@ async def index_google_drive_files( target_folder_id = folder_id target_folder_name = folder_name or "Selected Folder" - logger.info(f"Indexing Google Drive folder: {target_folder_name} ({target_folder_id})") + logger.info( + f"Indexing Google Drive folder: {target_folder_name} ({target_folder_id})" + ) folder_tokens = connector.config.get("folder_tokens", {}) start_page_token = folder_tokens.get(target_folder_id) - can_use_delta_sync = use_delta_sync and start_page_token and connector.last_indexed_at + can_use_delta_sync = ( + use_delta_sync and start_page_token and connector.last_indexed_at + ) if can_use_delta_sync: logger.info(f"Using delta sync for connector {connector_id}") @@ -151,9 +154,7 @@ async def index_google_drive_files( await update_connector_last_indexed(session, connector, update_last_indexed) await session.commit() - logger.info( - f"Successfully committed Google Drive indexing changes to database" - ) + logger.info("Successfully committed Google Drive indexing changes to database") await task_logger.log_task_success( log_entry, @@ -252,7 +253,9 @@ async def _index_full_scan( if documents_indexed % 10 == 0 and documents_indexed > 0: await session.commit() - logger.info(f"Committed batch: {documents_indexed} files indexed so far") + logger.info( + f"Committed batch: {documents_indexed} files indexed so far" + ) page_token = next_token if not page_token: @@ -391,9 +394,7 @@ async def _process_single_file( return 0, 1 -async def _remove_document( - session: AsyncSession, file_id: str, search_space_id: int -): +async def _remove_document(session: AsyncSession, file_id: str, search_space_id: int): """Remove a document that was deleted in Drive.""" unique_identifier_hash = generate_unique_identifier_hash( DocumentType.GOOGLE_DRIVE_FILE, file_id, search_space_id @@ -406,5 +407,3 @@ async def _remove_document( if existing_document: await session.delete(existing_document) logger.info(f"Removed deleted file document: {file_id}") - - diff --git a/surfsense_backend/app/tasks/document_processors/circleback_processor.py b/surfsense_backend/app/tasks/document_processors/circleback_processor.py new file mode 100644 index 000000000..0a1d91784 --- /dev/null +++ b/surfsense_backend/app/tasks/document_processors/circleback_processor.py @@ -0,0 +1,183 @@ +""" +Circleback meeting document processor. + +This module processes meeting data received from Circleback webhooks +and stores it as searchable documents in the database. +""" + +import logging +from typing import Any + +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import Document, DocumentType +from app.services.llm_service import get_document_summary_llm +from app.utils.document_converters import ( + create_document_chunks, + generate_content_hash, + generate_document_summary, + generate_unique_identifier_hash, +) + +from .base import ( + check_document_by_unique_identifier, + get_current_timestamp, +) + +logger = logging.getLogger(__name__) + + +async def add_circleback_meeting_document( + session: AsyncSession, + meeting_id: int, + meeting_name: str, + markdown_content: str, + metadata: dict[str, Any], + search_space_id: int, +) -> Document | None: + """ + Process and store a Circleback meeting document. + + Args: + session: Database session + meeting_id: Circleback meeting ID + meeting_name: Name of the meeting + markdown_content: Meeting content formatted as markdown + metadata: Meeting metadata dictionary + search_space_id: ID of the search space + + Returns: + Document object if successful, None if failed or duplicate + """ + try: + # Generate unique identifier hash using Circleback meeting ID + unique_identifier = f"circleback_{meeting_id}" + unique_identifier_hash = generate_unique_identifier_hash( + DocumentType.CIRCLEBACK, unique_identifier, search_space_id + ) + + # Generate content hash + content_hash = generate_content_hash(markdown_content, search_space_id) + + # Check if document with this unique identifier already exists + existing_document = await check_document_by_unique_identifier( + session, unique_identifier_hash + ) + + if existing_document: + # Document exists - check if content has changed + if existing_document.content_hash == content_hash: + logger.info(f"Circleback meeting {meeting_id} unchanged. Skipping.") + return existing_document + else: + # Content has changed - update the existing document + logger.info( + f"Content changed for Circleback meeting {meeting_id}. Updating document." + ) + + # Get LLM for generating summary + llm = await get_document_summary_llm(session, search_space_id) + if not llm: + logger.warning( + f"No LLM configured for search space {search_space_id}. Using content as summary." + ) + # Use first 1000 chars as summary if no LLM available + summary_content = ( + markdown_content[:1000] + "..." + if len(markdown_content) > 1000 + else markdown_content + ) + summary_embedding = None + else: + # Generate summary with metadata + document_metadata = { + "meeting_name": meeting_name, + "meeting_id": meeting_id, + "document_type": "Circleback Meeting", + **{ + k: v + for k, v in metadata.items() + if isinstance(v, str | int | float | bool) + }, + } + summary_content, summary_embedding = await generate_document_summary( + markdown_content, llm, document_metadata + ) + + # Process chunks + chunks = await create_document_chunks(markdown_content) + + # Convert to BlockNote JSON for editing capability + from app.utils.blocknote_converter import convert_markdown_to_blocknote + + blocknote_json = await convert_markdown_to_blocknote(markdown_content) + if not blocknote_json: + logger.warning( + f"Failed to convert Circleback meeting {meeting_id} to BlockNote JSON, document will not be editable" + ) + + # Prepare document metadata + document_metadata = { + "CIRCLEBACK_MEETING_ID": meeting_id, + "MEETING_NAME": meeting_name, + "SOURCE": "CIRCLEBACK_WEBHOOK", + **metadata, + } + + # Update or create document + if existing_document: + # Update existing document + existing_document.title = meeting_name + existing_document.content = summary_content + existing_document.content_hash = content_hash + if summary_embedding is not None: + existing_document.embedding = summary_embedding + existing_document.document_metadata = document_metadata + existing_document.chunks = chunks + existing_document.blocknote_document = blocknote_json + existing_document.content_needs_reindexing = False + existing_document.updated_at = get_current_timestamp() + + await session.commit() + await session.refresh(existing_document) + document = existing_document + logger.info( + f"Updated Circleback meeting document {meeting_id} in search space {search_space_id}" + ) + else: + # Create new document + document = Document( + search_space_id=search_space_id, + title=meeting_name, + document_type=DocumentType.CIRCLEBACK, + document_metadata=document_metadata, + content=summary_content, + embedding=summary_embedding, + chunks=chunks, + content_hash=content_hash, + unique_identifier_hash=unique_identifier_hash, + blocknote_document=blocknote_json, + content_needs_reindexing=False, + updated_at=get_current_timestamp(), + ) + + session.add(document) + await session.commit() + await session.refresh(document) + logger.info( + f"Created new Circleback meeting document {meeting_id} in search space {search_space_id}" + ) + + return document + + except SQLAlchemyError as db_error: + await session.rollback() + logger.error( + f"Database error processing Circleback meeting {meeting_id}: {db_error}" + ) + raise db_error + except Exception as e: + await session.rollback() + logger.error(f"Failed to process Circleback meeting {meeting_id}: {e!s}") + raise RuntimeError(f"Failed to process Circleback meeting: {e!s}") from e diff --git a/surfsense_backend/app/tasks/document_processors/file_processors.py b/surfsense_backend/app/tasks/document_processors/file_processors.py index 6a01db6a9..596cd9830 100644 --- a/surfsense_backend/app/tasks/document_processors/file_processors.py +++ b/surfsense_backend/app/tasks/document_processors/file_processors.py @@ -473,7 +473,8 @@ async def process_file_in_background( session: AsyncSession, task_logger: TaskLoggingService, log_entry: Log, - connector: dict | None = None, # Optional: {"type": "GOOGLE_DRIVE_FILE", "metadata": {...}} + connector: dict + | None = None, # Optional: {"type": "GOOGLE_DRIVE_FILE", "metadata": {...}} ): try: # Check if the file is a markdown or text file @@ -926,7 +927,9 @@ async def process_file_in_background( ) if connector: - await _update_document_from_connector(last_created_doc, connector, session) + await _update_document_from_connector( + last_created_doc, connector, session + ) await task_logger.log_task_success( log_entry, @@ -1053,7 +1056,9 @@ async def process_file_in_background( ) if connector: - await _update_document_from_connector(doc_result, connector, session) + await _update_document_from_connector( + doc_result, connector, session + ) await task_logger.log_task_success( log_entry, diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/circleback-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/circleback-connector/page.tsx new file mode 100644 index 000000000..ce28de5be --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/circleback-connector/page.tsx @@ -0,0 +1,363 @@ +"use client"; + +import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; +import { ArrowLeft, Check, Copy, ExternalLink, Loader2, Webhook } from "lucide-react"; +import { motion } from "motion/react"; +import Link from "next/link"; +import { useParams, useRouter } from "next/navigation"; +import { useEffect, useState } from "react"; +import { useForm } from "react-hook-form"; +import { toast } from "sonner"; +import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { EnumConnectorName } from "@/contracts/enums/connector"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; + +// Define the form schema with Zod +const circlebackConnectorFormSchema = z.object({ + name: z.string().min(3, { + message: "Connector name must be at least 3 characters.", + }), +}); + +// Define the type for the form values +type CirclebackConnectorFormValues = z.infer; + +export default function CirclebackConnectorPage() { + const router = useRouter(); + const params = useParams(); + const searchSpaceId = params.search_space_id as string; + const [isSubmitting, setIsSubmitting] = useState(false); + const [doesConnectorExist, setDoesConnectorExist] = useState(false); + const [copied, setCopied] = useState(false); + + const { data: connectors } = useAtomValue(connectorsAtom); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); + + // Construct the webhook URL + const apiBaseUrl = process.env.NEXT_PUBLIC_BACKEND_URL || "http://localhost:8000"; + const webhookUrl = `${apiBaseUrl}/api/v1/webhooks/circleback/${searchSpaceId}`; + + // Initialize the form + const form = useForm({ + resolver: zodResolver(circlebackConnectorFormSchema), + defaultValues: { + name: "Circleback Meetings", + }, + }); + + const { refetch: fetchConnectors } = useAtomValue(connectorsAtom); + + useEffect(() => { + fetchConnectors().then((data) => { + const connectors = data.data || []; + const connector = connectors.find( + (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.CIRCLEBACK_CONNECTOR + ); + if (connector) { + setDoesConnectorExist(true); + } + }); + }, []); + + // Copy webhook URL to clipboard + const copyToClipboard = async () => { + try { + await navigator.clipboard.writeText(webhookUrl); + setCopied(true); + toast.success("Webhook URL copied to clipboard!"); + setTimeout(() => setCopied(false), 2000); + } catch { + toast.error("Failed to copy to clipboard"); + } + }; + + // Handle form submission + const onSubmit = async (values: CirclebackConnectorFormValues) => { + setIsSubmitting(true); + try { + await createConnector({ + data: { + name: values.name, + connector_type: EnumConnectorName.CIRCLEBACK_CONNECTOR, + config: { + webhook_url: webhookUrl, + }, + is_indexable: false, // Webhooks push data, not indexed + last_indexed_at: null, + periodic_indexing_enabled: false, + indexing_frequency_minutes: null, + next_scheduled_at: null, + }, + queryParams: { + search_space_id: searchSpaceId, + }, + }); + + toast.success("Circleback connector created successfully!"); + + // Navigate back to connectors page + router.push(`/dashboard/${searchSpaceId}/connectors`); + } catch (error) { + console.error("Error creating connector:", error); + toast.error(error instanceof Error ? error.message : "Failed to create connector"); + } finally { + setIsSubmitting(false); + } + }; + + return ( +
+ + {/* Header */} +
+ + + Back to connectors + +
+
+ {getConnectorIcon(EnumConnectorName.CIRCLEBACK_CONNECTOR, "h-6 w-6")} +
+
+

Connect Circleback

+

+ Receive meeting notes and transcripts via webhook. +

+
+
+
+ + {/* Connection Card */} + {!doesConnectorExist ? ( + <> + + + Webhook Configuration + + Use this webhook URL in your Circleback automation to send meeting data to + SurfSense. + + + +
+ +
+ + +
+

+ Copy this URL and paste it in your Circleback automation settings. +

+
+ + + + How it works + + When you configure this webhook in Circleback, it will automatically send + meeting notes, transcripts, and action items to SurfSense after each meeting. + + + + + + + + Create Connector + + Register the Circleback connector to track incoming meeting data. + + +
+ + + ( + + Connector Name + + + + + A friendly name to identify this connector. + + + + )} + /> + +
+
+ + Automatic meeting notes import +
+
+ + Full transcripts with speaker identification +
+
+ + Action items and insights extraction +
+
+ + + + + +
+ + + + ) : ( + /* Success Card */ + + + ✅ Circleback connector is active! + + Your Circleback meetings will be automatically imported to this search space. + + + +
+ +
+ + +
+
+ + + )} + + {/* Help Section */} + + + Setup Instructions + + +
+

1. Copy the Webhook URL

+

+ Copy the webhook URL shown above. You'll need this for the next step. +

+
+
+

2. Open Circleback Automations

+

+ Go to{" "} + + Circleback Automations + + {" "} + and click "Create automation". +

+
+
+

3. Configure the Webhook

+

+ Set your automation conditions, then select "Send webhook request" and paste the + webhook URL. +

+
+
+

4. Select Meeting Outcomes

+

+ Choose which meeting data to include: notes, transcript, action items, and + insights. +

+
+
+

5. Create & Test

+

+ Give your automation a name and create it. You can send a test request to verify + the integration works. +

+
+ + + +
+ ); +} + diff --git a/surfsense_web/components/sources/connector-data.tsx b/surfsense_web/components/sources/connector-data.tsx index 7fca3e6b9..0ab696ceb 100644 --- a/surfsense_web/components/sources/connector-data.tsx +++ b/surfsense_web/components/sources/connector-data.tsx @@ -190,20 +190,27 @@ export const connectorCategories: ConnectorCategory[] = [ icon: getConnectorIcon(EnumConnectorName.GOOGLE_DRIVE_CONNECTOR, "h-6 w-6"), status: "available", }, - { - id: "luma-connector", - title: "Luma", - description: "luma_desc", - icon: getConnectorIcon(EnumConnectorName.LUMA_CONNECTOR, "h-6 w-6"), - status: "available", - }, - { - id: "zoom", - title: "Zoom", - description: "zoom_desc", - icon: , - status: "coming-soon", - }, - ], - }, + { + id: "luma-connector", + title: "Luma", + description: "luma_desc", + icon: getConnectorIcon(EnumConnectorName.LUMA_CONNECTOR, "h-6 w-6"), + status: "available", + }, + { + id: "circleback-connector", + title: "Circleback", + description: "circleback_desc", + icon: getConnectorIcon(EnumConnectorName.CIRCLEBACK_CONNECTOR, "h-6 w-6"), + status: "available", + }, + { + id: "zoom", + title: "Zoom", + description: "zoom_desc", + icon: , + status: "coming-soon", + }, + ], +}, ]; diff --git a/surfsense_web/contracts/enums/connector.ts b/surfsense_web/contracts/enums/connector.ts index eb2cf7ad8..22f6530da 100644 --- a/surfsense_web/contracts/enums/connector.ts +++ b/surfsense_web/contracts/enums/connector.ts @@ -19,4 +19,5 @@ export enum EnumConnectorName { LUMA_CONNECTOR = "LUMA_CONNECTOR", ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR", WEBCRAWLER_CONNECTOR = "WEBCRAWLER_CONNECTOR", + CIRCLEBACK_CONNECTOR = "CIRCLEBACK_CONNECTOR", } diff --git a/surfsense_web/contracts/enums/connectorIcons.tsx b/surfsense_web/contracts/enums/connectorIcons.tsx index 661be5253..9281c00e9 100644 --- a/surfsense_web/contracts/enums/connectorIcons.tsx +++ b/surfsense_web/contracts/enums/connectorIcons.tsx @@ -15,6 +15,7 @@ import { IconSparkles, IconTable, IconTicket, + IconUsersGroup, IconWorldWww, } from "@tabler/icons-react"; import { @@ -74,7 +75,11 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas return ; case EnumConnectorName.WEBCRAWLER_CONNECTOR: return ; + case EnumConnectorName.CIRCLEBACK_CONNECTOR: + return ; // Additional cases for non-enum connector types + case "CIRCLEBACK": + return ; case "CRAWLED_URL": return ; case "YOUTUBE_VIDEO": diff --git a/surfsense_web/messages/en.json b/surfsense_web/messages/en.json index 0fa6e461b..151556d3a 100644 --- a/surfsense_web/messages/en.json +++ b/surfsense_web/messages/en.json @@ -305,6 +305,7 @@ "bookstack_desc": "Connect to BookStack to search wiki pages and documentation.", "airtable_desc": "Connect to Airtable to search records, tables and database content.", "luma_desc": "Connect to Luma to search events, meetups and gatherings.", + "circleback_desc": "Receive meeting notes, transcripts and action items from Circleback via webhook.", "calendar_desc": "Connect to Google Calendar to search events, meetings and schedules.", "gmail_desc": "Connect to your Gmail account to search through your emails.", "google_drive_desc": "Connect to Google Drive to search and index your files and documents.", diff --git a/surfsense_web/messages/zh.json b/surfsense_web/messages/zh.json index 625c8a31e..67069cf55 100644 --- a/surfsense_web/messages/zh.json +++ b/surfsense_web/messages/zh.json @@ -305,6 +305,7 @@ "bookstack_desc": "连接到 BookStack 以搜索 Wiki 页面和文档。", "airtable_desc": "连接到 Airtable 以搜索记录、表格和数据库内容。", "luma_desc": "连接到 Luma 以搜索活动、聚会和集会。", + "circleback_desc": "通过 Webhook 从 Circleback 接收会议记录、转录和行动项目。", "calendar_desc": "连接到 Google 日历以搜索活动、会议和日程。", "gmail_desc": "连接到您的 Gmail 账户以搜索您的电子邮件。", "google_drive_desc": "连接到 Google 云端硬盘以搜索和索引您的文件和文档。", From 476c76461126d8479f2d6b1eb3aa0a09a5b7f3f2 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 30 Dec 2025 12:13:18 -0800 Subject: [PATCH 53/92] feat: fix Circleback connector and update related enums --- .../56_add_circleback_connector_enums.py | 39 ++----- .../agents/new_chat/tools/knowledge_base.py | 11 ++ .../app/services/connector_service.py | 102 ++++++++++++++++++ .../contracts/types/connector.types.ts | 1 + .../contracts/types/document.types.ts | 1 + 5 files changed, 123 insertions(+), 31 deletions(-) diff --git a/surfsense_backend/alembic/versions/56_add_circleback_connector_enums.py b/surfsense_backend/alembic/versions/56_add_circleback_connector_enums.py index 551b8180b..0c06ea139 100644 --- a/surfsense_backend/alembic/versions/56_add_circleback_connector_enums.py +++ b/surfsense_backend/alembic/versions/56_add_circleback_connector_enums.py @@ -19,40 +19,17 @@ depends_on: str | Sequence[str] | None = None def upgrade() -> None: """Safely add 'CIRCLEBACK' to documenttype and 'CIRCLEBACK_CONNECTOR' to searchsourceconnectortype enums if missing.""" + from sqlalchemy import text - # Add to documenttype enum - op.execute( - """ - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_type t - JOIN pg_enum e ON t.oid = e.enumtypid - WHERE t.typname = 'documenttype' AND e.enumlabel = 'CIRCLEBACK' - ) THEN - ALTER TYPE documenttype ADD VALUE 'CIRCLEBACK'; - END IF; - END - $$; - """ - ) + # Get connection and commit current transaction to allow ALTER TYPE + connection = op.get_bind() + connection.execute(text("COMMIT")) + + # Add to documenttype enum (must be outside transaction) + connection.execute(text("ALTER TYPE documenttype ADD VALUE IF NOT EXISTS 'CIRCLEBACK'")) # Add to searchsourceconnectortype enum - op.execute( - """ - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_type t - JOIN pg_enum e ON t.oid = e.enumtypid - WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'CIRCLEBACK_CONNECTOR' - ) THEN - ALTER TYPE searchsourceconnectortype ADD VALUE 'CIRCLEBACK_CONNECTOR'; - END IF; - END - $$; - """ - ) + connection.execute(text("ALTER TYPE searchsourceconnectortype ADD VALUE IF NOT EXISTS 'CIRCLEBACK_CONNECTOR'")) def downgrade() -> None: diff --git a/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py b/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py index 2096ce2b9..a3cdad359 100644 --- a/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py +++ b/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py @@ -497,6 +497,16 @@ async def search_knowledge_base_async( ) all_documents.extend(chunks) + elif connector == "CIRCLEBACK": + _, chunks = await connector_service.search_circleback( + user_query=query, + search_space_id=search_space_id, + top_k=top_k, + start_date=resolved_start_date, + end_date=resolved_end_date, + ) + all_documents.extend(chunks) + except Exception as e: print(f"Error searching connector {connector}: {e}") continue @@ -583,6 +593,7 @@ def create_search_knowledge_base_tool( - LUMA_CONNECTOR: "Luma events" - WEBCRAWLER_CONNECTOR: "Webpages indexed by SurfSense" (personally selected websites) - BOOKSTACK_CONNECTOR: "BookStack pages" (personal documentation) + - CIRCLEBACK: "Circleback meeting notes, transcripts, and action items" (personal meeting records) NOTE: `WEBCRAWLER_CONNECTOR` is mapped internally to the canonical document type `CRAWLED_URL`. diff --git a/surfsense_backend/app/services/connector_service.py b/surfsense_backend/app/services/connector_service.py index cf0a83dc8..26c687dd7 100644 --- a/surfsense_backend/app/services/connector_service.py +++ b/surfsense_backend/app/services/connector_service.py @@ -2606,3 +2606,105 @@ class ConnectorService: } return result_object, bookstack_docs + + async def search_circleback( + self, + user_query: str, + search_space_id: int, + top_k: int = 20, + start_date: datetime | None = None, + end_date: datetime | None = None, + ) -> tuple: + """ + Search for Circleback meeting notes and return both the source information and langchain documents. + + Uses combined chunk-level and document-level hybrid search with RRF fusion. + + Args: + user_query: The user's query + search_space_id: The search space ID to search in + top_k: Maximum number of results to return + start_date: Optional start date for filtering documents by updated_at + end_date: Optional end date for filtering documents by updated_at + + Returns: + tuple: (sources_info, langchain_documents) + """ + circleback_docs = await self._combined_rrf_search( + query_text=user_query, + search_space_id=search_space_id, + document_type="CIRCLEBACK", + top_k=top_k, + start_date=start_date, + end_date=end_date, + ) + + # Early return if no results + if not circleback_docs: + return { + "id": 52, + "name": "Circleback Meetings", + "type": "CIRCLEBACK", + "sources": [], + }, [] + + def _title_fn(doc_info: dict[str, Any], metadata: dict[str, Any]) -> str: + meeting_name = metadata.get("meeting_name", "") + meeting_date = metadata.get("meeting_date", "") + title = doc_info.get("title") or meeting_name or "Circleback Meeting" + if meeting_date: + title += f" ({meeting_date})" + return title + + def _url_fn(_doc_info: dict[str, Any], metadata: dict[str, Any]) -> str: + meeting_id = metadata.get("circleback_meeting_id", "") + return ( + f"https://app.circleback.ai/meetings/{meeting_id}" + if meeting_id + else "" + ) + + def _description_fn( + chunk: dict[str, Any], _doc_info: dict[str, Any], metadata: dict[str, Any] + ) -> str: + description = self._chunk_preview(chunk.get("content", ""), limit=200) + info_parts = [] + duration = metadata.get("duration_seconds") + attendee_count = metadata.get("attendee_count") + if duration: + minutes = int(duration) // 60 + info_parts.append(f"Duration: {minutes} min") + if attendee_count: + info_parts.append(f"Attendees: {attendee_count}") + if info_parts: + description = (description + " | " + " | ".join(info_parts)).strip(" |") + return description + + def _extra_fields_fn( + _chunk: dict[str, Any], _doc_info: dict[str, Any], metadata: dict[str, Any] + ) -> dict[str, Any]: + return { + "circleback_meeting_id": metadata.get("circleback_meeting_id", ""), + "meeting_name": metadata.get("meeting_name", ""), + "meeting_date": metadata.get("meeting_date", ""), + "duration_seconds": metadata.get("duration_seconds", 0), + "attendee_count": metadata.get("attendee_count", 0), + } + + sources_list = self._build_chunk_sources_from_documents( + circleback_docs, + title_fn=_title_fn, + url_fn=_url_fn, + description_fn=_description_fn, + extra_fields_fn=_extra_fields_fn, + ) + + # Create result object + result_object = { + "id": 52, + "name": "Circleback Meetings", + "type": "CIRCLEBACK", + "sources": sources_list, + } + + return result_object, circleback_docs \ No newline at end of file diff --git a/surfsense_web/contracts/types/connector.types.ts b/surfsense_web/contracts/types/connector.types.ts index c590f3941..bc7664777 100644 --- a/surfsense_web/contracts/types/connector.types.ts +++ b/surfsense_web/contracts/types/connector.types.ts @@ -23,6 +23,7 @@ export const searchSourceConnectorTypeEnum = z.enum([ "ELASTICSEARCH_CONNECTOR", "WEBCRAWLER_CONNECTOR", "BOOKSTACK_CONNECTOR", + "CIRCLEBACK_CONNECTOR", ]); export const searchSourceConnector = z.object({ diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts index 94ff27940..f7eb8f278 100644 --- a/surfsense_web/contracts/types/document.types.ts +++ b/surfsense_web/contracts/types/document.types.ts @@ -21,6 +21,7 @@ export const documentTypeEnum = z.enum([ "ELASTICSEARCH_CONNECTOR", "LINEAR_CONNECTOR", "NOTE", + "CIRCLEBACK", ]); export const document = z.object({ From ddfbb9509b405bf994b0d476729577523e053437 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 31 Dec 2025 02:00:11 +0530 Subject: [PATCH 54/92] feat: Implement connector editing functionality in the popup, including Google Drive folder selection, and enhance connector management with improved state handling and UI updates. --- .../google_drive_add_connector_route.py | 3 +- .../assistant-ui/connector-popup.tsx | 56 +- .../{ => components}/connector-card.tsx | 0 .../connector-dialog-header.tsx | 0 .../{ => components}/date-range-selector.tsx | 0 .../{ => components}/periodic-sync-config.tsx | 0 .../components/google-drive-config.tsx | 103 +++ .../connector-configs/index.tsx | 28 + .../views/connector-edit-view.tsx | 247 +++++++ .../views}/indexing-configuration-view.tsx | 42 +- .../{ => constants}/connector-constants.ts | 2 +- .../connector-popup.schemas.ts | 3 +- .../hooks/use-connector-dialog.ts | 637 ++++++++++++++++++ .../assistant-ui/connector-popup/index.ts | 25 +- .../{ => tabs}/active-connectors-tab.tsx | 8 +- .../{ => tabs}/all-connectors-tab.tsx | 29 +- .../connector-popup/use-connector-dialog.ts | 361 ---------- .../connectors/google-drive-folder-tree.tsx | 80 ++- surfsense_web/components/ui/switch.tsx | 4 +- 19 files changed, 1182 insertions(+), 446 deletions(-) rename surfsense_web/components/assistant-ui/connector-popup/{ => components}/connector-card.tsx (100%) rename surfsense_web/components/assistant-ui/connector-popup/{ => components}/connector-dialog-header.tsx (100%) rename surfsense_web/components/assistant-ui/connector-popup/{ => components}/date-range-selector.tsx (100%) rename surfsense_web/components/assistant-ui/connector-popup/{ => components}/periodic-sync-config.tsx (100%) create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx rename surfsense_web/components/assistant-ui/connector-popup/{ => connector-configs/views}/indexing-configuration-view.tsx (79%) rename surfsense_web/components/assistant-ui/connector-popup/{ => constants}/connector-constants.ts (98%) rename surfsense_web/components/assistant-ui/connector-popup/{ => constants}/connector-popup.schemas.ts (97%) create mode 100644 surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts rename surfsense_web/components/assistant-ui/connector-popup/{ => tabs}/active-connectors-tab.tsx (97%) rename surfsense_web/components/assistant-ui/connector-popup/{ => tabs}/all-connectors-tab.tsx (72%) delete mode 100644 surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts diff --git a/surfsense_backend/app/routes/google_drive_add_connector_route.py b/surfsense_backend/app/routes/google_drive_add_connector_route.py index d11404781..73ad76409 100644 --- a/surfsense_backend/app/routes/google_drive_add_connector_route.py +++ b/surfsense_backend/app/routes/google_drive_add_connector_route.py @@ -208,9 +208,8 @@ async def drive_callback( f"Successfully created Google Drive connector {db_connector.id} for user {user_id}" ) - # Redirect to connectors management page (not to folder selection) return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors?success=google-drive-connected" + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-drive-connector" ) except HTTPException: diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index 1e9e09869..c1ed38e7b 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -17,11 +17,12 @@ import { } from "@/components/ui/tabs"; import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { cn } from "@/lib/utils"; -import { AllConnectorsTab } from "./connector-popup/all-connectors-tab"; -import { ActiveConnectorsTab } from "./connector-popup/active-connectors-tab"; -import { ConnectorDialogHeader } from "./connector-popup/connector-dialog-header"; -import { IndexingConfigurationView } from "./connector-popup/indexing-configuration-view"; -import { useConnectorDialog } from "./connector-popup/use-connector-dialog"; +import { AllConnectorsTab } from "./connector-popup/tabs/all-connectors-tab"; +import { ActiveConnectorsTab } from "./connector-popup/tabs/active-connectors-tab"; +import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header"; +import { ConnectorEditView } from "./connector-popup/connector-configs/views/connector-edit-view"; +import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view"; +import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; export const ConnectorIndicator: FC = () => { @@ -67,9 +68,14 @@ export const ConnectorIndicator: FC = () => { isScrolled, searchQuery, indexingConfig, + indexingConnector, + indexingConnectorConfig, + editingConnector, startDate, endDate, isStartingIndexing, + isSaving, + isDisconnecting, periodicEnabled, frequencyMinutes, allConnectors, @@ -84,6 +90,13 @@ export const ConnectorIndicator: FC = () => { handleConnectOAuth, handleStartIndexing, handleSkipIndexing, + handleStartEdit, + handleSaveConnector, + handleDisconnectConnector, + handleBackFromEdit, + connectorConfig, + setConnectorConfig, + setIndexingConnectorConfig, } = useConnectorDialog(); // Get document types that have documents in the search space @@ -133,10 +146,35 @@ export const ConnectorIndicator: FC = () => { - {/* Indexing Configuration View - shown after OAuth success */} - {indexingConfig ? ( + {/* Connector Edit View - shown when editing existing connector */} + {editingConnector ? ( + handleSaveConnector(refreshConnectors)} + onDisconnect={() => handleDisconnectConnector(refreshConnectors)} + onBack={handleBackFromEdit} + onConfigChange={setConnectorConfig} + /> + ) : indexingConfig ? ( { onEndDateChange={setEndDate} onPeriodicEnabledChange={setPeriodicEnabled} onFrequencyChange={setFrequencyMinutes} + onConfigChange={setIndexingConnectorConfig} onStartIndexing={() => handleStartIndexing(refreshConnectors)} onSkip={handleSkipIndexing} /> @@ -171,7 +210,9 @@ export const ConnectorIndicator: FC = () => { searchSpaceId={searchSpaceId} connectedTypes={connectedTypes} connectingId={connectingId} + allConnectors={allConnectors} onConnectOAuth={handleConnectOAuth} + onManage={handleStartEdit} /> @@ -184,6 +225,7 @@ export const ConnectorIndicator: FC = () => { logsSummary={logsSummary} searchSpaceId={searchSpaceId} onTabChange={handleTabChange} + onManage={handleStartEdit} />
diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-card.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx similarity index 100% rename from surfsense_web/components/assistant-ui/connector-popup/connector-card.tsx rename to surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-dialog-header.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx similarity index 100% rename from surfsense_web/components/assistant-ui/connector-popup/connector-dialog-header.tsx rename to surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx diff --git a/surfsense_web/components/assistant-ui/connector-popup/date-range-selector.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/date-range-selector.tsx similarity index 100% rename from surfsense_web/components/assistant-ui/connector-popup/date-range-selector.tsx rename to surfsense_web/components/assistant-ui/connector-popup/components/date-range-selector.tsx diff --git a/surfsense_web/components/assistant-ui/connector-popup/periodic-sync-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx similarity index 100% rename from surfsense_web/components/assistant-ui/connector-popup/periodic-sync-config.tsx rename to surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx new file mode 100644 index 000000000..280d6ed23 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx @@ -0,0 +1,103 @@ +"use client"; + +import { Info } from "lucide-react"; +import { useState, useEffect } from "react"; +import type { FC } from "react"; +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { GoogleDriveFolderTree } from "@/components/connectors/google-drive-folder-tree"; +import type { ConnectorConfigProps } from "../index"; + +interface SelectedFolder { + id: string; + name: string; +} + +export const GoogleDriveConfig: FC = ({ + connector, + onConfigChange, +}) => { + // Initialize with existing selected folders from connector config + const existingFolders = (connector.config?.selected_folders as SelectedFolder[] | undefined) || []; + const [selectedFolders, setSelectedFolders] = useState(existingFolders); + const [showFolderSelector, setShowFolderSelector] = useState(false); + + // Update selected folders when connector config changes + useEffect(() => { + const folders = (connector.config?.selected_folders as SelectedFolder[] | undefined) || []; + setSelectedFolders(folders); + }, [connector.config]); + + const handleSelectFolders = (folders: SelectedFolder[]) => { + setSelectedFolders(folders); + if (onConfigChange) { + // Store folder IDs and names in config for indexing + onConfigChange({ + ...connector.config, + selected_folders: folders, + }); + } + }; + + return ( +
+
+

Folder Selection

+

+ Select specific folders to index. Only files directly in each folder will be processed—subfolders must be selected separately. +

+
+ + {selectedFolders.length > 0 && ( +
+

+ Selected {selectedFolders.length} folder{selectedFolders.length > 1 ? "s" : ""}: +

+
+ {selectedFolders.map((folder) => ( +

+ • {folder.name} +

+ ))} +
+
+ )} + + {showFolderSelector ? ( +
+ + +
+ ) : ( + + )} + + + + + Folder selection is used when indexing. You can change this selection when you start indexing. + + +
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx new file mode 100644 index 000000000..eb2594ad6 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx @@ -0,0 +1,28 @@ +"use client"; + +import type { FC } from "react"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import { GoogleDriveConfig } from "./components/google-drive-config"; + +export interface ConnectorConfigProps { + connector: SearchSourceConnector; + onConfigChange?: (config: Record) => void; +} + +export type ConnectorConfigComponent = FC; + +/** + * Factory function to get the appropriate config component for a connector type + */ +export function getConnectorConfigComponent( + connectorType: string +): ConnectorConfigComponent | null { + switch (connectorType) { + case "GOOGLE_DRIVE_CONNECTOR": + return GoogleDriveConfig; + // OAuth connectors (Gmail, Calendar, Airtable) and others don't need special config UI + default: + return null; + } +} + diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx new file mode 100644 index 000000000..cc4f3a815 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx @@ -0,0 +1,247 @@ +"use client"; + +import { ArrowLeft, Loader2, Trash2 } from "lucide-react"; +import { type FC, useState, useCallback, useRef, useEffect, useMemo } from "react"; +import { Button } from "@/components/ui/button"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import { cn } from "@/lib/utils"; +import { DateRangeSelector } from "../../components/date-range-selector"; +import { PeriodicSyncConfig } from "../../components/periodic-sync-config"; +import { getConnectorConfigComponent } from "../index"; + +interface ConnectorEditViewProps { + connector: SearchSourceConnector; + startDate: Date | undefined; + endDate: Date | undefined; + periodicEnabled: boolean; + frequencyMinutes: string; + isSaving: boolean; + isDisconnecting: boolean; + onStartDateChange: (date: Date | undefined) => void; + onEndDateChange: (date: Date | undefined) => void; + onPeriodicEnabledChange: (enabled: boolean) => void; + onFrequencyChange: (frequency: string) => void; + onSave: () => void; + onDisconnect: () => void; + onBack: () => void; + onConfigChange?: (config: Record) => void; +} + +export const ConnectorEditView: FC = ({ + connector, + startDate, + endDate, + periodicEnabled, + frequencyMinutes, + isSaving, + isDisconnecting, + onStartDateChange, + onEndDateChange, + onPeriodicEnabledChange, + onFrequencyChange, + onSave, + onDisconnect, + onBack, + onConfigChange, +}) => { + // Get connector-specific config component + const ConnectorConfigComponent = useMemo( + () => getConnectorConfigComponent(connector.connector_type), + [connector.connector_type] + ); + const [isScrolled, setIsScrolled] = useState(false); + const [hasMoreContent, setHasMoreContent] = useState(false); + const [showDisconnectConfirm, setShowDisconnectConfirm] = useState(false); + const scrollContainerRef = useRef(null); + + const checkScrollState = useCallback(() => { + if (!scrollContainerRef.current) return; + + const target = scrollContainerRef.current; + const scrolled = target.scrollTop > 0; + const hasMore = target.scrollHeight > target.clientHeight && + target.scrollTop + target.clientHeight < target.scrollHeight - 10; + + setIsScrolled(scrolled); + setHasMoreContent(hasMore); + }, []); + + const handleScroll = useCallback(() => { + checkScrollState(); + }, [checkScrollState]); + + // Check initial scroll state and on resize + useEffect(() => { + checkScrollState(); + const resizeObserver = new ResizeObserver(() => { + checkScrollState(); + }); + + if (scrollContainerRef.current) { + resizeObserver.observe(scrollContainerRef.current); + } + + return () => { + resizeObserver.disconnect(); + }; + }, [checkScrollState]); + + const handleDisconnectClick = () => { + setShowDisconnectConfirm(true); + }; + + const handleDisconnectConfirm = () => { + setShowDisconnectConfirm(false); + onDisconnect(); + }; + + const handleDisconnectCancel = () => { + setShowDisconnectConfirm(false); + }; + + return ( +
+ {/* Fixed Header */} +
+ {/* Back button */} + + + {/* Connector header */} +
+
+ {getConnectorIcon(connector.connector_type, "size-7")} +
+
+

+ {connector.name} +

+

+ Manage your connector settings and sync configuration +

+
+
+
+ + {/* Scrollable Content */} +
+
+
+ {/* Connector-specific configuration */} + {ConnectorConfigComponent && ( + + )} + + {/* Date range selector - not shown for Google Drive (uses folder selection instead) */} + {connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" && ( + + )} + + + + {/* Info box */} +
+
+ {getConnectorIcon(connector.connector_type, "size-4")} +
+
+

Re-indexing runs in the background

+

+ You can continue using SurfSense while we sync your data. Check the Active tab to see progress. +

+
+
+
+
+ {/* Top fade shadow - appears when scrolled */} + {isScrolled && ( +
+ )} + {/* Bottom fade shadow - appears when there's more content */} + {hasMoreContent && ( +
+ )} +
+ + {/* Fixed Footer - Action buttons */} +
+ {showDisconnectConfirm ? ( +
+ Are you sure? + + +
+ ) : ( + + )} + +
+
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/connector-popup/indexing-configuration-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx similarity index 79% rename from surfsense_web/components/assistant-ui/connector-popup/indexing-configuration-view.tsx rename to surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx index 3135c634b..2c11eb415 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/indexing-configuration-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx @@ -1,16 +1,19 @@ "use client"; import { ArrowLeft, Check, Loader2 } from "lucide-react"; -import { type FC, useState, useCallback, useRef, useEffect } from "react"; +import { type FC, useState, useCallback, useRef, useEffect, useMemo } from "react"; import { Button } from "@/components/ui/button"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { cn } from "@/lib/utils"; -import type { IndexingConfigState } from "./connector-constants"; -import { DateRangeSelector } from "./date-range-selector"; -import { PeriodicSyncConfig } from "./periodic-sync-config"; +import type { IndexingConfigState } from "../../constants/connector-constants"; +import { DateRangeSelector } from "../../components/date-range-selector"; +import { PeriodicSyncConfig } from "../../components/periodic-sync-config"; +import { getConnectorConfigComponent } from "../index"; interface IndexingConfigurationViewProps { config: IndexingConfigState; + connector?: SearchSourceConnector; startDate: Date | undefined; endDate: Date | undefined; periodicEnabled: boolean; @@ -20,12 +23,14 @@ interface IndexingConfigurationViewProps { onEndDateChange: (date: Date | undefined) => void; onPeriodicEnabledChange: (enabled: boolean) => void; onFrequencyChange: (frequency: string) => void; + onConfigChange?: (config: Record) => void; onStartIndexing: () => void; onSkip: () => void; } export const IndexingConfigurationView: FC = ({ config, + connector, startDate, endDate, periodicEnabled, @@ -35,9 +40,15 @@ export const IndexingConfigurationView: FC = ({ onEndDateChange, onPeriodicEnabledChange, onFrequencyChange, + onConfigChange, onStartIndexing, onSkip, }) => { + // Get connector-specific config component + const ConnectorConfigComponent = useMemo( + () => connector ? getConnectorConfigComponent(connector.connector_type) : null, + [connector] + ); const [isScrolled, setIsScrolled] = useState(false); const [hasMoreContent, setHasMoreContent] = useState(false); const scrollContainerRef = useRef(null); @@ -115,12 +126,23 @@ export const IndexingConfigurationView: FC = ({ onScroll={handleScroll} >
- + {/* Connector-specific configuration */} + {ConnectorConfigComponent && connector && ( + + )} + + {/* Date range selector - not shown for Google Drive (uses folder selection instead) */} + {config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && ( + + )} { + const router = useRouter(); + const searchParams = useSearchParams(); + const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); + const { data: allConnectors, refetch: refetchAllConnectors } = useAtomValue(connectorsAtom); + const { mutateAsync: indexConnector } = useAtomValue(indexConnectorMutationAtom); + const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom); + const { mutateAsync: deleteConnector } = useAtomValue(deleteConnectorMutationAtom); + + const [isOpen, setIsOpen] = useState(false); + const [activeTab, setActiveTab] = useState("all"); + const [connectingId, setConnectingId] = useState(null); + const [isScrolled, setIsScrolled] = useState(false); + const [searchQuery, setSearchQuery] = useState(""); + const [indexingConfig, setIndexingConfig] = useState(null); + const [indexingConnector, setIndexingConnector] = useState(null); + const [indexingConnectorConfig, setIndexingConnectorConfig] = useState | null>(null); + const [startDate, setStartDate] = useState(undefined); + const [endDate, setEndDate] = useState(undefined); + const [isStartingIndexing, setIsStartingIndexing] = useState(false); + const [periodicEnabled, setPeriodicEnabled] = useState(false); + const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); + + // Edit mode state + const [editingConnector, setEditingConnector] = useState(null); + const [isSaving, setIsSaving] = useState(false); + const [isDisconnecting, setIsDisconnecting] = useState(false); + const [connectorConfig, setConnectorConfig] = useState | null>(null); + + // Helper function to get frequency label + const getFrequencyLabel = useCallback((minutes: string): string => { + switch (minutes) { + case "15": return "15 minutes"; + case "60": return "hour"; + case "360": return "6 hours"; + case "720": return "12 hours"; + case "1440": return "day"; + case "10080": return "week"; + default: return `${minutes} minutes`; + } + }, []); + + // Synchronize state with URL query params + useEffect(() => { + try { + const params = parseConnectorPopupQueryParams(searchParams); + + if (params.modal === "connectors") { + setIsOpen(true); + + if (params.tab === "active" || params.tab === "all") { + setActiveTab(params.tab); + } + + // Clear indexing config if view is not "configure" anymore + if (params.view !== "configure" && indexingConfig) { + setIndexingConfig(null); + } + + // Clear editing connector if view is not "edit" anymore + if (params.view !== "edit" && editingConnector) { + setEditingConnector(null); + } + + if (params.view === "configure" && params.connector && !indexingConfig) { + const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === params.connector); + if (oauthConnector && allConnectors) { + const existingConnector = allConnectors.find( + (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType + ); + if (existingConnector) { + // Validate connector data before setting state + const connectorValidation = searchSourceConnector.safeParse(existingConnector); + if (connectorValidation.success) { + const config = validateIndexingConfigState({ + connectorType: oauthConnector.connectorType, + connectorId: existingConnector.id, + connectorTitle: oauthConnector.title, + }); + setIndexingConfig(config); + setIndexingConnector(existingConnector); + setIndexingConnectorConfig(existingConnector.config); + } + } + } + } + + // Handle edit view + if (params.view === "edit" && params.connectorId && allConnectors && !editingConnector) { + const connectorId = parseInt(params.connectorId, 10); + const connector = allConnectors.find((c: SearchSourceConnector) => c.id === connectorId); + if (connector) { + const connectorValidation = searchSourceConnector.safeParse(connector); + if (connectorValidation.success) { + setEditingConnector(connector); + setConnectorConfig(connector.config); + // Load existing periodic sync settings + setPeriodicEnabled(connector.periodic_indexing_enabled); + setFrequencyMinutes( + connector.indexing_frequency_minutes?.toString() || "1440" + ); + // Reset dates - user can set new ones for re-indexing + setStartDate(undefined); + setEndDate(undefined); + } + } + } + } else { + setIsOpen(false); + // Clear indexing config when modal is closed + if (indexingConfig) { + setIndexingConfig(null); + setIndexingConnector(null); + setIndexingConnectorConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + setIsScrolled(false); + setSearchQuery(""); + } + // Clear editing connector when modal is closed + if (editingConnector) { + setEditingConnector(null); + setConnectorConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + setIsScrolled(false); + setSearchQuery(""); + } + } + } catch (error) { + // Invalid query params - log but don't crash + console.warn("Invalid connector popup query params:", error); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [searchParams, allConnectors, editingConnector, indexingConfig]); + + // Detect OAuth success and transition to config view + useEffect(() => { + try { + const params = parseConnectorPopupQueryParams(searchParams); + + if (params.success === "true" && params.connector && searchSpaceId && params.modal === "connectors") { + const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === params.connector); + if (oauthConnector) { + refetchAllConnectors().then((result) => { + if (!result.data) return; + + const newConnector = result.data.find( + (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType + ); + if (newConnector) { + // Validate connector data before setting state + const connectorValidation = searchSourceConnector.safeParse(newConnector); + if (connectorValidation.success) { + const config = validateIndexingConfigState({ + connectorType: oauthConnector.connectorType, + connectorId: newConnector.id, + connectorTitle: oauthConnector.title, + }); + setIndexingConfig(config); + setIndexingConnector(newConnector); + setIndexingConnectorConfig(newConnector.config); + setIsOpen(true); + const url = new URL(window.location.href); + url.searchParams.delete("success"); + url.searchParams.set("view", "configure"); + window.history.replaceState({}, "", url.toString()); + } else { + console.warn("Invalid connector data after OAuth:", connectorValidation.error); + toast.error("Failed to validate connector data"); + } + } + }); + } + } + } catch (error) { + // Invalid query params - log but don't crash + console.warn("Invalid connector popup query params in OAuth success handler:", error); + } + }, [searchParams, searchSpaceId, refetchAllConnectors]); + + // Handle OAuth connection + const handleConnectOAuth = useCallback( + async (connector: (typeof OAUTH_CONNECTORS)[0]) => { + if (!searchSpaceId || !connector.authEndpoint) return; + + // Set connecting state immediately to disable button and show spinner + setConnectingId(connector.id); + + try { + const response = await authenticatedFetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}${connector.authEndpoint}?space_id=${searchSpaceId}`, + { method: "GET" } + ); + + if (!response.ok) { + throw new Error(`Failed to initiate ${connector.title} OAuth`); + } + + const data = await response.json(); + + // Validate OAuth response with Zod + const validatedData = parseOAuthAuthResponse(data); + + // Don't clear connectingId here - let the redirect happen with button still disabled + // The component will unmount on redirect anyway + window.location.href = validatedData.auth_url; + } catch (error) { + console.error(`Error connecting to ${connector.title}:`, error); + if (error instanceof Error && error.message.includes("Invalid auth URL")) { + toast.error(`Invalid response from ${connector.title} OAuth endpoint`); + } else { + toast.error(`Failed to connect to ${connector.title}`); + } + // Only clear connectingId on error so user can retry + setConnectingId(null); + } + }, + [searchSpaceId] + ); + + // Handle starting indexing + const handleStartIndexing = useCallback(async (refreshConnectors: () => void) => { + if (!indexingConfig || !searchSpaceId) return; + + // Validate date range + const dateRangeValidation = dateRangeSchema.safeParse({ startDate, endDate }); + if (!dateRangeValidation.success) { + const firstIssueMsg = + dateRangeValidation.error.issues && dateRangeValidation.error.issues.length > 0 + ? dateRangeValidation.error.issues[0].message + : "Invalid date range"; + toast.error(firstIssueMsg); + return; + } + + // Validate frequency minutes if periodic is enabled + if (periodicEnabled) { + const frequencyValidation = frequencyMinutesSchema.safeParse(frequencyMinutes); + if (!frequencyValidation.success) { + toast.error("Invalid frequency value"); + return; + } + } + + setIsStartingIndexing(true); + try { + const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined; + const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined; + + // Update connector with periodic sync settings and config changes + if (periodicEnabled || indexingConnectorConfig) { + const frequency = periodicEnabled ? parseInt(frequencyMinutes, 10) : undefined; + await updateConnector({ + id: indexingConfig.connectorId, + data: { + ...(periodicEnabled && { + periodic_indexing_enabled: true, + indexing_frequency_minutes: frequency, + }), + ...(indexingConnectorConfig && { + config: indexingConnectorConfig, + }), + }, + }); + } + + // Handle Google Drive folder selection + if (indexingConfig.connectorType === "GOOGLE_DRIVE_CONNECTOR" && indexingConnectorConfig) { + const selectedFolders = indexingConnectorConfig.selected_folders as Array<{ id: string; name: string }> | undefined; + if (selectedFolders && selectedFolders.length > 0) { + // Index with folder selection + const folderIds = selectedFolders.map((f) => f.id).join(","); + const folderNames = selectedFolders.map((f) => f.name).join(", "); + await indexConnector({ + connector_id: indexingConfig.connectorId, + queryParams: { + search_space_id: searchSpaceId, + folder_ids: folderIds, + folder_names: folderNames, + }, + }); + } else { + // Google Drive requires folder selection - show error if none selected + toast.error("Please select at least one folder to index"); + setIsStartingIndexing(false); + return; + } + } else { + await indexConnector({ + connector_id: indexingConfig.connectorId, + queryParams: { + search_space_id: searchSpaceId, + start_date: startDateStr, + end_date: endDateStr, + }, + }); + } + + toast.success(`${indexingConfig.connectorTitle} indexing started`, { + description: periodicEnabled + ? `Periodic sync enabled every ${getFrequencyLabel(frequencyMinutes)}.` + : "You can continue working while we sync your data.", + }); + + // Update URL - the effect will handle closing the modal and clearing state + const url = new URL(window.location.href); + url.searchParams.delete("modal"); + url.searchParams.delete("tab"); + url.searchParams.delete("success"); + url.searchParams.delete("connector"); + url.searchParams.delete("view"); + router.replace(url.pathname + url.search, { scroll: false }); + + refreshConnectors(); + queryClient.invalidateQueries({ + queryKey: cacheKeys.logs.summary(Number(searchSpaceId)), + }); + } catch (error) { + console.error("Error starting indexing:", error); + toast.error("Failed to start indexing"); + } finally { + setIsStartingIndexing(false); + } + }, [indexingConfig, searchSpaceId, startDate, endDate, indexConnector, updateConnector, periodicEnabled, frequencyMinutes, getFrequencyLabel, router, indexingConnectorConfig]); + + // Handle skipping indexing + const handleSkipIndexing = useCallback(() => { + // Update URL - the effect will handle closing the modal and clearing state + const url = new URL(window.location.href); + url.searchParams.delete("modal"); + url.searchParams.delete("tab"); + url.searchParams.delete("success"); + url.searchParams.delete("connector"); + url.searchParams.delete("view"); + router.replace(url.pathname + url.search, { scroll: false }); + }, [router]); + + // Handle starting edit mode + const handleStartEdit = useCallback((connector: SearchSourceConnector) => { + if (!searchSpaceId) return; + + // Check if this is an OAuth connector + const isOAuthConnector = OAUTH_CONNECTORS.some( + (oauthConnector) => oauthConnector.connectorType === connector.connector_type + ); + + // If not OAuth, redirect to old connector edit page + if (!isOAuthConnector) { + router.push(`/dashboard/${searchSpaceId}/connectors/${connector.id}/edit`); + return; + } + + // Validate connector data + const connectorValidation = searchSourceConnector.safeParse(connector); + if (!connectorValidation.success) { + toast.error("Invalid connector data"); + return; + } + + setEditingConnector(connector); + // Load existing periodic sync settings + setPeriodicEnabled(connector.periodic_indexing_enabled); + setFrequencyMinutes(connector.indexing_frequency_minutes?.toString() || "1440"); + // Reset dates - user can set new ones for re-indexing + setStartDate(undefined); + setEndDate(undefined); + + // Update URL + const url = new URL(window.location.href); + url.searchParams.set("modal", "connectors"); + url.searchParams.set("view", "edit"); + url.searchParams.set("connectorId", connector.id.toString()); + window.history.pushState({ modal: true }, "", url.toString()); + }, [searchSpaceId, router]); + + // Handle saving connector changes + const handleSaveConnector = useCallback(async (refreshConnectors: () => void) => { + if (!editingConnector || !searchSpaceId) return; + + // Validate date range (skip for Google Drive which uses folder selection) + if (editingConnector.connector_type !== "GOOGLE_DRIVE_CONNECTOR") { + const dateRangeValidation = dateRangeSchema.safeParse({ startDate, endDate }); + if (!dateRangeValidation.success) { + toast.error(dateRangeValidation.error.issues[0]?.message || "Invalid date range"); + return; + } + } + + // Validate frequency minutes if periodic is enabled + if (periodicEnabled) { + const frequencyValidation = frequencyMinutesSchema.safeParse(frequencyMinutes); + if (!frequencyValidation.success) { + toast.error("Invalid frequency value"); + return; + } + } + + setIsSaving(true); + try { + const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined; + const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined; + + // Update connector with periodic sync settings and config changes + const frequency = periodicEnabled ? parseInt(frequencyMinutes, 10) : null; + await updateConnector({ + id: editingConnector.id, + data: { + periodic_indexing_enabled: periodicEnabled, + indexing_frequency_minutes: frequency, + config: connectorConfig || editingConnector.config, + }, + }); + + // Re-index based on connector type + let indexingDescription = "Settings saved."; + if (editingConnector.connector_type === "GOOGLE_DRIVE_CONNECTOR") { + // Google Drive uses folder selection from config, not date ranges + const selectedFolders = (connectorConfig || editingConnector.config)?.selected_folders as Array<{ id: string; name: string }> | undefined; + if (selectedFolders && selectedFolders.length > 0) { + const folderIds = selectedFolders.map((f) => f.id).join(","); + const folderNames = selectedFolders.map((f) => f.name).join(", "); + await indexConnector({ + connector_id: editingConnector.id, + queryParams: { + search_space_id: searchSpaceId, + folder_ids: folderIds, + folder_names: folderNames, + }, + }); + indexingDescription = `Re-indexing started for ${selectedFolders.length} folder(s).`; + } + } else if (startDateStr || endDateStr) { + // Other connectors use date ranges + await indexConnector({ + connector_id: editingConnector.id, + queryParams: { + search_space_id: searchSpaceId, + start_date: startDateStr, + end_date: endDateStr, + }, + }); + indexingDescription = "Re-indexing started with new date range."; + } + + toast.success(`${editingConnector.name} updated successfully`, { + description: periodicEnabled + ? `Periodic sync ${frequency ? `enabled every ${getFrequencyLabel(frequencyMinutes)}` : "enabled"}. ${indexingDescription}` + : indexingDescription, + }); + + // Update URL - the effect will handle closing the modal and clearing state + const url = new URL(window.location.href); + url.searchParams.delete("modal"); + url.searchParams.delete("tab"); + url.searchParams.delete("view"); + url.searchParams.delete("connectorId"); + router.replace(url.pathname + url.search, { scroll: false }); + + refreshConnectors(); + queryClient.invalidateQueries({ + queryKey: cacheKeys.logs.summary(Number(searchSpaceId)), + }); + } catch (error) { + console.error("Error saving connector:", error); + toast.error("Failed to save connector changes"); + } finally { + setIsSaving(false); + } + }, [editingConnector, searchSpaceId, startDate, endDate, indexConnector, updateConnector, periodicEnabled, frequencyMinutes, getFrequencyLabel, router, connectorConfig]); + + // Handle disconnecting connector + const handleDisconnectConnector = useCallback(async (refreshConnectors: () => void) => { + if (!editingConnector || !searchSpaceId) return; + + setIsDisconnecting(true); + try { + await deleteConnector({ + id: editingConnector.id, + }); + + toast.success(`${editingConnector.name} disconnected successfully`); + + // Update URL - the effect will handle closing the modal and clearing state + const url = new URL(window.location.href); + url.searchParams.delete("modal"); + url.searchParams.delete("tab"); + url.searchParams.delete("view"); + url.searchParams.delete("connectorId"); + router.replace(url.pathname + url.search, { scroll: false }); + + refreshConnectors(); + queryClient.invalidateQueries({ + queryKey: cacheKeys.logs.summary(Number(searchSpaceId)), + }); + } catch (error) { + console.error("Error disconnecting connector:", error); + toast.error("Failed to disconnect connector"); + } finally { + setIsDisconnecting(false); + } + }, [editingConnector, searchSpaceId, deleteConnector, router]); + + // Handle going back from edit view + const handleBackFromEdit = useCallback(() => { + const url = new URL(window.location.href); + url.searchParams.set("modal", "connectors"); + url.searchParams.set("tab", "all"); + url.searchParams.delete("view"); + url.searchParams.delete("connectorId"); + router.replace(url.pathname + url.search, { scroll: false }); + }, [router]); + + // Handle dialog open/close + const handleOpenChange = useCallback( + (open: boolean) => { + setIsOpen(open); + + if (open) { + const url = new URL(window.location.href); + url.searchParams.set("modal", "connectors"); + url.searchParams.set("tab", activeTab); + window.history.pushState({ modal: true }, "", url.toString()); + } else { + const url = new URL(window.location.href); + url.searchParams.delete("modal"); + url.searchParams.delete("tab"); + url.searchParams.delete("success"); + url.searchParams.delete("connector"); + url.searchParams.delete("view"); + window.history.pushState({ modal: false }, "", url.toString()); + setIsScrolled(false); + setSearchQuery(""); + if (!isStartingIndexing && !isSaving && !isDisconnecting) { + setIndexingConfig(null); + setIndexingConnector(null); + setIndexingConnectorConfig(null); + setEditingConnector(null); + setConnectorConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + } + } + }, + [activeTab, isStartingIndexing, isDisconnecting, isSaving] + ); + + // Handle tab change + const handleTabChange = useCallback( + (value: string) => { + setActiveTab(value); + const url = new URL(window.location.href); + url.searchParams.set("tab", value); + window.history.replaceState({ modal: true }, "", url.toString()); + }, + [] + ); + + // Handle scroll + const handleScroll = useCallback((e: React.UIEvent) => { + setIsScrolled(e.currentTarget.scrollTop > 0); + }, []); + + return { + // State + isOpen, + activeTab, + connectingId, + isScrolled, + searchQuery, + indexingConfig, + indexingConnector, + indexingConnectorConfig, + editingConnector, + startDate, + endDate, + isStartingIndexing, + isSaving, + isDisconnecting, + periodicEnabled, + frequencyMinutes, + searchSpaceId, + allConnectors, + + // Setters + setSearchQuery, + setStartDate, + setEndDate, + setPeriodicEnabled, + setFrequencyMinutes, + + // Handlers + handleOpenChange, + handleTabChange, + handleScroll, + handleConnectOAuth, + handleStartIndexing, + handleSkipIndexing, + handleStartEdit, + handleSaveConnector, + handleDisconnectConnector, + handleBackFromEdit, + connectorConfig, + setConnectorConfig, + setIndexingConnectorConfig, + }; +}; + diff --git a/surfsense_web/components/assistant-ui/connector-popup/index.ts b/surfsense_web/components/assistant-ui/connector-popup/index.ts index 1aea4de95..1c5ebc471 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/index.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/index.ts @@ -2,17 +2,18 @@ export { ConnectorIndicator } from "../connector-popup"; // Sub-components (if needed for external use) -export { ConnectorCard } from "./connector-card"; -export { DateRangeSelector } from "./date-range-selector"; -export { PeriodicSyncConfig } from "./periodic-sync-config"; -export { IndexingConfigurationView } from "./indexing-configuration-view"; -export { ConnectorDialogHeader } from "./connector-dialog-header"; -export { AllConnectorsTab } from "./all-connectors-tab"; -export { ActiveConnectorsTab } from "./active-connectors-tab"; +export { ConnectorCard } from "./components/connector-card"; +export { DateRangeSelector } from "./components/date-range-selector"; +export { PeriodicSyncConfig } from "./components/periodic-sync-config"; +export { IndexingConfigurationView } from "./connector-configs/views/indexing-configuration-view"; +export { ConnectorEditView } from "./connector-configs/views/connector-edit-view"; +export { ConnectorDialogHeader } from "./components/connector-dialog-header"; +export { AllConnectorsTab } from "./tabs/all-connectors-tab"; +export { ActiveConnectorsTab } from "./tabs/active-connectors-tab"; // Constants and types -export { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "./connector-constants"; -export type { IndexingConfigState } from "./connector-constants"; +export { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "./constants/connector-constants"; +export type { IndexingConfigState } from "./constants/connector-constants"; // Schemas and validation export { @@ -24,14 +25,14 @@ export { parseConnectorPopupQueryParams, parseOAuthAuthResponse, validateIndexingConfigState, -} from "./connector-popup.schemas"; +} from "./constants/connector-popup.schemas"; export type { ConnectorPopupQueryParams, OAuthAuthResponse, FrequencyMinutes, DateRange, -} from "./connector-popup.schemas"; +} from "./constants/connector-popup.schemas"; // Hooks -export { useConnectorDialog } from "./use-connector-dialog"; +export { useConnectorDialog } from "./hooks/use-connector-dialog"; diff --git a/surfsense_web/components/assistant-ui/connector-popup/active-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx similarity index 97% rename from surfsense_web/components/assistant-ui/connector-popup/active-connectors-tab.tsx rename to surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx index 43deba278..fd364c3d1 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/active-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx @@ -24,6 +24,7 @@ interface ActiveConnectorsTabProps { logsSummary: LogSummary | undefined; searchSpaceId: string; onTabChange: (value: string) => void; + onManage?: (connector: SearchSourceConnector) => void; } export const ActiveConnectorsTab: FC = ({ @@ -34,6 +35,7 @@ export const ActiveConnectorsTab: FC = ({ logsSummary, searchSpaceId, onTabChange, + onManage, }) => { const router = useRouter(); @@ -119,11 +121,7 @@ export const ActiveConnectorsTab: FC = ({ variant="outline" size="sm" className="h-8 text-[11px] px-3 rounded-lg font-medium" - onClick={() => - router.push( - `/dashboard/${searchSpaceId}/connectors/add/${connector.id}` - ) - } + onClick={onManage ? () => onManage(connector) : undefined} disabled={isIndexing} > {isIndexing ? "Syncing..." : "Manage"} diff --git a/surfsense_web/components/assistant-ui/connector-popup/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx similarity index 72% rename from surfsense_web/components/assistant-ui/connector-popup/all-connectors-tab.tsx rename to surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index 4dd056c90..b06c5f274 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -2,15 +2,18 @@ import { useRouter } from "next/navigation"; import { type FC } from "react"; -import { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "./connector-constants"; -import { ConnectorCard } from "./connector-card"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants"; +import { ConnectorCard } from "../components/connector-card"; interface AllConnectorsTabProps { searchQuery: string; searchSpaceId: string; connectedTypes: Set; connectingId: string | null; + allConnectors: SearchSourceConnector[] | undefined; onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[0]) => void; + onManage?: (connector: SearchSourceConnector) => void; } export const AllConnectorsTab: FC = ({ @@ -18,7 +21,9 @@ export const AllConnectorsTab: FC = ({ searchSpaceId, connectedTypes, connectingId, + allConnectors, onConnectOAuth, + onManage, }) => { const router = useRouter(); @@ -49,6 +54,10 @@ export const AllConnectorsTab: FC = ({ {filteredOAuth.map((connector) => { const isConnected = connectedTypes.has(connector.connectorType); const isConnecting = connectingId === connector.id; + // Find the actual connector object if connected + const actualConnector = isConnected && allConnectors + ? allConnectors.find((c: SearchSourceConnector) => c.connector_type === connector.connectorType) + : undefined; return ( = ({ isConnected={isConnected} isConnecting={isConnecting} onConnect={() => onConnectOAuth(connector)} - onManage={() => - router.push( - `/dashboard/${searchSpaceId}/connectors/add/${connector.id}` - ) - } + onManage={actualConnector && onManage ? () => onManage(actualConnector) : undefined} /> ); })} @@ -83,6 +88,10 @@ export const AllConnectorsTab: FC = ({
{filteredOther.map((connector) => { const isConnected = connectedTypes.has(connector.connectorType); + // Find the actual connector object if connected + const actualConnector = isConnected && allConnectors + ? allConnectors.find((c: SearchSourceConnector) => c.connector_type === connector.connectorType) + : undefined; return ( = ({ `/dashboard/${searchSpaceId}/connectors/add/${connector.id}` ) } - onManage={() => - router.push( - `/dashboard/${searchSpaceId}/connectors/add/${connector.id}` - ) - } + onManage={actualConnector && onManage ? () => onManage(actualConnector) : undefined} /> ); })} diff --git a/surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts deleted file mode 100644 index 32a475300..000000000 --- a/surfsense_web/components/assistant-ui/connector-popup/use-connector-dialog.ts +++ /dev/null @@ -1,361 +0,0 @@ -import { useAtomValue } from "jotai"; -import { useRouter, useSearchParams } from "next/navigation"; -import { useCallback, useEffect, useMemo, useState } from "react"; -import { toast } from "sonner"; -import { indexConnectorMutationAtom, updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; -import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; -import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; -import { authenticatedFetch } from "@/lib/auth-utils"; -import { queryClient } from "@/lib/query-client/client"; -import { cacheKeys } from "@/lib/query-client/cache-keys"; -import { format } from "date-fns"; -import type { SearchSourceConnector } from "@/contracts/types/connector.types"; -import { searchSourceConnector } from "@/contracts/types/connector.types"; -import { OAUTH_CONNECTORS } from "./connector-constants"; -import type { IndexingConfigState } from "./connector-constants"; -import { - parseConnectorPopupQueryParams, - parseOAuthAuthResponse, - validateIndexingConfigState, - frequencyMinutesSchema, - dateRangeSchema, -} from "./connector-popup.schemas"; - -export const useConnectorDialog = () => { - const router = useRouter(); - const searchParams = useSearchParams(); - const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); - const { data: allConnectors, refetch: refetchAllConnectors } = useAtomValue(connectorsAtom); - const { mutateAsync: indexConnector } = useAtomValue(indexConnectorMutationAtom); - const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom); - - const [isOpen, setIsOpen] = useState(false); - const [activeTab, setActiveTab] = useState("all"); - const [connectingId, setConnectingId] = useState(null); - const [isScrolled, setIsScrolled] = useState(false); - const [searchQuery, setSearchQuery] = useState(""); - const [indexingConfig, setIndexingConfig] = useState(null); - const [startDate, setStartDate] = useState(undefined); - const [endDate, setEndDate] = useState(undefined); - const [isStartingIndexing, setIsStartingIndexing] = useState(false); - const [periodicEnabled, setPeriodicEnabled] = useState(false); - const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); - - // Helper function to get frequency label - const getFrequencyLabel = useCallback((minutes: string): string => { - switch (minutes) { - case "15": return "15 minutes"; - case "60": return "hour"; - case "360": return "6 hours"; - case "720": return "12 hours"; - case "1440": return "day"; - case "10080": return "week"; - default: return `${minutes} minutes`; - } - }, []); - - // Synchronize state with URL query params - useEffect(() => { - try { - const params = parseConnectorPopupQueryParams(searchParams); - - if (params.modal === "connectors") { - setIsOpen(true); - - if (params.tab === "active" || params.tab === "all") { - setActiveTab(params.tab); - } - - // Clear indexing config if view is not "configure" anymore - if (params.view !== "configure" && indexingConfig) { - setIndexingConfig(null); - } - - if (params.view === "configure" && params.connector && !indexingConfig) { - const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === params.connector); - if (oauthConnector && allConnectors) { - const existingConnector = allConnectors.find( - (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType - ); - if (existingConnector) { - // Validate connector data before setting state - const connectorValidation = searchSourceConnector.safeParse(existingConnector); - if (connectorValidation.success) { - const config = validateIndexingConfigState({ - connectorType: oauthConnector.connectorType, - connectorId: existingConnector.id, - connectorTitle: oauthConnector.title, - }); - setIndexingConfig(config); - } - } - } - } - } else { - setIsOpen(false); - // Clear indexing config when modal is closed - if (indexingConfig) { - setIndexingConfig(null); - setStartDate(undefined); - setEndDate(undefined); - setPeriodicEnabled(false); - setFrequencyMinutes("1440"); - setIsScrolled(false); - setSearchQuery(""); - } - } - } catch (error) { - // Invalid query params - log but don't crash - console.warn("Invalid connector popup query params:", error); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchParams, allConnectors]); - - // Detect OAuth success and transition to config view - useEffect(() => { - try { - const params = parseConnectorPopupQueryParams(searchParams); - - if (params.success === "true" && params.connector && searchSpaceId && params.modal === "connectors") { - const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === params.connector); - if (oauthConnector) { - refetchAllConnectors().then((result) => { - if (!result.data) return; - - const newConnector = result.data.find( - (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType - ); - if (newConnector) { - // Validate connector data before setting state - const connectorValidation = searchSourceConnector.safeParse(newConnector); - if (connectorValidation.success) { - const config = validateIndexingConfigState({ - connectorType: oauthConnector.connectorType, - connectorId: newConnector.id, - connectorTitle: oauthConnector.title, - }); - setIndexingConfig(config); - setIsOpen(true); - const url = new URL(window.location.href); - url.searchParams.delete("success"); - url.searchParams.set("view", "configure"); - window.history.replaceState({}, "", url.toString()); - } else { - console.warn("Invalid connector data after OAuth:", connectorValidation.error); - toast.error("Failed to validate connector data"); - } - } - }); - } - } - } catch (error) { - // Invalid query params - log but don't crash - console.warn("Invalid connector popup query params in OAuth success handler:", error); - } - }, [searchParams, searchSpaceId, refetchAllConnectors]); - - // Handle OAuth connection - const handleConnectOAuth = useCallback( - async (connector: (typeof OAUTH_CONNECTORS)[0]) => { - if (!searchSpaceId || !connector.authEndpoint) return; - - // Set connecting state immediately to disable button and show spinner - setConnectingId(connector.id); - - try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}${connector.authEndpoint}?space_id=${searchSpaceId}`, - { method: "GET" } - ); - - if (!response.ok) { - throw new Error(`Failed to initiate ${connector.title} OAuth`); - } - - const data = await response.json(); - - // Validate OAuth response with Zod - const validatedData = parseOAuthAuthResponse(data); - - // Don't clear connectingId here - let the redirect happen with button still disabled - // The component will unmount on redirect anyway - window.location.href = validatedData.auth_url; - } catch (error) { - console.error(`Error connecting to ${connector.title}:`, error); - if (error instanceof Error && error.message.includes("Invalid auth URL")) { - toast.error(`Invalid response from ${connector.title} OAuth endpoint`); - } else { - toast.error(`Failed to connect to ${connector.title}`); - } - // Only clear connectingId on error so user can retry - setConnectingId(null); - } - }, - [searchSpaceId] - ); - - // Handle starting indexing - const handleStartIndexing = useCallback(async (refreshConnectors: () => void) => { - if (!indexingConfig || !searchSpaceId) return; - - // Validate date range - const dateRangeValidation = dateRangeSchema.safeParse({ startDate, endDate }); - if (!dateRangeValidation.success) { - toast.error(dateRangeValidation.error.errors[0]?.message || "Invalid date range"); - return; - } - - // Validate frequency minutes if periodic is enabled - if (periodicEnabled) { - const frequencyValidation = frequencyMinutesSchema.safeParse(frequencyMinutes); - if (!frequencyValidation.success) { - toast.error("Invalid frequency value"); - return; - } - } - - setIsStartingIndexing(true); - try { - const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined; - const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined; - - if (periodicEnabled) { - const frequency = parseInt(frequencyMinutes, 10); - await updateConnector({ - id: indexingConfig.connectorId, - data: { - periodic_indexing_enabled: true, - indexing_frequency_minutes: frequency, - }, - }); - } - - await indexConnector({ - connector_id: indexingConfig.connectorId, - queryParams: { - search_space_id: searchSpaceId, - start_date: startDateStr, - end_date: endDateStr, - }, - }); - - toast.success(`${indexingConfig.connectorTitle} indexing started`, { - description: periodicEnabled - ? `Periodic sync enabled every ${getFrequencyLabel(frequencyMinutes)}.` - : "You can continue working while we sync your data.", - }); - - // Update URL - the effect will handle closing the modal and clearing state - const url = new URL(window.location.href); - url.searchParams.delete("modal"); - url.searchParams.delete("tab"); - url.searchParams.delete("success"); - url.searchParams.delete("connector"); - url.searchParams.delete("view"); - router.replace(url.pathname + url.search, { scroll: false }); - - refreshConnectors(); - queryClient.invalidateQueries({ - queryKey: cacheKeys.logs.summary(Number(searchSpaceId)), - }); - } catch (error) { - console.error("Error starting indexing:", error); - toast.error("Failed to start indexing"); - } finally { - setIsStartingIndexing(false); - } - }, [indexingConfig, searchSpaceId, startDate, endDate, indexConnector, updateConnector, periodicEnabled, frequencyMinutes, getFrequencyLabel, router]); - - // Handle skipping indexing - const handleSkipIndexing = useCallback(() => { - // Update URL - the effect will handle closing the modal and clearing state - const url = new URL(window.location.href); - url.searchParams.delete("modal"); - url.searchParams.delete("tab"); - url.searchParams.delete("success"); - url.searchParams.delete("connector"); - url.searchParams.delete("view"); - router.replace(url.pathname + url.search, { scroll: false }); - }, [router]); - - // Handle dialog open/close - const handleOpenChange = useCallback( - (open: boolean) => { - setIsOpen(open); - - if (open) { - const url = new URL(window.location.href); - url.searchParams.set("modal", "connectors"); - url.searchParams.set("tab", activeTab); - window.history.pushState({ modal: true }, "", url.toString()); - } else { - const url = new URL(window.location.href); - url.searchParams.delete("modal"); - url.searchParams.delete("tab"); - url.searchParams.delete("success"); - url.searchParams.delete("connector"); - url.searchParams.delete("view"); - window.history.pushState({ modal: false }, "", url.toString()); - setIsScrolled(false); - setSearchQuery(""); - if (!isStartingIndexing) { - setIndexingConfig(null); - setStartDate(undefined); - setEndDate(undefined); - setPeriodicEnabled(false); - setFrequencyMinutes("1440"); - } - } - }, - [activeTab, isStartingIndexing] - ); - - // Handle tab change - const handleTabChange = useCallback( - (value: string) => { - setActiveTab(value); - const url = new URL(window.location.href); - url.searchParams.set("tab", value); - window.history.replaceState({ modal: true }, "", url.toString()); - }, - [] - ); - - // Handle scroll - const handleScroll = useCallback((e: React.UIEvent) => { - setIsScrolled(e.currentTarget.scrollTop > 0); - }, []); - - return { - // State - isOpen, - activeTab, - connectingId, - isScrolled, - searchQuery, - indexingConfig, - startDate, - endDate, - isStartingIndexing, - periodicEnabled, - frequencyMinutes, - searchSpaceId, - allConnectors, - - // Setters - setSearchQuery, - setStartDate, - setEndDate, - setPeriodicEnabled, - setFrequencyMinutes, - - // Handlers - handleOpenChange, - handleTabChange, - handleScroll, - handleConnectOAuth, - handleStartIndexing, - handleSkipIndexing, - }; -}; - diff --git a/surfsense_web/components/connectors/google-drive-folder-tree.tsx b/surfsense_web/components/connectors/google-drive-folder-tree.tsx index cec207b2a..eed18f173 100644 --- a/surfsense_web/components/connectors/google-drive-folder-tree.tsx +++ b/surfsense_web/components/connectors/google-drive-folder-tree.tsx @@ -14,7 +14,6 @@ import { Presentation, } from "lucide-react"; import { useState } from "react"; -import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { ScrollArea } from "@/components/ui/scroll-area"; import { cn } from "@/lib/utils"; @@ -207,63 +206,74 @@ export function GoogleDriveFolderTree({ const childFolders = children?.filter((c) => c.isFolder) || []; const childFiles = children?.filter((c) => !c.isFolder) || []; + const indentSize = 0.75; // Smaller indent for mobile + return ( -
+
{isFolder ? ( - { e.stopPropagation(); toggleFolder(item); }} + aria-label={isExpanded ? `Collapse ${item.name}` : `Expand ${item.name}`} > {isLoading ? ( - + ) : isExpanded ? ( - + ) : ( - + )} - + ) : ( - + )} {isFolder && ( toggleFolderSelection(item.id, item.name)} - className="shrink-0" + className="shrink-0 h-3.5 w-3.5 sm:h-4 sm:w-4" onClick={(e) => e.stopPropagation()} /> )} -
+
{isFolder ? ( isExpanded ? ( - + ) : ( - + ) ) : ( - getFileIcon(item.mimeType, "h-4 w-4") + getFileIcon(item.mimeType, "h-3 w-3 sm:h-4 sm:w-4") )}
- isFolder && toggleFolder(item)} - > - {item.name} - + {isFolder ? ( + + ) : ( + + {item.name} + + )}
{isExpanded && isFolder && children && ( @@ -272,7 +282,7 @@ export function GoogleDriveFolderTree({ {childFiles.map((child) => renderItem(child, level + 1))} {children.length === 0 && ( -
Empty folder
+
Empty folder
)}
)} @@ -282,25 +292,29 @@ export function GoogleDriveFolderTree({ return (
- -
-
-
+ +
+
+
toggleFolderSelection("root", "My Drive")} - className="shrink-0" + className="shrink-0 h-3.5 w-3.5 sm:h-4 sm:w-4" /> - - toggleFolderSelection("root", "My Drive")}> + +
{isLoadingRoot && ( -
- +
+
)} @@ -309,7 +323,7 @@ export function GoogleDriveFolderTree({
{!isLoadingRoot && rootItems.length === 0 && ( -
+
No files or folders found in your Google Drive
)} diff --git a/surfsense_web/components/ui/switch.tsx b/surfsense_web/components/ui/switch.tsx index b64b32b73..de2c35fc0 100644 --- a/surfsense_web/components/ui/switch.tsx +++ b/surfsense_web/components/ui/switch.tsx @@ -11,7 +11,7 @@ const Switch = React.forwardRef< >(({ className, ...props }, ref) => ( From 5d1859db1748b80a7beb0320b8d6eb83e9dc16c9 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 31 Dec 2025 11:41:57 +0530 Subject: [PATCH 55/92] feat: Add Webcrawler and YouTube connector configurations, enhance connector dialog with creation functionality, and improve UI responsiveness and styling across components. --- .../assistant-ui/connector-popup.tsx | 4 + .../components/date-range-selector.tsx | 20 +-- .../components/periodic-sync-config.tsx | 22 +-- .../components/google-drive-config.tsx | 4 +- .../components/webcrawler-config.tsx | 127 ++++++++++++++ .../components/youtube-config.tsx | 148 ++++++++++++++++ .../connector-configs/index.tsx | 6 + .../views/connector-edit-view.tsx | 27 +-- .../views/indexing-configuration-view.tsx | 24 +-- .../constants/connector-constants.ts | 6 + .../hooks/use-connector-dialog.ts | 166 ++++++++++++++++-- .../tabs/all-connectors-tab.tsx | 21 ++- surfsense_web/contracts/enums/connector.ts | 1 + .../contracts/enums/connectorIcons.tsx | 2 + .../contracts/types/connector.types.ts | 1 + 15 files changed, 512 insertions(+), 67 deletions(-) create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/youtube-config.tsx diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index c1ed38e7b..55a11f420 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -88,6 +88,8 @@ export const ConnectorIndicator: FC = () => { handleTabChange, handleScroll, handleConnectOAuth, + handleCreateWebcrawler, + handleCreateYouTube, handleStartIndexing, handleSkipIndexing, handleStartEdit, @@ -212,6 +214,8 @@ export const ConnectorIndicator: FC = () => { connectingId={connectingId} allConnectors={allConnectors} onConnectOAuth={handleConnectOAuth} + onCreateWebcrawler={handleCreateWebcrawler} + onCreateYouTube={handleCreateYouTube} onManage={handleStartEdit} /> diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/date-range-selector.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/date-range-selector.tsx index 1112f3f36..5c7870639 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/components/date-range-selector.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/components/date-range-selector.tsx @@ -40,23 +40,23 @@ export const DateRangeSelector: FC = ({ }; return ( -
-

Select Date Range

-

+

+

Select Date Range

+

Choose how far back you want to sync your data. You can always re-index later with different dates.

{/* Start Date */}
- + @@ -120,7 +120,7 @@ export const DateRangeSelector: FC = ({ variant="outline" size="sm" onClick={handleLast30Days} - className="text-xs bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10" + className="text-xs sm:text-sm bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10" > Last 30 Days @@ -129,7 +129,7 @@ export const DateRangeSelector: FC = ({ variant="outline" size="sm" onClick={handleLastYear} - className="text-xs bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10" + className="text-xs sm:text-sm bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10" > Last Year diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx index 427a6ac86..f8b869a67 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx @@ -25,11 +25,11 @@ export const PeriodicSyncConfig: FC = ({ onFrequencyChange, }) => { return ( -
+
-

Enable Periodic Sync

-

+

Enable Periodic Sync

+

Automatically re-index at regular intervals

@@ -39,21 +39,21 @@ export const PeriodicSyncConfig: FC = ({ {enabled && (
- +
diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx index 280d6ed23..a30450dad 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx @@ -91,9 +91,9 @@ export const GoogleDriveConfig: FC = ({ )} - + - + Folder selection is used when indexing. You can change this selection when you start indexing. diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx new file mode 100644 index 000000000..4bb75c58d --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx @@ -0,0 +1,127 @@ +"use client"; + +import { Info } from "lucide-react"; +import { useState, useEffect } from "react"; +import type { FC } from "react"; +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Textarea } from "@/components/ui/textarea"; +import type { ConnectorConfigProps } from "../index"; + +export const WebcrawlerConfig: FC = ({ + connector, + onConfigChange, +}) => { + // Initialize with existing config values + const existingApiKey = (connector.config?.FIRECRAWL_API_KEY as string | undefined) || ""; + const existingUrls = (connector.config?.INITIAL_URLS as string | undefined) || ""; + + const [apiKey, setApiKey] = useState(existingApiKey); + const [initialUrls, setInitialUrls] = useState(existingUrls); + const [showApiKey, setShowApiKey] = useState(false); + + // Update state when connector config changes + useEffect(() => { + const apiKeyValue = (connector.config?.FIRECRAWL_API_KEY as string | undefined) || ""; + const urlsValue = (connector.config?.INITIAL_URLS as string | undefined) || ""; + setApiKey(apiKeyValue); + setInitialUrls(urlsValue); + }, [connector.config]); + + const handleApiKeyChange = (value: string) => { + setApiKey(value); + if (onConfigChange) { + onConfigChange({ + ...connector.config, + FIRECRAWL_API_KEY: value.trim() || undefined, + }); + } + }; + + const handleUrlsChange = (value: string) => { + setInitialUrls(value); + if (onConfigChange) { + onConfigChange({ + ...connector.config, + INITIAL_URLS: value.trim() || undefined, + }); + } + }; + + return ( +
+
+

Web Crawler Configuration

+

+ Configure your web crawler settings. You can add a Firecrawl API key for enhanced crawling or use the free fallback option. +

+
+ + {/* API Key Field */} +
+ +
+ handleApiKeyChange(e.target.value)} + className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 text-xs sm:text-sm pr-10" + /> + +
+

+ Get your API key from{" "} + + firecrawl.dev + + . If not provided, will use AsyncChromiumLoader as fallback. +

+
+ + {/* Initial URLs Field */} +
+ +