diff --git a/surfsense_backend/alembic/versions/74_add_composio_connector_enums.py b/surfsense_backend/alembic/versions/74_add_composio_connector_enums.py index 454b60754..cadf70cb6 100644 --- a/surfsense_backend/alembic/versions/74_add_composio_connector_enums.py +++ b/surfsense_backend/alembic/versions/74_add_composio_connector_enums.py @@ -1,16 +1,21 @@ -"""Add COMPOSIO_CONNECTOR to SearchSourceConnectorType and DocumentType enums +"""Add Composio connector types to SearchSourceConnectorType and DocumentType enums Revision ID: 74 Revises: 73 Create Date: 2026-01-21 -This migration adds the COMPOSIO_CONNECTOR enum value to both: +This migration adds the Composio connector enum values to both: - searchsourceconnectortype (for connector type tracking) - documenttype (for document type tracking) Composio is a managed OAuth integration service that allows connecting to various third-party services (Google Drive, Gmail, Calendar, etc.) without requiring separate OAuth app verification. + +This migration adds three specific connector types: +- COMPOSIO_GOOGLE_DRIVE_CONNECTOR +- COMPOSIO_GMAIL_CONNECTOR +- COMPOSIO_GOOGLE_CALENDAR_CONNECTOR """ from collections.abc import Sequence @@ -23,55 +28,65 @@ down_revision: str | None = "73" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None -# Define the ENUM type names and the new value +# Define the ENUM type names and the new values CONNECTOR_ENUM = "searchsourceconnectortype" -CONNECTOR_NEW_VALUE = "COMPOSIO_CONNECTOR" +CONNECTOR_NEW_VALUES = [ + "COMPOSIO_GOOGLE_DRIVE_CONNECTOR", + "COMPOSIO_GMAIL_CONNECTOR", + "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR", +] DOCUMENT_ENUM = "documenttype" -DOCUMENT_NEW_VALUE = "COMPOSIO_CONNECTOR" +DOCUMENT_NEW_VALUES = [ + "COMPOSIO_GOOGLE_DRIVE_CONNECTOR", + "COMPOSIO_GMAIL_CONNECTOR", + "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR", +] def upgrade() -> None: - """Upgrade schema - add COMPOSIO_CONNECTOR to connector and document enums safely.""" - # Add COMPOSIO_CONNECTOR to searchsourceconnectortype only if not exists - op.execute( - f""" - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_enum - WHERE enumlabel = '{CONNECTOR_NEW_VALUE}' - AND enumtypid = (SELECT oid FROM pg_type WHERE typname = '{CONNECTOR_ENUM}') - ) THEN - ALTER TYPE {CONNECTOR_ENUM} ADD VALUE '{CONNECTOR_NEW_VALUE}'; - END IF; - END$$; - """ - ) + """Upgrade schema - add Composio connector types to connector and document enums safely.""" + # Add each Composio connector type to searchsourceconnectortype only if not exists + for value in CONNECTOR_NEW_VALUES: + op.execute( + f""" + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum e + JOIN pg_type t ON e.enumtypid = t.oid + WHERE t.typname = '{CONNECTOR_ENUM}' AND e.enumlabel = '{value}' + ) THEN + ALTER TYPE {CONNECTOR_ENUM} ADD VALUE '{value}'; + END IF; + END$$; + """ + ) - # Add COMPOSIO_CONNECTOR to documenttype only if not exists - op.execute( - f""" - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_enum - WHERE enumlabel = '{DOCUMENT_NEW_VALUE}' - AND enumtypid = (SELECT oid FROM pg_type WHERE typname = '{DOCUMENT_ENUM}') - ) THEN - ALTER TYPE {DOCUMENT_ENUM} ADD VALUE '{DOCUMENT_NEW_VALUE}'; - END IF; - END$$; - """ - ) + # Add each Composio connector type to documenttype only if not exists + for value in DOCUMENT_NEW_VALUES: + op.execute( + f""" + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum e + JOIN pg_type t ON e.enumtypid = t.oid + WHERE t.typname = '{DOCUMENT_ENUM}' AND e.enumlabel = '{value}' + ) THEN + ALTER TYPE {DOCUMENT_ENUM} ADD VALUE '{value}'; + END IF; + END$$; + """ + ) def downgrade() -> None: - """Downgrade schema - remove COMPOSIO_CONNECTOR from connector and document enums. + """Downgrade schema - remove Composio connector types from connector and document enums. Note: PostgreSQL does not support removing enum values directly. To properly downgrade, you would need to: - 1. Delete any rows using the COMPOSIO_CONNECTOR value - 2. Create new enums without COMPOSIO_CONNECTOR + 1. Delete any rows using the Composio connector type values + 2. Create new enums without the Composio connector types 3. Alter the columns to use the new enums 4. Drop the old enums diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index b56f37373..705e89ea7 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -54,7 +54,9 @@ class DocumentType(str, Enum): BOOKSTACK_CONNECTOR = "BOOKSTACK_CONNECTOR" CIRCLEBACK = "CIRCLEBACK" NOTE = "NOTE" - COMPOSIO_CONNECTOR = "COMPOSIO_CONNECTOR" # Generic Composio integration + COMPOSIO_GOOGLE_DRIVE_CONNECTOR = "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" + COMPOSIO_GMAIL_CONNECTOR = "COMPOSIO_GMAIL_CONNECTOR" + COMPOSIO_GOOGLE_CALENDAR_CONNECTOR = "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR" class SearchSourceConnectorType(str, Enum): @@ -82,7 +84,9 @@ class SearchSourceConnectorType(str, Enum): BOOKSTACK_CONNECTOR = "BOOKSTACK_CONNECTOR" CIRCLEBACK_CONNECTOR = "CIRCLEBACK_CONNECTOR" MCP_CONNECTOR = "MCP_CONNECTOR" # Model Context Protocol - User-defined API tools - COMPOSIO_CONNECTOR = "COMPOSIO_CONNECTOR" # Generic Composio integration (Google, Slack, etc.) + COMPOSIO_GOOGLE_DRIVE_CONNECTOR = "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" + COMPOSIO_GMAIL_CONNECTOR = "COMPOSIO_GMAIL_CONNECTOR" + COMPOSIO_GOOGLE_CALENDAR_CONNECTOR = "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR" class LiteLLMProvider(str, Enum): diff --git a/surfsense_backend/app/routes/composio_routes.py b/surfsense_backend/app/routes/composio_routes.py index b6f418aa2..77891fc88 100644 --- a/surfsense_backend/app/routes/composio_routes.py +++ b/surfsense_backend/app/routes/composio_routes.py @@ -19,6 +19,7 @@ from fastapi.responses import RedirectResponse from pydantic import ValidationError from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select from app.config import config from app.db import ( @@ -30,15 +31,17 @@ from app.db import ( from app.services.composio_service import ( COMPOSIO_TOOLKIT_NAMES, INDEXABLE_TOOLKITS, + TOOLKIT_TO_CONNECTOR_TYPE, ComposioService, ) from app.users import current_active_user -from app.utils.connector_naming import ( - check_duplicate_connector, - generate_unique_connector_name, -) +from app.utils.connector_naming import generate_unique_connector_name from app.utils.oauth_security import OAuthStateManager +# Note: We no longer use check_duplicate_connector for Composio connectors because +# Composio generates a new connected_account_id each time, even for the same Google account. +# Instead, we check for existing connectors by type/space/user and update them. + logger = logging.getLogger(__name__) router = APIRouter() @@ -260,30 +263,65 @@ async def composio_callback( "is_indexable": toolkit_id in INDEXABLE_TOOLKITS, } - # Check for duplicate connector - # For Composio, we use toolkit_id + connected_account_id as unique identifier - identifier = final_connected_account_id or f"{toolkit_id}_{user_id}" - - is_duplicate = await check_duplicate_connector( - session, - SearchSourceConnectorType.COMPOSIO_CONNECTOR, - space_id, - user_id, - identifier, - ) - if is_duplicate: - logger.warning( - f"Duplicate Composio connector detected for user {user_id} with toolkit {toolkit_id}" + # Get the specific connector type for this toolkit + connector_type_str = TOOLKIT_TO_CONNECTOR_TYPE.get(toolkit_id) + if not connector_type_str: + raise HTTPException( + status_code=400, + detail=f"Unknown toolkit: {toolkit_id}. Available: {list(TOOLKIT_TO_CONNECTOR_TYPE.keys())}", ) + connector_type = SearchSourceConnectorType(connector_type_str) + + # Check for existing connector of the same type for this user/space + # When reconnecting, Composio gives a new connected_account_id, so we need to + # check by connector_type, user_id, and search_space_id instead of connected_account_id + existing_connector_result = await session.execute( + select(SearchSourceConnector).where( + SearchSourceConnector.connector_type == connector_type, + SearchSourceConnector.search_space_id == space_id, + SearchSourceConnector.user_id == user_id, + ) + ) + existing_connector = existing_connector_result.scalars().first() + + if existing_connector: + # Delete the old Composio connected account before updating + old_connected_account_id = existing_connector.config.get("composio_connected_account_id") + if old_connected_account_id and old_connected_account_id != final_connected_account_id: + try: + deleted = await service.delete_connected_account(old_connected_account_id) + if deleted: + logger.info( + f"Deleted old Composio connected account {old_connected_account_id} " + f"before updating connector {existing_connector.id}" + ) + else: + logger.warning( + f"Failed to delete old Composio connected account {old_connected_account_id}" + ) + except Exception as delete_error: + # Log but don't fail - the old account may already be deleted + logger.warning( + f"Error deleting old Composio connected account {old_connected_account_id}: {delete_error!s}" + ) + + # Update existing connector with new connected_account_id + logger.info( + f"Updating existing Composio connector {existing_connector.id} with new connected_account_id {final_connected_account_id}" + ) + existing_connector.config = connector_config + await session.commit() + await session.refresh(existing_connector) + return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=composio-connector" + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=composio-connector&connectorId={existing_connector.id}" ) try: # Generate a unique, user-friendly connector name connector_name = await generate_unique_connector_name( session, - SearchSourceConnectorType.COMPOSIO_CONNECTOR, + connector_type, space_id, user_id, f"{toolkit_name} (Composio)", @@ -291,7 +329,7 @@ async def composio_callback( db_connector = SearchSourceConnector( name=connector_name, - connector_type=SearchSourceConnectorType.COMPOSIO_CONNECTOR, + connector_type=connector_type, config=connector_config, search_space_id=space_id, user_id=user_id, diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index d60d08d57..9ad03fba8 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -37,6 +37,7 @@ from app.db import ( async_session_maker, get_async_session, ) +from app.services.composio_service import ComposioService from app.schemas import ( GoogleDriveIndexRequest, MCPConnectorCreate, @@ -529,6 +530,34 @@ async def delete_search_source_connector( f"Failed to delete periodic schedule for connector {connector_id}" ) + # For Composio connectors, also delete the connected account in Composio + composio_connector_types = [ + SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR, + SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, + SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR, + ] + if db_connector.connector_type in composio_connector_types: + composio_connected_account_id = db_connector.config.get("composio_connected_account_id") + if composio_connected_account_id and ComposioService.is_enabled(): + try: + service = ComposioService() + deleted = await service.delete_connected_account(composio_connected_account_id) + if deleted: + logger.info( + f"Successfully deleted Composio connected account {composio_connected_account_id} " + f"for connector {connector_id}" + ) + else: + logger.warning( + f"Failed to delete Composio connected account {composio_connected_account_id} " + f"for connector {connector_id}" + ) + except Exception as composio_error: + # Log but don't fail the deletion - Composio account may already be deleted + logger.warning( + f"Error deleting Composio connected account {composio_connected_account_id}: {composio_error!s}" + ) + await session.delete(db_connector) await session.commit() return {"message": "Search source connector deleted successfully"} @@ -868,7 +897,11 @@ async def index_connector_content( ) response_message = "Web page indexing started in the background." - elif connector.connector_type == SearchSourceConnectorType.COMPOSIO_CONNECTOR: + elif connector.connector_type in [ + SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR, + SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, + SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR, + ]: from app.tasks.celery_tasks.connector_tasks import ( index_composio_connector_task, ) @@ -2086,6 +2119,59 @@ async def run_bookstack_indexing( ) +async def run_composio_indexing_with_new_session( + connector_id: int, + search_space_id: int, + user_id: str, + start_date: str, + end_date: str, +): + """ + Create a new session and run the Composio indexing task. + This prevents session leaks by creating a dedicated session for the background task. + """ + async with async_session_maker() as session: + await run_composio_indexing( + session, connector_id, search_space_id, user_id, start_date, end_date + ) + + +async def run_composio_indexing( + session: AsyncSession, + connector_id: int, + search_space_id: int, + user_id: str, + start_date: str, + end_date: str, +): + """ + Run Composio connector indexing with real-time notifications. + + This wraps the Composio indexer with the notification system so that + Electric SQL can sync indexing progress to the frontend in real-time. + + Args: + session: Database session + connector_id: ID of the Composio connector + search_space_id: ID of the search space + user_id: ID of the user + start_date: Start date for indexing + end_date: End date for indexing + """ + from app.tasks.composio_indexer import index_composio_connector + + await _run_indexing_with_notifications( + session=session, + connector_id=connector_id, + search_space_id=search_space_id, + user_id=user_id, + start_date=start_date, + end_date=end_date, + indexing_function=index_composio_connector, + update_timestamp_func=_update_connector_timestamp_by_id, + ) + + # ============================================================================= # MCP Connector Routes # ============================================================================= diff --git a/surfsense_backend/app/services/composio_service.py b/surfsense_backend/app/services/composio_service.py index 4b6a32b03..17fbd64e0 100644 --- a/surfsense_backend/app/services/composio_service.py +++ b/surfsense_backend/app/services/composio_service.py @@ -39,6 +39,20 @@ COMPOSIO_TOOLKIT_NAMES = { # Toolkits that support indexing (Phase 1: Google services only) INDEXABLE_TOOLKITS = {"googledrive", "gmail", "googlecalendar"} +# Mapping of toolkit IDs to connector types +TOOLKIT_TO_CONNECTOR_TYPE = { + "googledrive": "COMPOSIO_GOOGLE_DRIVE_CONNECTOR", + "gmail": "COMPOSIO_GMAIL_CONNECTOR", + "googlecalendar": "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR", +} + +# Mapping of toolkit IDs to document types +TOOLKIT_TO_DOCUMENT_TYPE = { + "googledrive": "COMPOSIO_GOOGLE_DRIVE_CONNECTOR", + "gmail": "COMPOSIO_GMAIL_CONNECTOR", + "googlecalendar": "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR", +} + class ComposioService: """Service for interacting with Composio API.""" @@ -298,6 +312,26 @@ class ComposioService: logger.error(f"Failed to list connections for user {user_id}: {e!s}") return [] + async def delete_connected_account(self, connected_account_id: str) -> bool: + """ + Delete a connected account from Composio. + + This permanently removes the connected account and revokes access tokens. + + Args: + connected_account_id: The Composio connected account ID to delete. + + Returns: + True if deletion was successful, False otherwise. + """ + try: + self.client.connected_accounts.delete(connected_account_id) + logger.info(f"Successfully deleted Composio connected account: {connected_account_id}") + return True + except Exception as e: + logger.error(f"Failed to delete Composio connected account {connected_account_id}: {e!s}") + return False + async def execute_tool( self, connected_account_id: str, diff --git a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py index 72cedb40f..307b5a551 100644 --- a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py @@ -793,11 +793,13 @@ async def _index_composio_connector( start_date: str, end_date: str, ): - """Index Composio connector content with new session.""" - # Import from tasks folder (not connector_indexers) to avoid circular import - from app.tasks.composio_indexer import index_composio_connector + """Index Composio connector content with new session and real-time notifications.""" + # Import from routes to use the notification-wrapped version + from app.routes.search_source_connectors_routes import ( + run_composio_indexing, + ) async with get_celery_session_maker()() as session: - await index_composio_connector( + await run_composio_indexing( session, connector_id, search_space_id, user_id, start_date, end_date ) diff --git a/surfsense_backend/app/tasks/composio_indexer.py b/surfsense_backend/app/tasks/composio_indexer.py index 01d2cfce4..8762561ee 100644 --- a/surfsense_backend/app/tasks/composio_indexer.py +++ b/surfsense_backend/app/tasks/composio_indexer.py @@ -23,7 +23,7 @@ from app.db import ( SearchSourceConnector, SearchSourceConnectorType, ) -from app.services.composio_service import INDEXABLE_TOOLKITS +from app.services.composio_service import INDEXABLE_TOOLKITS, TOOLKIT_TO_DOCUMENT_TYPE from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( @@ -58,15 +58,13 @@ async def check_document_by_unique_identifier( async def get_connector_by_id( - session: AsyncSession, connector_id: int, connector_type: SearchSourceConnectorType + session: AsyncSession, connector_id: int, connector_type: SearchSourceConnectorType | None ) -> SearchSourceConnector | None: - """Get a connector by ID and type from the database.""" - result = await session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == connector_id, - SearchSourceConnector.connector_type == connector_type, - ) - ) + """Get a connector by ID and optionally by type from the database.""" + query = select(SearchSourceConnector).filter(SearchSourceConnector.id == connector_id) + if connector_type is not None: + query = query.filter(SearchSourceConnector.connector_type == connector_type) + result = await session.execute(query) return result.scalars().first() @@ -129,10 +127,23 @@ async def index_composio_connector( ) try: - # Get connector by id + # Get connector by id - accept any Composio connector type + # We'll check the actual type after loading connector = await get_connector_by_id( - session, connector_id, SearchSourceConnectorType.COMPOSIO_CONNECTOR + session, connector_id, None # Don't filter by type, we'll validate after ) + + # Validate it's a Composio connector + if connector and connector.connector_type not in [ + SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR, + SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, + SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR, + ]: + error_msg = f"Connector {connector_id} is not a Composio connector" + await task_logger.log_task_failure( + log_entry, error_msg, {"error_type": "InvalidConnectorType"} + ) + return 0, error_msg if not connector: error_msg = f"Composio connector with ID {connector_id} not found" @@ -276,7 +287,7 @@ async def _index_composio_google_drive( await task_logger.log_task_success( log_entry, success_msg, {"files_count": 0} ) - return 0, success_msg + return 0, None # Return None (not error) when no items found - this is success with 0 items logger.info(f"Found {len(all_files)} Google Drive files to index via Composio") @@ -299,8 +310,9 @@ async def _index_composio_google_drive( continue # Generate unique identifier hash + document_type = DocumentType(TOOLKIT_TO_DOCUMENT_TYPE["googledrive"]) unique_identifier_hash = generate_unique_identifier_hash( - DocumentType.COMPOSIO_CONNECTOR, f"drive_{file_id}", search_space_id + document_type, f"drive_{file_id}", search_space_id ) # Check if document exists @@ -394,7 +406,7 @@ async def _index_composio_google_drive( document = Document( search_space_id=search_space_id, title=f"Drive: {file_name}", - document_type=DocumentType.COMPOSIO_CONNECTOR, + document_type=DocumentType(TOOLKIT_TO_DOCUMENT_TYPE["googledrive"]), document_metadata={ "file_id": file_id, "file_name": file_name, @@ -489,7 +501,7 @@ async def _index_composio_gmail( await task_logger.log_task_success( log_entry, success_msg, {"messages_count": 0} ) - return 0, success_msg + return 0, None # Return None (not error) when no items found - this is success with 0 items logger.info(f"Found {len(messages)} Gmail messages to index via Composio") @@ -530,8 +542,9 @@ async def _index_composio_gmail( markdown_content = composio_connector.format_gmail_message_to_markdown(message) # Generate unique identifier + document_type = DocumentType(TOOLKIT_TO_DOCUMENT_TYPE["gmail"]) unique_identifier_hash = generate_unique_identifier_hash( - DocumentType.COMPOSIO_CONNECTOR, f"gmail_{message_id}", search_space_id + document_type, f"gmail_{message_id}", search_space_id ) content_hash = generate_content_hash(markdown_content, search_space_id) @@ -612,7 +625,7 @@ async def _index_composio_gmail( document = Document( search_space_id=search_space_id, title=f"Gmail: {subject}", - document_type=DocumentType.COMPOSIO_CONNECTOR, + document_type=DocumentType(TOOLKIT_TO_DOCUMENT_TYPE["gmail"]), document_metadata={ "message_id": message_id, "subject": subject, @@ -717,7 +730,7 @@ async def _index_composio_google_calendar( await task_logger.log_task_success( log_entry, success_msg, {"events_count": 0} ) - return 0, success_msg + return 0, None # Return None (not error) when no items found - this is success with 0 items logger.info(f"Found {len(events)} Google Calendar events to index via Composio") @@ -738,8 +751,9 @@ async def _index_composio_google_calendar( markdown_content = composio_connector.format_calendar_event_to_markdown(event) # Generate unique identifier + document_type = DocumentType(TOOLKIT_TO_DOCUMENT_TYPE["googlecalendar"]) unique_identifier_hash = generate_unique_identifier_hash( - DocumentType.COMPOSIO_CONNECTOR, f"calendar_{event_id}", search_space_id + document_type, f"calendar_{event_id}", search_space_id ) content_hash = generate_content_hash(markdown_content, search_space_id) @@ -828,7 +842,7 @@ async def _index_composio_google_calendar( document = Document( search_space_id=search_space_id, title=f"Calendar: {summary}", - document_type=DocumentType.COMPOSIO_CONNECTOR, + document_type=DocumentType(TOOLKIT_TO_DOCUMENT_TYPE["googlecalendar"]), document_metadata={ "event_id": event_id, "summary": summary, diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index 1f4341d07..228b12836 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -188,8 +188,18 @@ export const ConnectorIndicator: FC = () => { searchSpaceId={searchSpaceId} connectedToolkits={ (connectors || []) - .filter((c: SearchSourceConnector) => c.connector_type === "COMPOSIO_CONNECTOR") - .map((c: SearchSourceConnector) => c.config?.toolkit_id as string) + .filter((c: SearchSourceConnector) => + c.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" || + c.connector_type === "COMPOSIO_GMAIL_CONNECTOR" || + c.connector_type === "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR" + ) + .map((c: SearchSourceConnector) => { + // Map connector type back to toolkit_id + if (c.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR") return "googledrive"; + if (c.connector_type === "COMPOSIO_GMAIL_CONNECTOR") return "gmail"; + if (c.connector_type === "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR") return "googlecalendar"; + return c.config?.toolkit_id as string; + }) .filter(Boolean) } onBack={handleBackFromComposio} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/composio-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/composio-config.tsx index 6fe37e1e5..a96f906fe 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/composio-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/composio-config.tsx @@ -1,7 +1,5 @@ "use client"; -import { ExternalLink, Info, Zap } from "lucide-react"; -import Image from "next/image"; import type { FC } from "react"; import { Badge } from "@/components/ui/badge"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; @@ -13,92 +11,13 @@ interface ComposioConfigProps { onNameChange?: (name: string) => void; } -// Get toolkit display info -const getToolkitInfo = (toolkitId: string): { name: string; icon: string; description: string } => { - switch (toolkitId) { - case "googledrive": - return { - name: "Google Drive", - icon: "/connectors/google-drive.svg", - description: "Files and documents from Google Drive", - }; - case "gmail": - return { - name: "Gmail", - icon: "/connectors/google-gmail.svg", - description: "Emails from Gmail", - }; - case "googlecalendar": - return { - name: "Google Calendar", - icon: "/connectors/google-calendar.svg", - description: "Events from Google Calendar", - }; - case "slack": - return { - name: "Slack", - icon: "/connectors/slack.svg", - description: "Messages from Slack", - }; - case "notion": - return { - name: "Notion", - icon: "/connectors/notion.svg", - description: "Pages from Notion", - }; - case "github": - return { - name: "GitHub", - icon: "/connectors/github.svg", - description: "Repositories from GitHub", - }; - default: - return { - name: toolkitId, - icon: "/connectors/composio.svg", - description: "Connected via Composio", - }; - } -}; - export const ComposioConfig: FC = ({ connector }) => { const toolkitId = connector.config?.toolkit_id as string; - const toolkitName = connector.config?.toolkit_name as string; const isIndexable = connector.config?.is_indexable as boolean; const composioAccountId = connector.config?.composio_connected_account_id as string; - const toolkitInfo = getToolkitInfo(toolkitId); - return (
- {/* Toolkit Info Card */} -
-
-
- {toolkitInfo.name} -
-
-
-

{toolkitName || toolkitInfo.name}

- - - Composio - -
-

{toolkitInfo.description}

-
-
-
- {/* Connection Details */}

@@ -133,28 +52,6 @@ export const ComposioConfig: FC = ({ connector }) => { )}

- - {/* Info Banner */} -
-
- -
-

- This connection uses Composio's managed OAuth, which means you don't need to - wait for app verification. Your data is securely accessed through Composio. -

- - Learn more about Composio - - -
-
-
); }; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx index a7a92597c..160185b1e 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx @@ -74,7 +74,9 @@ export function getConnectorConfigComponent( return CirclebackConfig; case "MCP_CONNECTOR": return MCPConfig; - case "COMPOSIO_CONNECTOR": + case "COMPOSIO_GOOGLE_DRIVE_CONNECTOR": + case "COMPOSIO_GMAIL_CONNECTOR": + case "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": return ComposioConfig; // OAuth connectors (Gmail, Calendar, Airtable, Notion) and others don't need special config UI default: diff --git a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts index 7646d7a9b..11066f28a 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts @@ -168,14 +168,28 @@ export const OTHER_CONNECTORS = [ }, ] as const; -// Composio Connector (Single entry that opens toolkit selector) +// Composio Connectors - Individual entries for each supported toolkit export const COMPOSIO_CONNECTORS = [ { - id: "composio-connector", - title: "Composio", - description: "Connect 100+ apps via Composio (Google, Slack, Notion, etc.)", - connectorType: EnumConnectorName.COMPOSIO_CONNECTOR, - // No authEndpoint - handled via toolkit selector view + id: "composio-googledrive", + title: "Google Drive", + description: "Search your Drive files via Composio", + connectorType: EnumConnectorName.COMPOSIO_GOOGLE_DRIVE_CONNECTOR, + authEndpoint: "/api/v1/auth/composio/connector/add/?toolkit_id=googledrive", + }, + { + id: "composio-gmail", + title: "Gmail", + description: "Search through your emails via Composio", + connectorType: EnumConnectorName.COMPOSIO_GMAIL_CONNECTOR, + authEndpoint: "/api/v1/auth/composio/connector/add/?toolkit_id=gmail", + }, + { + id: "composio-googlecalendar", + title: "Google Calendar", + description: "Search through your events via Composio", + connectorType: EnumConnectorName.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR, + authEndpoint: "/api/v1/auth/composio/connector/add/?toolkit_id=googlecalendar", }, ] as const; diff --git a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-popup.schemas.ts b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-popup.schemas.ts index d74d66203..c7e77f666 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-popup.schemas.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-popup.schemas.ts @@ -7,7 +7,7 @@ import { searchSourceConnectorTypeEnum } from "@/contracts/types/connector.types export const connectorPopupQueryParamsSchema = z.object({ modal: z.enum(["connectors"]).optional(), tab: z.enum(["all", "active"]).optional(), - view: z.enum(["configure", "edit", "connect", "youtube", "accounts", "mcp-list"]).optional(), + view: z.enum(["configure", "edit", "connect", "youtube", "accounts", "mcp-list", "composio"]).optional(), connector: z.string().optional(), connectorId: z.string().optional(), connectorType: z.string().optional(), diff --git a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts index c6ef1a927..4a177ac36 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts @@ -26,7 +26,7 @@ import { import { cacheKeys } from "@/lib/query-client/cache-keys"; import { queryClient } from "@/lib/query-client/client"; import type { IndexingConfigState } from "../constants/connector-constants"; -import { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants"; +import { COMPOSIO_CONNECTORS, OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants"; import { dateRangeSchema, frequencyMinutesSchema, @@ -176,15 +176,24 @@ export const useConnectorDialog = () => { } // Handle accounts view - if (params.view === "accounts" && params.connectorType && !viewingAccountsType) { - const oauthConnector = OAUTH_CONNECTORS.find( - (c) => c.connectorType === params.connectorType - ); - if (oauthConnector) { - setViewingAccountsType({ - connectorType: oauthConnector.connectorType, - connectorTitle: oauthConnector.title, - }); + if (params.view === "accounts" && params.connectorType) { + // Update state if not set, or if connectorType has changed + const needsUpdate = !viewingAccountsType || + viewingAccountsType.connectorType !== params.connectorType; + + if (needsUpdate) { + // Check both OAUTH_CONNECTORS and COMPOSIO_CONNECTORS + const oauthConnector = OAUTH_CONNECTORS.find( + (c) => c.connectorType === params.connectorType + ) || COMPOSIO_CONNECTORS.find( + (c) => c.connectorType === params.connectorType + ); + if (oauthConnector) { + setViewingAccountsType({ + connectorType: oauthConnector.connectorType, + connectorTitle: oauthConnector.title, + }); + } } } @@ -293,6 +302,8 @@ export const useConnectorDialog = () => { indexingConfig, connectingConnectorType, viewingAccountsType, + viewingMCPList, + viewingComposio, ]); // Detect OAuth success / Failure and transition to config view @@ -389,15 +400,19 @@ export const useConnectorDialog = () => { // Handle OAuth connection const handleConnectOAuth = useCallback( - async (connector: (typeof OAUTH_CONNECTORS)[number]) => { + async (connector: (typeof OAUTH_CONNECTORS)[number] | (typeof COMPOSIO_CONNECTORS)[number]) => { if (!searchSpaceId || !connector.authEndpoint) return; // Set connecting state immediately to disable button and show spinner setConnectingId(connector.id); try { + // Check if authEndpoint already has query parameters + const separator = connector.authEndpoint.includes("?") ? "&" : "?"; + const url = `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}${connector.authEndpoint}${separator}space_id=${searchSpaceId}`; + const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}${connector.authEndpoint}?space_id=${searchSpaceId}`, + url, { method: "GET" } ); @@ -799,23 +814,19 @@ export const useConnectorDialog = () => { // Handle viewing accounts list for OAuth connector type const handleViewAccountsList = useCallback( - (connectorType: string, connectorTitle: string) => { + (connectorType: string, _connectorTitle?: string) => { if (!searchSpaceId) return; - setViewingAccountsType({ - connectorType, - connectorTitle, - }); - // Update URL to show accounts view, preserving current tab + // The useEffect will handle setting viewingAccountsType based on URL params const url = new URL(window.location.href); url.searchParams.set("modal", "connectors"); url.searchParams.set("view", "accounts"); url.searchParams.set("connectorType", connectorType); // Keep the current tab in URL so we can go back to it - window.history.pushState({ modal: true }, "", url.toString()); + router.replace(url.pathname + url.search, { scroll: false }); }, - [searchSpaceId] + [searchSpaceId, router] ); // Handle going back from accounts list view @@ -839,8 +850,8 @@ export const useConnectorDialog = () => { const url = new URL(window.location.href); url.searchParams.set("modal", "connectors"); url.searchParams.set("view", "mcp-list"); - window.history.pushState({ modal: true }, "", url.toString()); - }, [searchSpaceId]); + router.replace(url.pathname + url.search, { scroll: false }); + }, [searchSpaceId, router]); // Handle going back from MCP list view const handleBackFromMCPList = useCallback(() => { @@ -871,8 +882,8 @@ export const useConnectorDialog = () => { const url = new URL(window.location.href); url.searchParams.set("modal", "connectors"); url.searchParams.set("view", "composio"); - window.history.pushState({ modal: true }, "", url.toString()); - }, [searchSpaceId]); + router.replace(url.pathname + url.search, { scroll: false }); + }, [searchSpaceId, router]); // Handle going back from Composio view const handleBackFromComposio = useCallback(() => { @@ -1423,7 +1434,7 @@ export const useConnectorDialog = () => { setIsDisconnecting(false); } }, - [editingConnector, searchSpaceId, deleteConnector, router] + [editingConnector, searchSpaceId, deleteConnector, router, cameFromMCPList] ); // Handle quick index (index without date picker, uses backend defaults) @@ -1579,6 +1590,7 @@ export const useConnectorDialog = () => { viewingAccountsType, viewingMCPList, viewingComposio, + connectingComposioToolkit, // Setters setSearchQuery, @@ -1616,8 +1628,6 @@ export const useConnectorDialog = () => { setIndexingConnectorConfig, // Composio - viewingComposio, - connectingComposioToolkit, handleOpenComposio, handleBackFromComposio, handleConnectComposioToolkit, diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index 1b36b3b81..4a0680200 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -4,7 +4,6 @@ import type { FC } from "react"; import { EnumConnectorName } from "@/contracts/enums/connector"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { ConnectorCard } from "../components/connector-card"; -import { ComposioConnectorCard } from "../components/composio-connector-card"; import { CRAWLERS, OAUTH_CONNECTORS, OTHER_CONNECTORS, COMPOSIO_CONNECTORS } from "../constants/connector-constants"; import { getDocumentCountForConnector } from "../utils/connector-document-mapping"; @@ -29,13 +28,12 @@ interface AllConnectorsTabProps { allConnectors: SearchSourceConnector[] | undefined; documentTypeCounts?: Record; indexingConnectorIds?: Set; - onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[number]) => void; + onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[number] | (typeof COMPOSIO_CONNECTORS)[number]) => void; onConnectNonOAuth?: (connectorType: string) => void; onCreateWebcrawler?: () => void; onCreateYouTubeCrawler?: () => void; onManage?: (connector: SearchSourceConnector) => void; onViewAccountsList?: (connectorType: string, connectorTitle: string) => void; - onOpenComposio?: () => void; } export const AllConnectorsTab: FC = ({ @@ -51,7 +49,6 @@ export const AllConnectorsTab: FC = ({ onCreateYouTubeCrawler, onManage, onViewAccountsList, - onOpenComposio, }) => { // Filter connectors based on search const filteredOAuth = OAUTH_CONNECTORS.filter( @@ -79,23 +76,16 @@ export const AllConnectorsTab: FC = ({ c.description.toLowerCase().includes(searchQuery.toLowerCase()) ); - // Count Composio connectors - const composioConnectorCount = allConnectors - ? allConnectors.filter( - (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.COMPOSIO_CONNECTOR - ).length - : 0; - return (
- {/* Quick Connect */} - {filteredOAuth.length > 0 && ( + {/* Managed OAuth (Composio Integrations) */} + {filteredComposio.length > 0 && (
-

Quick Connect

+

Managed OAuth

- {filteredOAuth.map((connector) => { + {filteredComposio.map((connector) => { const isConnected = connectedTypes.has(connector.connectorType); const isConnecting = connectingId === connector.id; @@ -109,17 +99,6 @@ export const AllConnectorsTab: FC = ({ const accountCount = typeConnectors.length; - // Get the most recent last_indexed_at across all accounts - const mostRecentLastIndexed = typeConnectors.reduce( - (latest, c) => { - if (!c.last_indexed_at) return latest; - if (!latest) return c.last_indexed_at; - return new Date(c.last_indexed_at) > new Date(latest) - ? c.last_indexed_at - : latest; - }, - undefined - ); const documentCount = getDocumentCountForConnector( connector.connectorType, @@ -154,26 +133,57 @@ export const AllConnectorsTab: FC = ({
)} - {/* Composio Integrations */} - {filteredComposio.length > 0 && onOpenComposio && ( + {/* Quick Connect */} + {filteredOAuth.length > 0 && (
-

Managed OAuth

- - No verification needed - +

Quick Connect

- {filteredComposio.map((connector) => ( - - ))} + {filteredOAuth.map((connector) => { + const isConnected = connectedTypes.has(connector.connectorType); + const isConnecting = connectingId === connector.id; + + // Find all connectors of this type + const typeConnectors = + isConnected && allConnectors + ? allConnectors.filter( + (c: SearchSourceConnector) => c.connector_type === connector.connectorType + ) + : []; + + const accountCount = typeConnectors.length; + + + const documentCount = getDocumentCountForConnector( + connector.connectorType, + documentTypeCounts + ); + + // Check if any account is currently indexing + const isIndexing = typeConnectors.some((c) => indexingConnectorIds?.has(c.id)); + + return ( + onConnectOAuth(connector)} + onManage={ + isConnected && onViewAccountsList + ? () => onViewAccountsList(connector.connectorType, connector.title) + : undefined + } + /> + ); + })}
)} diff --git a/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts b/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts index 522e1763c..ded3bdcca 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts @@ -30,7 +30,10 @@ export const CONNECTOR_TO_DOCUMENT_TYPE: Record = { // Special mappings (connector type differs from document type) GOOGLE_DRIVE_CONNECTOR: "GOOGLE_DRIVE_FILE", WEBCRAWLER_CONNECTOR: "CRAWLED_URL", - COMPOSIO_CONNECTOR: "COMPOSIO_CONNECTOR", + // Composio connectors map to their own document types + COMPOSIO_GOOGLE_DRIVE_CONNECTOR: "COMPOSIO_GOOGLE_DRIVE_CONNECTOR", + COMPOSIO_GMAIL_CONNECTOR: "COMPOSIO_GMAIL_CONNECTOR", + COMPOSIO_GOOGLE_CALENDAR_CONNECTOR: "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR", }; /** diff --git a/surfsense_web/contracts/enums/connector.ts b/surfsense_web/contracts/enums/connector.ts index e1fb1e3f2..20d6093b6 100644 --- a/surfsense_web/contracts/enums/connector.ts +++ b/surfsense_web/contracts/enums/connector.ts @@ -24,5 +24,7 @@ export enum EnumConnectorName { YOUTUBE_CONNECTOR = "YOUTUBE_CONNECTOR", CIRCLEBACK_CONNECTOR = "CIRCLEBACK_CONNECTOR", MCP_CONNECTOR = "MCP_CONNECTOR", - COMPOSIO_CONNECTOR = "COMPOSIO_CONNECTOR", + COMPOSIO_GOOGLE_DRIVE_CONNECTOR = "COMPOSIO_GOOGLE_DRIVE_CONNECTOR", + COMPOSIO_GMAIL_CONNECTOR = "COMPOSIO_GMAIL_CONNECTOR", + COMPOSIO_GOOGLE_CALENDAR_CONNECTOR = "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR", } diff --git a/surfsense_web/contracts/enums/connectorIcons.tsx b/surfsense_web/contracts/enums/connectorIcons.tsx index 947c886d5..a1e6c9040 100644 --- a/surfsense_web/contracts/enums/connectorIcons.tsx +++ b/surfsense_web/contracts/enums/connectorIcons.tsx @@ -66,8 +66,12 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas return ; case EnumConnectorName.MCP_CONNECTOR: return MCP; - case EnumConnectorName.COMPOSIO_CONNECTOR: - return Composio; + case EnumConnectorName.COMPOSIO_GOOGLE_DRIVE_CONNECTOR: + return Google Drive; + case EnumConnectorName.COMPOSIO_GMAIL_CONNECTOR: + return Gmail; + case EnumConnectorName.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR: + return Google Calendar; // Additional cases for non-enum connector types case "YOUTUBE_CONNECTOR": return YouTube; @@ -87,8 +91,12 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas return ; case "GOOGLE_DRIVE_FILE": return ; - case "COMPOSIO_CONNECTOR": - return Composio; + case "COMPOSIO_GOOGLE_DRIVE_CONNECTOR": + return Google Drive; + case "COMPOSIO_GMAIL_CONNECTOR": + return Gmail; + case "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": + return Google Calendar; case "NOTE": return ; case "EXTENSION": diff --git a/surfsense_web/contracts/types/connector.types.ts b/surfsense_web/contracts/types/connector.types.ts index 861bf1758..d52469ce9 100644 --- a/surfsense_web/contracts/types/connector.types.ts +++ b/surfsense_web/contracts/types/connector.types.ts @@ -27,7 +27,9 @@ export const searchSourceConnectorTypeEnum = z.enum([ "BOOKSTACK_CONNECTOR", "CIRCLEBACK_CONNECTOR", "MCP_CONNECTOR", - "COMPOSIO_CONNECTOR", + "COMPOSIO_GOOGLE_DRIVE_CONNECTOR", + "COMPOSIO_GMAIL_CONNECTOR", + "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR", ]); export const searchSourceConnector = z.object({ @@ -149,6 +151,13 @@ export const googleDriveIndexBody = z.object({ name: z.string(), }) ), + indexing_options: z + .object({ + max_files_per_folder: z.number().int().min(1).max(1000), + incremental_sync: z.boolean(), + include_subfolders: z.boolean(), + }) + .optional(), }); /** diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts index a8f3a3b38..01a58173e 100644 --- a/surfsense_web/contracts/types/document.types.ts +++ b/surfsense_web/contracts/types/document.types.ts @@ -25,7 +25,9 @@ export const documentTypeEnum = z.enum([ "CIRCLEBACK", "SURFSENSE_DOCS", "NOTE", - "COMPOSIO_CONNECTOR", + "COMPOSIO_GOOGLE_DRIVE_CONNECTOR", + "COMPOSIO_GMAIL_CONNECTOR", + "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR", ]); export const document = z.object({