diff --git a/surfsense_backend/alembic/versions/74_add_composio_connector_enums.py b/surfsense_backend/alembic/versions/74_add_composio_connector_enums.py deleted file mode 100644 index 454b60754..000000000 --- a/surfsense_backend/alembic/versions/74_add_composio_connector_enums.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Add COMPOSIO_CONNECTOR to SearchSourceConnectorType and DocumentType enums - -Revision ID: 74 -Revises: 73 -Create Date: 2026-01-21 - -This migration adds the COMPOSIO_CONNECTOR enum value to both: -- searchsourceconnectortype (for connector type tracking) -- documenttype (for document type tracking) - -Composio is a managed OAuth integration service that allows connecting -to various third-party services (Google Drive, Gmail, Calendar, etc.) -without requiring separate OAuth app verification. -""" - -from collections.abc import Sequence - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "74" -down_revision: str | None = "73" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - -# Define the ENUM type names and the new value -CONNECTOR_ENUM = "searchsourceconnectortype" -CONNECTOR_NEW_VALUE = "COMPOSIO_CONNECTOR" -DOCUMENT_ENUM = "documenttype" -DOCUMENT_NEW_VALUE = "COMPOSIO_CONNECTOR" - - -def upgrade() -> None: - """Upgrade schema - add COMPOSIO_CONNECTOR to connector and document enums safely.""" - # Add COMPOSIO_CONNECTOR to searchsourceconnectortype only if not exists - op.execute( - f""" - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_enum - WHERE enumlabel = '{CONNECTOR_NEW_VALUE}' - AND enumtypid = (SELECT oid FROM pg_type WHERE typname = '{CONNECTOR_ENUM}') - ) THEN - ALTER TYPE {CONNECTOR_ENUM} ADD VALUE '{CONNECTOR_NEW_VALUE}'; - END IF; - END$$; - """ - ) - - # Add COMPOSIO_CONNECTOR to documenttype only if not exists - op.execute( - f""" - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_enum - WHERE enumlabel = '{DOCUMENT_NEW_VALUE}' - AND enumtypid = (SELECT oid FROM pg_type WHERE typname = '{DOCUMENT_ENUM}') - ) THEN - ALTER TYPE {DOCUMENT_ENUM} ADD VALUE '{DOCUMENT_NEW_VALUE}'; - END IF; - END$$; - """ - ) - - -def downgrade() -> None: - """Downgrade schema - remove COMPOSIO_CONNECTOR from connector and document enums. - - Note: PostgreSQL does not support removing enum values directly. - To properly downgrade, you would need to: - 1. Delete any rows using the COMPOSIO_CONNECTOR value - 2. Create new enums without COMPOSIO_CONNECTOR - 3. Alter the columns to use the new enums - 4. Drop the old enums - - This is left as a no-op since removing enum values is complex - and typically not needed in practice. - """ - pass diff --git a/surfsense_backend/alembic/versions/74_no_op.py b/surfsense_backend/alembic/versions/74_no_op.py new file mode 100644 index 000000000..a5ee99b29 --- /dev/null +++ b/surfsense_backend/alembic/versions/74_no_op.py @@ -0,0 +1,29 @@ +"""No-op migration for Composio support + +Revision ID: 74 +Revises: 73 +Create Date: 2026-01-21 + +NOTE: This migration is a no-op since Composio is not supported yet. +""" + +from collections.abc import Sequence + +# revision identifiers, used by Alembic. +revision: str = "74" +down_revision: str | None = "73" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """No-op upgrade for Composio support.""" + pass + + +def downgrade() -> None: + """No-op downgrade for Composio support. + + Note: PostgreSQL does not support removing enum values directly. + """ + pass diff --git a/surfsense_backend/alembic/versions/77_add_thread_id_to_chat_comments.py b/surfsense_backend/alembic/versions/77_add_thread_id_to_chat_comments.py index 0a2615e84..86886eacf 100644 --- a/surfsense_backend/alembic/versions/77_add_thread_id_to_chat_comments.py +++ b/surfsense_backend/alembic/versions/77_add_thread_id_to_chat_comments.py @@ -64,5 +64,7 @@ def upgrade() -> None: def downgrade() -> None: """Remove thread_id column from chat_comments.""" op.execute("DROP INDEX IF EXISTS idx_chat_comments_thread_id") - op.execute("ALTER TABLE chat_comments DROP CONSTRAINT IF EXISTS fk_chat_comments_thread_id") + op.execute( + "ALTER TABLE chat_comments DROP CONSTRAINT IF EXISTS fk_chat_comments_thread_id" + ) op.execute("ALTER TABLE chat_comments DROP COLUMN IF EXISTS thread_id") diff --git a/surfsense_backend/app/connectors/composio_connector.py b/surfsense_backend/app/connectors/composio_connector.py index 18fd9564c..fdf57d8ea 100644 --- a/surfsense_backend/app/connectors/composio_connector.py +++ b/surfsense_backend/app/connectors/composio_connector.py @@ -12,7 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from app.db import SearchSourceConnector -from app.services.composio_service import ComposioService, INDEXABLE_TOOLKITS +from app.services.composio_service import INDEXABLE_TOOLKITS, ComposioService logger = logging.getLogger(__name__) @@ -268,7 +268,9 @@ class ComposioConnector: from_email = header_dict.get("from", "Unknown Sender") to_email = header_dict.get("to", "Unknown Recipient") # Composio provides messageTimestamp directly - date_str = message.get("messageTimestamp", "") or header_dict.get("date", "Unknown Date") + date_str = message.get("messageTimestamp", "") or header_dict.get( + "date", "Unknown Date" + ) # Build markdown content markdown_content = f"# {subject}\n\n" diff --git a/surfsense_backend/app/connectors/github_connector.py b/surfsense_backend/app/connectors/github_connector.py index 6f04ccdba..9d4b98c4b 100644 --- a/surfsense_backend/app/connectors/github_connector.py +++ b/surfsense_backend/app/connectors/github_connector.py @@ -58,7 +58,9 @@ class GitHubConnector: if self.token: logger.info("GitHub connector initialized with authentication token.") else: - logger.info("GitHub connector initialized without token (public repos only).") + logger.info( + "GitHub connector initialized without token (public repos only)." + ) def ingest_repository( self, @@ -95,17 +97,27 @@ class GitHubConnector: cmd = [ "gitingest", repo_url, - "--output", output_path, - "--max-size", str(max_file_size), + "--output", + output_path, + "--max-size", + str(max_file_size), # Common exclude patterns - "-e", "node_modules/*", - "-e", "vendor/*", - "-e", ".git/*", - "-e", "__pycache__/*", - "-e", "dist/*", - "-e", "build/*", - "-e", "*.lock", - "-e", "package-lock.json", + "-e", + "node_modules/*", + "-e", + "vendor/*", + "-e", + ".git/*", + "-e", + "__pycache__/*", + "-e", + "dist/*", + "-e", + "build/*", + "-e", + "*.lock", + "-e", + "package-lock.json", ] # Add branch if specified @@ -147,7 +159,9 @@ class GitHubConnector: os.unlink(output_path) if not full_content or not full_content.strip(): - logger.warning(f"No content retrieved from repository: {repo_full_name}") + logger.warning( + f"No content retrieved from repository: {repo_full_name}" + ) return None # Parse the gitingest output @@ -171,11 +185,11 @@ class GitHubConnector: logger.error(f"gitingest timed out for repository: {repo_full_name}") return None except FileNotFoundError: - logger.error( - "gitingest CLI not found. Falling back to Python library." - ) + logger.error("gitingest CLI not found. Falling back to Python library.") # Fall back to Python library - return self._ingest_with_python_library(repo_full_name, branch, max_file_size) + return self._ingest_with_python_library( + repo_full_name, branch, max_file_size + ) except Exception as e: logger.error(f"Failed to ingest repository {repo_full_name}: {e}") return None diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 4b9be6f4a..db04009d2 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -82,7 +82,9 @@ class SearchSourceConnectorType(str, Enum): BOOKSTACK_CONNECTOR = "BOOKSTACK_CONNECTOR" CIRCLEBACK_CONNECTOR = "CIRCLEBACK_CONNECTOR" MCP_CONNECTOR = "MCP_CONNECTOR" # Model Context Protocol - User-defined API tools - COMPOSIO_CONNECTOR = "COMPOSIO_CONNECTOR" # Generic Composio integration (Google, Slack, etc.) + COMPOSIO_CONNECTOR = ( + "COMPOSIO_CONNECTOR" # Generic Composio integration (Google, Slack, etc.) + ) class LiteLLMProvider(str, Enum): diff --git a/surfsense_backend/app/routes/composio_routes.py b/surfsense_backend/app/routes/composio_routes.py index b6f418aa2..eecbaf598 100644 --- a/surfsense_backend/app/routes/composio_routes.py +++ b/surfsense_backend/app/routes/composio_routes.py @@ -10,7 +10,6 @@ Endpoints: - GET /auth/composio/connector/callback - Handle OAuth callback """ -import asyncio import logging from uuid import UUID @@ -85,7 +84,9 @@ async def list_composio_toolkits(user: User = Depends(current_active_user)): @router.get("/auth/composio/connector/add") async def initiate_composio_auth( space_id: int, - toolkit_id: str = Query(..., description="Composio toolkit ID (e.g., 'googledrive', 'gmail')"), + toolkit_id: str = Query( + ..., description="Composio toolkit ID (e.g., 'googledrive', 'gmail')" + ), user: User = Depends(current_active_user), ): """ @@ -166,7 +167,9 @@ async def initiate_composio_auth( @router.get("/auth/composio/connector/callback") async def composio_callback( state: str | None = None, - connectedAccountId: str | None = None, # Composio sends camelCase + composio_connected_account_id: str | None = Query( + None, alias="connectedAccountId" + ), # Composio sends camelCase connected_account_id: str | None = None, # Fallback snake_case error: str | None = None, session: AsyncSession = Depends(get_async_session), @@ -233,15 +236,18 @@ async def composio_callback( ) # Initialize Composio service - service = ComposioService() - entity_id = f"surfsense_{user_id}" - + ComposioService() + # Use camelCase param if provided (Composio's format), fallback to snake_case - final_connected_account_id = connectedAccountId or connected_account_id - + final_connected_account_id = ( + composio_connected_account_id or connected_account_id + ) + # DEBUG: Log all query parameters received - logger.info(f"DEBUG: Callback received - connectedAccountId: {connectedAccountId}, connected_account_id: {connected_account_id}, using: {final_connected_account_id}") - + logger.info( + f"DEBUG: Callback received - connectedAccountId: {composio_connected_account_id}, connected_account_id: {connected_account_id}, using: {final_connected_account_id}" + ) + # If we still don't have a connected_account_id, warn but continue # (the connector will be created but indexing won't work until updated) if not final_connected_account_id: @@ -250,7 +256,9 @@ async def composio_callback( "The connector will be created but indexing may not work." ) else: - logger.info(f"Successfully got connected_account_id: {final_connected_account_id}") + logger.info( + f"Successfully got connected_account_id: {final_connected_account_id}" + ) # Build connector config connector_config = { diff --git a/surfsense_backend/app/services/composio_service.py b/surfsense_backend/app/services/composio_service.py index 4b6a32b03..6046ea2d8 100644 --- a/surfsense_backend/app/services/composio_service.py +++ b/surfsense_backend/app/services/composio_service.py @@ -97,7 +97,7 @@ class ComposioService: config_toolkit = getattr(auth_config, "toolkit", None) if config_toolkit is None: continue - + # Extract toolkit name/slug from the object toolkit_name = None if isinstance(config_toolkit, str): @@ -108,18 +108,22 @@ class ComposioService: toolkit_name = config_toolkit.name elif hasattr(config_toolkit, "id"): toolkit_name = config_toolkit.id - + # Compare case-insensitively if toolkit_name and toolkit_name.lower() == toolkit_id.lower(): - logger.info(f"Found auth config {auth_config.id} for toolkit {toolkit_id}") + logger.info( + f"Found auth config {auth_config.id} for toolkit {toolkit_id}" + ) return auth_config.id - + # Log available auth configs for debugging - logger.warning(f"No auth config found for toolkit '{toolkit_id}'. Available auth configs:") + logger.warning( + f"No auth config found for toolkit '{toolkit_id}'. Available auth configs:" + ) for auth_config in auth_configs.items: config_toolkit = getattr(auth_config, "toolkit", None) logger.warning(f" - {auth_config.id}: toolkit={config_toolkit}") - + return None except Exception as e: logger.error(f"Failed to list auth configs: {e!s}") @@ -148,7 +152,7 @@ class ComposioService: try: # First, get the auth_config_id for this toolkit auth_config_id = self._get_auth_config_for_toolkit(toolkit_id) - + if not auth_config_id: raise ValueError( f"No auth config found for toolkit '{toolkit_id}'. " @@ -200,7 +204,9 @@ class ComposioService: "user_id": getattr(account, "user_id", None), } except Exception as e: - logger.error(f"Failed to get connected account {connected_account_id}: {e!s}") + logger.error( + f"Failed to get connected account {connected_account_id}: {e!s}" + ) return None async def list_all_connections(self) -> list[dict[str, Any]]: @@ -212,15 +218,17 @@ class ComposioService: """ try: accounts_response = self.client.connected_accounts.list() - + if hasattr(accounts_response, "items"): accounts = accounts_response.items elif hasattr(accounts_response, "__iter__"): accounts = accounts_response else: - logger.warning(f"Unexpected accounts response type: {type(accounts_response)}") + logger.warning( + f"Unexpected accounts response type: {type(accounts_response)}" + ) return [] - + result = [] for acc in accounts: toolkit_raw = getattr(acc, "toolkit", None) @@ -234,14 +242,16 @@ class ComposioService: toolkit_info = toolkit_raw.name else: toolkit_info = str(toolkit_raw) - - result.append({ - "id": acc.id, - "status": getattr(acc, "status", None), - "toolkit": toolkit_info, - "user_id": getattr(acc, "user_id", None), - }) - + + result.append( + { + "id": acc.id, + "status": getattr(acc, "status", None), + "toolkit": toolkit_info, + "user_id": getattr(acc, "user_id", None), + } + ) + logger.info(f"DEBUG: Found {len(result)} TOTAL connections in Composio") return result except Exception as e: @@ -261,16 +271,18 @@ class ComposioService: try: logger.info(f"DEBUG: Calling connected_accounts.list(user_id='{user_id}')") accounts_response = self.client.connected_accounts.list(user_id=user_id) - + # Handle paginated response (may have .items attribute) or direct list if hasattr(accounts_response, "items"): accounts = accounts_response.items elif hasattr(accounts_response, "__iter__"): accounts = accounts_response else: - logger.warning(f"Unexpected accounts response type: {type(accounts_response)}") + logger.warning( + f"Unexpected accounts response type: {type(accounts_response)}" + ) return [] - + result = [] for acc in accounts: # Extract toolkit info - might be string or object @@ -285,13 +297,15 @@ class ComposioService: toolkit_info = toolkit_raw.name else: toolkit_info = toolkit_raw - - result.append({ - "id": acc.id, - "status": getattr(acc, "status", None), - "toolkit": toolkit_info, - }) - + + result.append( + { + "id": acc.id, + "status": getattr(acc, "status", None), + "toolkit": toolkit_info, + } + ) + logger.info(f"Found {len(result)} connections for user {user_id}: {result}") return result except Exception as e: @@ -383,18 +397,24 @@ class ComposioService: return [], None, result.get("error", "Unknown error") data = result.get("data", {}) - logger.info(f"DEBUG: Drive data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}") - + logger.info( + f"DEBUG: Drive data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}" + ) + # Handle nested response structure from Composio files = [] next_token = None if isinstance(data, dict): # Try direct access first, then nested files = data.get("files", []) or data.get("data", {}).get("files", []) - next_token = data.get("nextPageToken") or data.get("next_page_token") or data.get("data", {}).get("nextPageToken") + next_token = ( + data.get("nextPageToken") + or data.get("next_page_token") + or data.get("data", {}).get("nextPageToken") + ) elif isinstance(data, list): files = data - + logger.info(f"DEBUG: Extracted {len(files)} drive files") return files, next_token, None @@ -475,16 +495,22 @@ class ComposioService: return [], result.get("error", "Unknown error") data = result.get("data", {}) - logger.info(f"DEBUG: Gmail data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}") + logger.info( + f"DEBUG: Gmail data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}" + ) logger.info(f"DEBUG: Gmail full data: {data}") - + # Try different possible response structures messages = [] if isinstance(data, dict): - messages = data.get("messages", []) or data.get("data", {}).get("messages", []) or data.get("emails", []) + messages = ( + data.get("messages", []) + or data.get("data", {}).get("messages", []) + or data.get("emails", []) + ) elif isinstance(data, list): messages = data - + logger.info(f"DEBUG: Extracted {len(messages)} messages") return messages, None @@ -569,16 +595,22 @@ class ComposioService: return [], result.get("error", "Unknown error") data = result.get("data", {}) - logger.info(f"DEBUG: Calendar data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}") + logger.info( + f"DEBUG: Calendar data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}" + ) logger.info(f"DEBUG: Calendar full data: {data}") - + # Try different possible response structures events = [] if isinstance(data, dict): - events = data.get("items", []) or data.get("data", {}).get("items", []) or data.get("events", []) + events = ( + data.get("items", []) + or data.get("data", {}).get("items", []) + or data.get("events", []) + ) elif isinstance(data, list): events = data - + logger.info(f"DEBUG: Extracted {len(events)} calendar events") return events, None diff --git a/surfsense_backend/app/services/notification_service.py b/surfsense_backend/app/services/notification_service.py index 28807e783..836daeb9e 100644 --- a/surfsense_backend/app/services/notification_service.py +++ b/surfsense_backend/app/services/notification_service.py @@ -726,7 +726,10 @@ class MentionNotificationHandler(BaseNotificationHandler): except Exception as e: # Handle race condition - if duplicate key error, try to fetch existing await session.rollback() - if "duplicate key" in str(e).lower() or "unique constraint" in str(e).lower(): + if ( + "duplicate key" in str(e).lower() + or "unique constraint" in str(e).lower() + ): logger.warning( f"Duplicate notification detected for mention {mention_id}, fetching existing" ) diff --git a/surfsense_backend/app/tasks/composio_indexer.py b/surfsense_backend/app/tasks/composio_indexer.py index 01d2cfce4..abb238924 100644 --- a/surfsense_backend/app/tasks/composio_indexer.py +++ b/surfsense_backend/app/tasks/composio_indexer.py @@ -144,7 +144,9 @@ async def index_composio_connector( # Get toolkit ID from config toolkit_id = connector.config.get("toolkit_id") if not toolkit_id: - error_msg = f"Composio connector {connector_id} has no toolkit_id configured" + error_msg = ( + f"Composio connector {connector_id} has no toolkit_id configured" + ) await task_logger.log_task_failure( log_entry, error_msg, {"error_type": "MissingToolkitId"} ) @@ -287,8 +289,14 @@ async def _index_composio_google_drive( try: # Handle both standard Google API and potential Composio variations file_id = file_info.get("id", "") or file_info.get("fileId", "") - file_name = file_info.get("name", "") or file_info.get("fileName", "") or "Untitled" - mime_type = file_info.get("mimeType", "") or file_info.get("mime_type", "") + file_name = ( + file_info.get("name", "") + or file_info.get("fileName", "") + or "Untitled" + ) + mime_type = file_info.get("mimeType", "") or file_info.get( + "mime_type", "" + ) if not file_id: documents_skipped += 1 @@ -309,12 +317,15 @@ async def _index_composio_google_drive( ) # Get file content - content, content_error = await composio_connector.get_drive_file_content( - file_id - ) + ( + content, + content_error, + ) = await composio_connector.get_drive_file_content(file_id) if content_error or not content: - logger.warning(f"Could not get content for file {file_name}: {content_error}") + logger.warning( + f"Could not get content for file {file_name}: {content_error}" + ) # Use metadata as content fallback markdown_content = f"# {file_name}\n\n" markdown_content += f"**File ID:** {file_id}\n" @@ -344,12 +355,19 @@ async def _index_composio_google_drive( "mime_type": mime_type, "document_type": "Google Drive File (Composio)", } - summary_content, summary_embedding = await generate_document_summary( + ( + summary_content, + summary_embedding, + ) = await generate_document_summary( markdown_content, user_llm, document_metadata ) else: - summary_content = f"Google Drive File: {file_name}\n\nType: {mime_type}" - summary_embedding = config.embedding_model_instance.embed(summary_content) + summary_content = ( + f"Google Drive File: {file_name}\n\nType: {mime_type}" + ) + summary_embedding = config.embedding_model_instance.embed( + summary_content + ) chunks = await create_document_chunks(markdown_content) @@ -382,12 +400,19 @@ async def _index_composio_google_drive( "mime_type": mime_type, "document_type": "Google Drive File (Composio)", } - summary_content, summary_embedding = await generate_document_summary( + ( + summary_content, + summary_embedding, + ) = await generate_document_summary( markdown_content, user_llm, document_metadata ) else: - summary_content = f"Google Drive File: {file_name}\n\nType: {mime_type}" - summary_embedding = config.embedding_model_instance.embed(summary_content) + summary_content = ( + f"Google Drive File: {file_name}\n\nType: {mime_type}" + ) + summary_embedding = config.embedding_model_instance.embed( + summary_content + ) chunks = await create_document_chunks(markdown_content) @@ -527,11 +552,15 @@ async def _index_composio_gmail( date_str = value # Format to markdown using the full message data - markdown_content = composio_connector.format_gmail_message_to_markdown(message) + markdown_content = composio_connector.format_gmail_message_to_markdown( + message + ) # Generate unique identifier unique_identifier_hash = generate_unique_identifier_hash( - DocumentType.COMPOSIO_CONNECTOR, f"gmail_{message_id}", search_space_id + DocumentType.COMPOSIO_CONNECTOR, + f"gmail_{message_id}", + search_space_id, ) content_hash = generate_content_hash(markdown_content, search_space_id) @@ -560,12 +589,19 @@ async def _index_composio_gmail( "sender": sender, "document_type": "Gmail Message (Composio)", } - summary_content, summary_embedding = await generate_document_summary( + ( + summary_content, + summary_embedding, + ) = await generate_document_summary( markdown_content, user_llm, document_metadata ) else: - summary_content = f"Gmail: {subject}\n\nFrom: {sender}\nDate: {date_str}" - summary_embedding = config.embedding_model_instance.embed(summary_content) + summary_content = ( + f"Gmail: {subject}\n\nFrom: {sender}\nDate: {date_str}" + ) + summary_embedding = config.embedding_model_instance.embed( + summary_content + ) chunks = await create_document_chunks(markdown_content) @@ -600,12 +636,19 @@ async def _index_composio_gmail( "sender": sender, "document_type": "Gmail Message (Composio)", } - summary_content, summary_embedding = await generate_document_summary( + ( + summary_content, + summary_embedding, + ) = await generate_document_summary( markdown_content, user_llm, document_metadata ) else: - summary_content = f"Gmail: {subject}\n\nFrom: {sender}\nDate: {date_str}" - summary_embedding = config.embedding_model_instance.embed(summary_content) + summary_content = ( + f"Gmail: {subject}\n\nFrom: {sender}\nDate: {date_str}" + ) + summary_embedding = config.embedding_model_instance.embed( + summary_content + ) chunks = await create_document_chunks(markdown_content) @@ -728,18 +771,24 @@ async def _index_composio_google_calendar( try: # Handle both standard Google API and potential Composio variations event_id = event.get("id", "") or event.get("eventId", "") - summary = event.get("summary", "") or event.get("title", "") or "No Title" + summary = ( + event.get("summary", "") or event.get("title", "") or "No Title" + ) if not event_id: documents_skipped += 1 continue # Format to markdown - markdown_content = composio_connector.format_calendar_event_to_markdown(event) + markdown_content = composio_connector.format_calendar_event_to_markdown( + event + ) # Generate unique identifier unique_identifier_hash = generate_unique_identifier_hash( - DocumentType.COMPOSIO_CONNECTOR, f"calendar_{event_id}", search_space_id + DocumentType.COMPOSIO_CONNECTOR, + f"calendar_{event_id}", + search_space_id, ) content_hash = generate_content_hash(markdown_content, search_space_id) @@ -772,14 +821,19 @@ async def _index_composio_google_calendar( "start_time": start_time, "document_type": "Google Calendar Event (Composio)", } - summary_content, summary_embedding = await generate_document_summary( + ( + summary_content, + summary_embedding, + ) = await generate_document_summary( markdown_content, user_llm, document_metadata ) else: summary_content = f"Calendar: {summary}\n\nStart: {start_time}\nEnd: {end_time}" if location: summary_content += f"\nLocation: {location}" - summary_embedding = config.embedding_model_instance.embed(summary_content) + summary_embedding = config.embedding_model_instance.embed( + summary_content + ) chunks = await create_document_chunks(markdown_content) @@ -814,14 +868,21 @@ async def _index_composio_google_calendar( "start_time": start_time, "document_type": "Google Calendar Event (Composio)", } - summary_content, summary_embedding = await generate_document_summary( + ( + summary_content, + summary_embedding, + ) = await generate_document_summary( markdown_content, user_llm, document_metadata ) else: - summary_content = f"Calendar: {summary}\n\nStart: {start_time}\nEnd: {end_time}" + summary_content = ( + f"Calendar: {summary}\n\nStart: {start_time}\nEnd: {end_time}" + ) if location: summary_content += f"\nLocation: {location}" - summary_embedding = config.embedding_model_instance.embed(summary_content) + summary_embedding = config.embedding_model_instance.embed( + summary_content + ) chunks = await create_document_chunks(markdown_content) @@ -874,5 +935,7 @@ async def _index_composio_google_calendar( return documents_indexed, None except Exception as e: - logger.error(f"Failed to index Google Calendar via Composio: {e!s}", exc_info=True) + logger.error( + f"Failed to index Google Calendar via Composio: {e!s}", exc_info=True + ) return 0, f"Failed to index Google Calendar via Composio: {e!s}" diff --git a/surfsense_backend/app/tasks/connector_indexers/__init__.py b/surfsense_backend/app/tasks/connector_indexers/__init__.py index 35b5fde4c..8f25e6fdd 100644 --- a/surfsense_backend/app/tasks/connector_indexers/__init__.py +++ b/surfsense_backend/app/tasks/connector_indexers/__init__.py @@ -26,6 +26,7 @@ Available indexers: # Calendar and scheduling from .airtable_indexer import index_airtable_records from .bookstack_indexer import index_bookstack_pages + # Note: composio_indexer is imported directly in connector_tasks.py to avoid circular imports from .clickup_indexer import index_clickup_tasks from .confluence_indexer import index_confluence_pages diff --git a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py index f16ee0156..4a8df4918 100644 --- a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py @@ -128,7 +128,9 @@ async def index_github_repos( if github_pat: logger.info("Using GitHub PAT for authentication (private repos supported)") else: - logger.info("No GitHub PAT provided - only public repositories can be indexed") + logger.info( + "No GitHub PAT provided - only public repositories can be indexed" + ) # 3. Initialize GitHub connector with gitingest backend await task_logger.log_task_progress( @@ -308,9 +310,7 @@ async def _process_repository_digest( if existing_document: # Document exists - check if content has changed if existing_document.content_hash == content_hash: - logger.info( - f"Repository {repo_full_name} unchanged. Skipping." - ) + logger.info(f"Repository {repo_full_name} unchanged. Skipping.") return 0 else: logger.info( @@ -341,7 +341,7 @@ async def _process_repository_digest( summary_content = ( f"# Repository: {repo_full_name}\n\n" f"## File Structure\n\n{digest.tree}\n\n" - f"## File Contents (truncated)\n\n{digest.content[:MAX_DIGEST_CHARS - len(digest.tree) - 200]}..." + f"## File Contents (truncated)\n\n{digest.content[: MAX_DIGEST_CHARS - len(digest.tree) - 200]}..." ) summary_text, summary_embedding = await generate_document_summary( @@ -362,9 +362,7 @@ async def _process_repository_digest( # This preserves file-level granularity in search chunks_data = await create_document_chunks(digest.content) except Exception as chunk_err: - logger.error( - f"Failed to chunk repository {repo_full_name}: {chunk_err}" - ) + logger.error(f"Failed to chunk repository {repo_full_name}: {chunk_err}") # Fall back to a simpler chunking approach chunks_data = await _simple_chunk_content(digest.content) diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index c61dad660..4d43b7f64 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -33,7 +33,9 @@ import { DisplayImageToolUI } from "@/components/tool-ui/display-image"; import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast"; import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview"; import { ScrapeWebpageToolUI } from "@/components/tool-ui/scrape-webpage"; -import { SaveMemoryToolUI, RecallMemoryToolUI } from "@/components/tool-ui/user-memory"; +import { RecallMemoryToolUI, SaveMemoryToolUI } from "@/components/tool-ui/user-memory"; +import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; +import { useMessagesElectric } from "@/hooks/use-messages-electric"; // import { WriteTodosToolUI } from "@/components/tool-ui/write-todos"; import { getBearerToken } from "@/lib/auth-utils"; import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter"; @@ -51,8 +53,6 @@ import { type MessageRecord, type ThreadRecord, } from "@/lib/chat/thread-persistence"; -import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; -import { useMessagesElectric } from "@/hooks/use-messages-electric"; import { trackChatCreated, trackChatError, @@ -266,7 +266,16 @@ export default function NewChatPage() { const { data: membersData } = useAtomValue(membersAtom); const handleElectricMessagesUpdate = useCallback( - (electricMessages: { id: number; thread_id: number; role: string; content: unknown; author_id: string | null; created_at: string }[]) => { + ( + electricMessages: { + id: number; + thread_id: number; + role: string; + content: unknown; + author_id: string | null; + created_at: string; + }[] + ) => { if (isRunning) { return; } diff --git a/surfsense_web/components/assistant-ui/chat-session-status.tsx b/surfsense_web/components/assistant-ui/chat-session-status.tsx index 62f7c33ce..88fea6b8c 100644 --- a/surfsense_web/components/assistant-ui/chat-session-status.tsx +++ b/surfsense_web/components/assistant-ui/chat-session-status.tsx @@ -1,7 +1,7 @@ "use client"; -import type { FC } from "react"; import { Loader2 } from "lucide-react"; +import type { FC } from "react"; import { cn } from "@/lib/utils"; interface ChatSessionStatusProps { @@ -32,7 +32,8 @@ export const ChatSessionStatus: FC = ({ } const respondingUser = members.find((m) => m.user_id === respondingToUserId); - const displayName = respondingUser?.user_display_name || respondingUser?.user_email || "another user"; + const displayName = + respondingUser?.user_display_name || respondingUser?.user_email || "another user"; return (
{ ) : viewingComposio && searchSpaceId ? ( c.connector_type === "COMPOSIO_CONNECTOR") - .map((c: SearchSourceConnector) => c.config?.toolkit_id as string) - .filter(Boolean) - } + connectedToolkits={(connectors || []) + .filter((c: SearchSourceConnector) => c.connector_type === "COMPOSIO_CONNECTOR") + .map((c: SearchSourceConnector) => c.config?.toolkit_id as string) + .filter(Boolean)} onBack={handleBackFromComposio} onConnectToolkit={handleConnectComposioToolkit} isConnecting={connectingComposioToolkit !== null} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx index a7a92597c..4884584e6 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx @@ -5,8 +5,8 @@ import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { BaiduSearchApiConfig } from "./components/baidu-search-api-config"; import { BookStackConfig } from "./components/bookstack-config"; import { CirclebackConfig } from "./components/circleback-config"; -import { ComposioConfig } from "./components/composio-config"; import { ClickUpConfig } from "./components/clickup-config"; +import { ComposioConfig } from "./components/composio-config"; import { ConfluenceConfig } from "./components/confluence-config"; import { DiscordConfig } from "./components/discord-config"; import { ElasticsearchConfig } from "./components/elasticsearch-config"; diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index 55c66c392..c5f5448bd 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -3,9 +3,14 @@ import type { FC } from "react"; import { EnumConnectorName } from "@/contracts/enums/connector"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; -import { ConnectorCard } from "../components/connector-card"; import { ComposioConnectorCard } from "../components/composio-connector-card"; -import { CRAWLERS, OAUTH_CONNECTORS, OTHER_CONNECTORS, COMPOSIO_CONNECTORS } from "../constants/connector-constants"; +import { ConnectorCard } from "../components/connector-card"; +import { + COMPOSIO_CONNECTORS, + CRAWLERS, + OAUTH_CONNECTORS, + OTHER_CONNECTORS, +} from "../constants/connector-constants"; import { getDocumentCountForConnector } from "../utils/connector-document-mapping"; /** diff --git a/surfsense_web/components/assistant-ui/connector-popup/views/composio-toolkit-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/views/composio-toolkit-view.tsx index 456835597..9c0bd7223 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/views/composio-toolkit-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/views/composio-toolkit-view.tsx @@ -5,12 +5,12 @@ import { Calendar, Check, ExternalLink, + FileText, Github, + HardDrive, Loader2, Mail, - HardDrive, MessageSquare, - FileText, Zap, } from "lucide-react"; import Image from "next/image"; @@ -82,17 +82,65 @@ const getToolkitIcon = (toolkitId: string, className?: string) => { switch (toolkitId) { case "googledrive": - return Google Drive; + return ( + Google Drive + ); case "gmail": - return Gmail; + return ( + Gmail + ); case "googlecalendar": - return Google Calendar; + return ( + Google Calendar + ); case "slack": - return Slack; + return ( + Slack + ); case "notion": - return Notion; + return ( + Notion + ); case "github": - return GitHub; + return ( + GitHub + ); default: return ; } @@ -139,9 +187,7 @@ export const ComposioToolkitView: FC = ({ />
-

- Composio -

+

Composio

Connect 100+ apps with managed OAuth - no verification needed

@@ -165,12 +211,16 @@ export const ComposioToolkitView: FC = ({

Google Services

- + Indexable

- Connect Google services via Composio's verified OAuth app. Your data will be indexed and searchable. + Connect Google services via Composio's verified OAuth app. Your data will be + indexed and searchable.

{indexableToolkits.map((toolkit) => { @@ -201,16 +251,17 @@ export const ComposioToolkitView: FC = ({ {getToolkitIcon(toolkit.id, "size-5")}
{isConnected && ( - + Connected )}

{toolkit.name}

-

- {toolkit.description} -

+

{toolkit.description}