diff --git a/README.md b/README.md index d7845fdfc..ae8220625 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,7 @@ Check out our public roadmap and contribute your ideas or feedback: **Linux/macOS:** ```bash -docker run -d -p 3000:3000 -p 8000:8000 \ +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ -v surfsense-data:/data \ --name surfsense \ --restart unless-stopped \ @@ -168,7 +168,7 @@ docker run -d -p 3000:3000 -p 8000:8000 \ **Windows (PowerShell):** ```powershell -docker run -d -p 3000:3000 -p 8000:8000 ` +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 ` -v surfsense-data:/data ` --name surfsense ` --restart unless-stopped ` @@ -180,7 +180,7 @@ docker run -d -p 3000:3000 -p 8000:8000 ` You can pass any environment variable using `-e` flags: ```bash -docker run -d -p 3000:3000 -p 8000:8000 \ +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ -v surfsense-data:/data \ -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \ -e OPENAI_API_KEY=your_openai_api_key \ @@ -201,6 +201,7 @@ After starting, access SurfSense at: - **Frontend**: [http://localhost:3000](http://localhost:3000) - **Backend API**: [http://localhost:8000](http://localhost:8000) - **API Docs**: [http://localhost:8000/docs](http://localhost:8000/docs) +- **Electric-SQL**: [http://localhost:5133](http://localhost:5133) **Useful Commands:** diff --git a/README.zh-CN.md b/README.zh-CN.md index 5eb369287..19a49ed05 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -165,7 +165,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 **Linux/macOS:** ```bash -docker run -d -p 3000:3000 -p 8000:8000 \ +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ -v surfsense-data:/data \ --name surfsense \ --restart unless-stopped \ @@ -175,7 +175,7 @@ docker run -d -p 3000:3000 -p 8000:8000 \ **Windows (PowerShell):** ```powershell -docker run -d -p 3000:3000 -p 8000:8000 ` +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 ` -v surfsense-data:/data ` --name surfsense ` --restart unless-stopped ` @@ -187,7 +187,7 @@ docker run -d -p 3000:3000 -p 8000:8000 ` 您可以使用 `-e` 标志传递任何环境变量： ```bash -docker run -d -p 3000:3000 -p 8000:8000 \ +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ -v surfsense-data:/data \ -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \ -e OPENAI_API_KEY=your_openai_api_key \ @@ -208,6 +208,7 @@ docker run -d -p 3000:3000 -p 8000:8000 \ - **前端**: [http://localhost:3000](http://localhost:3000) - **后端 API**: [http://localhost:8000](http://localhost:8000) - **API 文档**: [http://localhost:8000/docs](http://localhost:8000/docs) +- **Electric-SQL**: [http://localhost:5133](http://localhost:5133) **常用命令：** diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 6ac7c55de..1dd825cd3 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -4,6 +4,10 @@ DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense CELERY_BROKER_URL=redis://localhost:6379/0 CELERY_RESULT_BACKEND=redis://localhost:6379/0 +#Electric(for migrations only) +ELECTRIC_DB_USER=electric +ELECTRIC_DB_PASSWORD=electric_password + # Periodic task interval # # Run every minute (default) # SCHEDULE_CHECKER_INTERVAL=1m diff --git a/surfsense_backend/alembic/versions/72_simplify_rbac_roles.py b/surfsense_backend/alembic/versions/72_simplify_rbac_roles.py new file mode 100644 index 000000000..e7d5ff019 --- /dev/null +++ b/surfsense_backend/alembic/versions/72_simplify_rbac_roles.py @@ -0,0 +1,300 @@ +"""Simplify RBAC roles - Remove Admin role, keep only Owner, Editor, Viewer + +Revision ID: 72 +Revises: 71 +Create Date: 2025-01-20 + +This migration: +1. Moves any users with Admin role to Editor role +2. Updates invites that reference Admin role to use Editor role +3. Deletes the Admin role from all search spaces +4. Updates Editor permissions to the new simplified set (everything except delete) +5. Updates Viewer permissions to the new simplified set (read-only + comments) +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "72" +down_revision = "71" +branch_labels = None +depends_on = None + +# New Editor permissions (can do everything except delete, manage roles, and update settings) +NEW_EDITOR_PERMISSIONS = [ + "documents:create", + "documents:read", + "documents:update", + "chats:create", + "chats:read", + "chats:update", + "comments:create", + "comments:read", + "llm_configs:create", + "llm_configs:read", + "llm_configs:update", + "podcasts:create", + "podcasts:read", + "podcasts:update", + "connectors:create", + "connectors:read", + "connectors:update", + "logs:read", + "members:invite", + "members:view", + "roles:read", + "settings:view", +] + +# New Viewer permissions (read-only + comments) +NEW_VIEWER_PERMISSIONS = [ + "documents:read", + "chats:read", + "comments:create", + "comments:read", + "llm_configs:read", + "podcasts:read", + "connectors:read", + "logs:read", + "members:view", + "roles:read", + "settings:view", +] + + +def upgrade(): + connection = op.get_bind() + + # Step 1: For each search space, get the Editor role ID and Admin role ID + search_spaces = connection.execute( + sa.text("SELECT id FROM searchspaces") + ).fetchall() + + for (ss_id,) in search_spaces: + # Get Admin and Editor role IDs for this search space + admin_role = connection.execute( + sa.text(""" + SELECT id FROM search_space_roles + WHERE search_space_id = :ss_id AND name = 'Admin' + """), + {"ss_id": ss_id}, + ).fetchone() + + editor_role = connection.execute( + sa.text(""" + SELECT id FROM search_space_roles + WHERE search_space_id = :ss_id AND name = 'Editor' + """), + {"ss_id": ss_id}, + ).fetchone() + + if admin_role and editor_role: + admin_role_id = admin_role[0] + editor_role_id = editor_role[0] + + # Step 2: Move all memberships from Admin to Editor + connection.execute( + sa.text(""" + UPDATE search_space_memberships + SET role_id = :editor_role_id + WHERE role_id = :admin_role_id + """), + {"editor_role_id": editor_role_id, "admin_role_id": admin_role_id}, + ) + + # Step 3: Move all invites from Admin to Editor + connection.execute( + sa.text(""" + UPDATE search_space_invites + SET role_id = :editor_role_id + WHERE role_id = :admin_role_id + """), + {"editor_role_id": editor_role_id, "admin_role_id": admin_role_id}, + ) + + # Step 4: Delete the Admin role + connection.execute( + sa.text(""" + DELETE FROM search_space_roles + WHERE id = :admin_role_id + """), + {"admin_role_id": admin_role_id}, + ) + + # Step 5: Update Editor permissions for all search spaces + editor_perms_literal = ( + "ARRAY[" + ",".join(f"'{p}'" for p in NEW_EDITOR_PERMISSIONS) + "]::TEXT[]" + ) + connection.execute( + sa.text(f""" + UPDATE search_space_roles + SET permissions = {editor_perms_literal}, + description = 'Can create and update content (no delete, role management, or settings access)' + WHERE name = 'Editor' AND is_system_role = TRUE + """) + ) + + # Step 6: Update Viewer permissions for all search spaces + viewer_perms_literal = ( + "ARRAY[" + ",".join(f"'{p}'" for p in NEW_VIEWER_PERMISSIONS) + "]::TEXT[]" + ) + connection.execute( + sa.text(f""" + UPDATE search_space_roles + SET permissions = {viewer_perms_literal} + WHERE name = 'Viewer' AND is_system_role = TRUE + """) + ) + + +def downgrade(): + """ + Downgrade recreates the Admin role and restores original permissions. + Note: Users who were moved from Admin to Editor will remain as Editor. + """ + connection = op.get_bind() + + # Old Admin permissions + old_admin_permissions = [ + "documents:create", + "documents:read", + "documents:update", + "documents:delete", + "chats:create", + "chats:read", + "chats:update", + "chats:delete", + "comments:create", + "comments:read", + "comments:delete", + "llm_configs:create", + "llm_configs:read", + "llm_configs:update", + "llm_configs:delete", + "podcasts:create", + "podcasts:read", + "podcasts:update", + "podcasts:delete", + "connectors:create", + "connectors:read", + "connectors:update", + "connectors:delete", + "logs:read", + "logs:delete", + "members:invite", + "members:view", + "members:remove", + "members:manage_roles", + "roles:create", + "roles:read", + "roles:update", + "roles:delete", + "settings:view", + "settings:update", + ] + + # Old Editor permissions + old_editor_permissions = [ + "documents:create", + "documents:read", + "documents:update", + "documents:delete", + "chats:create", + "chats:read", + "chats:update", + "chats:delete", + "comments:create", + "comments:read", + "llm_configs:read", + "llm_configs:create", + "llm_configs:update", + "podcasts:create", + "podcasts:read", + "podcasts:update", + "podcasts:delete", + "connectors:create", + "connectors:read", + "connectors:update", + "logs:read", + "members:view", + "roles:read", + "settings:view", + ] + + # Old Viewer permissions + old_viewer_permissions = [ + "documents:read", + "chats:read", + "comments:create", + "comments:read", + "llm_configs:read", + "podcasts:read", + "connectors:read", + "logs:read", + "members:view", + "roles:read", + "settings:view", + ] + + # Recreate Admin role for each search space + search_spaces = connection.execute( + sa.text("SELECT id FROM searchspaces") + ).fetchall() + + admin_perms_literal = ( + "ARRAY[" + ",".join(f"'{p}'" for p in old_admin_permissions) + "]::TEXT[]" + ) + + for (ss_id,) in search_spaces: + # Check if Admin role already exists + existing = connection.execute( + sa.text(""" + SELECT id FROM search_space_roles + WHERE search_space_id = :ss_id AND name = 'Admin' + """), + {"ss_id": ss_id}, + ).fetchone() + + if not existing: + connection.execute( + sa.text(f""" + INSERT INTO search_space_roles + (name, description, permissions, is_default, is_system_role, search_space_id) + VALUES ( + 'Admin', + 'Can manage most resources except deleting the search space', + {admin_perms_literal}, + FALSE, + TRUE, + :ss_id + ) + """), + {"ss_id": ss_id}, + ) + + # Restore old Editor permissions + editor_perms_literal = ( + "ARRAY[" + ",".join(f"'{p}'" for p in old_editor_permissions) + "]::TEXT[]" + ) + connection.execute( + sa.text(f""" + UPDATE search_space_roles + SET permissions = {editor_perms_literal}, + description = 'Can create and edit documents, chats, and podcasts' + WHERE name = 'Editor' AND is_system_role = TRUE + """) + ) + + # Restore old Viewer permissions + viewer_perms_literal = ( + "ARRAY[" + ",".join(f"'{p}'" for p in old_viewer_permissions) + "]::TEXT[]" + ) + connection.execute( + sa.text(f""" + UPDATE search_space_roles + SET permissions = {viewer_perms_literal} + WHERE name = 'Viewer' AND is_system_role = TRUE + """) + ) diff --git a/surfsense_backend/app/agents/new_chat/tools/mcp_client.py b/surfsense_backend/app/agents/new_chat/tools/mcp_client.py index 56f85b361..44c48344c 100644 --- a/surfsense_backend/app/agents/new_chat/tools/mcp_client.py +++ b/surfsense_backend/app/agents/new_chat/tools/mcp_client.py @@ -77,7 +77,7 @@ class MCPClient: # Initialize the connection await session.initialize() self.session = session - + if attempt > 0: logger.info( "Connected to MCP server on attempt %d: %s %s", @@ -267,30 +267,38 @@ async def test_mcp_http_connection( """ try: - logger.info("Testing HTTP MCP connection to: %s (transport: %s)", url, transport) - + logger.info( + "Testing HTTP MCP connection to: %s (transport: %s)", url, transport + ) + # Use streamable HTTP client for all HTTP-based transports - async with streamablehttp_client(url, headers=headers or {}) as (read, write, _): - async with ClientSession(read, write) as session: - await session.initialize() - - # List available tools - response = await session.list_tools() - tools = [] - for tool in response.tools: - tools.append({ + async with ( + streamablehttp_client(url, headers=headers or {}) as (read, write, _), + ClientSession(read, write) as session, + ): + await session.initialize() + + # List available tools + response = await session.list_tools() + tools = [] + for tool in response.tools: + tools.append( + { "name": tool.name, "description": tool.description or "", - "input_schema": tool.inputSchema if hasattr(tool, "inputSchema") else {}, - }) - - logger.info("HTTP MCP connection successful. Found %d tools.", len(tools)) - return { - "status": "success", - "message": f"Connected successfully. Found {len(tools)} tools.", - "tools": tools, - } - + "input_schema": tool.inputSchema + if hasattr(tool, "inputSchema") + else {}, + } + ) + + logger.info("HTTP MCP connection successful. Found %d tools.", len(tools)) + return { + "status": "success", + "message": f"Connected successfully. Found {len(tools)} tools.", + "tools": tools, + } + except Exception as e: logger.error("Failed to connect to HTTP MCP server: %s", e, exc_info=True) return { diff --git a/surfsense_backend/app/agents/new_chat/tools/mcp_tool.py b/surfsense_backend/app/agents/new_chat/tools/mcp_tool.py index 4cb85f4cc..5ccd2e749 100644 --- a/surfsense_backend/app/agents/new_chat/tools/mcp_tool.py +++ b/surfsense_backend/app/agents/new_chat/tools/mcp_tool.py @@ -160,27 +160,31 @@ async def _create_mcp_tool_from_definition_http( logger.info(f"MCP HTTP tool '{tool_name}' called with params: {kwargs}") try: - async with streamablehttp_client(url, headers=headers) as (read, write, _): - async with ClientSession(read, write) as session: - await session.initialize() - - # Call the tool - response = await session.call_tool(tool_name, arguments=kwargs) - - # Extract content from response - result = [] - for content in response.content: - if hasattr(content, "text"): - result.append(content.text) - elif hasattr(content, "data"): - result.append(str(content.data)) - else: - result.append(str(content)) - - result_str = "\n".join(result) if result else "" - logger.info(f"MCP HTTP tool '{tool_name}' succeeded: {result_str[:200]}") - return result_str - + async with ( + streamablehttp_client(url, headers=headers) as (read, write, _), + ClientSession(read, write) as session, + ): + await session.initialize() + + # Call the tool + response = await session.call_tool(tool_name, arguments=kwargs) + + # Extract content from response + result = [] + for content in response.content: + if hasattr(content, "text"): + result.append(content.text) + elif hasattr(content, "data"): + result.append(str(content.data)) + else: + result.append(str(content)) + + result_str = "\n".join(result) if result else "" + logger.info( + f"MCP HTTP tool '{tool_name}' succeeded: {result_str[:200]}" + ) + return result_str + except Exception as e: error_msg = f"MCP HTTP tool '{tool_name}' execution failed: {e!s}" logger.exception(error_msg) @@ -192,7 +196,11 @@ async def _create_mcp_tool_from_definition_http( description=tool_description, coroutine=mcp_http_tool_call, args_schema=input_model, - metadata={"mcp_input_schema": input_schema, "mcp_transport": "http", "mcp_url": url}, + metadata={ + "mcp_input_schema": input_schema, + "mcp_transport": "http", + "mcp_url": url, + }, ) logger.info(f"Created MCP tool (HTTP): '{tool_name}'") @@ -205,17 +213,17 @@ async def _load_stdio_mcp_tools( server_config: dict[str, Any], ) -> list[StructuredTool]: """Load tools from a stdio-based MCP server. - + Args: connector_id: Connector ID for logging connector_name: Connector name for logging server_config: Server configuration with command, args, env - + Returns: List of tools from the MCP server """ tools: list[StructuredTool] = [] - + # Validate required command field command = server_config.get("command") if not command or not isinstance(command, str): @@ -262,7 +270,7 @@ async def _load_stdio_mcp_tools( f"Failed to create tool '{tool_def.get('name')}' " f"from connector {connector_id}: {e!s}" ) - + return tools @@ -272,17 +280,17 @@ async def _load_http_mcp_tools( server_config: dict[str, Any], ) -> list[StructuredTool]: """Load tools from an HTTP-based MCP server. - + Args: connector_id: Connector ID for logging connector_name: Connector name for logging server_config: Server configuration with url, headers - + Returns: List of tools from the MCP server """ tools: list[StructuredTool] = [] - + # Validate required url field url = server_config.get("url") if not url or not isinstance(url, str): @@ -301,41 +309,49 @@ async def _load_http_mcp_tools( # Connect and discover tools via HTTP try: - async with streamablehttp_client(url, headers=headers) as (read, write, _): - async with ClientSession(read, write) as session: - await session.initialize() - - # List available tools - response = await session.list_tools() - tool_definitions = [] - for tool in response.tools: - tool_definitions.append({ + async with ( + streamablehttp_client(url, headers=headers) as (read, write, _), + ClientSession(read, write) as session, + ): + await session.initialize() + + # List available tools + response = await session.list_tools() + tool_definitions = [] + for tool in response.tools: + tool_definitions.append( + { "name": tool.name, "description": tool.description or "", - "input_schema": tool.inputSchema if hasattr(tool, "inputSchema") else {}, - }) - - logger.info( - f"Discovered {len(tool_definitions)} tools from HTTP MCP server " - f"'{url}' (connector {connector_id})" + "input_schema": tool.inputSchema + if hasattr(tool, "inputSchema") + else {}, + } ) + logger.info( + f"Discovered {len(tool_definitions)} tools from HTTP MCP server " + f"'{url}' (connector {connector_id})" + ) + # Create LangChain tools from definitions for tool_def in tool_definitions: try: - tool = await _create_mcp_tool_from_definition_http(tool_def, url, headers) + tool = await _create_mcp_tool_from_definition_http( + tool_def, url, headers + ) tools.append(tool) except Exception as e: logger.exception( f"Failed to create HTTP tool '{tool_def.get('name')}' " f"from connector {connector_id}: {e!s}" ) - + except Exception as e: logger.exception( f"Failed to connect to HTTP MCP server at '{url}' (connector {connector_id}): {e!s}" ) - + return tools @@ -372,7 +388,7 @@ async def load_mcp_tools( # Early validation: Extract and validate connector config config = connector.config or {} server_config = config.get("server_config", {}) - + # Validate server_config exists and is a dict if not server_config or not isinstance(server_config, dict): logger.warning( @@ -382,7 +398,7 @@ async def load_mcp_tools( # Determine transport type transport = server_config.get("transport", "stdio") - + if transport in ("streamable-http", "http", "sse"): # HTTP-based MCP server connector_tools = await _load_http_mcp_tools( @@ -393,9 +409,9 @@ async def load_mcp_tools( connector_tools = await _load_stdio_mcp_tools( connector.id, connector.name, server_config ) - + tools.extend(connector_tools) - + except Exception as e: logger.exception( f"Failed to load tools from MCP connector {connector.id}: {e!s}" diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 2b514483a..38e27ecf2 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -201,89 +201,42 @@ class Permission(str, Enum): # Predefined role permission sets for convenience +# Note: Only Owner, Editor, and Viewer roles are supported. +# Owner has full access (*), Editor can do everything except delete, Viewer has read-only access. DEFAULT_ROLE_PERMISSIONS = { "Owner": [Permission.FULL_ACCESS.value], - "Admin": [ - # Documents - Permission.DOCUMENTS_CREATE.value, - Permission.DOCUMENTS_READ.value, - Permission.DOCUMENTS_UPDATE.value, - Permission.DOCUMENTS_DELETE.value, - # Chats - Permission.CHATS_CREATE.value, - Permission.CHATS_READ.value, - Permission.CHATS_UPDATE.value, - Permission.CHATS_DELETE.value, - # Comments - Permission.COMMENTS_CREATE.value, - Permission.COMMENTS_READ.value, - Permission.COMMENTS_DELETE.value, - # LLM Configs - Permission.LLM_CONFIGS_CREATE.value, - Permission.LLM_CONFIGS_READ.value, - Permission.LLM_CONFIGS_UPDATE.value, - Permission.LLM_CONFIGS_DELETE.value, - # Podcasts - Permission.PODCASTS_CREATE.value, - Permission.PODCASTS_READ.value, - Permission.PODCASTS_UPDATE.value, - Permission.PODCASTS_DELETE.value, - # Connectors - Permission.CONNECTORS_CREATE.value, - Permission.CONNECTORS_READ.value, - Permission.CONNECTORS_UPDATE.value, - Permission.CONNECTORS_DELETE.value, - # Logs - Permission.LOGS_READ.value, - Permission.LOGS_DELETE.value, - # Members - Permission.MEMBERS_INVITE.value, - Permission.MEMBERS_VIEW.value, - Permission.MEMBERS_REMOVE.value, - Permission.MEMBERS_MANAGE_ROLES.value, - # Roles - Permission.ROLES_CREATE.value, - Permission.ROLES_READ.value, - Permission.ROLES_UPDATE.value, - Permission.ROLES_DELETE.value, - # Settings (no delete) - Permission.SETTINGS_VIEW.value, - Permission.SETTINGS_UPDATE.value, - ], "Editor": [ - # Documents + # Documents (no delete) Permission.DOCUMENTS_CREATE.value, Permission.DOCUMENTS_READ.value, Permission.DOCUMENTS_UPDATE.value, - Permission.DOCUMENTS_DELETE.value, - # Chats + # Chats (no delete) Permission.CHATS_CREATE.value, Permission.CHATS_READ.value, Permission.CHATS_UPDATE.value, - Permission.CHATS_DELETE.value, # Comments (no delete) Permission.COMMENTS_CREATE.value, Permission.COMMENTS_READ.value, - # LLM Configs (read only) - Permission.LLM_CONFIGS_READ.value, + # LLM Configs (no delete) Permission.LLM_CONFIGS_CREATE.value, + Permission.LLM_CONFIGS_READ.value, Permission.LLM_CONFIGS_UPDATE.value, - # Podcasts + # Podcasts (no delete) Permission.PODCASTS_CREATE.value, Permission.PODCASTS_READ.value, Permission.PODCASTS_UPDATE.value, - Permission.PODCASTS_DELETE.value, - # Connectors (full access for editors) + # Connectors (no delete) Permission.CONNECTORS_CREATE.value, Permission.CONNECTORS_READ.value, Permission.CONNECTORS_UPDATE.value, - # Logs + # Logs (read only) Permission.LOGS_READ.value, - # Members (view only) + # Members (can invite and view only, cannot manage roles or remove) + Permission.MEMBERS_INVITE.value, Permission.MEMBERS_VIEW.value, - # Roles (read only) + # Roles (read only - cannot create, update, or delete) Permission.ROLES_READ.value, - # Settings (view only) + # Settings (view only, no update or delete) Permission.SETTINGS_VIEW.value, ], "Viewer": [ @@ -291,7 +244,7 @@ DEFAULT_ROLE_PERMISSIONS = { Permission.DOCUMENTS_READ.value, # Chats (read only) Permission.CHATS_READ.value, - # Comments (no delete) + # Comments (can create and read, but not delete) Permission.COMMENTS_CREATE.value, Permission.COMMENTS_READ.value, # LLM Configs (read only) @@ -865,7 +818,7 @@ class SearchSpaceRole(BaseModel, TimestampMixin): permissions = Column(ARRAY(String), nullable=False, default=[]) # Whether this role is assigned to new members by default when they join via invite is_default = Column(Boolean, nullable=False, default=False) - # System roles (Owner, Admin, Editor, Viewer) cannot be deleted + # System roles (Owner, Editor, Viewer) cannot be deleted is_system_role = Column(Boolean, nullable=False, default=False) search_space_id = Column( @@ -1221,6 +1174,11 @@ def get_default_roles_config() -> list[dict]: Get the configuration for default system roles. These roles are created automatically when a search space is created. + Only 3 roles are supported: + - Owner: Full access to everything (assigned to search space creator) + - Editor: Can create/update content but cannot delete, manage roles, or change settings + - Viewer: Read-only access to resources (can add comments) + Returns: List of role configurations with name, description, permissions, and flags """ @@ -1232,16 +1190,9 @@ def get_default_roles_config() -> list[dict]: "is_default": False, "is_system_role": True, }, - { - "name": "Admin", - "description": "Can manage most resources except deleting the search space", - "permissions": DEFAULT_ROLE_PERMISSIONS["Admin"], - "is_default": False, - "is_system_role": True, - }, { "name": "Editor", - "description": "Can create and edit documents, chats, and podcasts", + "description": "Can create and update content (no delete, role management, or settings access)", "permissions": DEFAULT_ROLE_PERMISSIONS["Editor"], "is_default": True, # Default role for new members via invite "is_system_role": True, diff --git a/surfsense_backend/app/routes/new_chat_routes.py b/surfsense_backend/app/routes/new_chat_routes.py index 7a5224ba6..8fddc55c4 100644 --- a/surfsense_backend/app/routes/new_chat_routes.py +++ b/surfsense_backend/app/routes/new_chat_routes.py @@ -67,16 +67,15 @@ async def check_thread_access( Access is granted if: - User is the creator of the thread - - Thread visibility is SEARCH_SPACE (any member can access) + - Thread visibility is SEARCH_SPACE (any member can access) - for read/update operations only - Thread is a legacy thread (created_by_id is NULL) - only if user is search space owner Args: session: Database session thread: The thread to check access for user: The user requesting access - require_ownership: If True, only the creator can access (for edit/delete operations) - For SEARCH_SPACE threads, any member with permission can access - Legacy threads (NULL creator) are accessible by search space owner + require_ownership: If True, ONLY the creator can perform this action (e.g., changing visibility). + This is checked FIRST, before visibility rules. Returns: True if access is granted @@ -87,11 +86,18 @@ async def check_thread_access( is_owner = thread.created_by_id == user.id is_legacy = thread.created_by_id is None - # Shared threads (SEARCH_SPACE) are accessible by any member - # This check comes first so shared threads are always accessible + # If ownership is required (e.g., changing visibility), ONLY the creator can do it + # This check comes first to ensure ownership-required operations are always creator-only + if require_ownership: + if not is_owner: + raise HTTPException( + status_code=403, + detail="Only the creator of this chat can perform this action", + ) + return True + + # Shared threads (SEARCH_SPACE) are accessible by any member for read/update operations if thread.visibility == ChatVisibility.SEARCH_SPACE: - # For ownership-required operations on shared threads, any member can proceed - # (permission check is done at route level) return True # For legacy threads (created before visibility feature), @@ -112,15 +118,6 @@ async def check_thread_access( detail="You don't have access to this chat", ) - # If ownership is required, only the creator can access - if require_ownership: - if not is_owner: - raise HTTPException( - status_code=403, - detail="Only the creator of this chat can perform this action", - ) - return True - # For read access: owner can access their own private threads if is_owner: return True diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index dedef0ea9..f6319653f 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -2403,25 +2403,29 @@ async def test_mcp_server_connection( ) transport = server_config.get("transport", "stdio") - + # HTTP transport (streamable-http, http, sse) if transport in ("streamable-http", "http", "sse"): url = server_config.get("url") headers = server_config.get("headers", {}) - + if not url: - raise HTTPException(status_code=400, detail="Server URL is required for HTTP transport") - + raise HTTPException( + status_code=400, detail="Server URL is required for HTTP transport" + ) + result = await test_mcp_http_connection(url, headers, transport) return result - + # stdio transport (default) command = server_config.get("command") args = server_config.get("args", []) env = server_config.get("env", {}) if not command: - raise HTTPException(status_code=400, detail="Server command is required for stdio transport") + raise HTTPException( + status_code=400, detail="Server command is required for stdio transport" + ) # Test the connection result = await test_mcp_connection(command, args, env) diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index ddffdc969..b8ff3e649 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -84,7 +84,7 @@ class SearchSourceConnectorRead(SearchSourceConnectorBase, IDModel, TimestampMod class MCPServerConfig(BaseModel): """Configuration for an MCP server connection. - + Supports two transport types: - stdio: Local process (command, args, env) - streamable-http/http/sse: Remote HTTP server (url, headers) @@ -94,13 +94,13 @@ class MCPServerConfig(BaseModel): command: str | None = None # e.g., "uvx", "node", "python" args: list[str] = [] # e.g., ["mcp-server-git", "--repository", "/path"] env: dict[str, str] = {} # Environment variables for the server process - + # HTTP transport fields url: str | None = None # e.g., "https://mcp-server.com/mcp" headers: dict[str, str] = {} # HTTP headers for authentication - + transport: str = "stdio" # "stdio" | "streamable-http" | "http" | "sse" - + def is_http_transport(self) -> bool: """Check if this config uses HTTP transport.""" return self.transport in ("streamable-http", "http", "sse") diff --git a/surfsense_web/app/dashboard/[search_space_id]/team/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/team/page.tsx index b8ba2d77b..f00982555 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/team/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/team/page.tsx @@ -767,20 +767,18 @@ function RolesTab({ className={cn( "h-10 w-10 rounded-lg flex items-center justify-center", role.name === "Owner" && "bg-amber-500/20", - role.name === "Admin" && "bg-red-500/20", role.name === "Editor" && "bg-blue-500/20", role.name === "Viewer" && "bg-gray-500/20", - !["Owner", "Admin", "Editor", "Viewer"].includes(role.name) && "bg-primary/20" + !["Owner", "Editor", "Viewer"].includes(role.name) && "bg-primary/20" )} > @@ -1310,6 +1308,49 @@ function CreateInviteDialog({ // ============ Create Role Dialog ============ +// Preset permission sets for quick role creation +// Editor: can create/read/update content, but cannot manage roles, remove members, or change settings +// Viewer: read-only access with ability to create comments +const PRESET_PERMISSIONS = { + editor: [ + "documents:create", + "documents:read", + "documents:update", + "chats:create", + "chats:read", + "chats:update", + "comments:create", + "comments:read", + "llm_configs:create", + "llm_configs:read", + "llm_configs:update", + "podcasts:create", + "podcasts:read", + "podcasts:update", + "connectors:create", + "connectors:read", + "connectors:update", + "logs:read", + "members:invite", + "members:view", + "roles:read", + "settings:view", + ], + viewer: [ + "documents:read", + "chats:read", + "comments:create", + "comments:read", + "llm_configs:read", + "podcasts:read", + "connectors:read", + "logs:read", + "members:view", + "roles:read", + "settings:view", + ], +}; + function CreateRoleDialog({ groupedPermissions, onCreateRole, @@ -1369,6 +1410,11 @@ function CreateRoleDialog({ } }; + const applyPreset = (preset: "editor" | "viewer") => { + setSelectedPermissions(PRESET_PERMISSIONS[preset]); + toast.success(`Applied ${preset === "editor" ? "Editor" : "Viewer"} preset permissions`); + }; + return ( @@ -1416,7 +1462,34 @@ function CreateRoleDialog({ />
- +
+ +
+ + +
+
+

+ Use presets to quickly apply Editor (create/read/update) or Viewer (read-only) permissions +

{Object.entries(groupedPermissions).map(([category, perms]) => { @@ -1427,10 +1500,8 @@ function CreateRoleDialog({ return (
- +
{perms.map((perm) => ( - + ))}
diff --git a/surfsense_web/components/assistant-ui/assistant-message.tsx b/surfsense_web/components/assistant-ui/assistant-message.tsx index 106596403..681dc315a 100644 --- a/surfsense_web/components/assistant-ui/assistant-message.tsx +++ b/surfsense_web/components/assistant-ui/assistant-message.tsx @@ -204,7 +204,9 @@ export const AssistantMessage: FC = () => { > {hasComments ? ( - {commentCount} {commentCount === 1 ? "comment" : "comments"} + + {commentCount} {commentCount === 1 ? "comment" : "comments"} + ) : ( Add comment )} diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index b946c153f..5ad520b05 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -22,7 +22,6 @@ import { useIndexingConnectors } from "./connector-popup/hooks/use-indexing-conn import { ActiveConnectorsTab } from "./connector-popup/tabs/active-connectors-tab"; import { AllConnectorsTab } from "./connector-popup/tabs/all-connectors-tab"; import { ConnectorAccountsListView } from "./connector-popup/views/connector-accounts-list-view"; -import { MCPConnectorListView } from "./connector-popup/views/mcp-connector-list-view"; import { YouTubeCrawlerView } from "./connector-popup/views/youtube-crawler-view"; export const ConnectorIndicator: FC = () => { @@ -178,18 +177,16 @@ export const ConnectorIndicator: FC = () => { {isYouTubeView && searchSpaceId ? ( ) : viewingMCPList ? ( -
- c.connector_type === "MCP_CONNECTOR" - ) as SearchSourceConnector[] - } - onAddNew={handleAddNewMCPFromList} - onManageConnector={handleStartEdit} - onBack={handleBackFromMCPList} - /> -
+ ) : viewingAccountsType ? ( void; onManage?: () => void; @@ -46,10 +48,12 @@ export const ConnectorCard: FC = ({ isConnecting = false, documentCount, accountCount, + connectorCount, isIndexing = false, onConnect, onManage, }) => { + const isMCP = connectorType === EnumConnectorName.MCP_CONNECTOR; // Get connector status const { getConnectorStatus, isConnectorEnabled, getConnectorStatusMessage, shouldShowWarnings } = useConnectorStatus(); @@ -112,13 +116,21 @@ export const ConnectorCard: FC = ({

) : isConnected ? (

- {formatDocumentCount(documentCount)} - {accountCount !== undefined && accountCount > 0 && ( + {isMCP && connectorCount !== undefined ? ( + + {connectorCount} {connectorCount === 1 ? "server" : "servers"} + + ) : ( <> - • - - {accountCount} {accountCount === 1 ? "Account" : "Accounts"} - + {formatDocumentCount(documentCount)} + {accountCount !== undefined && accountCount > 0 && ( + <> + • + + {accountCount} {accountCount === 1 ? "Account" : "Accounts"} + + + )} )}

diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx index 1d52f0182..b7c68734a 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx @@ -1,7 +1,7 @@ "use client"; -import type { FC } from "react"; import { AlertCircle } from "lucide-react"; +import type { FC } from "react"; import { Label } from "@/components/ui/label"; import { Select, diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/mcp-connect-form.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/mcp-connect-form.tsx index b0f6c8078..9ece079f3 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/mcp-connect-form.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/mcp-connect-form.tsx @@ -4,18 +4,16 @@ import { CheckCircle2, ChevronDown, ChevronUp, Server, XCircle } from "lucide-re import { type FC, useRef, useState } from "react"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Textarea } from "@/components/ui/textarea"; import { EnumConnectorName } from "@/contracts/enums/connector"; -import type { MCPToolDefinition } from "@/contracts/types/mcp.types"; -import type { ConnectFormProps } from ".."; import { extractServerName, + type MCPConnectionTestResult, parseMCPConfig, testMCPConnection, - type MCPConnectionTestResult, } from "../../utils/mcp-config-validator"; +import type { ConnectFormProps } from ".."; export const MCPConnectForm: FC = ({ onSubmit, isSubmitting }) => { const isSubmittingRef = useRef(false); @@ -46,7 +44,7 @@ export const MCPConnectForm: FC = ({ onSubmit, isSubmitting }) name: "My Remote MCP Server", url: "https://your-mcp-server.com/mcp", headers: { - "API_KEY": "your_api_key_here", + API_KEY: "your_api_key_here", }, transport: "streamable-http", }, @@ -178,29 +176,47 @@ export const MCPConnectForm: FC = ({ onSubmit, isSubmitting }) id="config" value={configJson} onChange={(e) => handleConfigChange(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Tab") { + e.preventDefault(); + const target = e.target as HTMLTextAreaElement; + const start = target.selectionStart; + const end = target.selectionEnd; + const indent = " "; // 2 spaces for JSON + const newValue = + configJson.substring(0, start) + indent + configJson.substring(end); + handleConfigChange(newValue); + // Set cursor position after the inserted tab + requestAnimationFrame(() => { + target.selectionStart = target.selectionEnd = start + indent.length; + }); + } + }} placeholder={DEFAULT_CONFIG} rows={16} className={`font-mono text-xs ${jsonError ? "border-red-500" : ""}`} /> {jsonError &&

JSON Error: {jsonError}

}

- Local (stdio): command, args, env, transport: "stdio"
- Remote (HTTP): url, headers, transport: "streamable-http" + Paste a single MCP server configuration. Must include: name, command, args (optional), + env (optional), transport (optional).

+ {/* Test Connection */}
+ {/* Test Result */} {testResult && ( = ({ onSubmit, isSubmitting }) type="button" variant="ghost" size="sm" - className="h-6 px-2" + className="h-6 px-2 self-start sm:self-auto text-xs" onClick={(e) => { e.preventDefault(); e.stopPropagation(); @@ -236,18 +252,20 @@ export const MCPConnectForm: FC = ({ onSubmit, isSubmitting }) {showDetails ? ( <> - Hide Details + Hide Details + Hide ) : ( <> - Show Details + Show Details + Show )} )}
- + {testResult.message} {showDetails && testResult.tools.length > 0 && (
diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx index d0bc96872..17f4a49a5 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx @@ -1,6 +1,6 @@ "use client"; -import { File, FileText, FileSpreadsheet, FolderClosed, Image, Presentation } from "lucide-react"; +import { File, FileSpreadsheet, FileText, FolderClosed, Image, Presentation } from "lucide-react"; import type { FC } from "react"; import { useEffect, useState } from "react"; import { GoogleDriveFolderTree } from "@/components/connectors/google-drive-folder-tree"; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-config.tsx index 54ba3ed3a..ac450677e 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-config.tsx @@ -10,12 +10,12 @@ import { Label } from "@/components/ui/label"; import { Textarea } from "@/components/ui/textarea"; import { EnumConnectorName } from "@/contracts/enums/connector"; import type { MCPServerConfig } from "@/contracts/types/mcp.types"; -import type { ConnectorConfigProps } from "../index"; import { + type MCPConnectionTestResult, parseMCPConfig, testMCPConnection, - type MCPConnectionTestResult, } from "../../utils/mcp-config-validator"; +import type { ConnectorConfigProps } from "../index"; interface MCPConfigProps extends ConnectorConfigProps { onNameChange?: (name: string) => void; @@ -47,10 +47,10 @@ export const MCPConfig: FC = ({ connector, onConfigChange, onNam const serverConfig = connector.config?.server_config as MCPServerConfig | undefined; if (serverConfig) { const transport = serverConfig.transport || "stdio"; - + // Build config object based on transport type let configObj: Record; - + if (transport === "streamable-http" || transport === "http" || transport === "sse") { // HTTP transport - use url and headers configObj = { @@ -67,7 +67,7 @@ export const MCPConfig: FC = ({ connector, onConfigChange, onNam transport: transport, }; } - + setConfigJson(JSON.stringify(configObj, null, 2)); } }, [isValidConnector, connector.name, connector.config?.server_config]); @@ -148,15 +148,23 @@ export const MCPConfig: FC = ({ connector, onConfigChange, onNam return (
{/* Server Name */} -
- - handleNameChange(e.target.value)} - placeholder="e.g., Filesystem Server" - required - /> +
+
+ + handleNameChange(e.target.value)} + placeholder="e.g., Filesystem Server" + className="border-slate-400/20 focus-visible:border-slate-400/40" + required + /> +

+ A friendly name to identify this connector. +

+
{/* Server Configuration */} @@ -173,12 +181,29 @@ export const MCPConfig: FC = ({ connector, onConfigChange, onNam id="config" value={configJson} onChange={(e) => handleConfigChange(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Tab") { + e.preventDefault(); + const target = e.target as HTMLTextAreaElement; + const start = target.selectionStart; + const end = target.selectionEnd; + const indent = " "; // 2 spaces for JSON + const newValue = + configJson.substring(0, start) + indent + configJson.substring(end); + handleConfigChange(newValue); + // Set cursor position after the inserted tab + requestAnimationFrame(() => { + target.selectionStart = target.selectionEnd = start + indent.length; + }); + } + }} rows={16} className={`font-mono text-xs ${jsonError ? "border-red-500" : ""}`} /> {jsonError &&

JSON Error: {jsonError}

}

- Local (stdio): command, args, env, transport: "stdio"
+ Local (stdio): command, args, env, transport: "stdio" +
Remote (HTTP): url, headers, transport: "streamable-http"

@@ -189,10 +214,10 @@ export const MCPConfig: FC = ({ connector, onConfigChange, onNam type="button" onClick={handleTestConnection} disabled={isTesting} - variant="outline" - className="w-full" + variant="secondary" + className="w-full h-8 text-[13px] px-3 rounded-lg font-medium bg-white text-slate-700 hover:bg-slate-50 border-0 shadow-xs dark:bg-secondary dark:text-secondary-foreground dark:hover:bg-secondary/80" > - {isTesting ? "Testing Connection..." : "Test Connection"} + {isTesting ? "Testing Connection" : "Test Connection"}
@@ -211,7 +236,7 @@ export const MCPConfig: FC = ({ connector, onConfigChange, onNam )}
-
+
{testResult.status === "success" ? "Connection Successful" @@ -222,7 +247,7 @@ export const MCPConfig: FC = ({ connector, onConfigChange, onNam type="button" variant="ghost" size="sm" - className="h-6 px-2" + className="h-6 px-2 self-start sm:self-auto text-xs" onClick={(e) => { e.preventDefault(); e.stopPropagation(); @@ -232,12 +257,14 @@ export const MCPConfig: FC = ({ connector, onConfigChange, onNam {showDetails ? ( <> - Hide Details + Hide Details + Hide ) : ( <> - Show Details + Show Details + Show )} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx index 89c36ffc2..5433acbf7 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx @@ -151,7 +151,7 @@ export const ConnectorEditView: FC = ({

- {connector.connector_type === "MCP_CONNECTOR" ? "MCP Server" : connector.name} + {connector.name}

Manage your connector settings and sync configuration diff --git a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts index 7a2243705..4f56f588d 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts @@ -646,7 +646,7 @@ export const useConnectorDialog = () => { const successMessage = currentConnectorType === "MCP_CONNECTOR" - ? `${connector.name} MCP server added successfully` + ? `${connector.name} added successfully` : `${connectorTitle} connected and indexing started!`; toast.success(successMessage, { description: periodicEnabledForIndexing @@ -711,7 +711,7 @@ export const useConnectorDialog = () => { // Other non-indexable connectors - just show success message and close const successMessage = currentConnectorType === "MCP_CONNECTOR" - ? `${connector.name} MCP server added successfully` + ? `${connector.name} added successfully` : `${connectorTitle} connected successfully!`; toast.success(successMessage); diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index 6152504fc..2487b7276 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -1,6 +1,7 @@ "use client"; import type { FC } from "react"; +import { EnumConnectorName } from "@/contracts/enums/connector"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { ConnectorCard } from "../components/connector-card"; import { CRAWLERS, OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants"; @@ -161,6 +162,16 @@ export const AllConnectorsTab: FC = ({ ); const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id); + // For MCP connectors, count total MCP connectors instead of document count + const isMCP = connector.connectorType === EnumConnectorName.MCP_CONNECTOR; + const mcpConnectorCount = + isMCP && allConnectors + ? allConnectors.filter( + (c: SearchSourceConnector) => + c.connector_type === EnumConnectorName.MCP_CONNECTOR + ).length + : undefined; + const handleConnect = onConnectNonOAuth ? () => onConnectNonOAuth(connector.connectorType) : () => {}; // Fallback - connector popup should handle all connector types @@ -175,6 +186,7 @@ export const AllConnectorsTab: FC = ({ isConnected={isConnected} isConnecting={isConnecting} documentCount={documentCount} + connectorCount={mcpConnectorCount} isIndexing={isIndexing} onConnect={handleConnect} onManage={ diff --git a/surfsense_web/components/assistant-ui/connector-popup/utils/mcp-config-validator.ts b/surfsense_web/components/assistant-ui/connector-popup/utils/mcp-config-validator.ts index e03d76445..650a95e3d 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/utils/mcp-config-validator.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/utils/mcp-config-validator.ts @@ -138,35 +138,37 @@ export const parseMCPConfig = (configJson: string): MCPConfigValidationResult => // Replace technical error messages with user-friendly ones if (errorMsg.includes("expected string, received undefined")) { - errorMsg = "This field is required"; + errorMsg = fieldPath ? `The '${fieldPath}' field is required` : "This field is required"; } else if (errorMsg.includes("Invalid input")) { - errorMsg = "Invalid value"; + errorMsg = fieldPath ? `The '${fieldPath}' field has an invalid value` : "Invalid value"; + } else if (fieldPath && !errorMsg.toLowerCase().includes(fieldPath.toLowerCase())) { + // If error message doesn't mention the field name, prepend it + errorMsg = `The '${fieldPath}' field: ${errorMsg}`; } - const formattedError = fieldPath ? `${fieldPath}: ${errorMsg}` : errorMsg; - - console.error("[MCP Validator] ❌ Validation error:", formattedError); + console.error("[MCP Validator] ❌ Validation error:", errorMsg); console.error("[MCP Validator] Full Zod errors:", result.error.issues); return { config: null, - error: formattedError, + error: errorMsg, }; } // Build config based on transport type - const config: MCPServerConfig = result.data.transport === "stdio" || !result.data.transport - ? { - command: (result.data as z.infer).command, - args: (result.data as z.infer).args, - env: (result.data as z.infer).env, - transport: "stdio" as const, - } - : { - url: (result.data as z.infer).url, - headers: (result.data as z.infer).headers, - transport: result.data.transport as "streamable-http" | "http" | "sse", - }; + const config: MCPServerConfig = + result.data.transport === "stdio" || !result.data.transport + ? { + command: (result.data as z.infer).command, + args: (result.data as z.infer).args, + env: (result.data as z.infer).env, + transport: "stdio" as const, + } + : { + url: (result.data as z.infer).url, + headers: (result.data as z.infer).headers, + transport: result.data.transport as "streamable-http" | "http" | "sse", + }; // Cache the successfully parsed config configCache.set(configJson, { diff --git a/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx index 5f8c1f3ed..a48ca02e6 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx @@ -1,9 +1,10 @@ "use client"; import { differenceInDays, differenceInMinutes, format, isToday, isYesterday } from "date-fns"; -import { ArrowLeft, Loader2, Plus } from "lucide-react"; +import { ArrowLeft, Loader2, Plus, Server } from "lucide-react"; import type { FC } from "react"; import { Button } from "@/components/ui/button"; +import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { cn } from "@/lib/utils"; @@ -19,6 +20,7 @@ interface ConnectorAccountsListViewProps { onManage: (connector: SearchSourceConnector) => void; onAddAccount: () => void; isConnecting?: boolean; + addButtonText?: string; } /** @@ -70,6 +72,7 @@ export const ConnectorAccountsListView: FC = ({ onManage, onAddAccount, isConnecting = false, + addButtonText, }) => { // Get connector status const { isConnectorEnabled, getConnectorStatusMessage } = useConnectorStatus(); @@ -80,6 +83,22 @@ export const ConnectorAccountsListView: FC = ({ // Filter connectors to only show those of this type const typeConnectors = connectors.filter((c) => c.connector_type === connectorType); + // Determine button text - default to "Add Account" unless specified + const buttonText = + addButtonText || + (connectorType === EnumConnectorName.MCP_CONNECTOR ? "Add New MCP Server" : "Add Account"); + const isMCP = connectorType === EnumConnectorName.MCP_CONNECTOR; + + // Helper to get display name for connector (handles MCP server name extraction) + const getDisplayName = (connector: SearchSourceConnector): string => { + if (isMCP) { + // For MCP, extract server name from config if available + const serverName = connector.config?.server_config?.name || connector.name; + return serverName; + } + return getConnectorDisplayName(connector.name); + }; + return (

{/* Header */} @@ -115,22 +134,22 @@ export const ConnectorAccountsListView: FC = ({ onClick={onAddAccount} disabled={isConnecting || !isEnabled} className={cn( - "flex items-center gap-1.5 sm:gap-2 px-2 sm:px-3 py-1.5 sm:py-2 rounded-lg border-2 border-dashed text-left transition-all duration-200 shrink-0 self-center sm:self-auto sm:w-auto", + "flex items-center justify-center gap-1.5 h-8 px-3 rounded-md border-2 border-dashed text-xs sm:text-sm transition-all duration-200 shrink-0 w-full sm:w-auto", !isEnabled ? "border-border/30 opacity-50 cursor-not-allowed" - : "border-primary/50 hover:bg-primary/5", + : "border-slate-400/20 dark:border-white/20 hover:bg-primary/5", isConnecting && "opacity-50 cursor-not-allowed" )} > -
+
{isConnecting ? ( - + ) : ( - + )}
- - {isConnecting ? "Connecting" : "Add Account"} + + {isConnecting ? "Connecting" : buttonText}
@@ -139,61 +158,81 @@ export const ConnectorAccountsListView: FC = ({ {/* Content */}
{/* Connected Accounts Grid */} -
- {typeConnectors.map((connector) => { - const isIndexing = indexingConnectorIds.has(connector.id); + {typeConnectors.length === 0 ? ( +
+
+ {isMCP ? ( + + ) : ( + getConnectorIcon(connectorType, "size-8") + )} +
+

+ {isMCP ? "No MCP Servers" : `No ${connectorTitle} Accounts`} +

+

+ {isMCP + ? "Get started by adding your first Model Context Protocol server" + : `Get started by connecting your first ${connectorTitle} account`} +

+
+ ) : ( +
+ {typeConnectors.map((connector) => { + const isIndexing = indexingConnectorIds.has(connector.id); - return ( -
+ return (
- {getConnectorIcon(connector.connector_type, "size-6")} -
-
-

- {getConnectorDisplayName(connector.name)} -

- {isIndexing ? ( -

- - Syncing +

+ {getConnectorIcon(connector.connector_type, "size-6")} +
+
+

+ {getDisplayName(connector)}

- ) : ( -

- {isIndexableConnector(connector.connector_type) - ? connector.last_indexed_at - ? `Last indexed: ${formatLastIndexedDate(connector.last_indexed_at)}` - : "Never indexed" - : "Active"} -

- )} + {isIndexing ? ( +

+ + Syncing +

+ ) : ( +

+ {isIndexableConnector(connector.connector_type) + ? connector.last_indexed_at + ? `Last indexed: ${formatLastIndexedDate(connector.last_indexed_at)}` + : "Never indexed" + : "Active"} +

+ )} +
+
- -
- ); - })} -
+ ); + })} +
+ )}
); diff --git a/surfsense_web/components/assistant-ui/connector-popup/views/mcp-connector-list-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/views/mcp-connector-list-view.tsx deleted file mode 100644 index 78a0b0b0c..000000000 --- a/surfsense_web/components/assistant-ui/connector-popup/views/mcp-connector-list-view.tsx +++ /dev/null @@ -1,134 +0,0 @@ -"use client"; - -import { Plus, Server, XCircle } from "lucide-react"; -import type { FC } from "react"; -import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; -import { Button } from "@/components/ui/button"; -import { EnumConnectorName } from "@/contracts/enums/connector"; -import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import type { SearchSourceConnector } from "@/contracts/types/connector.types"; -import { cn } from "@/lib/utils"; - -interface MCPConnectorListViewProps { - mcpConnectors: SearchSourceConnector[]; - onAddNew: () => void; - onManageConnector: (connector: SearchSourceConnector) => void; - onBack: () => void; -} - -export const MCPConnectorListView: FC = ({ - mcpConnectors, - onAddNew, - onManageConnector, - onBack, -}) => { - // Validate that all connectors are MCP connectors - const invalidConnectors = mcpConnectors.filter( - (c) => c.connector_type !== EnumConnectorName.MCP_CONNECTOR - ); - - if (invalidConnectors.length > 0) { - console.error( - "MCPConnectorListView received non-MCP connectors:", - invalidConnectors.map((c) => c.connector_type) - ); - return ( - - - Invalid Connector Type - - This view can only display MCP connectors. Found {invalidConnectors.length} invalid - connector(s). - - - ); - } - return ( -
- {/* Header */} -
-
- -
-

MCP Connectors

-

- Manage your Model Context Protocol servers -

-
-
-
- - {/* Add New Button */} -
- -
- - {/* MCP Connectors List */} -
- {mcpConnectors.length === 0 ? ( -
-
- -
-

No MCP Servers

-

- Get started by adding your first Model Context Protocol server -

-
- ) : ( - mcpConnectors.map((connector) => { - // Extract server name from config - const serverName = connector.config?.server_config?.name || connector.name; - - return ( -
-
- {getConnectorIcon("MCP_CONNECTOR", "size-6")} -
-
-

{serverName}

-
- -
- ); - }) - )} -
-
- ); -}; diff --git a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx index 570440f6a..ae8fe9b8d 100644 --- a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx +++ b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx @@ -11,8 +11,8 @@ import { useState, } from "react"; import ReactDOMServer from "react-dom/server"; -import type { Document } from "@/contracts/types/document.types"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import type { Document } from "@/contracts/types/document.types"; import { cn } from "@/lib/utils"; export interface MentionedDocument { diff --git a/surfsense_web/components/chat-comments/comment-panel/comment-panel.tsx b/surfsense_web/components/chat-comments/comment-panel/comment-panel.tsx index 2ec960614..12f336d85 100644 --- a/surfsense_web/components/chat-comments/comment-panel/comment-panel.tsx +++ b/surfsense_web/components/chat-comments/comment-panel/comment-panel.tsx @@ -36,10 +36,12 @@ export function CommentPanel({ if (isLoading) { return ( -
+
Loading comments... @@ -57,10 +59,7 @@ export function CommentPanel({ return (
{hasThreads && ( @@ -92,11 +91,7 @@ export function CommentPanel({
)} -
+
{isComposerOpen ? ( {/* Drag handle indicator - only for bottom sheet */} @@ -37,10 +30,7 @@ export function CommentSheet({
)} - + Comments @@ -52,11 +42,7 @@ export function CommentSheet({
- +
diff --git a/surfsense_web/components/connectors/google-drive-folder-tree.tsx b/surfsense_web/components/connectors/google-drive-folder-tree.tsx index 894564167..30df2d788 100644 --- a/surfsense_web/components/connectors/google-drive-folder-tree.tsx +++ b/surfsense_web/components/connectors/google-drive-folder-tree.tsx @@ -4,6 +4,7 @@ import { ChevronDown, ChevronRight, File, + FileSpreadsheet, FileText, FolderClosed, FolderOpen, @@ -11,7 +12,6 @@ import { Image, Loader2, Presentation, - FileSpreadsheet, } from "lucide-react"; import { useState } from "react"; import { Checkbox } from "@/components/ui/checkbox"; diff --git a/surfsense_web/components/layout/ui/header/Header.tsx b/surfsense_web/components/layout/ui/header/Header.tsx index 1981bba68..f4d324cfb 100644 --- a/surfsense_web/components/layout/ui/header/Header.tsx +++ b/surfsense_web/components/layout/ui/header/Header.tsx @@ -28,21 +28,22 @@ export function Header({ const hasThread = isChatPage && currentThreadState.id !== null; // Create minimal thread object for ChatShareButton (used for API calls) - const threadForButton: ThreadRecord | null = hasThread - ? { - id: currentThreadState.id!, - visibility: currentThreadState.visibility ?? "PRIVATE", - // These fields are not used by ChatShareButton for display, only for checks - created_by_id: null, - search_space_id: 0, - title: "", - archived: false, - created_at: "", - updated_at: "", - } - : null; + const threadForButton: ThreadRecord | null = + hasThread && currentThreadState.id !== null + ? { + id: currentThreadState.id, + visibility: currentThreadState.visibility ?? "PRIVATE", + // These fields are not used by ChatShareButton for display, only for checks + created_by_id: null, + search_space_id: 0, + title: "", + archived: false, + created_at: "", + updated_at: "", + } + : null; - const handleVisibilityChange = (visibility: ChatVisibility) => { + const handleVisibilityChange = (_visibility: ChatVisibility) => { // Visibility change is handled by ChatShareButton internally via Jotai // This callback can be used for additional side effects if needed }; diff --git a/surfsense_web/components/notifications/NotificationButton.tsx b/surfsense_web/components/notifications/NotificationButton.tsx index e9f5db2dc..acecc06af 100644 --- a/surfsense_web/components/notifications/NotificationButton.tsx +++ b/surfsense_web/components/notifications/NotificationButton.tsx @@ -1,16 +1,16 @@ "use client"; -import { useState } from "react"; +import { useAtomValue } from "jotai"; import { Bell } from "lucide-react"; +import { useParams } from "next/navigation"; +import { useState } from "react"; +import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { Button } from "@/components/ui/button"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { useNotifications } from "@/hooks/use-notifications"; -import { useAtomValue } from "jotai"; -import { currentUserAtom } from "@/atoms/user/user-query.atoms"; -import { NotificationPopup } from "./NotificationPopup"; import { cn } from "@/lib/utils"; -import { useParams } from "next/navigation"; +import { NotificationPopup } from "./NotificationPopup"; export function NotificationButton() { const [open, setOpen] = useState(false); diff --git a/surfsense_web/components/notifications/NotificationPopup.tsx b/surfsense_web/components/notifications/NotificationPopup.tsx index 9196ceaa4..50deadf03 100644 --- a/surfsense_web/components/notifications/NotificationPopup.tsx +++ b/surfsense_web/components/notifications/NotificationPopup.tsx @@ -1,14 +1,14 @@ "use client"; -import { Bell, CheckCheck, Loader2, AlertCircle, CheckCircle2 } from "lucide-react"; +import { formatDistanceToNow } from "date-fns"; +import { AlertCircle, Bell, CheckCheck, CheckCircle2, Loader2 } from "lucide-react"; import { useRouter } from "next/navigation"; +import { convertRenderedToDisplay } from "@/components/chat-comments/comment-item/comment-item"; import { Button } from "@/components/ui/button"; import { ScrollArea } from "@/components/ui/scroll-area"; import { Separator } from "@/components/ui/separator"; import type { Notification } from "@/hooks/use-notifications"; -import { formatDistanceToNow } from "date-fns"; import { cn } from "@/lib/utils"; -import { convertRenderedToDisplay } from "@/components/chat-comments/comment-item/comment-item"; interface NotificationPopupProps { notifications: Notification[]; diff --git a/surfsense_web/components/providers/ElectricProvider.tsx b/surfsense_web/components/providers/ElectricProvider.tsx index af3046a64..e31885973 100644 --- a/surfsense_web/components/providers/ElectricProvider.tsx +++ b/surfsense_web/components/providers/ElectricProvider.tsx @@ -1,13 +1,13 @@ "use client"; -import { useEffect, useState, useRef } from "react"; import { useAtomValue } from "jotai"; +import { useEffect, useRef, useState } from "react"; import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { - initElectric, cleanupElectric, - isElectricInitialized, type ElectricClient, + initElectric, + isElectricInitialized, } from "@/lib/electric/client"; import { ElectricContext } from "@/lib/electric/context"; diff --git a/surfsense_web/content/docs/docker-installation.mdx b/surfsense_web/content/docs/docker-installation.mdx index 6501c7783..ec8cb246b 100644 --- a/surfsense_web/content/docs/docker-installation.mdx +++ b/surfsense_web/content/docs/docker-installation.mdx @@ -26,7 +26,7 @@ Make sure to include the `-v surfsense-data:/data` in your Docker command. This **Linux/macOS:** ```bash -docker run -d -p 3000:3000 -p 8000:8000 \ +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ -v surfsense-data:/data \ --name surfsense \ --restart unless-stopped \ @@ -36,7 +36,7 @@ docker run -d -p 3000:3000 -p 8000:8000 \ **Windows (PowerShell):** ```powershell -docker run -d -p 3000:3000 -p 8000:8000 ` +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 ` -v surfsense-data:/data ` --name surfsense ` --restart unless-stopped ` @@ -50,7 +50,7 @@ docker run -d -p 3000:3000 -p 8000:8000 ` You can pass any [environment variable](/docs/manual-installation#backend-environment-variables) using `-e` flags: ```bash -docker run -d -p 3000:3000 -p 8000:8000 \ +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ -v surfsense-data:/data \ -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \ -e OPENAI_API_KEY=your_openai_api_key \ @@ -93,6 +93,7 @@ After starting, access SurfSense at: - **Frontend**: [http://localhost:3000](http://localhost:3000) - **Backend API**: [http://localhost:8000](http://localhost:8000) - **API Docs**: [http://localhost:8000/docs](http://localhost:8000/docs) +- **Electric-SQL**: [http://localhost:5133](http://localhost:5133) ### Quick Start Environment Variables @@ -195,6 +196,11 @@ Before you begin, ensure you have: | NEXT_PUBLIC_FASTAPI_BACKEND_URL | URL of the backend API (used by frontend during build and runtime) | http://localhost:8000 | | NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE | Authentication method for frontend: `LOCAL` or `GOOGLE` | LOCAL | | NEXT_PUBLIC_ETL_SERVICE | Document parsing service for frontend UI: `UNSTRUCTURED`, `LLAMACLOUD`, or `DOCLING` | DOCLING | +| ELECTRIC_PORT | Port for Electric-SQL service | 5133 | +| POSTGRES_HOST | PostgreSQL host for Electric connection (`db` for Docker PostgreSQL, `host.docker.internal` for local PostgreSQL) | db | +| ELECTRIC_DB_USER | PostgreSQL username for Electric connection | electric | +| ELECTRIC_DB_PASSWORD | PostgreSQL password for Electric connection | electric_password | +| NEXT_PUBLIC_ELECTRIC_URL | URL for Electric-SQL service (used by frontend) | http://localhost:5133 | **Note:** Frontend environment variables with the `NEXT_PUBLIC_` prefix are embedded into the Next.js production build at build time. Since the frontend now runs as a production build in Docker, these variables must be set in the root `.env` file (Docker-specific configuration) and will be passed as build arguments during the Docker build process. @@ -209,7 +215,8 @@ Before you begin, ensure you have: | AUTH_TYPE | Authentication method: `GOOGLE` for OAuth with Google, `LOCAL` for email/password authentication | | GOOGLE_OAUTH_CLIENT_ID | (Optional) Client ID from Google Cloud Console (required if AUTH_TYPE=GOOGLE) | | GOOGLE_OAUTH_CLIENT_SECRET | (Optional) Client secret from Google Cloud Console (required if AUTH_TYPE=GOOGLE) | -| GOOGLE_DRIVE_REDIRECT_URI | (Optional) Redirect URI for Google Drive connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/drive/connector/callback`). Required when using Google Drive connector. | +| ELECTRIC_DB_USER | (Optional) PostgreSQL username for Electric-SQL connection (default: `electric`) | +| ELECTRIC_DB_PASSWORD | (Optional) PostgreSQL password for Electric-SQL connection (default: `electric_password`) | | EMBEDDING_MODEL | Name of the embedding model (e.g., `sentence-transformers/all-MiniLM-L6-v2`, `openai://text-embedding-ada-002`) | | RERANKERS_ENABLED | (Optional) Enable or disable document reranking for improved search results (e.g., `TRUE` or `FALSE`, default: `FALSE`) | | RERANKERS_MODEL_NAME | Name of the reranker model (e.g., `ms-marco-MiniLM-L-12-v2`) (required if RERANKERS_ENABLED=TRUE) | @@ -230,6 +237,44 @@ Before you begin, ensure you have: | REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) | | PAGES_LIMIT | (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) | +**Google Connector OAuth Configuration:** +| ENV VARIABLE | DESCRIPTION | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| GOOGLE_CALENDAR_REDIRECT_URI | (Optional) Redirect URI for Google Calendar connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/calendar/connector/callback`) | +| GOOGLE_GMAIL_REDIRECT_URI | (Optional) Redirect URI for Gmail connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/gmail/connector/callback`) | +| GOOGLE_DRIVE_REDIRECT_URI | (Optional) Redirect URI for Google Drive connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/drive/connector/callback`) | + +**Connector OAuth Configurations (Optional):** + +| ENV VARIABLE | DESCRIPTION | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| AIRTABLE_CLIENT_ID | (Optional) Airtable OAuth client ID from [Airtable Developer Hub](https://airtable.com/create/oauth) | +| AIRTABLE_CLIENT_SECRET | (Optional) Airtable OAuth client secret | +| AIRTABLE_REDIRECT_URI | (Optional) Redirect URI for Airtable connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/airtable/connector/callback`) | +| CLICKUP_CLIENT_ID | (Optional) ClickUp OAuth client ID | +| CLICKUP_CLIENT_SECRET | (Optional) ClickUp OAuth client secret | +| CLICKUP_REDIRECT_URI | (Optional) Redirect URI for ClickUp connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/clickup/connector/callback`) | +| DISCORD_CLIENT_ID | (Optional) Discord OAuth client ID | +| DISCORD_CLIENT_SECRET | (Optional) Discord OAuth client secret | +| DISCORD_REDIRECT_URI | (Optional) Redirect URI for Discord connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/discord/connector/callback`) | +| DISCORD_BOT_TOKEN | (Optional) Discord bot token from Developer Portal | +| ATLASSIAN_CLIENT_ID | (Optional) Atlassian OAuth client ID (for Jira and Confluence) | +| ATLASSIAN_CLIENT_SECRET | (Optional) Atlassian OAuth client secret | +| JIRA_REDIRECT_URI | (Optional) Redirect URI for Jira connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/jira/connector/callback`) | +| CONFLUENCE_REDIRECT_URI | (Optional) Redirect URI for Confluence connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/confluence/connector/callback`) | +| LINEAR_CLIENT_ID | (Optional) Linear OAuth client ID | +| LINEAR_CLIENT_SECRET | (Optional) Linear OAuth client secret | +| LINEAR_REDIRECT_URI | (Optional) Redirect URI for Linear connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/linear/connector/callback`) | +| NOTION_CLIENT_ID | (Optional) Notion OAuth client ID | +| NOTION_CLIENT_SECRET | (Optional) Notion OAuth client secret | +| NOTION_REDIRECT_URI | (Optional) Redirect URI for Notion connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/notion/connector/callback`) | +| SLACK_CLIENT_ID | (Optional) Slack OAuth client ID | +| SLACK_CLIENT_SECRET | (Optional) Slack OAuth client secret | +| SLACK_REDIRECT_URI | (Optional) Redirect URI for Slack connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/slack/connector/callback`) | +| TEAMS_CLIENT_ID | (Optional) Microsoft Teams OAuth client ID | +| TEAMS_CLIENT_SECRET | (Optional) Microsoft Teams OAuth client secret | +| TEAMS_REDIRECT_URI | (Optional) Redirect URI for Teams connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/teams/connector/callback`) | + **Optional Backend LangSmith Observability:** | ENV VARIABLE | DESCRIPTION | @@ -282,6 +327,8 @@ For more details, see the [Uvicorn documentation](https://www.uvicorn.org/#comma - `NEXT_PUBLIC_FASTAPI_BACKEND_URL` - URL of the backend service - `NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE` - Authentication method (`LOCAL` or `GOOGLE`) - `NEXT_PUBLIC_ETL_SERVICE` - Document parsing service (should match backend `ETL_SERVICE`) +- `NEXT_PUBLIC_ELECTRIC_URL` - URL for Electric-SQL service (default: `http://localhost:5133`) +- `NEXT_PUBLIC_ELECTRIC_AUTH_MODE` - Electric-SQL authentication mode (default: `insecure`) These variables are embedded into the application during the Docker build process and affect the frontend's behavior and available features. @@ -312,6 +359,7 @@ These variables are embedded into the application during the Docker build proces - Frontend: [http://localhost:3000](http://localhost:3000) - Backend API: [http://localhost:8000](http://localhost:8000) - API Documentation: [http://localhost:8000/docs](http://localhost:8000/docs) + - Electric-SQL: [http://localhost:5133](http://localhost:5133) - pgAdmin: [http://localhost:5050](http://localhost:5050) ## Docker Services Overview @@ -322,6 +370,7 @@ The Docker setup includes several services that work together: - **Frontend**: Next.js web application - **PostgreSQL (db)**: Database with pgvector extension - **Redis**: Message broker for Celery +- **Electric-SQL**: Real-time sync service for database operations - **Celery Worker**: Handles background tasks (document processing, indexing, etc.) - **Celery Beat**: Scheduler for periodic tasks (enables scheduled connector indexing) - The schedule interval can be configured using the `SCHEDULE_CHECKER_INTERVAL` environment variable in your backend `.env` file diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index 0dd703758..b4da781ba 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -72,7 +72,8 @@ Edit the `.env` file and set the following variables: | AUTH_TYPE | Authentication method: `GOOGLE` for OAuth with Google, `LOCAL` for email/password authentication | | GOOGLE_OAUTH_CLIENT_ID | (Optional) Client ID from Google Cloud Console (required if AUTH_TYPE=GOOGLE) | | GOOGLE_OAUTH_CLIENT_SECRET | (Optional) Client secret from Google Cloud Console (required if AUTH_TYPE=GOOGLE) | -| GOOGLE_DRIVE_REDIRECT_URI | (Optional) Redirect URI for Google Drive connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/drive/connector/callback`). Required when using Google Drive connector. | +| ELECTRIC_DB_USER | (Optional) PostgreSQL username for Electric-SQL connection (default: `electric`) | +| ELECTRIC_DB_PASSWORD | (Optional) PostgreSQL password for Electric-SQL connection (default: `electric_password`) | | EMBEDDING_MODEL | Name of the embedding model (e.g., `sentence-transformers/all-MiniLM-L6-v2`, `openai://text-embedding-ada-002`) | | RERANKERS_ENABLED | (Optional) Enable or disable document reranking for improved search results (e.g., `TRUE` or `FALSE`, default: `FALSE`) | | RERANKERS_MODEL_NAME | Name of the reranker model (e.g., `ms-marco-MiniLM-L-12-v2`) (required if RERANKERS_ENABLED=TRUE) | @@ -83,6 +84,7 @@ Edit the `.env` file and set the following variables: | STT_SERVICE | Speech-to-Text API provider for Audio Files (e.g., `local/base`, `openai/whisper-1`). See [supported providers](https://docs.litellm.ai/docs/audio_transcription#supported-providers) | | STT_SERVICE_API_KEY | (Optional if local) API key for the Speech-to-Text service | | STT_SERVICE_API_BASE | (Optional) Custom API base URL for the Speech-to-Text service | +| FIRECRAWL_API_KEY | (Optional) API key for Firecrawl service for web crawling | | ETL_SERVICE | Document parsing service: `UNSTRUCTURED` (supports 34+ formats), `LLAMACLOUD` (supports 50+ formats including legacy document types), or `DOCLING` (local processing, supports PDF, Office docs, images, HTML, CSV) | | UNSTRUCTURED_API_KEY | API key for Unstructured.io service for document parsing (required if ETL_SERVICE=UNSTRUCTURED) | | LLAMA_CLOUD_API_KEY | API key for LlamaCloud service for document parsing (required if ETL_SERVICE=LLAMACLOUD) | @@ -92,6 +94,43 @@ Edit the `.env` file and set the following variables: | REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) | | PAGES_LIMIT | (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) | +**Google Connector OAuth Configuration:** +| ENV VARIABLE | DESCRIPTION | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| GOOGLE_CALENDAR_REDIRECT_URI | (Optional) Redirect URI for Google Calendar connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/calendar/connector/callback`) | +| GOOGLE_GMAIL_REDIRECT_URI | (Optional) Redirect URI for Gmail connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/gmail/connector/callback`) | +| GOOGLE_DRIVE_REDIRECT_URI | (Optional) Redirect URI for Google Drive connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/drive/connector/callback`) | + +**Connector OAuth Configurations (Optional):** + +| ENV VARIABLE | DESCRIPTION | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| AIRTABLE_CLIENT_ID | (Optional) Airtable OAuth client ID from [Airtable Developer Hub](https://airtable.com/create/oauth) | +| AIRTABLE_CLIENT_SECRET | (Optional) Airtable OAuth client secret | +| AIRTABLE_REDIRECT_URI | (Optional) Redirect URI for Airtable connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/airtable/connector/callback`) | +| CLICKUP_CLIENT_ID | (Optional) ClickUp OAuth client ID | +| CLICKUP_CLIENT_SECRET | (Optional) ClickUp OAuth client secret | +| CLICKUP_REDIRECT_URI | (Optional) Redirect URI for ClickUp connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/clickup/connector/callback`) | +| DISCORD_CLIENT_ID | (Optional) Discord OAuth client ID | +| DISCORD_CLIENT_SECRET | (Optional) Discord OAuth client secret | +| DISCORD_REDIRECT_URI | (Optional) Redirect URI for Discord connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/discord/connector/callback`) | +| DISCORD_BOT_TOKEN | (Optional) Discord bot token from Developer Portal | +| ATLASSIAN_CLIENT_ID | (Optional) Atlassian OAuth client ID (for Jira and Confluence) | +| ATLASSIAN_CLIENT_SECRET | (Optional) Atlassian OAuth client secret | +| JIRA_REDIRECT_URI | (Optional) Redirect URI for Jira connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/jira/connector/callback`) | +| CONFLUENCE_REDIRECT_URI | (Optional) Redirect URI for Confluence connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/confluence/connector/callback`) | +| LINEAR_CLIENT_ID | (Optional) Linear OAuth client ID | +| LINEAR_CLIENT_SECRET | (Optional) Linear OAuth client secret | +| LINEAR_REDIRECT_URI | (Optional) Redirect URI for Linear connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/linear/connector/callback`) | +| NOTION_CLIENT_ID | (Optional) Notion OAuth client ID | +| NOTION_CLIENT_SECRET | (Optional) Notion OAuth client secret | +| NOTION_REDIRECT_URI | (Optional) Redirect URI for Notion connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/notion/connector/callback`) | +| SLACK_CLIENT_ID | (Optional) Slack OAuth client ID | +| SLACK_CLIENT_SECRET | (Optional) Slack OAuth client secret | +| SLACK_REDIRECT_URI | (Optional) Redirect URI for Slack connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/slack/connector/callback`) | +| TEAMS_CLIENT_ID | (Optional) Microsoft Teams OAuth client ID | +| TEAMS_CLIENT_SECRET | (Optional) Microsoft Teams OAuth client secret | +| TEAMS_REDIRECT_URI | (Optional) Redirect URI for Teams connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/teams/connector/callback`) | **(Optional) Backend LangSmith Observability:** | ENV VARIABLE | DESCRIPTION | @@ -368,6 +407,8 @@ Edit the `.env` file and set: | NEXT_PUBLIC_FASTAPI_BACKEND_URL | Backend URL (e.g., `http://localhost:8000`) | | NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE | Same value as set in backend AUTH_TYPE i.e `GOOGLE` for OAuth with Google, `LOCAL` for email/password authentication | | NEXT_PUBLIC_ETL_SERVICE | Document parsing service (should match backend ETL_SERVICE): `UNSTRUCTURED`, `LLAMACLOUD`, or `DOCLING` - affects supported file formats in upload interface | +| NEXT_PUBLIC_ELECTRIC_URL | URL for Electric-SQL service (e.g., `http://localhost:5133`) | +| NEXT_PUBLIC_ELECTRIC_AUTH_MODE | Electric-SQL authentication mode (default: `insecure`) | ### 2. Install Dependencies diff --git a/surfsense_web/contracts/enums/connectorIcons.tsx b/surfsense_web/contracts/enums/connectorIcons.tsx index 5f147b63b..9350b6a1e 100644 --- a/surfsense_web/contracts/enums/connectorIcons.tsx +++ b/surfsense_web/contracts/enums/connectorIcons.tsx @@ -65,7 +65,7 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas case EnumConnectorName.CIRCLEBACK_CONNECTOR: return ; case EnumConnectorName.MCP_CONNECTOR: - return ; + return MCP; // Additional cases for non-enum connector types case "YOUTUBE_CONNECTOR": return YouTube; diff --git a/surfsense_web/hooks/use-connectors-electric.ts b/surfsense_web/hooks/use-connectors-electric.ts index 94d5062c9..08ef0621d 100644 --- a/surfsense_web/hooks/use-connectors-electric.ts +++ b/surfsense_web/hooks/use-connectors-electric.ts @@ -1,9 +1,9 @@ "use client"; -import { useEffect, useState, useCallback, useRef } from "react"; -import { useElectricClient } from "@/lib/electric/context"; -import type { SyncHandle } from "@/lib/electric/client"; +import { useCallback, useEffect, useRef, useState } from "react"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import type { SyncHandle } from "@/lib/electric/client"; +import { useElectricClient } from "@/lib/electric/context"; /** * Hook for managing connectors with Electric SQL real-time sync diff --git a/surfsense_web/hooks/use-documents-electric.ts b/surfsense_web/hooks/use-documents-electric.ts index 74d9e91e7..43809499e 100644 --- a/surfsense_web/hooks/use-documents-electric.ts +++ b/surfsense_web/hooks/use-documents-electric.ts @@ -1,8 +1,8 @@ "use client"; -import { useEffect, useState, useRef, useMemo } from "react"; -import { useElectricClient } from "@/lib/electric/context"; +import { useEffect, useMemo, useRef, useState } from "react"; import type { SyncHandle } from "@/lib/electric/client"; +import { useElectricClient } from "@/lib/electric/context"; interface Document { id: number; diff --git a/surfsense_web/hooks/use-notifications.ts b/surfsense_web/hooks/use-notifications.ts index 7a3b49861..fbdf421de 100644 --- a/surfsense_web/hooks/use-notifications.ts +++ b/surfsense_web/hooks/use-notifications.ts @@ -1,10 +1,10 @@ "use client"; -import { useEffect, useState, useCallback, useRef } from "react"; -import { useElectricClient } from "@/lib/electric/context"; -import type { SyncHandle } from "@/lib/electric/client"; +import { useCallback, useEffect, useRef, useState } from "react"; import type { Notification } from "@/contracts/types/notification.types"; import { authenticatedFetch } from "@/lib/auth-utils"; +import type { SyncHandle } from "@/lib/electric/client"; +import { useElectricClient } from "@/lib/electric/context"; export type { Notification } from "@/contracts/types/notification.types"; diff --git a/surfsense_web/lib/electric/client.ts b/surfsense_web/lib/electric/client.ts index c33969914..514185d23 100644 --- a/surfsense_web/lib/electric/client.ts +++ b/surfsense_web/lib/electric/client.ts @@ -13,8 +13,8 @@ */ import { PGlite } from "@electric-sql/pglite"; -import { electricSync } from "@electric-sql/pglite-sync"; import { live } from "@electric-sql/pglite/live"; +import { electricSync } from "@electric-sql/pglite-sync"; // Types export interface ElectricClient { @@ -270,365 +270,375 @@ export async function initElectric(userId: string): Promise { // Create and track the sync promise to prevent race conditions const syncPromise = (async (): Promise => { // Build params for the shape request - // Electric SQL expects params as URL query parameters - const params: Record = { table }; + // Electric SQL expects params as URL query parameters + const params: Record = { table }; - // Validate and fix WHERE clause to ensure string literals are properly quoted - let validatedWhere = where; - if (where) { - // Check if where uses positional parameters - if (where.includes("$1")) { - // Extract the value from the where clause if it's embedded - // For now, we'll use the where clause as-is and let Electric handle it - params.where = where; - validatedWhere = where; - } else { - // Validate that string literals are properly quoted - // Count single quotes - should be even (pairs) for properly quoted strings - const singleQuoteCount = (where.match(/'/g) || []).length; - - if (singleQuoteCount % 2 !== 0) { - // Odd number of quotes means unterminated string literal - console.warn("Where clause has unmatched quotes, fixing:", where); - // Add closing quote at the end - validatedWhere = `${where}'`; - params.where = validatedWhere; - } else { - // Use the where clause directly (already formatted) + // Validate and fix WHERE clause to ensure string literals are properly quoted + let validatedWhere = where; + if (where) { + // Check if where uses positional parameters + if (where.includes("$1")) { + // Extract the value from the where clause if it's embedded + // For now, we'll use the where clause as-is and let Electric handle it params.where = where; validatedWhere = where; + } else { + // Validate that string literals are properly quoted + // Count single quotes - should be even (pairs) for properly quoted strings + const singleQuoteCount = (where.match(/'/g) || []).length; + + if (singleQuoteCount % 2 !== 0) { + // Odd number of quotes means unterminated string literal + console.warn("Where clause has unmatched quotes, fixing:", where); + // Add closing quote at the end + validatedWhere = `${where}'`; + params.where = validatedWhere; + } else { + // Use the where clause directly (already formatted) + params.where = where; + validatedWhere = where; + } } } - } - if (columns) params.columns = columns.join(","); + if (columns) params.columns = columns.join(","); - console.log("[Electric] Syncing shape with params:", params); - console.log("[Electric] Electric URL:", `${electricUrl}/v1/shape`); - console.log("[Electric] Where clause:", where, "Validated:", validatedWhere); + console.log("[Electric] Syncing shape with params:", params); + console.log("[Electric] Electric URL:", `${electricUrl}/v1/shape`); + console.log("[Electric] Where clause:", where, "Validated:", validatedWhere); - try { - // Debug: Test Electric SQL connection directly first - // Use validatedWhere to ensure proper URL encoding - const testUrl = `${electricUrl}/v1/shape?table=${table}&offset=-1${validatedWhere ? `&where=${encodeURIComponent(validatedWhere)}` : ""}`; - console.log("[Electric] Testing Electric SQL directly:", testUrl); try { - const testResponse = await fetch(testUrl); - const testHeaders = { - handle: testResponse.headers.get("electric-handle"), - offset: testResponse.headers.get("electric-offset"), - upToDate: testResponse.headers.get("electric-up-to-date"), - }; - console.log("[Electric] Direct Electric SQL response headers:", testHeaders); - const testData = await testResponse.json(); - console.log( - "[Electric] Direct Electric SQL data count:", - Array.isArray(testData) ? testData.length : "not array", - testData - ); - } catch (testErr) { - console.error("[Electric] Direct Electric SQL test failed:", testErr); - } - - // Use PGlite's electric sync plugin to sync the shape - // According to Electric SQL docs, the shape config uses params for table, where, columns - // Note: mapColumns is OPTIONAL per pglite-sync types.ts - - // Create a promise that resolves when initial sync is complete - // Using recommended approach: check isUpToDate immediately, watch stream, shorter timeout - // IMPORTANT: We don't unsubscribe from the stream - it must stay active for real-time updates - let syncResolved = false; - // Initialize with no-op functions to satisfy TypeScript - let resolveInitialSync: () => void = () => {}; - let rejectInitialSync: (error: Error) => void = () => {}; - - const initialSyncPromise = new Promise((resolve, reject) => { - resolveInitialSync = () => { - if (!syncResolved) { - syncResolved = true; - // DON'T unsubscribe from stream - it needs to stay active for real-time updates - resolve(); - } - }; - rejectInitialSync = (error: Error) => { - if (!syncResolved) { - syncResolved = true; - // DON'T unsubscribe from stream even on error - let Electric handle it - reject(error); - } - }; - - // Shorter timeout (5 seconds) as fallback - setTimeout(() => { - if (!syncResolved) { - console.warn( - `[Electric] ⚠️ Sync timeout for ${table} - checking isUpToDate one more time...` - ); - // Check isUpToDate one more time before resolving - // This will be checked after shape is created - setTimeout(() => { - if (!syncResolved) { - console.warn( - `[Electric] ⚠️ Sync timeout for ${table} - resolving anyway after 5s` - ); - resolveInitialSync(); - } - }, 100); - } - }, 5000); - }); - - // Include userId in shapeKey for user-specific sync state - const shapeConfig = { - shape: { - url: `${electricUrl}/v1/shape`, - params: { - table, - ...(validatedWhere ? { where: validatedWhere } : {}), - ...(columns ? { columns: columns.join(",") } : {}), - }, - }, - table, - primaryKey, - shapeKey: `${userId}_v${SYNC_VERSION}_${table}_${where?.replace(/[^a-zA-Z0-9]/g, "_") || "all"}`, // User-specific versioned key - onInitialSync: () => { + // Debug: Test Electric SQL connection directly first + // Use validatedWhere to ensure proper URL encoding + const testUrl = `${electricUrl}/v1/shape?table=${table}&offset=-1${validatedWhere ? `&where=${encodeURIComponent(validatedWhere)}` : ""}`; + console.log("[Electric] Testing Electric SQL directly:", testUrl); + try { + const testResponse = await fetch(testUrl); + const testHeaders = { + handle: testResponse.headers.get("electric-handle"), + offset: testResponse.headers.get("electric-offset"), + upToDate: testResponse.headers.get("electric-up-to-date"), + }; + console.log("[Electric] Direct Electric SQL response headers:", testHeaders); + const testData = await testResponse.json(); console.log( - `[Electric] ✅ Initial sync complete for ${table} - data should now be in PGlite` + "[Electric] Direct Electric SQL data count:", + Array.isArray(testData) ? testData.length : "not array", + testData + ); + } catch (testErr) { + console.error("[Electric] Direct Electric SQL test failed:", testErr); + } + + // Use PGlite's electric sync plugin to sync the shape + // According to Electric SQL docs, the shape config uses params for table, where, columns + // Note: mapColumns is OPTIONAL per pglite-sync types.ts + + // Create a promise that resolves when initial sync is complete + // Using recommended approach: check isUpToDate immediately, watch stream, shorter timeout + // IMPORTANT: We don't unsubscribe from the stream - it must stay active for real-time updates + let syncResolved = false; + // Initialize with no-op functions to satisfy TypeScript + let resolveInitialSync: () => void = () => {}; + let rejectInitialSync: (error: Error) => void = () => {}; + + const initialSyncPromise = new Promise((resolve, reject) => { + resolveInitialSync = () => { + if (!syncResolved) { + syncResolved = true; + // DON'T unsubscribe from stream - it needs to stay active for real-time updates + resolve(); + } + }; + rejectInitialSync = (error: Error) => { + if (!syncResolved) { + syncResolved = true; + // DON'T unsubscribe from stream even on error - let Electric handle it + reject(error); + } + }; + + // Shorter timeout (5 seconds) as fallback + setTimeout(() => { + if (!syncResolved) { + console.warn( + `[Electric] ⚠️ Sync timeout for ${table} - checking isUpToDate one more time...` + ); + // Check isUpToDate one more time before resolving + // This will be checked after shape is created + setTimeout(() => { + if (!syncResolved) { + console.warn( + `[Electric] ⚠️ Sync timeout for ${table} - resolving anyway after 5s` + ); + resolveInitialSync(); + } + }, 100); + } + }, 5000); + }); + + // Include userId in shapeKey for user-specific sync state + const shapeConfig = { + shape: { + url: `${electricUrl}/v1/shape`, + params: { + table, + ...(validatedWhere ? { where: validatedWhere } : {}), + ...(columns ? { columns: columns.join(",") } : {}), + }, + }, + table, + primaryKey, + shapeKey: `${userId}_v${SYNC_VERSION}_${table}_${where?.replace(/[^a-zA-Z0-9]/g, "_") || "all"}`, // User-specific versioned key + onInitialSync: () => { + console.log( + `[Electric] ✅ Initial sync complete for ${table} - data should now be in PGlite` + ); + resolveInitialSync(); + }, + onError: (error: Error) => { + console.error(`[Electric] ❌ Shape sync error for ${table}:`, error); + console.error( + "[Electric] Error details:", + JSON.stringify(error, Object.getOwnPropertyNames(error)) + ); + rejectInitialSync(error); + }, + }; + + console.log( + "[Electric] syncShapeToTable config:", + JSON.stringify(shapeConfig, null, 2) + ); + + // Type assertion to PGlite with electric extension + const pgWithElectric = db as PGlite & { + electric: { + syncShapeToTable: ( + config: typeof shapeConfig + ) => Promise<{ unsubscribe: () => void; isUpToDate: boolean; stream: unknown }>; + }; + }; + + let shape: { unsubscribe: () => void; isUpToDate: boolean; stream: unknown }; + try { + shape = await pgWithElectric.electric.syncShapeToTable(shapeConfig); + } catch (syncError) { + // Handle "Already syncing" error - pglite-sync might not have fully cleaned up yet + const errorMessage = + syncError instanceof Error ? syncError.message : String(syncError); + if (errorMessage.includes("Already syncing")) { + console.warn( + `[Electric] Already syncing ${table}, waiting for existing sync to settle...` + ); + + // Wait a short time for pglite-sync to settle + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Check if an active handle now exists (another sync might have completed) + const existingHandle = activeSyncHandles.get(cacheKey); + if (existingHandle) { + console.log(`[Electric] Found existing handle after waiting: ${cacheKey}`); + return existingHandle; + } + + // Retry once after waiting + console.log(`[Electric] Retrying sync for ${table}...`); + try { + shape = await pgWithElectric.electric.syncShapeToTable(shapeConfig); + } catch (retryError) { + const retryMessage = + retryError instanceof Error ? retryError.message : String(retryError); + if (retryMessage.includes("Already syncing")) { + // Still syncing - create a placeholder handle that indicates the table is being synced + console.warn( + `[Electric] ${table} still syncing, creating placeholder handle` + ); + const placeholderHandle: SyncHandle = { + unsubscribe: () => { + console.log(`[Electric] Placeholder unsubscribe for: ${cacheKey}`); + activeSyncHandles.delete(cacheKey); + }, + get isUpToDate() { + return false; // We don't know the real state + }, + stream: undefined, + initialSyncPromise: Promise.resolve(), // Already syncing means data should be coming + }; + activeSyncHandles.set(cacheKey, placeholderHandle); + return placeholderHandle; + } + throw retryError; + } + } else { + throw syncError; + } + } + + if (!shape) { + throw new Error("syncShapeToTable returned undefined"); + } + + // Log the actual shape result structure + console.log("[Electric] Shape sync result (initial):", { + hasUnsubscribe: typeof shape?.unsubscribe === "function", + isUpToDate: shape?.isUpToDate, + hasStream: !!shape?.stream, + streamType: typeof shape?.stream, + }); + + // Recommended Approach Step 1: Check isUpToDate immediately + if (shape.isUpToDate) { + console.log( + `[Electric] ✅ Sync already up-to-date for ${table} (resuming from previous state)` ); resolveInitialSync(); - }, - onError: (error: Error) => { - console.error(`[Electric] ❌ Shape sync error for ${table}:`, error); - console.error( - "[Electric] Error details:", - JSON.stringify(error, Object.getOwnPropertyNames(error)) - ); - rejectInitialSync(error); - }, - }; - - console.log( - "[Electric] syncShapeToTable config:", - JSON.stringify(shapeConfig, null, 2) - ); - - // Type assertion to PGlite with electric extension - const pgWithElectric = db as PGlite & { - electric: { - syncShapeToTable: ( - config: typeof shapeConfig - ) => Promise<{ unsubscribe: () => void; isUpToDate: boolean; stream: unknown }>; - }; - }; - - let shape: { unsubscribe: () => void; isUpToDate: boolean; stream: unknown }; - try { - shape = await pgWithElectric.electric.syncShapeToTable(shapeConfig); - } catch (syncError) { - // Handle "Already syncing" error - pglite-sync might not have fully cleaned up yet - const errorMessage = syncError instanceof Error ? syncError.message : String(syncError); - if (errorMessage.includes("Already syncing")) { - console.warn(`[Electric] Already syncing ${table}, waiting for existing sync to settle...`); - - // Wait a short time for pglite-sync to settle - await new Promise(resolve => setTimeout(resolve, 100)); - - // Check if an active handle now exists (another sync might have completed) - const existingHandle = activeSyncHandles.get(cacheKey); - if (existingHandle) { - console.log(`[Electric] Found existing handle after waiting: ${cacheKey}`); - return existingHandle; - } - - // Retry once after waiting - console.log(`[Electric] Retrying sync for ${table}...`); - try { - shape = await pgWithElectric.electric.syncShapeToTable(shapeConfig); - } catch (retryError) { - const retryMessage = retryError instanceof Error ? retryError.message : String(retryError); - if (retryMessage.includes("Already syncing")) { - // Still syncing - create a placeholder handle that indicates the table is being synced - console.warn(`[Electric] ${table} still syncing, creating placeholder handle`); - const placeholderHandle: SyncHandle = { - unsubscribe: () => { - console.log(`[Electric] Placeholder unsubscribe for: ${cacheKey}`); - activeSyncHandles.delete(cacheKey); - }, - get isUpToDate() { - return false; // We don't know the real state - }, - stream: undefined, - initialSyncPromise: Promise.resolve(), // Already syncing means data should be coming - }; - activeSyncHandles.set(cacheKey, placeholderHandle); - return placeholderHandle; - } - throw retryError; - } } else { - throw syncError; - } - } - - if (!shape) { - throw new Error("syncShapeToTable returned undefined"); - } - - // Log the actual shape result structure - console.log("[Electric] Shape sync result (initial):", { - hasUnsubscribe: typeof shape?.unsubscribe === "function", - isUpToDate: shape?.isUpToDate, - hasStream: !!shape?.stream, - streamType: typeof shape?.stream, - }); - - // Recommended Approach Step 1: Check isUpToDate immediately - if (shape.isUpToDate) { - console.log( - `[Electric] ✅ Sync already up-to-date for ${table} (resuming from previous state)` - ); - resolveInitialSync(); - } else { - // Recommended Approach Step 2: Subscribe to stream and watch for "up-to-date" message - if (shape?.stream) { - const stream = shape.stream as any; - console.log("[Electric] Shape stream details:", { - shapeHandle: stream?.shapeHandle, - lastOffset: stream?.lastOffset, - isUpToDate: stream?.isUpToDate, - error: stream?.error, - hasSubscribe: typeof stream?.subscribe === "function", - hasUnsubscribe: typeof stream?.unsubscribe === "function", - }); - - // Subscribe to the stream to watch for "up-to-date" control message - // NOTE: We keep this subscription active - don't unsubscribe! - // The stream is what Electric SQL uses for real-time updates - if (typeof stream?.subscribe === "function") { - console.log( - "[Electric] Subscribing to shape stream to watch for up-to-date message..." - ); - // Subscribe but don't store unsubscribe - we want it to stay active - stream.subscribe((messages: unknown[]) => { - // Continue receiving updates even after sync is resolved - if (!syncResolved) { - console.log( - "[Electric] 🔵 Shape stream received messages:", - messages?.length || 0 - ); - } - - // Check if any message indicates sync is complete - if (messages && messages.length > 0) { - for (const message of messages) { - const msg = message as any; - // Check for "up-to-date" control message - if ( - msg?.headers?.control === "up-to-date" || - msg?.headers?.electric_up_to_date === "true" || - (typeof msg === "object" && "up-to-date" in msg) - ) { - if (!syncResolved) { - console.log(`[Electric] ✅ Received up-to-date message for ${table}`); - resolveInitialSync(); - } - // Continue listening for real-time updates - don't return! - } - } - if (!syncResolved && messages.length > 0) { - console.log( - "[Electric] First message:", - JSON.stringify(messages[0], null, 2) - ); - } - } - - // Also check stream's isUpToDate property after receiving messages - if (!syncResolved && stream?.isUpToDate) { - console.log(`[Electric] ✅ Stream isUpToDate is true for ${table}`); - resolveInitialSync(); - } + // Recommended Approach Step 2: Subscribe to stream and watch for "up-to-date" message + if (shape?.stream) { + const stream = shape.stream as any; + console.log("[Electric] Shape stream details:", { + shapeHandle: stream?.shapeHandle, + lastOffset: stream?.lastOffset, + isUpToDate: stream?.isUpToDate, + error: stream?.error, + hasSubscribe: typeof stream?.subscribe === "function", + hasUnsubscribe: typeof stream?.unsubscribe === "function", }); - // Also check stream's isUpToDate property immediately - if (stream?.isUpToDate) { - console.log(`[Electric] ✅ Stream isUpToDate is true immediately for ${table}`); - resolveInitialSync(); + // Subscribe to the stream to watch for "up-to-date" control message + // NOTE: We keep this subscription active - don't unsubscribe! + // The stream is what Electric SQL uses for real-time updates + if (typeof stream?.subscribe === "function") { + console.log( + "[Electric] Subscribing to shape stream to watch for up-to-date message..." + ); + // Subscribe but don't store unsubscribe - we want it to stay active + stream.subscribe((messages: unknown[]) => { + // Continue receiving updates even after sync is resolved + if (!syncResolved) { + console.log( + "[Electric] 🔵 Shape stream received messages:", + messages?.length || 0 + ); + } + + // Check if any message indicates sync is complete + if (messages && messages.length > 0) { + for (const message of messages) { + const msg = message as any; + // Check for "up-to-date" control message + if ( + msg?.headers?.control === "up-to-date" || + msg?.headers?.electric_up_to_date === "true" || + (typeof msg === "object" && "up-to-date" in msg) + ) { + if (!syncResolved) { + console.log(`[Electric] ✅ Received up-to-date message for ${table}`); + resolveInitialSync(); + } + // Continue listening for real-time updates - don't return! + } + } + if (!syncResolved && messages.length > 0) { + console.log( + "[Electric] First message:", + JSON.stringify(messages[0], null, 2) + ); + } + } + + // Also check stream's isUpToDate property after receiving messages + if (!syncResolved && stream?.isUpToDate) { + console.log(`[Electric] ✅ Stream isUpToDate is true for ${table}`); + resolveInitialSync(); + } + }); + + // Also check stream's isUpToDate property immediately + if (stream?.isUpToDate) { + console.log( + `[Electric] ✅ Stream isUpToDate is true immediately for ${table}` + ); + resolveInitialSync(); + } } + + // Also poll isUpToDate periodically as a backup (every 200ms) + const pollInterval = setInterval(() => { + if (syncResolved) { + clearInterval(pollInterval); + return; + } + + if (shape.isUpToDate || stream?.isUpToDate) { + console.log( + `[Electric] ✅ Sync completed (detected via polling) for ${table}` + ); + clearInterval(pollInterval); + resolveInitialSync(); + } + }, 200); + + // Clean up polling when promise resolves + initialSyncPromise.finally(() => { + clearInterval(pollInterval); + }); + } else { + console.warn( + `[Electric] ⚠️ No stream available for ${table}, relying on callback and timeout` + ); } - - // Also poll isUpToDate periodically as a backup (every 200ms) - const pollInterval = setInterval(() => { - if (syncResolved) { - clearInterval(pollInterval); - return; - } - - if (shape.isUpToDate || stream?.isUpToDate) { - console.log(`[Electric] ✅ Sync completed (detected via polling) for ${table}`); - clearInterval(pollInterval); - resolveInitialSync(); - } - }, 200); - - // Clean up polling when promise resolves - initialSyncPromise.finally(() => { - clearInterval(pollInterval); - }); - } else { - console.warn( - `[Electric] ⚠️ No stream available for ${table}, relying on callback and timeout` - ); } - } - // Create the sync handle with proper cleanup - const syncHandle: SyncHandle = { - unsubscribe: () => { - console.log(`[Electric] Unsubscribing from: ${cacheKey}`); - // Remove from cache first - activeSyncHandles.delete(cacheKey); - // Then unsubscribe from the shape - if (shape && typeof shape.unsubscribe === "function") { - shape.unsubscribe(); - } - }, - // Use getter to always return current state - get isUpToDate() { - return shape?.isUpToDate ?? false; - }, - stream: shape?.stream, - initialSyncPromise, // Expose promise so callers can wait for sync - }; + // Create the sync handle with proper cleanup + const syncHandle: SyncHandle = { + unsubscribe: () => { + console.log(`[Electric] Unsubscribing from: ${cacheKey}`); + // Remove from cache first + activeSyncHandles.delete(cacheKey); + // Then unsubscribe from the shape + if (shape && typeof shape.unsubscribe === "function") { + shape.unsubscribe(); + } + }, + // Use getter to always return current state + get isUpToDate() { + return shape?.isUpToDate ?? false; + }, + stream: shape?.stream, + initialSyncPromise, // Expose promise so callers can wait for sync + }; - // Cache the sync handle for reuse (memory optimization) - activeSyncHandles.set(cacheKey, syncHandle); - console.log( - `[Electric] Cached sync handle for: ${cacheKey} (total cached: ${activeSyncHandles.size})` - ); - - return syncHandle; - } catch (error) { - console.error("[Electric] Failed to sync shape:", error); - // Check if Electric SQL server is reachable - try { - const response = await fetch(`${electricUrl}/v1/shape?table=${table}&offset=-1`, { - method: "GET", - }); + // Cache the sync handle for reuse (memory optimization) + activeSyncHandles.set(cacheKey, syncHandle); console.log( - "[Electric] Electric SQL server response:", - response.status, - response.statusText + `[Electric] Cached sync handle for: ${cacheKey} (total cached: ${activeSyncHandles.size})` ); - if (!response.ok) { - console.error("[Electric] Electric SQL server error:", await response.text()); + + return syncHandle; + } catch (error) { + console.error("[Electric] Failed to sync shape:", error); + // Check if Electric SQL server is reachable + try { + const response = await fetch(`${electricUrl}/v1/shape?table=${table}&offset=-1`, { + method: "GET", + }); + console.log( + "[Electric] Electric SQL server response:", + response.status, + response.statusText + ); + if (!response.ok) { + console.error("[Electric] Electric SQL server error:", await response.text()); + } + } catch (fetchError) { + console.error("[Electric] Cannot reach Electric SQL server:", fetchError); + console.error("[Electric] Make sure Electric SQL is running at:", electricUrl); } - } catch (fetchError) { - console.error("[Electric] Cannot reach Electric SQL server:", fetchError); - console.error("[Electric] Make sure Electric SQL is running at:", electricUrl); + throw error; } - throw error; - } })(); // Track the sync promise to prevent concurrent syncs for the same shape diff --git a/surfsense_web/public/connectors/modelcontextprotocol.svg b/surfsense_web/public/connectors/modelcontextprotocol.svg new file mode 100644 index 000000000..e9c3fa46e --- /dev/null +++ b/surfsense_web/public/connectors/modelcontextprotocol.svg @@ -0,0 +1 @@ + \ No newline at end of file