diff --git a/.vscode/launch.json b/.vscode/launch.json index dfe20d832..ad7f04bd0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,7 +5,7 @@ "version": "0.2.0", "configurations": [ { - "name": "Python Debugger: UV Run with Reload", + "name": "Backend: FastAPI", "type": "debugpy", "request": "launch", "module": "uvicorn", @@ -25,7 +25,7 @@ "python": "${command:python.interpreterPath}" }, { - "name": "Python Debugger: main.py (direct)", + "name": "Backend: FastAPI (main.py)", "type": "debugpy", "request": "launch", "program": "${workspaceFolder}/surfsense_backend/main.py", @@ -34,17 +34,95 @@ "cwd": "${workspaceFolder}/surfsense_backend" }, { - "name": "Python Debugger: Chat DeepAgent", + "name": "Frontend: Next.js", + "type": "node", + "request": "launch", + "cwd": "${workspaceFolder}/surfsense_web", + "runtimeExecutable": "npm", + "runtimeArgs": ["run", "dev"], + "console": "integratedTerminal", + "serverReadyAction": { + "pattern": "- Local:.+(https?://.+)", + "uriFormat": "%s", + "action": "debugWithChrome" + } + }, + { + "name": "Frontend: Next.js (Server-Side Debug)", + "type": "node", + "request": "launch", + "cwd": "${workspaceFolder}/surfsense_web", + "runtimeExecutable": "npm", + "runtimeArgs": ["run", "debug:server"], + "console": "integratedTerminal", + "serverReadyAction": { + "pattern": "- Local:.+(https?://.+)", + "uriFormat": "%s", + "action": "debugWithChrome" + } + }, + { + "name": "Celery: Worker", "type": "debugpy", "request": "launch", - "module": "app.agents.new_chat.chat_deepagent", + "module": "celery", + "args": [ + "-A", + "app.celery_app:celery_app", + "worker", + "--loglevel=info", + "--pool=solo" + ], "console": "integratedTerminal", "justMyCode": false, "cwd": "${workspaceFolder}/surfsense_backend", - "python": "${command:python.interpreterPath}", - "env": { - "PYTHONPATH": "${workspaceFolder}/surfsense_backend" + "python": "${command:python.interpreterPath}" + }, + { + "name": "Celery: Beat Scheduler", + "type": "debugpy", + "request": "launch", + "module": "celery", + "args": [ + "-A", + "app.celery_app:celery_app", + "beat", + "--loglevel=info" + ], + "console": "integratedTerminal", + "justMyCode": false, + "cwd": "${workspaceFolder}/surfsense_backend", + "python": "${command:python.interpreterPath}" + } + ], + "compounds": [ + { + "name": "Full Stack: Backend + Frontend + Celery", + "configurations": [ + "Backend: FastAPI", + "Frontend: Next.js", + "Celery: Worker", + "Celery: Beat Scheduler" + ], + "stopAll": true, + "presentation": { + "hidden": false, + "group": "Full Stack", + "order": 1 + } + }, + { + "name": "Full Stack: Backend + Frontend", + "configurations": [ + "Backend: FastAPI", + "Frontend: Next.js" + ], + "stopAll": true, + "presentation": { + "hidden": false, + "group": "Full Stack", + "order": 2 } } ] -} \ No newline at end of file +} diff --git a/Dockerfile.allinone b/Dockerfile.allinone index 12eee5c90..1c04ffb99 100644 --- a/Dockerfile.allinone +++ b/Dockerfile.allinone @@ -37,14 +37,11 @@ COPY surfsense_web/ ./ # Run fumadocs-mdx postinstall now that source files are available RUN pnpm fumadocs-mdx -# Build args for frontend -ARG NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000 -ARG NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL -ARG NEXT_PUBLIC_ETL_SERVICE=DOCLING - -ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=$NEXT_PUBLIC_FASTAPI_BACKEND_URL -ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=$NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE -ENV NEXT_PUBLIC_ETL_SERVICE=$NEXT_PUBLIC_ETL_SERVICE +# Build with placeholder values that will be replaced at runtime +# These unique strings allow runtime substitution via entrypoint script +ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__ +ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__ +ENV NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__ # Build RUN pnpm run build @@ -233,6 +230,12 @@ ENV AUTH_TYPE=LOCAL ENV ETL_SERVICE=DOCLING ENV EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 +# Frontend configuration (can be overridden at runtime) +# These are injected into the Next.js build at container startup +ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000 +ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL +ENV NEXT_PUBLIC_ETL_SERVICE=DOCLING + # Data volume VOLUME ["/data"] diff --git a/README.md b/README.md index acd900588..4f2ce4332 100644 --- a/README.md +++ b/README.md @@ -174,44 +174,27 @@ docker run -d -p 3000:3000 -p 8000:8000 ` ghcr.io/modsetter/surfsense:latest ``` -**With Custom Configuration (e.g., OpenAI Embeddings):** +**With Custom Configuration:** + +You can pass any environment variable using `-e` flags: ```bash docker run -d -p 3000:3000 -p 8000:8000 \ -v surfsense-data:/data \ -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \ -e OPENAI_API_KEY=your_openai_api_key \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -**With OAuth-based Connectors (Google Calendar, Gmail, Drive, Airtable):** - -To use OAuth-based connectors, you need to configure the respective client credentials: - -```bash -docker run -d -p 3000:3000 -p 8000:8000 \ - -v surfsense-data:/data \ - # Google Connectors (Calendar, Gmail, Drive) + -e AUTH_TYPE=GOOGLE \ -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \ -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \ - -e GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback \ - -e GOOGLE_GMAIL_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/gmail/connector/callback \ - -e GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback \ - # Airtable Connector - -e AIRTABLE_CLIENT_ID=your_airtable_client_id \ - -e AIRTABLE_CLIENT_SECRET=your_airtable_client_secret \ - -e AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback \ + -e ETL_SERVICE=LLAMACLOUD \ + -e LLAMA_CLOUD_API_KEY=your_llama_cloud_key \ --name surfsense \ --restart unless-stopped \ ghcr.io/modsetter/surfsense:latest ``` > [!NOTE] -> - For Google connectors, create OAuth 2.0 credentials in the [Google Cloud Console](https://console.cloud.google.com/apis/credentials) -> - For Airtable connector, create an OAuth integration in the [Airtable Developer Hub](https://airtable.com/create/oauth) -> - If deploying behind a reverse proxy with HTTPS, add `-e BACKEND_URL=https://api.yourdomain.com` and update the redirect URIs accordingly +> - If deploying behind a reverse proxy with HTTPS, add `-e BACKEND_URL=https://api.yourdomain.com` After starting, access SurfSense at: - **Frontend**: [http://localhost:3000](http://localhost:3000) diff --git a/README.zh-CN.md b/README.zh-CN.md index 4e4b0174b..fe6ec8e30 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -181,44 +181,27 @@ docker run -d -p 3000:3000 -p 8000:8000 ` ghcr.io/modsetter/surfsense:latest ``` -**使用自定义配置(例如 OpenAI 嵌入):** +**使用自定义配置:** + +您可以使用 `-e` 标志传递任何环境变量: ```bash docker run -d -p 3000:3000 -p 8000:8000 \ -v surfsense-data:/data \ -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \ -e OPENAI_API_KEY=your_openai_api_key \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -**使用 OAuth 连接器(Google 日历、Gmail、云端硬盘、Airtable):** - -要使用基于 OAuth 的连接器,您需要配置相应的客户端凭据: - -```bash -docker run -d -p 3000:3000 -p 8000:8000 \ - -v surfsense-data:/data \ - # Google 连接器(日历、Gmail、云端硬盘) + -e AUTH_TYPE=GOOGLE \ -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \ -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \ - -e GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback \ - -e GOOGLE_GMAIL_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/gmail/connector/callback \ - -e GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback \ - # Airtable 连接器 - -e AIRTABLE_CLIENT_ID=your_airtable_client_id \ - -e AIRTABLE_CLIENT_SECRET=your_airtable_client_secret \ - -e AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback \ + -e ETL_SERVICE=LLAMACLOUD \ + -e LLAMA_CLOUD_API_KEY=your_llama_cloud_key \ --name surfsense \ --restart unless-stopped \ ghcr.io/modsetter/surfsense:latest ``` > [!NOTE] -> - 对于 Google 连接器,请在 [Google Cloud Console](https://console.cloud.google.com/apis/credentials) 中创建 OAuth 2.0 凭据 -> - 对于 Airtable 连接器,请在 [Airtable 开发者中心](https://airtable.com/create/oauth) 中创建 OAuth 集成 -> - 如果部署在带有 HTTPS 的反向代理后面,请添加 `-e BACKEND_URL=https://api.yourdomain.com` 并相应地更新重定向 URI +> - 如果部署在带有 HTTPS 的反向代理后面,请添加 `-e BACKEND_URL=https://api.yourdomain.com` 启动后,访问 SurfSense: - **前端**: [http://localhost:3000](http://localhost:3000) diff --git a/scripts/docker/entrypoint-allinone.sh b/scripts/docker/entrypoint-allinone.sh index 427256f6d..8248968ab 100644 --- a/scripts/docker/entrypoint-allinone.sh +++ b/scripts/docker/entrypoint-allinone.sh @@ -96,6 +96,30 @@ if [ -d /app/frontend/.next/standalone ]; then cp -r /app/frontend/.next/static /app/frontend/.next/static 2>/dev/null || true fi +# ================================================ +# Runtime Environment Variable Replacement +# ================================================ +# Next.js NEXT_PUBLIC_* vars are baked in at build time. +# This replaces placeholder values with actual runtime env vars. +echo "🔧 Applying runtime environment configuration..." + +# Set defaults if not provided +NEXT_PUBLIC_FASTAPI_BACKEND_URL="${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000}" +NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE="${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL}" +NEXT_PUBLIC_ETL_SERVICE="${NEXT_PUBLIC_ETL_SERVICE:-DOCLING}" + +# Replace placeholders in all JS files +find /app/frontend -type f \( -name "*.js" -o -name "*.json" \) -exec sed -i \ + -e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_URL__|${NEXT_PUBLIC_FASTAPI_BACKEND_URL}|g" \ + -e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__|${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}|g" \ + -e "s|__NEXT_PUBLIC_ETL_SERVICE__|${NEXT_PUBLIC_ETL_SERVICE}|g" \ + {} + + +echo "✅ Environment configuration applied" +echo " Backend URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}" +echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}" +echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}" + # ================================================ # Run database migrations # ================================================ @@ -135,10 +159,10 @@ echo "===========================================" echo " 📋 Configuration" echo "===========================================" echo " Frontend URL: http://localhost:3000" -echo " Backend API: http://localhost:8000" -echo " API Docs: http://localhost:8000/docs" -echo " Auth Type: ${AUTH_TYPE:-LOCAL}" -echo " ETL Service: ${ETL_SERVICE:-DOCLING}" +echo " Backend API: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}" +echo " API Docs: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}/docs" +echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}" +echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}" echo " TTS Service: ${TTS_SERVICE}" echo " STT Service: ${STT_SERVICE}" echo "===========================================" diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 91a0cb42f..2c2fec48b 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -34,17 +34,48 @@ REGISTRATION_ENABLED=TRUE or FALSE GOOGLE_OAUTH_CLIENT_ID=924507538m GOOGLE_OAUTH_CLIENT_SECRET=GOCSV -# Connector Specific Configs +# Google Connector Specific Configurations GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback GOOGLE_GMAIL_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/gmail/connector/callback GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback -GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback -# Airtable OAuth for Aitable Connector -AIRTABLE_CLIENT_ID=your_airtable_client_id -AIRTABLE_CLIENT_SECRET=your_airtable_client_secret +# Aitable OAuth Configuration +AIRTABLE_CLIENT_ID=your_airtable_client_id_here +AIRTABLE_CLIENT_SECRET=your_airtable_client_secret_here AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback +# ClickUp OAuth Configuration +CLICKUP_CLIENT_ID=your_clickup_client_id_here +CLICKUP_CLIENT_SECRET=your_clickup_client_secret_here +CLICKUP_REDIRECT_URI=http://localhost:8000/api/v1/auth/clickup/connector/callback + +# Discord OAuth Configuration +DISCORD_CLIENT_ID=your_discord_client_id_here +DISCORD_CLIENT_SECRET=your_discord_client_secret_here +DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback +DISCORD_BOT_TOKEN=your_bot_token_from_developer_portal + +# Atlassian OAuth Configuration +ATLASSIAN_CLIENT_ID=your_atlassian_client_id_here +ATLASSIAN_CLIENT_SECRET=your_atlassian_client_secret_here +JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback +CONFLUENCE_REDIRECT_URI=http://localhost:8000/api/v1/auth/confluence/connector/callback + +# Linear OAuth Configuration +LINEAR_CLIENT_ID=your_linear_client_id_here +LINEAR_CLIENT_SECRET=your_linear_client_secret_here +LINEAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/linear/connector/callback + +# Notion OAuth Configuration +NOTION_CLIENT_ID=your_notion_client_id_here +NOTION_CLIENT_SECRET=your_notion_client_secret_here +NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback + +# Slack OAuth Configuration +SLACK_CLIENT_ID=your_slack_client_id_here +SLACK_CLIENT_SECRET=your_slack_client_secret_here +SLACK_REDIRECT_URI=http://localhost:8000/api/v1/auth/slack/connector/callback + # Embedding Model # Examples: # # Get sentence transformers embeddings diff --git a/surfsense_backend/alembic/versions/57_allow_multiple_connectors_per_type.py b/surfsense_backend/alembic/versions/57_allow_multiple_connectors_per_type.py new file mode 100644 index 000000000..a1482ee4b --- /dev/null +++ b/surfsense_backend/alembic/versions/57_allow_multiple_connectors_per_type.py @@ -0,0 +1,55 @@ +"""Allow multiple connectors of same type per search space + +Revision ID: 57 +Revises: 56 +Create Date: 2026-01-06 12:00:00.000000 + +""" + +from collections.abc import Sequence + +from sqlalchemy import text + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "57" +down_revision: str | None = "56" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + connection = op.get_bind() + constraint_exists = connection.execute( + text(""" + SELECT 1 FROM information_schema.table_constraints + WHERE table_name='search_source_connectors' + AND constraint_type='UNIQUE' + AND constraint_name='uq_searchspace_user_connector_type' + """) + ).scalar() + if constraint_exists: + op.drop_constraint( + "uq_searchspace_user_connector_type", + "search_source_connectors", + type_="unique", + ) + + +def downgrade() -> None: + connection = op.get_bind() + constraint_exists = connection.execute( + text(""" + SELECT 1 FROM information_schema.table_constraints + WHERE table_name='search_source_connectors' + AND constraint_type='UNIQUE' + AND constraint_name='uq_searchspace_user_connector_type' + """) + ).scalar() + if not constraint_exists: + op.create_unique_constraint( + "uq_searchspace_user_connector_type", + "search_source_connectors", + ["search_space_id", "user_id", "connector_type"], + ) diff --git a/surfsense_backend/alembic/versions/58_unique_connector_name_per_space_user.py b/surfsense_backend/alembic/versions/58_unique_connector_name_per_space_user.py new file mode 100644 index 000000000..4dd8d7b70 --- /dev/null +++ b/surfsense_backend/alembic/versions/58_unique_connector_name_per_space_user.py @@ -0,0 +1,55 @@ +""" +Add unique constraint for (search_space_id, user_id, name) on search_source_connectors. + +Revision ID: 58 +Revises: 57 +Create Date: 2026-01-06 14:00:00.000000 + +""" + +from collections.abc import Sequence + +from sqlalchemy import text + +from alembic import op + +revision: str = "58" +down_revision: str | None = "57" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + connection = op.get_bind() + constraint_exists = connection.execute( + text(""" + SELECT 1 FROM information_schema.table_constraints + WHERE table_name='search_source_connectors' + AND constraint_type='UNIQUE' + AND constraint_name='uq_searchspace_user_connector_name' + """) + ).scalar() + if not constraint_exists: + op.create_unique_constraint( + "uq_searchspace_user_connector_name", + "search_source_connectors", + ["search_space_id", "user_id", "name"], + ) + + +def downgrade() -> None: + connection = op.get_bind() + constraint_exists = connection.execute( + text(""" + SELECT 1 FROM information_schema.table_constraints + WHERE table_name='search_source_connectors' + AND constraint_type='UNIQUE' + AND constraint_name='uq_searchspace_user_connector_name' + """) + ).scalar() + if constraint_exists: + op.drop_constraint( + "uq_searchspace_user_connector_name", + "search_source_connectors", + type_="unique", + ) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 9c503fb18..e76e69e94 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -90,6 +90,38 @@ class Config: AIRTABLE_CLIENT_SECRET = os.getenv("AIRTABLE_CLIENT_SECRET") AIRTABLE_REDIRECT_URI = os.getenv("AIRTABLE_REDIRECT_URI") + # Notion OAuth + NOTION_CLIENT_ID = os.getenv("NOTION_CLIENT_ID") + NOTION_CLIENT_SECRET = os.getenv("NOTION_CLIENT_SECRET") + NOTION_REDIRECT_URI = os.getenv("NOTION_REDIRECT_URI") + + # Atlassian OAuth (shared for Jira and Confluence) + ATLASSIAN_CLIENT_ID = os.getenv("ATLASSIAN_CLIENT_ID") + ATLASSIAN_CLIENT_SECRET = os.getenv("ATLASSIAN_CLIENT_SECRET") + JIRA_REDIRECT_URI = os.getenv("JIRA_REDIRECT_URI") + CONFLUENCE_REDIRECT_URI = os.getenv("CONFLUENCE_REDIRECT_URI") + + # Linear OAuth + LINEAR_CLIENT_ID = os.getenv("LINEAR_CLIENT_ID") + LINEAR_CLIENT_SECRET = os.getenv("LINEAR_CLIENT_SECRET") + LINEAR_REDIRECT_URI = os.getenv("LINEAR_REDIRECT_URI") + + # Slack OAuth + SLACK_CLIENT_ID = os.getenv("SLACK_CLIENT_ID") + SLACK_CLIENT_SECRET = os.getenv("SLACK_CLIENT_SECRET") + SLACK_REDIRECT_URI = os.getenv("SLACK_REDIRECT_URI") + + # Discord OAuth + DISCORD_CLIENT_ID = os.getenv("DISCORD_CLIENT_ID") + DISCORD_CLIENT_SECRET = os.getenv("DISCORD_CLIENT_SECRET") + DISCORD_REDIRECT_URI = os.getenv("DISCORD_REDIRECT_URI") + DISCORD_BOT_TOKEN = os.getenv("DISCORD_BOT_TOKEN") + + # ClickUp OAuth + CLICKUP_CLIENT_ID = os.getenv("CLICKUP_CLIENT_ID") + CLICKUP_CLIENT_SECRET = os.getenv("CLICKUP_CLIENT_SECRET") + CLICKUP_REDIRECT_URI = os.getenv("CLICKUP_REDIRECT_URI") + # LLM instances are now managed per-user through the LLMConfig system # Legacy environment variables removed in favor of user-specific configurations diff --git a/surfsense_backend/app/connectors/airtable_connector.py b/surfsense_backend/app/connectors/airtable_connector.py index 840b2276c..ec873e3fe 100644 --- a/surfsense_backend/app/connectors/airtable_connector.py +++ b/surfsense_backend/app/connectors/airtable_connector.py @@ -294,6 +294,12 @@ class AirtableConnector: Tuple of (records, error_message) """ try: + # Validate date strings before parsing + if not start_date or start_date.lower() in ("undefined", "null", "none"): + return [], "Invalid start_date: date string is required" + if not end_date or end_date.lower() in ("undefined", "null", "none"): + return [], "Invalid end_date: date string is required" + # Parse and validate dates start_dt = isoparse(start_date) end_dt = isoparse(end_date) @@ -382,3 +388,43 @@ class AirtableConnector: markdown_parts.append("") return "\n".join(markdown_parts) + + +# --- OAuth User Info --- + +AIRTABLE_WHOAMI_URL = "https://api.airtable.com/v0/meta/whoami" + + +async def fetch_airtable_user_email(access_token: str) -> str | None: + """ + Fetch user email from Airtable whoami API. + + Args: + access_token: The Airtable OAuth access token + + Returns: + User's email address or None if fetch fails + """ + try: + async with httpx.AsyncClient() as client: + response = await client.get( + AIRTABLE_WHOAMI_URL, + headers={"Authorization": f"Bearer {access_token}"}, + timeout=10.0, + ) + + if response.status_code == 200: + data = response.json() + email = data.get("email") + if email: + logger.debug(f"Fetched Airtable user email: {email}") + return email + + logger.warning( + f"Failed to fetch Airtable user info: {response.status_code}" + ) + return None + + except Exception as e: + logger.warning(f"Error fetching Airtable user email: {e!s}") + return None diff --git a/surfsense_backend/app/connectors/airtable_history.py b/surfsense_backend/app/connectors/airtable_history.py new file mode 100644 index 000000000..64f6465fe --- /dev/null +++ b/surfsense_backend/app/connectors/airtable_history.py @@ -0,0 +1,175 @@ +""" +Airtable OAuth Connector. + +Handles OAuth-based authentication and token refresh for Airtable API access. +""" + +import logging + +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.connectors.airtable_connector import AirtableConnector +from app.db import SearchSourceConnector +from app.routes.airtable_add_connector_route import refresh_airtable_token +from app.schemas.airtable_auth_credentials import AirtableAuthCredentialsBase +from app.utils.oauth_security import TokenEncryption + +logger = logging.getLogger(__name__) + + +class AirtableHistoryConnector: + """ + Airtable connector with OAuth support and automatic token refresh. + + This connector uses OAuth 2.0 access tokens to authenticate with the + Airtable API. It automatically refreshes expired tokens when needed. + """ + + def __init__( + self, + session: AsyncSession, + connector_id: int, + credentials: AirtableAuthCredentialsBase | None = None, + ): + """ + Initialize the AirtableHistoryConnector with auto-refresh capability. + + Args: + session: Database session for updating connector + connector_id: Connector ID for direct updates + credentials: Airtable OAuth credentials (optional, will be loaded from DB if not provided) + """ + self._session = session + self._connector_id = connector_id + self._credentials = credentials + self._airtable_connector: AirtableConnector | None = None + + async def _get_valid_token(self) -> str: + """ + Get valid Airtable access token, refreshing if needed. + + Returns: + Valid access token + + Raises: + ValueError: If credentials are missing or invalid + Exception: If token refresh fails + """ + # Load credentials from DB if not provided + if self._credentials is None: + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise ValueError(f"Connector {self._connector_id} not found") + + config_data = connector.config.copy() + + # Decrypt credentials if they are encrypted + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt sensitive fields + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + logger.info( + f"Decrypted Airtable credentials for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to decrypt Airtable credentials for connector {self._connector_id}: {e!s}" + ) + raise ValueError( + f"Failed to decrypt Airtable credentials: {e!s}" + ) from e + + try: + self._credentials = AirtableAuthCredentialsBase.from_dict(config_data) + except Exception as e: + raise ValueError(f"Invalid Airtable credentials: {e!s}") from e + + # Check if token is expired and refreshable + if self._credentials.is_expired and self._credentials.is_refreshable: + try: + logger.info( + f"Airtable token expired for connector {self._connector_id}, refreshing..." + ) + + # Get connector for refresh + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise RuntimeError( + f"Connector {self._connector_id} not found; cannot refresh token." + ) + + # Refresh token + connector = await refresh_airtable_token(self._session, connector) + + # Reload credentials after refresh + config_data = connector.config.copy() + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + token_encryption = TokenEncryption(config.SECRET_KEY) + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + self._credentials = AirtableAuthCredentialsBase.from_dict(config_data) + + # Invalidate cached connector so it's recreated with new token + self._airtable_connector = None + + logger.info( + f"Successfully refreshed Airtable token for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to refresh Airtable token for connector {self._connector_id}: {e!s}" + ) + raise Exception( + f"Failed to refresh Airtable OAuth credentials: {e!s}" + ) from e + + return self._credentials.access_token + + async def _get_connector(self) -> AirtableConnector: + """ + Get or create AirtableConnector with valid token. + + Returns: + AirtableConnector instance + """ + if self._airtable_connector is None: + # Ensure we have valid credentials (this will refresh if needed) + await self._get_valid_token() + # Use the credentials object which is now guaranteed to be valid + if not self._credentials: + raise ValueError("Credentials not loaded") + self._airtable_connector = AirtableConnector(self._credentials) + return self._airtable_connector diff --git a/surfsense_backend/app/connectors/clickup_history.py b/surfsense_backend/app/connectors/clickup_history.py new file mode 100644 index 000000000..70e90028b --- /dev/null +++ b/surfsense_backend/app/connectors/clickup_history.py @@ -0,0 +1,349 @@ +""" +ClickUp History Module + +A module for retrieving data from ClickUp with OAuth support and backward compatibility. +Allows fetching tasks from workspaces and lists with automatic token refresh. +""" + +import logging +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.connectors.clickup_connector import ClickUpConnector +from app.db import SearchSourceConnector +from app.routes.clickup_add_connector_route import refresh_clickup_token +from app.schemas.clickup_auth_credentials import ClickUpAuthCredentialsBase +from app.utils.oauth_security import TokenEncryption + +logger = logging.getLogger(__name__) + + +class ClickUpHistoryConnector: + """ + Class for retrieving data from ClickUp with OAuth support and backward compatibility. + """ + + def __init__( + self, + session: AsyncSession, + connector_id: int, + credentials: ClickUpAuthCredentialsBase | None = None, + api_token: str | None = None, # For backward compatibility + ): + """ + Initialize the ClickUpHistoryConnector. + + Args: + session: Database session for token refresh + connector_id: Connector ID for direct updates + credentials: ClickUp OAuth credentials (optional, will be loaded from DB if not provided) + api_token: Legacy API token for backward compatibility (optional) + """ + self._session = session + self._connector_id = connector_id + self._credentials = credentials + self._api_token = api_token # Legacy API token + self._use_oauth = False + self._use_legacy = api_token is not None + self._clickup_client: ClickUpConnector | None = None + + async def _get_valid_token(self) -> str: + """ + Get valid ClickUp access token, refreshing if needed. + For legacy API tokens, returns the token directly. + + Returns: + Valid access token or API token + + Raises: + ValueError: If credentials are missing or invalid + Exception: If token refresh fails + """ + # If using legacy API token, return it directly + if self._use_legacy and self._api_token: + return self._api_token + + # Load credentials from DB if not provided + if self._credentials is None: + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise ValueError(f"Connector {self._connector_id} not found") + + config_data = connector.config.copy() + + # Check if using OAuth or legacy API token + is_oauth = config_data.get("_token_encrypted", False) or config_data.get( + "access_token" + ) + has_legacy_token = config_data.get("CLICKUP_API_TOKEN") is not None + + if is_oauth: + # OAuth 2.0 authentication + self._use_oauth = True + # Decrypt credentials if they are encrypted + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt sensitive fields + if config_data.get("access_token"): + config_data["access_token"] = ( + token_encryption.decrypt_token( + config_data["access_token"] + ) + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = ( + token_encryption.decrypt_token( + config_data["refresh_token"] + ) + ) + + logger.info( + f"Decrypted ClickUp OAuth credentials for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to decrypt ClickUp OAuth credentials for connector {self._connector_id}: {e!s}" + ) + raise ValueError( + f"Failed to decrypt ClickUp OAuth credentials: {e!s}" + ) from e + + try: + self._credentials = ClickUpAuthCredentialsBase.from_dict( + config_data + ) + except Exception as e: + raise ValueError(f"Invalid ClickUp OAuth credentials: {e!s}") from e + elif has_legacy_token: + # Legacy API token authentication (backward compatibility) + self._use_legacy = True + self._api_token = config_data.get("CLICKUP_API_TOKEN") + + # Decrypt token if it's encrypted (legacy tokens might be encrypted) + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY and self._api_token: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + self._api_token = token_encryption.decrypt_token( + self._api_token + ) + logger.info( + f"Decrypted legacy ClickUp API token for connector {self._connector_id}" + ) + except Exception as e: + logger.warning( + f"Failed to decrypt legacy ClickUp API token for connector {self._connector_id}: {e!s}. " + "Trying to use token as-is (might be unencrypted)." + ) + # Continue with token as-is - might be unencrypted legacy token + + if not self._api_token: + raise ValueError("ClickUp API token not found in connector config") + + # Return legacy token directly (no refresh needed) + return self._api_token + else: + raise ValueError( + "ClickUp credentials not found in connector config (neither OAuth nor API token)" + ) + + # Check if token is expired and refreshable (only for OAuth) + if ( + self._use_oauth + and self._credentials.is_expired + and self._credentials.is_refreshable + ): + try: + logger.info( + f"ClickUp token expired for connector {self._connector_id}, refreshing..." + ) + + # Get connector for refresh + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise RuntimeError( + f"Connector {self._connector_id} not found; cannot refresh token." + ) + + # Refresh token + connector = await refresh_clickup_token(self._session, connector) + + # Reload credentials after refresh + config_data = connector.config.copy() + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + token_encryption = TokenEncryption(config.SECRET_KEY) + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + self._credentials = ClickUpAuthCredentialsBase.from_dict(config_data) + + # Invalidate cached client so it's recreated with new token + self._clickup_client = None + + logger.info( + f"Successfully refreshed ClickUp token for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to refresh ClickUp token for connector {self._connector_id}: {e!s}" + ) + raise Exception( + f"Failed to refresh ClickUp OAuth credentials: {e!s}" + ) from e + + if self._use_oauth: + return self._credentials.access_token + else: + return self._api_token + + async def _get_client(self) -> ClickUpConnector: + """ + Get or create ClickUpConnector with valid token. + + Returns: + ClickUpConnector instance + """ + if self._clickup_client is None: + token = await self._get_valid_token() + # ClickUp API uses Bearer token for OAuth, or direct token for legacy + if self._use_oauth: + # For OAuth, use Bearer token format (ClickUp OAuth expects "Bearer {token}") + self._clickup_client = ClickUpConnector(api_token=f"Bearer {token}") + else: + # For legacy API token, use token directly (format: "pk_...") + self._clickup_client = ClickUpConnector(api_token=token) + return self._clickup_client + + async def close(self): + """Close any open connections.""" + self._clickup_client = None + + async def __aenter__(self): + """Async context manager entry.""" + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit.""" + await self.close() + + async def get_authorized_workspaces(self) -> dict[str, Any]: + """ + Fetch authorized workspaces (teams) from ClickUp. + + Returns: + Dictionary containing teams data + + Raises: + ValueError: If credentials have not been set + Exception: If the API request fails + """ + client = await self._get_client() + return client.get_authorized_workspaces() + + async def get_workspace_tasks( + self, workspace_id: str, include_closed: bool = False + ) -> list[dict[str, Any]]: + """ + Fetch all tasks from a ClickUp workspace. + + Args: + workspace_id: ClickUp workspace (team) ID + include_closed: Whether to include closed tasks (default: False) + + Returns: + List of task objects + + Raises: + ValueError: If credentials have not been set + Exception: If the API request fails + """ + client = await self._get_client() + return client.get_workspace_tasks( + workspace_id=workspace_id, include_closed=include_closed + ) + + async def get_tasks_in_date_range( + self, + workspace_id: str, + start_date: str, + end_date: str, + include_closed: bool = False, + ) -> tuple[list[dict[str, Any]], str | None]: + """ + Fetch tasks from ClickUp within a specific date range. + + Args: + workspace_id: ClickUp workspace (team) ID + start_date: Start date in YYYY-MM-DD format + end_date: End date in YYYY-MM-DD format + include_closed: Whether to include closed tasks (default: False) + + Returns: + Tuple containing (tasks list, error message or None) + """ + client = await self._get_client() + return client.get_tasks_in_date_range( + workspace_id=workspace_id, + start_date=start_date, + end_date=end_date, + include_closed=include_closed, + ) + + async def get_task_details(self, task_id: str) -> dict[str, Any]: + """ + Fetch detailed information about a specific task. + + Args: + task_id: ClickUp task ID + + Returns: + Task details + + Raises: + ValueError: If credentials have not been set + Exception: If the API request fails + """ + client = await self._get_client() + return client.get_task_details(task_id) + + async def get_task_comments(self, task_id: str) -> dict[str, Any]: + """ + Fetch comments for a specific task. + + Args: + task_id: ClickUp task ID + + Returns: + Task comments + + Raises: + ValueError: If credentials have not been set + Exception: If the API request fails + """ + client = await self._get_client() + return client.get_task_comments(task_id) diff --git a/surfsense_backend/app/connectors/confluence_history.py b/surfsense_backend/app/connectors/confluence_history.py new file mode 100644 index 000000000..9e10ffcf1 --- /dev/null +++ b/surfsense_backend/app/connectors/confluence_history.py @@ -0,0 +1,592 @@ +""" +Confluence OAuth Connector. + +Handles OAuth-based authentication and token refresh for Confluence API access. +""" + +import logging +from typing import Any + +import httpx +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.connectors.confluence_connector import ConfluenceConnector +from app.db import SearchSourceConnector +from app.routes.confluence_add_connector_route import refresh_confluence_token +from app.schemas.atlassian_auth_credentials import AtlassianAuthCredentialsBase +from app.utils.oauth_security import TokenEncryption + +logger = logging.getLogger(__name__) + + +class ConfluenceHistoryConnector: + """ + Confluence connector with OAuth support and automatic token refresh. + + This connector uses OAuth 2.0 access tokens to authenticate with the + Confluence API. It automatically refreshes expired tokens when needed. + Also supports legacy API token authentication for backward compatibility. + """ + + def __init__( + self, + session: AsyncSession, + connector_id: int, + credentials: AtlassianAuthCredentialsBase | None = None, + ): + """ + Initialize the ConfluenceHistoryConnector with auto-refresh capability. + + Args: + session: Database session for updating connector + connector_id: Connector ID for direct updates + credentials: Confluence OAuth credentials (optional, will be loaded from DB if not provided) + """ + self._session = session + self._connector_id = connector_id + self._credentials = credentials + self._cloud_id: str | None = None + self._base_url: str | None = None + self._http_client: httpx.AsyncClient | None = None + self._use_oauth = True + self._legacy_email: str | None = None + self._legacy_api_token: str | None = None + self._legacy_confluence_client: ConfluenceConnector | None = None + + async def _get_valid_token(self) -> str: + """ + Get valid Confluence access token, refreshing if needed. + + Returns: + Valid access token + + Raises: + ValueError: If credentials are missing or invalid + Exception: If token refresh fails + """ + # Load credentials from DB if not provided + if self._credentials is None: + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise ValueError(f"Connector {self._connector_id} not found") + + config_data = connector.config.copy() + + # Check if using OAuth or legacy API token + is_oauth = config_data.get("_token_encrypted", False) or config_data.get( + "access_token" + ) + + if is_oauth: + # OAuth 2.0 authentication + # Decrypt credentials if they are encrypted + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt sensitive fields + if config_data.get("access_token"): + config_data["access_token"] = ( + token_encryption.decrypt_token( + config_data["access_token"] + ) + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = ( + token_encryption.decrypt_token( + config_data["refresh_token"] + ) + ) + + logger.info( + f"Decrypted Confluence credentials for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to decrypt Confluence credentials for connector {self._connector_id}: {e!s}" + ) + raise ValueError( + f"Failed to decrypt Confluence credentials: {e!s}" + ) from e + + try: + self._credentials = AtlassianAuthCredentialsBase.from_dict( + config_data + ) + # Store cloud_id and base_url for API calls (with backward compatibility for site_url) + self._cloud_id = config_data.get("cloud_id") + self._base_url = config_data.get("base_url") or config_data.get( + "site_url" + ) + self._use_oauth = True + except Exception as e: + raise ValueError( + f"Invalid Confluence OAuth credentials: {e!s}" + ) from e + else: + # Legacy API token authentication + self._legacy_email = config_data.get("CONFLUENCE_EMAIL") + self._legacy_api_token = config_data.get("CONFLUENCE_API_TOKEN") + self._base_url = config_data.get("CONFLUENCE_BASE_URL") + self._use_oauth = False + + if ( + not self._legacy_email + or not self._legacy_api_token + or not self._base_url + ): + raise ValueError( + "Confluence credentials not found in connector config" + ) + + # Check if token is expired and refreshable (only for OAuth) + if ( + self._use_oauth + and self._credentials.is_expired + and self._credentials.is_refreshable + ): + try: + logger.info( + f"Confluence token expired for connector {self._connector_id}, refreshing..." + ) + + # Get connector for refresh + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise RuntimeError( + f"Connector {self._connector_id} not found; cannot refresh token." + ) + + # Refresh token + connector = await refresh_confluence_token(self._session, connector) + + # Reload credentials after refresh + config_data = connector.config.copy() + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + token_encryption = TokenEncryption(config.SECRET_KEY) + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + self._credentials = AtlassianAuthCredentialsBase.from_dict(config_data) + self._cloud_id = config_data.get("cloud_id") + # Handle backward compatibility: check both base_url and site_url + self._base_url = config_data.get("base_url") or config_data.get( + "site_url" + ) + + # Invalidate cached client so it's recreated with new token + if self._http_client: + await self._http_client.aclose() + self._http_client = None + + logger.info( + f"Successfully refreshed Confluence token for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to refresh Confluence token for connector {self._connector_id}: {e!s}" + ) + raise Exception( + f"Failed to refresh Confluence OAuth credentials: {e!s}" + ) from e + + if self._use_oauth: + return self._credentials.access_token + else: + # For legacy auth, return empty string (not used for token-based auth) + return "" + + async def _get_client(self) -> httpx.AsyncClient: + """ + Get or create HTTP client with valid token. + + Returns: + httpx.AsyncClient instance + """ + if self._http_client is None: + self._http_client = httpx.AsyncClient(timeout=30.0) + return self._http_client + + async def _get_legacy_client(self) -> ConfluenceConnector: + """ + Get or create ConfluenceConnector with legacy credentials. + + Returns: + ConfluenceConnector instance + """ + if self._legacy_confluence_client is None: + self._legacy_confluence_client = ConfluenceConnector( + base_url=self._base_url, + email=self._legacy_email, + api_token=self._legacy_api_token, + ) + return self._legacy_confluence_client + + async def _get_base_url(self) -> str: + """ + Get the base URL for Confluence API calls. + + Returns: + Base URL string + """ + if not self._use_oauth: + # For legacy auth, use the base_url directly + return self._base_url or "" + + if not self._cloud_id: + raise ValueError("Cloud ID not available. Cannot construct API URL.") + + # Use the Atlassian API format: https://api.atlassian.com/ex/confluence/{cloudid} + return f"https://api.atlassian.com/ex/confluence/{self._cloud_id}" + + async def _make_api_request( + self, endpoint: str, params: dict[str, Any] | None = None + ) -> dict[str, Any]: + """ + Make a request to the Confluence API. + + Args: + endpoint: API endpoint (without base URL) + params: Query parameters for the request (optional) + + Returns: + Response data from the API + + Raises: + ValueError: If credentials have not been set + Exception: If the API request fails + """ + if not self._use_oauth: + # Use legacy ConfluenceConnector for API requests + client = await self._get_legacy_client() + # ConfluenceConnector uses synchronous requests, so we need to handle this differently + # For now, we'll use the legacy client's make_api_request method + # But since it's sync, we'll need to wrap it + import asyncio + + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, client.make_api_request, endpoint, params + ) + + # OAuth flow + token = await self._get_valid_token() + base_url = await self._get_base_url() + http_client = await self._get_client() + + url = f"{base_url}/wiki/api/v2/{endpoint}" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + "Accept": "application/json", + } + + try: + response = await http_client.get(url, headers=headers, params=params) + response.raise_for_status() + return response.json() + except httpx.HTTPStatusError as e: + # Enhanced error logging to see the actual error + error_detail = { + "status_code": e.response.status_code, + "url": str(e.request.url), + "response_text": e.response.text, + "headers": dict(e.response.headers), + } + logger.error(f"Confluence API HTTP error: {error_detail}") + raise Exception( + f"Confluence API request failed (HTTP {e.response.status_code}): {e.response.text}" + ) from e + except httpx.RequestError as e: + logger.error(f"Confluence API request error: {e!s}", exc_info=True) + raise Exception(f"Confluence API request failed: {e!s}") from e + + async def get_all_spaces(self) -> list[dict[str, Any]]: + """ + Fetch all spaces from Confluence. + + Returns: + List of space objects + + Raises: + ValueError: If credentials have not been set + Exception: If the API request fails + """ + params = { + "limit": 100, + } + + all_spaces = [] + cursor = None + + while True: + if cursor: + params["cursor"] = cursor + + result = await self._make_api_request("spaces", params) + + if not isinstance(result, dict) or "results" not in result: + raise Exception("Invalid response from Confluence API") + + spaces = result["results"] + all_spaces.extend(spaces) + + # Check if there are more spaces to fetch + links = result.get("_links", {}) + if "next" not in links: + break + + # Extract cursor from next link if available + next_link = links["next"] + if "cursor=" in next_link: + cursor = next_link.split("cursor=")[1].split("&")[0] + else: + break + + return all_spaces + + async def get_pages_in_space( + self, space_id: str, include_body: bool = True + ) -> list[dict[str, Any]]: + """ + Fetch all pages in a specific space. + + Args: + space_id: The ID of the space to fetch pages from + include_body: Whether to include page body content + + Returns: + List of page objects + + Raises: + ValueError: If credentials have not been set + Exception: If the API request fails + """ + params = { + "limit": 100, + } + + if include_body: + params["body-format"] = "storage" + + all_pages = [] + cursor = None + + while True: + if cursor: + params["cursor"] = cursor + + result = await self._make_api_request(f"spaces/{space_id}/pages", params) + + if not isinstance(result, dict) or "results" not in result: + raise Exception("Invalid response from Confluence API") + + pages = result["results"] + all_pages.extend(pages) + + # Check if there are more pages to fetch + links = result.get("_links", {}) + if "next" not in links: + break + + # Extract cursor from next link if available + next_link = links["next"] + if "cursor=" in next_link: + cursor = next_link.split("cursor=")[1].split("&")[0] + else: + break + + return all_pages + + async def get_page_comments(self, page_id: str) -> list[dict[str, Any]]: + """ + Fetch all comments for a specific page (both footer and inline comments). + + Args: + page_id: The ID of the page to fetch comments from + + Returns: + List of comment objects + + Raises: + ValueError: If credentials have not been set + Exception: If the API request fails + """ + all_comments = [] + + # Get footer comments + footer_comments = await self._get_comments_for_page(page_id, "footer-comments") + all_comments.extend(footer_comments) + + # Get inline comments + inline_comments = await self._get_comments_for_page(page_id, "inline-comments") + all_comments.extend(inline_comments) + + return all_comments + + async def _get_comments_for_page( + self, page_id: str, comment_type: str + ) -> list[dict[str, Any]]: + """ + Helper method to fetch comments of a specific type for a page. + + Args: + page_id: The ID of the page + comment_type: Type of comments ('footer-comments' or 'inline-comments') + + Returns: + List of comment objects + """ + params = { + "limit": 100, + "body-format": "storage", + } + + all_comments = [] + cursor = None + + while True: + if cursor: + params["cursor"] = cursor + + result = await self._make_api_request( + f"pages/{page_id}/{comment_type}", params + ) + + if not isinstance(result, dict) or "results" not in result: + break # No comments or invalid response + + comments = result["results"] + all_comments.extend(comments) + + # Check if there are more comments to fetch + links = result.get("_links", {}) + if "next" not in links: + break + + # Extract cursor from next link if available + next_link = links["next"] + if "cursor=" in next_link: + cursor = next_link.split("cursor=")[1].split("&")[0] + else: + break + + return all_comments + + async def get_pages_by_date_range( + self, + start_date: str, + end_date: str, + space_ids: list[str] | None = None, + include_comments: bool = True, + ) -> tuple[list[dict[str, Any]], str | None]: + """ + Fetch pages within a date range, optionally filtered by spaces. + + Args: + start_date: Start date in YYYY-MM-DD format + end_date: End date in YYYY-MM-DD format (inclusive) + space_ids: Optional list of space IDs to filter pages + include_comments: Whether to include comments for each page + + Returns: + Tuple containing (pages list with comments, error message or None) + """ + try: + if not self._use_oauth: + # Use legacy ConfluenceConnector for API requests + client = await self._get_legacy_client() + # Ensure credentials are loaded + await self._get_valid_token() + # ConfluenceConnector.get_pages_by_date_range is synchronous + import asyncio + + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + client.get_pages_by_date_range, + start_date, + end_date, + space_ids, + include_comments, + ) + + # OAuth flow + all_pages = [] + + if space_ids: + # Fetch pages from specific spaces + for space_id in space_ids: + pages = await self.get_pages_in_space(space_id, include_body=True) + all_pages.extend(pages) + else: + # Fetch all pages (this might be expensive for large instances) + params = { + "limit": 100, + "body-format": "storage", + } + + cursor = None + while True: + if cursor: + params["cursor"] = cursor + + result = await self._make_api_request("pages", params) + if not isinstance(result, dict) or "results" not in result: + break + + pages = result["results"] + all_pages.extend(pages) + + links = result.get("_links", {}) + if "next" not in links: + break + + next_link = links["next"] + if "cursor=" in next_link: + cursor = next_link.split("cursor=")[1].split("&")[0] + else: + break + + return all_pages, None + + except Exception as e: + return [], f"Error fetching pages: {e!s}" + + async def close(self): + """Close the HTTP client connection.""" + if self._http_client: + await self._http_client.aclose() + self._http_client = None + # Legacy client doesn't need explicit closing + self._legacy_confluence_client = None + + async def __aenter__(self): + """Async context manager entry.""" + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit.""" + await self.close() diff --git a/surfsense_backend/app/connectors/discord_connector.py b/surfsense_backend/app/connectors/discord_connector.py index 506b463a5..1e12cb9a4 100644 --- a/surfsense_backend/app/connectors/discord_connector.py +++ b/surfsense_backend/app/connectors/discord_connector.py @@ -3,7 +3,7 @@ Discord Connector A module for interacting with Discord's HTTP API to retrieve guilds, channels, and message history. -Requires a Discord bot token. +Supports both direct bot token and OAuth-based authentication with token refresh. """ import asyncio @@ -12,6 +12,14 @@ import logging import discord from discord.ext import commands +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.db import SearchSourceConnector +from app.routes.discord_add_connector_route import refresh_discord_token +from app.schemas.discord_auth_credentials import DiscordAuthCredentialsBase +from app.utils.oauth_security import TokenEncryption logger = logging.getLogger(__name__) @@ -19,12 +27,21 @@ logger = logging.getLogger(__name__) class DiscordConnector(commands.Bot): """Class for retrieving guild, channel, and message history from Discord.""" - def __init__(self, token: str | None = None): + def __init__( + self, + token: str | None = None, + session: AsyncSession | None = None, + connector_id: int | None = None, + credentials: DiscordAuthCredentialsBase | None = None, + ): """ - Initialize the DiscordConnector with a bot token. + Initialize the DiscordConnector with a bot token or OAuth credentials. Args: - token (str): The Discord bot token. + token: Discord bot token (optional, for backward compatibility) + session: Database session for token refresh (optional) + connector_id: Connector ID for token refresh (optional) + credentials: Discord OAuth credentials (optional, will be loaded from DB if not provided) """ intents = discord.Intents.default() intents.guilds = True # Required to fetch guilds and channels @@ -34,7 +51,14 @@ class DiscordConnector(commands.Bot): super().__init__( command_prefix="!", intents=intents ) # command_prefix is required but not strictly used here - self.token = token + self._session = session + self._connector_id = connector_id + self._credentials = credentials + # For backward compatibility, if token is provided directly, use it + if token: + self.token = token + else: + self.token = None self._bot_task = None # Holds the async bot task self._is_running = False # Flag to track if the bot is running @@ -57,12 +81,143 @@ class DiscordConnector(commands.Bot): async def on_resumed(): logger.debug("Bot resumed connection to Discord gateway.") + async def _get_valid_token(self) -> str: + """ + Get valid Discord bot token, refreshing if needed. + + Returns: + Valid bot token + + Raises: + ValueError: If credentials are missing or invalid + Exception: If token refresh fails + """ + # If we have a direct token (backward compatibility), use it + if ( + self.token + and self._session is None + and self._connector_id is None + and self._credentials is None + ): + # This means it was initialized with a direct token, use it + return self.token + + # Load credentials from DB if not provided + if self._credentials is None: + if not self._session or not self._connector_id: + raise ValueError( + "Cannot load credentials: session and connector_id required" + ) + + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise ValueError(f"Connector {self._connector_id} not found") + + config_data = connector.config.copy() + + # Decrypt credentials if they are encrypted + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt sensitive fields + if config_data.get("bot_token"): + config_data["bot_token"] = token_encryption.decrypt_token( + config_data["bot_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + logger.info( + f"Decrypted Discord credentials for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to decrypt Discord credentials for connector {self._connector_id}: {e!s}" + ) + raise ValueError( + f"Failed to decrypt Discord credentials: {e!s}" + ) from e + + try: + self._credentials = DiscordAuthCredentialsBase.from_dict(config_data) + except Exception as e: + raise ValueError(f"Invalid Discord credentials: {e!s}") from e + + # Check if token is expired and refreshable + if self._credentials.is_expired and self._credentials.is_refreshable: + try: + logger.info( + f"Discord token expired for connector {self._connector_id}, refreshing..." + ) + + # Get connector for refresh + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise RuntimeError( + f"Connector {self._connector_id} not found; cannot refresh token." + ) + + # Refresh token + connector = await refresh_discord_token(self._session, connector) + + # Reload credentials after refresh + config_data = connector.config.copy() + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + token_encryption = TokenEncryption(config.SECRET_KEY) + if config_data.get("bot_token"): + config_data["bot_token"] = token_encryption.decrypt_token( + config_data["bot_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + self._credentials = DiscordAuthCredentialsBase.from_dict(config_data) + + logger.info( + f"Successfully refreshed Discord token for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to refresh Discord token for connector {self._connector_id}: {e!s}" + ) + raise Exception( + f"Failed to refresh Discord OAuth credentials: {e!s}" + ) from e + + return self._credentials.bot_token + async def start_bot(self): """Starts the bot to connect to Discord.""" logger.info("Starting Discord bot...") + # Get valid token (with auto-refresh if using OAuth) if not self.token: - raise ValueError("Discord bot token not set. Call set_token(token) first.") + # Try to get token from credentials + try: + self.token = await self._get_valid_token() + except ValueError as e: + raise ValueError( + f"Discord bot token not set. {e!s} Please authenticate via OAuth or provide a token." + ) from e try: if self._is_running: @@ -107,7 +262,7 @@ class DiscordConnector(commands.Bot): def set_token(self, token: str) -> None: """ - Set the discord bot token. + Set the discord bot token (for backward compatibility). Args: token (str): The Discord bot token. diff --git a/surfsense_backend/app/connectors/google_calendar_connector.py b/surfsense_backend/app/connectors/google_calendar_connector.py index 164d230e0..6d389ddd5 100644 --- a/surfsense_backend/app/connectors/google_calendar_connector.py +++ b/surfsense_backend/app/connectors/google_calendar_connector.py @@ -109,7 +109,36 @@ class GoogleCalendarConnector: raise RuntimeError( "GOOGLE_CALENDAR_CONNECTOR connector not found; cannot persist refreshed token." ) - connector.config = json.loads(self._credentials.to_json()) + + # Encrypt sensitive credentials before storing + from app.config import config + from app.utils.oauth_security import TokenEncryption + + creds_dict = json.loads(self._credentials.to_json()) + token_encrypted = connector.config.get("_token_encrypted", False) + + if token_encrypted and config.SECRET_KEY: + token_encryption = TokenEncryption(config.SECRET_KEY) + # Encrypt sensitive fields + if creds_dict.get("token"): + creds_dict["token"] = token_encryption.encrypt_token( + creds_dict["token"] + ) + if creds_dict.get("refresh_token"): + creds_dict["refresh_token"] = ( + token_encryption.encrypt_token( + creds_dict["refresh_token"] + ) + ) + if creds_dict.get("client_secret"): + creds_dict["client_secret"] = ( + token_encryption.encrypt_token( + creds_dict["client_secret"] + ) + ) + creds_dict["_token_encrypted"] = True + + connector.config = creds_dict flag_modified(connector, "config") await self._session.commit() except Exception as e: @@ -182,6 +211,18 @@ class GoogleCalendarConnector: Tuple containing (events list, error message or None) """ try: + # Validate date strings + if not start_date or start_date.lower() in ("undefined", "null", "none"): + return ( + [], + "Invalid start_date: must be a valid date string in YYYY-MM-DD format", + ) + if not end_date or end_date.lower() in ("undefined", "null", "none"): + return ( + [], + "Invalid end_date: must be a valid date string in YYYY-MM-DD format", + ) + service = await self._get_service() # Parse both dates diff --git a/surfsense_backend/app/connectors/google_drive/credentials.py b/surfsense_backend/app/connectors/google_drive/credentials.py index f88486468..7e6335f6d 100644 --- a/surfsense_backend/app/connectors/google_drive/credentials.py +++ b/surfsense_backend/app/connectors/google_drive/credentials.py @@ -1,6 +1,7 @@ """Google Drive OAuth credential management.""" import json +import logging from datetime import datetime from google.auth.transport.requests import Request @@ -9,7 +10,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.orm.attributes import flag_modified +from app.config import config from app.db import SearchSourceConnector +from app.utils.oauth_security import TokenEncryption + +logger = logging.getLogger(__name__) async def get_valid_credentials( @@ -38,7 +43,41 @@ async def get_valid_credentials( if not connector: raise ValueError(f"Connector {connector_id} not found") - config_data = connector.config + config_data = ( + connector.config.copy() + ) # Work with a copy to avoid modifying original + + # Decrypt credentials if they are encrypted + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt sensitive fields + if config_data.get("token"): + config_data["token"] = token_encryption.decrypt_token( + config_data["token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + if config_data.get("client_secret"): + config_data["client_secret"] = token_encryption.decrypt_token( + config_data["client_secret"] + ) + + logger.info( + f"Decrypted Google Drive credentials for connector {connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to decrypt Google Drive credentials for connector {connector_id}: {e!s}" + ) + raise ValueError( + f"Failed to decrypt Google Drive credentials: {e!s}" + ) from e + exp = config_data.get("expiry", "").replace("Z", "") if not all( @@ -66,7 +105,29 @@ async def get_valid_credentials( try: credentials.refresh(Request()) - connector.config = json.loads(credentials.to_json()) + creds_dict = json.loads(credentials.to_json()) + + # Encrypt sensitive credentials before storing + token_encrypted = connector.config.get("_token_encrypted", False) + + if token_encrypted and config.SECRET_KEY: + token_encryption = TokenEncryption(config.SECRET_KEY) + # Encrypt sensitive fields + if creds_dict.get("token"): + creds_dict["token"] = token_encryption.encrypt_token( + creds_dict["token"] + ) + if creds_dict.get("refresh_token"): + creds_dict["refresh_token"] = token_encryption.encrypt_token( + creds_dict["refresh_token"] + ) + if creds_dict.get("client_secret"): + creds_dict["client_secret"] = token_encryption.encrypt_token( + creds_dict["client_secret"] + ) + creds_dict["_token_encrypted"] = True + + connector.config = creds_dict flag_modified(connector, "config") await session.commit() diff --git a/surfsense_backend/app/connectors/google_gmail_connector.py b/surfsense_backend/app/connectors/google_gmail_connector.py index 402337448..10008ad73 100644 --- a/surfsense_backend/app/connectors/google_gmail_connector.py +++ b/surfsense_backend/app/connectors/google_gmail_connector.py @@ -6,6 +6,7 @@ Allows fetching emails from Gmail mailbox using Google OAuth credentials. import base64 import json +import logging import re from typing import Any @@ -21,6 +22,34 @@ from app.db import ( SearchSourceConnectorType, ) +logger = logging.getLogger(__name__) + + +def fetch_google_user_email(credentials: Credentials) -> str | None: + """ + Fetch user email from Gmail API using Google credentials. + + Uses the Gmail users.getProfile endpoint which returns the authenticated + user's email address. + + Args: + credentials: Google OAuth Credentials object (not encrypted) + + Returns: + User's email address or None if fetch fails + """ + try: + service = build("gmail", "v1", credentials=credentials) + profile = service.users().getProfile(userId="me").execute() + email = profile.get("emailAddress") + if email: + logger.debug(f"Fetched Google user email: {email}") + return email + return None + except Exception as e: + logger.warning(f"Error fetching Google user email: {e!s}") + return None + class GoogleGmailConnector: """Class for retrieving emails from Gmail using Google OAuth credentials.""" diff --git a/surfsense_backend/app/connectors/jira_connector.py b/surfsense_backend/app/connectors/jira_connector.py index e73198e79..370460e04 100644 --- a/surfsense_backend/app/connectors/jira_connector.py +++ b/surfsense_backend/app/connectors/jira_connector.py @@ -3,6 +3,7 @@ Jira Connector Module A module for retrieving data from Jira. Allows fetching issue lists and their comments, projects and more. +Supports both OAuth 2.0 (preferred) and legacy API token authentication. """ import base64 @@ -18,6 +19,8 @@ class JiraConnector: def __init__( self, base_url: str | None = None, + access_token: str | None = None, + cloud_id: str | None = None, email: str | None = None, api_token: str | None = None, ): @@ -25,18 +28,39 @@ class JiraConnector: Initialize the JiraConnector class. Args: - base_url: Jira instance base URL (e.g., 'https://yourcompany.atlassian.net') (optional) - email: Jira account email address (optional) - api_token: Jira API token (optional) + base_url: Jira instance base URL (e.g., 'https://yourcompany.atlassian.net') + access_token: OAuth 2.0 access token (preferred method) + cloud_id: Atlassian cloud ID (used with OAuth for API URL construction) + email: Jira account email address (legacy method, used with api_token) + api_token: Jira API token (legacy method, used with email) """ self.base_url = base_url.rstrip("/") if base_url else None + self.access_token = access_token + self.cloud_id = cloud_id self.email = email self.api_token = api_token self.api_version = "3" # Jira Cloud API version + self._use_oauth = access_token is not None + + def set_oauth_credentials( + self, base_url: str, access_token: str, cloud_id: str | None = None + ) -> None: + """ + Set OAuth 2.0 credentials (preferred method). + + Args: + base_url: Jira instance base URL + access_token: OAuth 2.0 access token + cloud_id: Atlassian cloud ID (optional, used for API URL construction) + """ + self.base_url = base_url.rstrip("/") + self.access_token = access_token + self.cloud_id = cloud_id + self._use_oauth = True def set_credentials(self, base_url: str, email: str, api_token: str) -> None: """ - Set the Jira credentials. + Set the Jira credentials (legacy method using API token). Args: base_url: Jira instance base URL @@ -46,50 +70,69 @@ class JiraConnector: self.base_url = base_url.rstrip("/") self.email = email self.api_token = api_token + self._use_oauth = False def set_email(self, email: str) -> None: """ - Set the Jira account email. + Set the Jira account email (legacy method). Args: email: Jira account email address """ self.email = email + self._use_oauth = False def set_api_token(self, api_token: str) -> None: """ - Set the Jira API token. + Set the Jira API token (legacy method). Args: api_token: Jira API token """ self.api_token = api_token + self._use_oauth = False def get_headers(self) -> dict[str, str]: """ - Get headers for Jira API requests using Basic Authentication. + Get headers for Jira API requests. + + Uses OAuth Bearer token if available, otherwise falls back to Basic Auth. Returns: Dictionary of headers Raises: - ValueError: If email, api_token, or base_url have not been set + ValueError: If credentials have not been set """ - if not all([self.base_url, self.email, self.api_token]): - raise ValueError( - "Jira credentials not initialized. Call set_credentials() first." - ) + if self._use_oauth: + # OAuth 2.0 authentication + if not self.base_url or not self.access_token: + raise ValueError( + "Jira OAuth credentials not initialized. Call set_oauth_credentials() first." + ) - # Create Basic Auth header using email:api_token - auth_str = f"{self.email}:{self.api_token}" - auth_bytes = auth_str.encode("utf-8") - auth_header = "Basic " + base64.b64encode(auth_bytes).decode("ascii") + return { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.access_token}", + "Accept": "application/json", + } + else: + # Legacy Basic Auth + if not all([self.base_url, self.email, self.api_token]): + raise ValueError( + "Jira credentials not initialized. Call set_credentials() first." + ) - return { - "Content-Type": "application/json", - "Authorization": auth_header, - "Accept": "application/json", - } + # Create Basic Auth header using email:api_token + auth_str = f"{self.email}:{self.api_token}" + auth_bytes = auth_str.encode("utf-8") + auth_header = "Basic " + base64.b64encode(auth_bytes).decode("ascii") + + return { + "Content-Type": "application/json", + "Authorization": auth_header, + "Accept": "application/json", + } def make_api_request( self, @@ -104,22 +147,26 @@ class JiraConnector: Args: endpoint: API endpoint (without base URL) params: Query parameters for the request (optional) + method: HTTP method (GET or POST) + json_payload: JSON payload for POST requests (optional) Returns: Response data from the API Raises: - ValueError: If email, api_token, or base_url have not been set + ValueError: If credentials have not been set Exception: If the API request fails """ - if not all([self.base_url, self.email, self.api_token]): - raise ValueError( - "Jira credentials not initialized. Call set_credentials() first." - ) - - url = f"{self.base_url}/rest/api/{self.api_version}/{endpoint}" headers = self.get_headers() + # Construct API URL based on authentication method + if self._use_oauth and self.cloud_id: + # Use Atlassian API gateway with cloud_id for OAuth + url = f"https://api.atlassian.com/ex/jira/{self.cloud_id}/rest/api/{self.api_version}/{endpoint}" + else: + # Use direct base URL (works for both OAuth and legacy) + url = f"{self.base_url}/rest/api/{self.api_version}/{endpoint}" + if method.upper() == "POST": response = requests.post( url, headers=headers, json=json_payload, timeout=500 @@ -234,16 +281,23 @@ class JiraConnector: try: # Build JQL query for date range # Query issues that were either created OR updated within the date range - date_filter = ( - f"(createdDate >= '{start_date}' AND createdDate <= '{end_date}')" - ) - # TODO : This JQL needs some improvement to work as expected + # Use end_date + 1 day with < operator to include the full end date + from datetime import datetime, timedelta - jql = f"{date_filter}" + # Parse end_date and add 1 day for inclusive end date + end_date_obj = datetime.strptime(end_date, "%Y-%m-%d") + end_date_next = (end_date_obj + timedelta(days=1)).strftime("%Y-%m-%d") + + # Check both created and updated dates to catch all relevant issues + # Use 'created' and 'updated' (standard JQL field names) + date_filter = ( + f"(created >= '{start_date}' AND created < '{end_date_next}') " + f"OR (updated >= '{start_date}' AND updated < '{end_date_next}')" + ) + + jql = f"{date_filter} ORDER BY created DESC" if project_key: - jql = ( - f'project = "{project_key}" AND {date_filter} ORDER BY created DESC' - ) + jql = f'project = "{project_key}" AND ({date_filter}) ORDER BY created DESC' # Define fields to retrieve fields = [ diff --git a/surfsense_backend/app/connectors/jira_history.py b/surfsense_backend/app/connectors/jira_history.py new file mode 100644 index 000000000..6e04ec2a4 --- /dev/null +++ b/surfsense_backend/app/connectors/jira_history.py @@ -0,0 +1,331 @@ +""" +Jira OAuth Connector. + +Handles OAuth-based authentication and token refresh for Jira API access. +Supports both OAuth 2.0 (preferred) and legacy API token authentication. +""" + +import logging +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.connectors.jira_connector import JiraConnector +from app.db import SearchSourceConnector +from app.routes.jira_add_connector_route import refresh_jira_token +from app.schemas.atlassian_auth_credentials import AtlassianAuthCredentialsBase +from app.utils.oauth_security import TokenEncryption + +logger = logging.getLogger(__name__) + + +class JiraHistoryConnector: + """ + Jira connector with OAuth support and automatic token refresh. + + This connector uses OAuth 2.0 access tokens to authenticate with the + Jira API. It automatically refreshes expired tokens when needed. + Also supports legacy API token authentication for backward compatibility. + """ + + def __init__( + self, + session: AsyncSession, + connector_id: int, + credentials: AtlassianAuthCredentialsBase | None = None, + ): + """ + Initialize the JiraHistoryConnector with auto-refresh capability. + + Args: + session: Database session for updating connector + connector_id: Connector ID for direct updates + credentials: Jira OAuth credentials (optional, will be loaded from DB if not provided) + """ + self._session = session + self._connector_id = connector_id + self._credentials = credentials + self._cloud_id: str | None = None + self._base_url: str | None = None + self._jira_client: JiraConnector | None = None + self._use_oauth = True + self._legacy_email: str | None = None + self._legacy_api_token: str | None = None + + async def _get_valid_token(self) -> str: + """ + Get valid Jira access token, refreshing if needed. + + Returns: + Valid access token + + Raises: + ValueError: If credentials are missing or invalid + Exception: If token refresh fails + """ + # Load credentials from DB if not provided + if self._credentials is None: + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise ValueError(f"Connector {self._connector_id} not found") + + config_data = connector.config.copy() + + # Check if using OAuth or legacy API token + is_oauth = config_data.get("_token_encrypted", False) or config_data.get( + "access_token" + ) + + if is_oauth: + # OAuth 2.0 authentication + if not config.SECRET_KEY: + raise ValueError( + "SECRET_KEY not configured but tokens are marked as encrypted" + ) + + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt access_token + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + logger.info( + f"Decrypted Jira access token for connector {self._connector_id}" + ) + + # Decrypt refresh_token if present + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + logger.info( + f"Decrypted Jira refresh token for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to decrypt Jira credentials for connector {self._connector_id}: {e!s}" + ) + raise ValueError( + f"Failed to decrypt Jira credentials: {e!s}" + ) from e + + try: + self._credentials = AtlassianAuthCredentialsBase.from_dict( + config_data + ) + self._cloud_id = config_data.get("cloud_id") + self._base_url = config_data.get("base_url") + self._use_oauth = True + except Exception as e: + raise ValueError(f"Invalid Jira OAuth credentials: {e!s}") from e + else: + # Legacy API token authentication + self._legacy_email = config_data.get("JIRA_EMAIL") + self._legacy_api_token = config_data.get("JIRA_API_TOKEN") + self._base_url = config_data.get("JIRA_BASE_URL") + self._use_oauth = False + + if ( + not self._legacy_email + or not self._legacy_api_token + or not self._base_url + ): + raise ValueError("Jira credentials not found in connector config") + + # Check if token is expired and refreshable (only for OAuth) + if ( + self._use_oauth + and self._credentials.is_expired + and self._credentials.is_refreshable + ): + try: + logger.info( + f"Jira token expired for connector {self._connector_id}, refreshing..." + ) + + # Get connector for refresh + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise RuntimeError( + f"Connector {self._connector_id} not found; cannot refresh token." + ) + + # Refresh token + connector = await refresh_jira_token(self._session, connector) + + # Reload credentials after refresh + config_data = connector.config.copy() + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + token_encryption = TokenEncryption(config.SECRET_KEY) + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + self._credentials = AtlassianAuthCredentialsBase.from_dict(config_data) + self._cloud_id = config_data.get("cloud_id") + self._base_url = config_data.get("base_url") + + # Invalidate cached client so it's recreated with new token + self._jira_client = None + + logger.info( + f"Successfully refreshed Jira token for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to refresh Jira token for connector {self._connector_id}: {e!s}" + ) + raise Exception( + f"Failed to refresh Jira OAuth credentials: {e!s}" + ) from e + + if self._use_oauth: + return self._credentials.access_token + else: + # For legacy auth, return empty string (not used for token-based auth) + return "" + + async def _get_jira_client(self) -> JiraConnector: + """ + Get or create JiraConnector with valid credentials. + + Returns: + JiraConnector instance + """ + if self._jira_client is None: + if self._use_oauth: + # Ensure we have valid token (will refresh if needed) + await self._get_valid_token() + + self._jira_client = JiraConnector( + base_url=self._base_url, + access_token=self._credentials.access_token, + cloud_id=self._cloud_id, + ) + else: + # Legacy API token authentication + self._jira_client = JiraConnector( + base_url=self._base_url, + email=self._legacy_email, + api_token=self._legacy_api_token, + ) + else: + # If OAuth, refresh token if expired before returning client + if self._use_oauth: + await self._get_valid_token() + # Update client with new token if it was refreshed + if self._credentials: + self._jira_client.set_oauth_credentials( + base_url=self._base_url or "", + access_token=self._credentials.access_token, + cloud_id=self._cloud_id, + ) + + return self._jira_client + + async def get_issues_by_date_range( + self, + start_date: str, + end_date: str, + include_comments: bool = True, + project_key: str | None = None, + ) -> tuple[list[dict[str, Any]], str | None]: + """ + Fetch issues within a date range. + This method wraps JiraConnector.get_issues_by_date_range() with automatic token refresh. + + Args: + start_date: Start date in YYYY-MM-DD format + end_date: End date in YYYY-MM-DD format (inclusive) + include_comments: Whether to include comments in the response + project_key: Optional project key to filter issues + + Returns: + Tuple containing (issues list, error message or None) + """ + # Ensure token is valid (will refresh if needed) + if self._use_oauth: + await self._get_valid_token() + + # Get client with valid credentials + client = await self._get_jira_client() + + # JiraConnector methods are synchronous, so we call them directly + # Token refresh has already been handled above + return client.get_issues_by_date_range( + start_date=start_date, + end_date=end_date, + include_comments=include_comments, + project_key=project_key, + ) + + def format_issue(self, issue: dict[str, Any]) -> dict[str, Any]: + """ + Format an issue for easier consumption. + Wraps JiraConnector.format_issue(). + + Args: + issue: The issue object from Jira API + + Returns: + Formatted issue dictionary + """ + # This is a synchronous method that doesn't need token refresh + # since it just formats data that's already been fetched + if self._jira_client is None: + # Create a minimal client just for formatting (doesn't need credentials) + self._jira_client = JiraConnector() + return self._jira_client.format_issue(issue) + + def format_issue_to_markdown(self, issue: dict[str, Any]) -> str: + """ + Convert an issue to markdown format. + Wraps JiraConnector.format_issue_to_markdown(). + + Args: + issue: The issue object (either raw or formatted) + + Returns: + Markdown string representation of the issue + """ + # This is a synchronous method that doesn't need token refresh + # since it just formats data that's already been fetched + if self._jira_client is None: + # Create a minimal client just for formatting (doesn't need credentials) + self._jira_client = JiraConnector() + return self._jira_client.format_issue_to_markdown(issue) + + async def close(self): + """Close any resources (currently no-op for JiraConnector).""" + # JiraConnector doesn't maintain persistent connections, so nothing to close + self._jira_client = None + + async def __aenter__(self): + """Async context manager entry.""" + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit.""" + await self.close() diff --git a/surfsense_backend/app/connectors/linear_connector.py b/surfsense_backend/app/connectors/linear_connector.py index b4c54fda3..b8206a40d 100644 --- a/surfsense_backend/app/connectors/linear_connector.py +++ b/surfsense_backend/app/connectors/linear_connector.py @@ -5,33 +5,203 @@ A module for retrieving issues and comments from Linear. Allows fetching issue lists and their comments with date range filtering. """ +import logging from datetime import datetime from typing import Any +import httpx import requests +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.db import SearchSourceConnector +from app.schemas.linear_auth_credentials import LinearAuthCredentialsBase +from app.utils.oauth_security import TokenEncryption + +logger = logging.getLogger(__name__) + +LINEAR_GRAPHQL_URL = "https://api.linear.app/graphql" + +ORGANIZATION_QUERY = """ +query { + organization { + name + } +} +""" + + +async def fetch_linear_organization_name(access_token: str) -> str | None: + """ + Fetch organization/workspace name from Linear GraphQL API. + + Args: + access_token: The Linear OAuth access token + + Returns: + Organization name or None if fetch fails + """ + try: + async with httpx.AsyncClient() as client: + response = await client.post( + LINEAR_GRAPHQL_URL, + headers={ + "Authorization": access_token, + "Content-Type": "application/json", + }, + json={"query": ORGANIZATION_QUERY}, + timeout=10.0, + ) + + if response.status_code == 200: + data = response.json() + org_name = data.get("data", {}).get("organization", {}).get("name") + if org_name: + logger.debug(f"Fetched Linear organization name: {org_name}") + return org_name + + logger.warning(f"Failed to fetch Linear org info: {response.status_code}") + return None + + except Exception as e: + logger.warning(f"Error fetching Linear organization name: {e!s}") + return None class LinearConnector: """Class for retrieving issues and comments from Linear.""" - def __init__(self, token: str | None = None): + def __init__( + self, + session: AsyncSession, + connector_id: int, + credentials: LinearAuthCredentialsBase | None = None, + ): """ - Initialize the LinearConnector class. + Initialize the LinearConnector class with auto-refresh capability. Args: - token: Linear API token (optional, can be set later with set_token) + session: Database session for updating connector + connector_id: Connector ID for direct updates + credentials: Linear OAuth credentials (optional, will be loaded from DB if not provided) """ - self.token = token + self._session = session + self._connector_id = connector_id + self._credentials = credentials self.api_url = "https://api.linear.app/graphql" - def set_token(self, token: str) -> None: + async def _get_valid_token(self) -> str: """ - Set the Linear API token. + Get valid Linear access token, refreshing if needed. - Args: - token: Linear API token + Returns: + Valid access token + + Raises: + ValueError: If credentials are missing or invalid + Exception: If token refresh fails """ - self.token = token + # Load credentials from DB if not provided + if self._credentials is None: + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise ValueError(f"Connector {self._connector_id} not found") + + config_data = connector.config.copy() + + # Decrypt credentials if they are encrypted + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt sensitive fields + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + logger.info( + f"Decrypted Linear credentials for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to decrypt Linear credentials for connector {self._connector_id}: {e!s}" + ) + raise ValueError( + f"Failed to decrypt Linear credentials: {e!s}" + ) from e + + try: + self._credentials = LinearAuthCredentialsBase.from_dict(config_data) + except Exception as e: + raise ValueError(f"Invalid Linear credentials: {e!s}") from e + + # Check if token is expired and refreshable + if self._credentials.is_expired and self._credentials.is_refreshable: + try: + logger.info( + f"Linear token expired for connector {self._connector_id}, refreshing..." + ) + + # Get connector for refresh + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise RuntimeError( + f"Connector {self._connector_id} not found; cannot refresh token." + ) + + # Lazy import to avoid circular dependency + from app.routes.linear_add_connector_route import refresh_linear_token + + # Refresh token + connector = await refresh_linear_token(self._session, connector) + + # Reload credentials after refresh + config_data = connector.config.copy() + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + token_encryption = TokenEncryption(config.SECRET_KEY) + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + self._credentials = LinearAuthCredentialsBase.from_dict(config_data) + + logger.info( + f"Successfully refreshed Linear token for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to refresh Linear token for connector {self._connector_id}: {e!s}" + ) + raise Exception( + f"Failed to refresh Linear OAuth credentials: {e!s}" + ) from e + + return self._credentials.access_token def get_headers(self) -> dict[str, str]: """ @@ -41,18 +211,26 @@ class LinearConnector: Dictionary of headers Raises: - ValueError: If no Linear token has been set + ValueError: If no Linear access token has been set """ - if not self.token: - raise ValueError("Linear token not initialized. Call set_token() first.") + # This is a synchronous method, but we need async token refresh + # For now, we'll raise an error if called directly + # All API calls should go through execute_graphql_query which handles async refresh + if not self._credentials or not self._credentials.access_token: + raise ValueError( + "Linear access token not initialized. Use execute_graphql_query() method." + ) - return {"Content-Type": "application/json", "Authorization": self.token} + return { + "Content-Type": "application/json", + "Authorization": f"Bearer {self._credentials.access_token}", + } - def execute_graphql_query( + async def execute_graphql_query( self, query: str, variables: dict[str, Any] | None = None ) -> dict[str, Any]: """ - Execute a GraphQL query against the Linear API. + Execute a GraphQL query against the Linear API with automatic token refresh. Args: query: GraphQL query string @@ -62,13 +240,17 @@ class LinearConnector: Response data from the API Raises: - ValueError: If no Linear token has been set + ValueError: If no Linear access token has been set Exception: If the API request fails """ - if not self.token: - raise ValueError("Linear token not initialized. Call set_token() first.") + # Get valid token (refreshes if needed) + access_token = await self._get_valid_token() + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {access_token}", + } - headers = self.get_headers() payload = {"query": query} if variables: @@ -83,7 +265,9 @@ class LinearConnector: f"Query failed with status code {response.status_code}: {response.text}" ) - def get_all_issues(self, include_comments: bool = True) -> list[dict[str, Any]]: + async def get_all_issues( + self, include_comments: bool = True + ) -> list[dict[str, Any]]: """ Fetch all issues from Linear. @@ -94,7 +278,7 @@ class LinearConnector: List of issue objects Raises: - ValueError: If no Linear token has been set + ValueError: If no Linear access token has been set Exception: If the API request fails """ comments_query = "" @@ -146,7 +330,7 @@ class LinearConnector: }} """ - result = self.execute_graphql_query(query) + result = await self.execute_graphql_query(query) # Extract issues from the response if ( @@ -158,7 +342,7 @@ class LinearConnector: return [] - def get_issues_by_date_range( + async def get_issues_by_date_range( self, start_date: str, end_date: str, include_comments: bool = True ) -> tuple[list[dict[str, Any]], str | None]: """ @@ -172,6 +356,18 @@ class LinearConnector: Returns: Tuple containing (issues list, error message or None) """ + # Validate date strings + if not start_date or start_date.lower() in ("undefined", "null", "none"): + return ( + [], + "Invalid start_date: must be a valid date string in YYYY-MM-DD format", + ) + if not end_date or end_date.lower() in ("undefined", "null", "none"): + return ( + [], + "Invalid end_date: must be a valid date string in YYYY-MM-DD format", + ) + # Convert date strings to ISO format try: # For Linear API: we need to use a more specific format for the filter @@ -258,7 +454,7 @@ class LinearConnector: # Handle pagination to get all issues while has_next_page: variables = {"after": cursor} if cursor else {} - result = self.execute_graphql_query(query, variables) + result = await self.execute_graphql_query(query, variables) # Check for errors if "errors" in result: @@ -446,37 +642,3 @@ class LinearConnector: return dt.strftime("%Y-%m-%d %H:%M:%S") except ValueError: return iso_date - - -# Example usage (uncomment to use): -""" -if __name__ == "__main__": - # Set your token here - token = "YOUR_LINEAR_API_KEY" - - linear = LinearConnector(token) - - try: - # Get all issues with comments - issues = linear.get_all_issues() - print(f"Retrieved {len(issues)} issues") - - # Format and print the first issue as markdown - if issues: - issue_md = linear.format_issue_to_markdown(issues[0]) - print("\nSample Issue in Markdown:\n") - print(issue_md) - - # Get issues by date range - start_date = "2023-01-01" - end_date = "2023-01-31" - date_issues, error = linear.get_issues_by_date_range(start_date, end_date) - - if error: - print(f"Error: {error}") - else: - print(f"\nRetrieved {len(date_issues)} issues from {start_date} to {end_date}") - - except Exception as e: - print(f"Error: {e}") -""" diff --git a/surfsense_backend/app/connectors/notion_history.py b/surfsense_backend/app/connectors/notion_history.py index 81f6642f1..e38218a6e 100644 --- a/surfsense_backend/app/connectors/notion_history.py +++ b/surfsense_backend/app/connectors/notion_history.py @@ -1,19 +1,167 @@ +import logging + from notion_client import AsyncClient +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.db import SearchSourceConnector +from app.routes.notion_add_connector_route import refresh_notion_token +from app.schemas.notion_auth_credentials import NotionAuthCredentialsBase +from app.utils.oauth_security import TokenEncryption + +logger = logging.getLogger(__name__) class NotionHistoryConnector: - def __init__(self, token): + def __init__( + self, + session: AsyncSession, + connector_id: int, + credentials: NotionAuthCredentialsBase | None = None, + ): """ - Initialize the NotionPageFetcher with a token. + Initialize the NotionHistoryConnector with auto-refresh capability. Args: - token (str): Notion integration token + session: Database session for updating connector + connector_id: Connector ID for direct updates + credentials: Notion OAuth credentials (optional, will be loaded from DB if not provided) """ - self.notion = AsyncClient(auth=token) + self._session = session + self._connector_id = connector_id + self._credentials = credentials + self._notion_client: AsyncClient | None = None + + async def _get_valid_token(self) -> str: + """ + Get valid Notion access token, refreshing if needed. + + Returns: + Valid access token + + Raises: + ValueError: If credentials are missing or invalid + Exception: If token refresh fails + """ + # Load credentials from DB if not provided + if self._credentials is None: + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise ValueError(f"Connector {self._connector_id} not found") + + config_data = connector.config.copy() + + # Decrypt credentials if they are encrypted + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt sensitive fields + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + logger.info( + f"Decrypted Notion credentials for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to decrypt Notion credentials for connector {self._connector_id}: {e!s}" + ) + raise ValueError( + f"Failed to decrypt Notion credentials: {e!s}" + ) from e + + try: + self._credentials = NotionAuthCredentialsBase.from_dict(config_data) + except Exception as e: + raise ValueError(f"Invalid Notion credentials: {e!s}") from e + + # Check if token is expired and refreshable + if self._credentials.is_expired and self._credentials.is_refreshable: + try: + logger.info( + f"Notion token expired for connector {self._connector_id}, refreshing..." + ) + + # Get connector for refresh + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise RuntimeError( + f"Connector {self._connector_id} not found; cannot refresh token." + ) + + # Refresh token + connector = await refresh_notion_token(self._session, connector) + + # Reload credentials after refresh + config_data = connector.config.copy() + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + token_encryption = TokenEncryption(config.SECRET_KEY) + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + self._credentials = NotionAuthCredentialsBase.from_dict(config_data) + + # Invalidate cached client so it's recreated with new token + self._notion_client = None + + logger.info( + f"Successfully refreshed Notion token for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to refresh Notion token for connector {self._connector_id}: {e!s}" + ) + raise Exception( + f"Failed to refresh Notion OAuth credentials: {e!s}" + ) from e + + return self._credentials.access_token + + async def _get_client(self) -> AsyncClient: + """ + Get or create Notion AsyncClient with valid token. + + Returns: + Notion AsyncClient instance + """ + if self._notion_client is None: + token = await self._get_valid_token() + self._notion_client = AsyncClient(auth=token) + return self._notion_client async def close(self): """Close the async client connection.""" - await self.notion.aclose() + if self._notion_client: + await self._notion_client.aclose() + self._notion_client = None async def __aenter__(self): """Async context manager entry.""" @@ -34,6 +182,8 @@ class NotionHistoryConnector: Returns: list: List of dictionaries containing page data """ + notion = await self._get_client() + # Build the filter for the search # Note: Notion API requires specific filter structure search_params = {} @@ -67,7 +217,7 @@ class NotionHistoryConnector: if cursor: search_params["start_cursor"] = cursor - search_results = await self.notion.search(**search_params) + search_results = await notion.search(**search_params) pages.extend(search_results["results"]) has_more = search_results.get("has_more", False) @@ -125,6 +275,8 @@ class NotionHistoryConnector: Returns: list: List of processed blocks from the page """ + notion = await self._get_client() + blocks = [] has_more = True cursor = None @@ -132,11 +284,11 @@ class NotionHistoryConnector: # Paginate through all blocks while has_more: if cursor: - response = await self.notion.blocks.children.list( + response = await notion.blocks.children.list( block_id=page_id, start_cursor=cursor ) else: - response = await self.notion.blocks.children.list(block_id=page_id) + response = await notion.blocks.children.list(block_id=page_id) blocks.extend(response["results"]) has_more = response["has_more"] @@ -162,6 +314,8 @@ class NotionHistoryConnector: Returns: dict: Processed block with content and children """ + notion = await self._get_client() + block_id = block["id"] block_type = block["type"] @@ -174,9 +328,7 @@ class NotionHistoryConnector: if has_children: # Fetch and process child blocks - children_response = await self.notion.blocks.children.list( - block_id=block_id - ) + children_response = await notion.blocks.children.list(block_id=block_id) for child_block in children_response["results"]: child_blocks.append(await self.process_block(child_block)) diff --git a/surfsense_backend/app/connectors/slack_history.py b/surfsense_backend/app/connectors/slack_history.py index 36160c30b..2b36b9f96 100644 --- a/surfsense_backend/app/connectors/slack_history.py +++ b/surfsense_backend/app/connectors/slack_history.py @@ -12,6 +12,14 @@ from typing import Any from slack_sdk import WebClient from slack_sdk.errors import SlackApiError +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.db import SearchSourceConnector +from app.routes.slack_add_connector_route import refresh_slack_token +from app.schemas.slack_auth_credentials import SlackAuthCredentialsBase +from app.utils.oauth_security import TokenEncryption logger = logging.getLogger(__name__) # Added logger @@ -19,25 +27,199 @@ logger = logging.getLogger(__name__) # Added logger class SlackHistory: """Class for retrieving conversation history from Slack channels.""" - def __init__(self, token: str | None = None): + def __init__( + self, + token: str | None = None, + session: AsyncSession | None = None, + connector_id: int | None = None, + credentials: SlackAuthCredentialsBase | None = None, + ): """ Initialize the SlackHistory class. Args: - token: Slack API token (optional, can be set later with set_token) + token: Slack API token (optional, for backward compatibility) + session: Database session for token refresh (optional) + connector_id: Connector ID for token refresh (optional) + credentials: Slack OAuth credentials (optional, will be loaded from DB if not provided) """ - self.client = WebClient(token=token) if token else None + self._session = session + self._connector_id = connector_id + self._credentials = credentials + # For backward compatibility, if token is provided directly, use it + if token: + self.client = WebClient(token=token) + else: + self.client = None + + async def _get_valid_token(self) -> str: + """ + Get valid Slack bot token, refreshing if needed. + + Returns: + Valid bot token + + Raises: + ValueError: If credentials are missing or invalid + Exception: If token refresh fails + """ + # If we have a direct token (backward compatibility), use it + # Check if client was initialized with a token directly (not via credentials) + if ( + self.client + and self._session is None + and self._connector_id is None + and self._credentials is None + ): + # This means it was initialized with a direct token, extract it + # WebClient stores token internally, we need to get it from the client + # For backward compatibility, we'll use the client directly + # But we can't easily extract the token, so we'll just use the client + # In this case, we'll skip refresh logic + # This is the old pattern - just use the client as-is + # We can't extract token easily, so we'll raise an error + # asking to use the new pattern + raise ValueError( + "Cannot refresh token: Please use session and connector_id for auto-refresh support" + ) + + # Load credentials from DB if not provided + if self._credentials is None: + if not self._session or not self._connector_id: + raise ValueError( + "Cannot load credentials: session and connector_id required" + ) + + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise ValueError(f"Connector {self._connector_id} not found") + + config_data = connector.config.copy() + + # Decrypt credentials if they are encrypted + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt sensitive fields + if config_data.get("bot_token"): + config_data["bot_token"] = token_encryption.decrypt_token( + config_data["bot_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + logger.info( + f"Decrypted Slack credentials for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to decrypt Slack credentials for connector {self._connector_id}: {e!s}" + ) + raise ValueError( + f"Failed to decrypt Slack credentials: {e!s}" + ) from e + + try: + self._credentials = SlackAuthCredentialsBase.from_dict(config_data) + except Exception as e: + raise ValueError(f"Invalid Slack credentials: {e!s}") from e + + # Check if token is expired and refreshable + if self._credentials.is_expired and self._credentials.is_refreshable: + try: + logger.info( + f"Slack token expired for connector {self._connector_id}, refreshing..." + ) + + # Get connector for refresh + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise RuntimeError( + f"Connector {self._connector_id} not found; cannot refresh token." + ) + + # Refresh token + connector = await refresh_slack_token(self._session, connector) + + # Reload credentials after refresh + config_data = connector.config.copy() + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + token_encryption = TokenEncryption(config.SECRET_KEY) + if config_data.get("bot_token"): + config_data["bot_token"] = token_encryption.decrypt_token( + config_data["bot_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + self._credentials = SlackAuthCredentialsBase.from_dict(config_data) + + # Invalidate cached client so it's recreated with new token + self.client = None + + logger.info( + f"Successfully refreshed Slack token for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to refresh Slack token for connector {self._connector_id}: {e!s}" + ) + raise Exception( + f"Failed to refresh Slack OAuth credentials: {e!s}" + ) from e + + return self._credentials.bot_token + + async def _ensure_client(self) -> WebClient: + """ + Ensure Slack client is initialized with valid token. + + Returns: + WebClient instance + """ + # If client was initialized with direct token (backward compatibility), use it + if self.client and (self._session is None or self._connector_id is None): + return self.client + + # Otherwise, initialize with token from credentials (with auto-refresh) + if self.client is None: + token = await self._get_valid_token() + # Skip if it's the placeholder for direct token initialization + if token != "direct_token_initialized": + self.client = WebClient(token=token) + return self.client def set_token(self, token: str) -> None: """ - Set the Slack API token. + Set the Slack API token (for backward compatibility). Args: token: Slack API token """ self.client = WebClient(token=token) - def get_all_channels(self, include_private: bool = True) -> list[dict[str, Any]]: + async def get_all_channels( + self, include_private: bool = True + ) -> list[dict[str, Any]]: """ Fetch all channels that the bot has access to, with rate limit handling. @@ -52,8 +234,7 @@ class SlackHistory: SlackApiError: If there's an unrecoverable error calling the Slack API RuntimeError: For unexpected errors during channel fetching. """ - if not self.client: - raise ValueError("Slack client not initialized. Call set_token() first.") + client = await self._ensure_client() channels_list = [] # Changed from dict to list types = "public_channel" @@ -72,7 +253,7 @@ class SlackHistory: time.sleep(3) current_limit = 1000 # Max limit - api_result = self.client.conversations_list( + api_result = client.conversations_list( types=types, cursor=next_cursor, limit=current_limit ) @@ -129,7 +310,7 @@ class SlackHistory: return channels_list - def get_conversation_history( + async def get_conversation_history( self, channel_id: str, limit: int = 1000, @@ -152,8 +333,7 @@ class SlackHistory: ValueError: If no Slack client has been initialized SlackApiError: If there's an error calling the Slack API """ - if not self.client: - raise ValueError("Slack client not initialized. Call set_token() first.") + client = await self._ensure_client() messages = [] next_cursor = None @@ -177,7 +357,7 @@ class SlackHistory: current_api_call_successful = False result = None # Ensure result is defined try: - result = self.client.conversations_history(**kwargs) + result = client.conversations_history(**kwargs) current_api_call_successful = True except SlackApiError as e_history: if ( @@ -197,7 +377,7 @@ class SlackHistory: else: raise # Re-raise to outer handler for not_in_channel or other SlackApiErrors - if not current_api_call_successful: + if not current_api_call_successful or result is None: continue # Retry the current page fetch due to handled rate limit # Process result if successful @@ -252,7 +432,7 @@ class SlackHistory: except ValueError: return None - def get_history_by_date_range( + async def get_history_by_date_range( self, channel_id: str, start_date: str, end_date: str, limit: int = 1000 ) -> tuple[list[dict[str, Any]], str | None]: """ @@ -282,7 +462,7 @@ class SlackHistory: latest += 86400 # seconds in a day try: - messages = self.get_conversation_history( + messages = await self.get_conversation_history( channel_id=channel_id, limit=limit, oldest=oldest, latest=latest ) return messages, None @@ -291,7 +471,7 @@ class SlackHistory: except ValueError as e: return [], str(e) - def get_user_info(self, user_id: str) -> dict[str, Any]: + async def get_user_info(self, user_id: str) -> dict[str, Any]: """ Get information about a user. @@ -305,8 +485,7 @@ class SlackHistory: ValueError: If no Slack client has been initialized SlackApiError: If there's an error calling the Slack API """ - if not self.client: - raise ValueError("Slack client not initialized. Call set_token() first.") + client = await self._ensure_client() while True: try: @@ -314,7 +493,7 @@ class SlackHistory: # For now, we are only adding Retry-After as per plan. # time.sleep(0.6) # Optional: ~100 req/min if ever needed. - result = self.client.users_info(user=user_id) + result = client.users_info(user=user_id) return result["user"] # Success, return and exit loop implicitly except SlackApiError as e_user_info: @@ -343,7 +522,7 @@ class SlackHistory: ) raise general_error from general_error # Re-raise unexpected errors - def format_message( + async def format_message( self, msg: dict[str, Any], include_user_info: bool = False ) -> dict[str, Any]: """ @@ -369,9 +548,9 @@ class SlackHistory: "is_thread": "thread_ts" in msg, } - if include_user_info and "user" in msg and self.client: + if include_user_info and "user" in msg: try: - user_info = self.get_user_info(msg["user"]) + user_info = await self.get_user_info(msg["user"]) formatted["user_name"] = user_info.get("real_name", "Unknown") formatted["user_email"] = user_info.get("profile", {}).get("email", "") except Exception: diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 3c18650ae..47d540e7d 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -4,6 +4,9 @@ from .airtable_add_connector_route import ( router as airtable_add_connector_router, ) from .circleback_webhook_route import router as circleback_webhook_router +from .clickup_add_connector_route import router as clickup_add_connector_router +from .confluence_add_connector_route import router as confluence_add_connector_router +from .discord_add_connector_route import router as discord_add_connector_router from .documents_routes import router as documents_router from .editor_routes import router as editor_router from .google_calendar_add_connector_route import ( @@ -15,15 +18,19 @@ from .google_drive_add_connector_route import ( from .google_gmail_add_connector_route import ( router as google_gmail_add_connector_router, ) +from .jira_add_connector_route import router as jira_add_connector_router +from .linear_add_connector_route import router as linear_add_connector_router from .logs_routes import router as logs_router from .luma_add_connector_route import router as luma_add_connector_router from .new_chat_routes import router as new_chat_router from .new_llm_config_routes import router as new_llm_config_router from .notes_routes import router as notes_router +from .notion_add_connector_route import router as notion_add_connector_router from .podcasts_routes import router as podcasts_router from .rbac_routes import router as rbac_router from .search_source_connectors_routes import router as search_source_connectors_router from .search_spaces_routes import router as search_spaces_router +from .slack_add_connector_route import router as slack_add_connector_router router = APIRouter() @@ -39,7 +46,14 @@ router.include_router(google_calendar_add_connector_router) router.include_router(google_gmail_add_connector_router) router.include_router(google_drive_add_connector_router) router.include_router(airtable_add_connector_router) +router.include_router(linear_add_connector_router) router.include_router(luma_add_connector_router) +router.include_router(notion_add_connector_router) +router.include_router(slack_add_connector_router) +router.include_router(discord_add_connector_router) +router.include_router(jira_add_connector_router) +router.include_router(confluence_add_connector_router) +router.include_router(clickup_add_connector_router) router.include_router(new_llm_config_router) # LLM configs with prompt configuration router.include_router(logs_router) router.include_router(circleback_webhook_router) # Circleback meeting webhooks diff --git a/surfsense_backend/app/routes/airtable_add_connector_route.py b/surfsense_backend/app/routes/airtable_add_connector_route.py index 3bcbe4dc0..64fa104d8 100644 --- a/surfsense_backend/app/routes/airtable_add_connector_route.py +++ b/surfsense_backend/app/routes/airtable_add_connector_route.py @@ -1,6 +1,5 @@ import base64 import hashlib -import json import logging import secrets from datetime import UTC, datetime, timedelta @@ -12,9 +11,9 @@ from fastapi.responses import RedirectResponse from pydantic import ValidationError from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select from app.config import config +from app.connectors.airtable_connector import fetch_airtable_user_email from app.db import ( SearchSourceConnector, SearchSourceConnectorType, @@ -23,6 +22,11 @@ from app.db import ( ) from app.schemas.airtable_auth_credentials import AirtableAuthCredentialsBase from app.users import current_active_user +from app.utils.connector_naming import ( + check_duplicate_connector, + generate_unique_connector_name, +) +from app.utils.oauth_security import OAuthStateManager, TokenEncryption logger = logging.getLogger(__name__) @@ -40,6 +44,30 @@ SCOPES = [ "user.email:read", ] +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + def make_basic_auth_header(client_id: str, client_secret: str) -> str: credentials = f"{client_id}:{client_secret}".encode() @@ -90,18 +118,19 @@ async def connect_airtable(space_id: int, user: User = Depends(current_active_us status_code=500, detail="Airtable OAuth not configured." ) + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + # Generate PKCE parameters code_verifier, code_challenge = generate_pkce_pair() - # Generate state parameter - state_payload = json.dumps( - { - "space_id": space_id, - "user_id": str(user.id), - "code_verifier": code_verifier, - } + # Generate secure state parameter with HMAC signature (including code_verifier for PKCE) + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state( + space_id, user.id, code_verifier=code_verifier ) - state_encoded = base64.urlsafe_b64encode(state_payload.encode()).decode() # Build authorization URL auth_params = { @@ -134,8 +163,9 @@ async def connect_airtable(space_id: int, user: User = Depends(current_active_us @router.get("/auth/airtable/connector/callback") async def airtable_callback( request: Request, - code: str, - state: str, + code: str | None = None, + error: str | None = None, + state: str | None = None, session: AsyncSession = Depends(get_async_session), ): """ @@ -143,7 +173,8 @@ async def airtable_callback( Args: request: FastAPI request object - code: Authorization code from Airtable + code: Authorization code from Airtable (if user granted access) + error: Error code from Airtable (if user denied access or error occurred) state: State parameter containing user/space info session: Database session @@ -151,10 +182,42 @@ async def airtable_callback( Redirect response to frontend """ try: - # Decode and parse the state + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Airtable OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=airtable_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=airtable_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() try: - decoded_state = base64.urlsafe_b64decode(state.encode()).decode() - data = json.loads(decoded_state) + data = state_manager.validate_state(state) + except HTTPException: + raise except Exception as e: raise HTTPException( status_code=400, detail=f"Invalid state parameter: {e!s}" @@ -162,7 +225,12 @@ async def airtable_callback( user_id = UUID(data["user_id"]) space_id = data["space_id"] - code_verifier = data["code_verifier"] + code_verifier = data.get("code_verifier") + + if not code_verifier: + raise HTTPException( + status_code=400, detail="Missing code_verifier in state parameter" + ) auth_header = make_basic_auth_header( config.AIRTABLE_CLIENT_ID, config.AIRTABLE_CLIENT_SECRET ) @@ -201,56 +269,78 @@ async def airtable_callback( token_json = token_response.json() + # Encrypt sensitive tokens before storing + token_encryption = get_token_encryption() + access_token = token_json.get("access_token") + refresh_token = token_json.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from Airtable" + ) + + user_email = await fetch_airtable_user_email(access_token) + # Calculate expiration time (UTC, tz-aware) expires_at = None if token_json.get("expires_in"): now_utc = datetime.now(UTC) expires_at = now_utc + timedelta(seconds=int(token_json["expires_in"])) - # Create credentials object + # Create credentials object with encrypted tokens credentials = AirtableAuthCredentialsBase( - access_token=token_json["access_token"], - refresh_token=token_json.get("refresh_token"), + access_token=token_encryption.encrypt_token(access_token), + refresh_token=token_encryption.encrypt_token(refresh_token) + if refresh_token + else None, token_type=token_json.get("token_type", "Bearer"), expires_in=token_json.get("expires_in"), expires_at=expires_at, scope=token_json.get("scope"), ) - # Check if connector already exists for this search space and user - existing_connector_result = await session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.AIRTABLE_CONNECTOR, - ) - ) - existing_connector = existing_connector_result.scalars().first() + # Mark that tokens are encrypted for backward compatibility + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True - if existing_connector: - # Update existing connector - existing_connector.config = credentials.to_dict() - existing_connector.name = "Airtable Connector" - existing_connector.is_indexable = True - logger.info( - f"Updated existing Airtable connector for user {user_id} in space {space_id}" + # Check for duplicate connector (same account already connected) + is_duplicate = await check_duplicate_connector( + session, + SearchSourceConnectorType.AIRTABLE_CONNECTOR, + space_id, + user_id, + user_email, + ) + if is_duplicate: + logger.warning( + f"Duplicate Airtable connector detected for user {user_id} with email {user_email}" ) - else: - # Create new connector - new_connector = SearchSourceConnector( - name="Airtable Connector", - connector_type=SearchSourceConnectorType.AIRTABLE_CONNECTOR, - is_indexable=True, - config=credentials.to_dict(), - search_space_id=space_id, - user_id=user_id, - ) - session.add(new_connector) - logger.info( - f"Created new Airtable connector for user {user_id} in space {space_id}" + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=airtable-connector" ) + # Generate a unique, user-friendly connector name + connector_name = await generate_unique_connector_name( + session, + SearchSourceConnectorType.AIRTABLE_CONNECTOR, + space_id, + user_id, + user_email, + ) + # Create new connector + new_connector = SearchSourceConnector( + name=connector_name, + connector_type=SearchSourceConnectorType.AIRTABLE_CONNECTOR, + is_indexable=True, + config=credentials_dict, + search_space_id=space_id, + user_id=user_id, + ) + session.add(new_connector) + logger.info( + f"Created new Airtable connector for user {user_id} in space {space_id}" + ) + try: await session.commit() logger.info(f"Successfully saved Airtable connector for user {user_id}") @@ -258,7 +348,7 @@ async def airtable_callback( # Redirect to the frontend with success params for indexing config # Using query params to auto-open the popup with config view on new-chat page return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=airtable-connector" + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=airtable-connector&connectorId={new_connector.id}" ) except ValidationError as e: @@ -270,7 +360,7 @@ async def airtable_callback( await session.rollback() raise HTTPException( status_code=409, - detail=f"Integrity error: A connector with this type already exists. {e!s}", + detail=f"Database integrity error: {e!s}", ) from e except Exception as e: logger.error(f"Failed to create search source connector: {e!s}") @@ -291,7 +381,7 @@ async def airtable_callback( async def refresh_airtable_token( session: AsyncSession, connector: SearchSourceConnector -): +) -> SearchSourceConnector: """ Refresh the Airtable access token for a connector. @@ -306,6 +396,27 @@ async def refresh_airtable_token( logger.info(f"Refreshing Airtable token for connector {connector.id}") credentials = AirtableAuthCredentialsBase.from_dict(connector.config) + + # Decrypt tokens if they are encrypted + token_encryption = get_token_encryption() + is_encrypted = connector.config.get("_token_encrypted", False) + + refresh_token = credentials.refresh_token + if is_encrypted and refresh_token: + try: + refresh_token = token_encryption.decrypt_token(refresh_token) + except Exception as e: + logger.error(f"Failed to decrypt refresh token: {e!s}") + raise HTTPException( + status_code=500, detail="Failed to decrypt stored refresh token" + ) from e + + if not refresh_token: + raise HTTPException( + status_code=400, + detail="No refresh token available. Please re-authenticate.", + ) + auth_header = make_basic_auth_header( config.AIRTABLE_CLIENT_ID, config.AIRTABLE_CLIENT_SECRET ) @@ -313,7 +424,7 @@ async def refresh_airtable_token( # Prepare token refresh data refresh_data = { "grant_type": "refresh_token", - "refresh_token": credentials.refresh_token, + "refresh_token": refresh_token, "client_id": config.AIRTABLE_CLIENT_ID, "client_secret": config.AIRTABLE_CLIENT_SECRET, } @@ -330,8 +441,14 @@ async def refresh_airtable_token( ) if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error_description", error_detail) + except Exception: + pass raise HTTPException( - status_code=400, detail="Token refresh failed: {token_response.text}" + status_code=400, detail=f"Token refresh failed: {error_detail}" ) token_json = token_response.json() @@ -342,14 +459,29 @@ async def refresh_airtable_token( now_utc = datetime.now(UTC) expires_at = now_utc + timedelta(seconds=int(token_json["expires_in"])) - # Update credentials object - credentials.access_token = token_json["access_token"] + # Encrypt new tokens before storing + access_token = token_json.get("access_token") + new_refresh_token = token_json.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from Airtable refresh" + ) + + # Update credentials object with encrypted tokens + credentials.access_token = token_encryption.encrypt_token(access_token) + if new_refresh_token: + credentials.refresh_token = token_encryption.encrypt_token( + new_refresh_token + ) credentials.expires_in = token_json.get("expires_in") credentials.expires_at = expires_at credentials.scope = token_json.get("scope") - # Update connector config - connector.config = credentials.to_dict() + # Update connector config with encrypted tokens + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict await session.commit() await session.refresh(connector) @@ -358,6 +490,8 @@ async def refresh_airtable_token( ) return connector + except HTTPException: + raise except Exception as e: raise HTTPException( status_code=500, detail=f"Failed to refresh Airtable token: {e!s}" diff --git a/surfsense_backend/app/routes/clickup_add_connector_route.py b/surfsense_backend/app/routes/clickup_add_connector_route.py new file mode 100644 index 000000000..f962f65fb --- /dev/null +++ b/surfsense_backend/app/routes/clickup_add_connector_route.py @@ -0,0 +1,481 @@ +""" +ClickUp Connector OAuth Routes. + +Handles OAuth 2.0 authentication flow for ClickUp connector. +""" + +import logging +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.db import ( + SearchSourceConnector, + SearchSourceConnectorType, + User, + get_async_session, +) +from app.schemas.clickup_auth_credentials import ClickUpAuthCredentialsBase +from app.users import current_active_user +from app.utils.oauth_security import OAuthStateManager, TokenEncryption + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# ClickUp OAuth endpoints +AUTHORIZATION_URL = "https://app.clickup.com/api" +TOKEN_URL = "https://api.clickup.com/api/v2/oauth/token" + +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + +@router.get("/auth/clickup/connector/add") +async def connect_clickup(space_id: int, user: User = Depends(current_active_user)): + """ + Initiate ClickUp OAuth flow. + + Args: + space_id: The search space ID + user: Current authenticated user + + Returns: + Authorization URL for redirect + """ + try: + if not space_id: + raise HTTPException(status_code=400, detail="space_id is required") + + if not config.CLICKUP_CLIENT_ID: + raise HTTPException(status_code=500, detail="ClickUp OAuth not configured.") + + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) + + # Build authorization URL + from urllib.parse import urlencode + + auth_params = { + "client_id": config.CLICKUP_CLIENT_ID, + "redirect_uri": config.CLICKUP_REDIRECT_URI, + "state": state_encoded, + } + + auth_url = f"{AUTHORIZATION_URL}?{urlencode(auth_params)}" + + logger.info(f"Generated ClickUp OAuth URL for user {user.id}, space {space_id}") + return {"auth_url": auth_url} + + except Exception as e: + logger.error(f"Failed to initiate ClickUp OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to initiate ClickUp OAuth: {e!s}" + ) from e + + +@router.get("/auth/clickup/connector/callback") +async def clickup_callback( + request: Request, + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +): + """ + Handle ClickUp OAuth callback. + + Args: + request: FastAPI request object + code: Authorization code from ClickUp (if user granted access) + error: Error code from ClickUp (if user denied access or error occurred) + state: State parameter containing user/space info + session: Database session + + Returns: + Redirect response to frontend + """ + try: + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"ClickUp OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=clickup_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=clickup_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e + + user_id = UUID(data["user_id"]) + space_id = data["space_id"] + + # Validate redirect URI (security: ensure it matches configured value) + if not config.CLICKUP_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="CLICKUP_REDIRECT_URI not configured" + ) + + # Exchange authorization code for access token + token_data = { + "client_id": config.CLICKUP_CLIENT_ID, + "client_secret": config.CLICKUP_CLIENT_SECRET, + "code": code, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + json=token_data, + headers={"Content-Type": "application/json"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error", error_detail) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token exchange failed: {error_detail}" + ) + + token_json = token_response.json() + + # Extract access token + access_token = token_json.get("access_token") + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from ClickUp" + ) + + # Extract refresh token if available + refresh_token = token_json.get("refresh_token") + + # Encrypt sensitive tokens before storing + token_encryption = get_token_encryption() + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Get user information and workspace information from ClickUp API + user_info = {} + workspace_info = {} + try: + async with httpx.AsyncClient() as client: + # Get user info + user_response = await client.get( + "https://api.clickup.com/api/v2/user", + headers={"Authorization": f"Bearer {access_token}"}, + timeout=30.0, + ) + if user_response.status_code == 200: + user_data = user_response.json().get("user", {}) + user_info = { + "user_id": str(user_data.get("id")) + if user_data.get("id") is not None + else None, + "user_email": user_data.get("email"), + "user_name": user_data.get("username"), + } + + # Get workspace (team) info - get the first workspace + team_response = await client.get( + "https://api.clickup.com/api/v2/team", + headers={"Authorization": f"Bearer {access_token}"}, + timeout=30.0, + ) + if team_response.status_code == 200: + teams_data = team_response.json().get("teams", []) + if teams_data and len(teams_data) > 0: + first_team = teams_data[0] + workspace_info = { + "workspace_id": str(first_team.get("id")) + if first_team.get("id") is not None + else None, + "workspace_name": first_team.get("name"), + } + except Exception as e: + logger.warning(f"Failed to fetch user/workspace info from ClickUp: {e!s}") + + # Store the encrypted tokens and user/workspace info in connector config + connector_config = { + "access_token": token_encryption.encrypt_token(access_token), + "refresh_token": token_encryption.encrypt_token(refresh_token) + if refresh_token + else None, + "expires_in": expires_in, + "expires_at": expires_at.isoformat() if expires_at else None, + "user_id": user_info.get("user_id"), + "user_email": user_info.get("user_email"), + "user_name": user_info.get("user_name"), + "workspace_id": workspace_info.get("workspace_id"), + "workspace_name": workspace_info.get("workspace_name"), + # Mark that token is encrypted for backward compatibility + "_token_encrypted": True, + } + + # Check if connector already exists for this search space and user + existing_connector_result = await session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == space_id, + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.CLICKUP_CONNECTOR, + ) + ) + existing_connector = existing_connector_result.scalars().first() + + if existing_connector: + # Update existing connector + existing_connector.config = connector_config + existing_connector.name = "ClickUp Connector" + existing_connector.is_indexable = True + logger.info( + f"Updated existing ClickUp connector for user {user_id} in space {space_id}" + ) + else: + # Create new connector + new_connector = SearchSourceConnector( + name="ClickUp Connector", + connector_type=SearchSourceConnectorType.CLICKUP_CONNECTOR, + is_indexable=True, + config=connector_config, + search_space_id=space_id, + user_id=user_id, + ) + session.add(new_connector) + logger.info( + f"Created new ClickUp connector for user {user_id} in space {space_id}" + ) + + try: + await session.commit() + logger.info(f"Successfully saved ClickUp connector for user {user_id}") + + # Redirect to the frontend with success params + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=clickup-connector" + ) + + except ValidationError as e: + await session.rollback() + raise HTTPException( + status_code=422, detail=f"Validation error: {e!s}" + ) from e + except IntegrityError as e: + await session.rollback() + raise HTTPException( + status_code=409, + detail=f"Integrity error: A connector with this type already exists. {e!s}", + ) from e + except Exception as e: + logger.error(f"Failed to create search source connector: {e!s}") + await session.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to create search source connector: {e!s}", + ) from e + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to complete ClickUp OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to complete ClickUp OAuth: {e!s}" + ) from e + + +async def refresh_clickup_token( + session: AsyncSession, connector: SearchSourceConnector +) -> SearchSourceConnector: + """ + Refresh the ClickUp access token for a connector. + + Args: + session: Database session + connector: ClickUp connector to refresh + + Returns: + Updated connector object + """ + try: + logger.info(f"Refreshing ClickUp token for connector {connector.id}") + + credentials = ClickUpAuthCredentialsBase.from_dict(connector.config) + + # Decrypt tokens if they are encrypted + token_encryption = get_token_encryption() + is_encrypted = connector.config.get("_token_encrypted", False) + + refresh_token = credentials.refresh_token + if is_encrypted and refresh_token: + try: + refresh_token = token_encryption.decrypt_token(refresh_token) + except Exception as e: + logger.error(f"Failed to decrypt refresh token: {e!s}") + raise HTTPException( + status_code=500, detail="Failed to decrypt stored refresh token" + ) from e + + if not refresh_token: + raise HTTPException( + status_code=400, + detail="No refresh token available. Please re-authenticate.", + ) + + # Prepare token refresh data + refresh_data = { + "client_id": config.CLICKUP_CLIENT_ID, + "client_secret": config.CLICKUP_CLIENT_SECRET, + "refresh_token": refresh_token, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + json=refresh_data, + headers={"Content-Type": "application/json"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error", error_detail) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token refresh failed: {error_detail}" + ) + + token_json = token_response.json() + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Encrypt new tokens before storing + access_token = token_json.get("access_token") + new_refresh_token = token_json.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from ClickUp refresh" + ) + + # Update credentials object with encrypted tokens + credentials.access_token = token_encryption.encrypt_token(access_token) + if new_refresh_token: + credentials.refresh_token = token_encryption.encrypt_token( + new_refresh_token + ) + credentials.expires_in = expires_in + credentials.expires_at = expires_at + + # Preserve user and workspace info + if not credentials.user_id: + credentials.user_id = connector.config.get("user_id") + if not credentials.user_email: + credentials.user_email = connector.config.get("user_email") + if not credentials.user_name: + credentials.user_name = connector.config.get("user_name") + if not credentials.workspace_id: + credentials.workspace_id = connector.config.get("workspace_id") + if not credentials.workspace_name: + credentials.workspace_name = connector.config.get("workspace_name") + + # Update connector config with encrypted tokens + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + + logger.info( + f"Successfully refreshed ClickUp token for connector {connector.id}" + ) + + return connector + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to refresh ClickUp token: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to refresh ClickUp token: {e!s}" + ) from e diff --git a/surfsense_backend/app/routes/confluence_add_connector_route.py b/surfsense_backend/app/routes/confluence_add_connector_route.py new file mode 100644 index 000000000..6c5830b17 --- /dev/null +++ b/surfsense_backend/app/routes/confluence_add_connector_route.py @@ -0,0 +1,498 @@ +""" +Confluence Connector OAuth Routes. + +Handles OAuth 2.0 authentication flow for Confluence connector. +""" + +import logging +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import ( + SearchSourceConnector, + SearchSourceConnectorType, + User, + get_async_session, +) +from app.schemas.atlassian_auth_credentials import AtlassianAuthCredentialsBase +from app.users import current_active_user +from app.utils.connector_naming import ( + check_duplicate_connector, + extract_identifier_from_credentials, + generate_unique_connector_name, +) +from app.utils.oauth_security import OAuthStateManager, TokenEncryption + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# Atlassian OAuth endpoints +AUTHORIZATION_URL = "https://auth.atlassian.com/authorize" +TOKEN_URL = "https://auth.atlassian.com/oauth/token" +RESOURCES_URL = "https://api.atlassian.com/oauth/token/accessible-resources" + +# OAuth scopes for Confluence +SCOPES = [ + "read:confluence-user", + "read:space:confluence", + "read:page:confluence", + "read:comment:confluence", + "offline_access", # Required for refresh tokens +] + +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + +@router.get("/auth/confluence/connector/add") +async def connect_confluence(space_id: int, user: User = Depends(current_active_user)): + """ + Initiate Confluence OAuth flow. + + Args: + space_id: The search space ID + user: Current authenticated user + + Returns: + Authorization URL for redirect + """ + try: + if not space_id: + raise HTTPException(status_code=400, detail="space_id is required") + + if not config.ATLASSIAN_CLIENT_ID: + raise HTTPException( + status_code=500, detail="Atlassian OAuth not configured." + ) + + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) + + # Build authorization URL + from urllib.parse import urlencode + + auth_params = { + "audience": "api.atlassian.com", + "client_id": config.ATLASSIAN_CLIENT_ID, + "scope": " ".join(SCOPES), + "redirect_uri": config.CONFLUENCE_REDIRECT_URI, + "state": state_encoded, + "response_type": "code", + "prompt": "consent", + } + + auth_url = f"{AUTHORIZATION_URL}?{urlencode(auth_params)}" + + logger.info( + f"Generated Confluence OAuth URL for user {user.id}, space {space_id}" + ) + return {"auth_url": auth_url} + + except Exception as e: + logger.error(f"Failed to initiate Confluence OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to initiate Confluence OAuth: {e!s}" + ) from e + + +@router.get("/auth/confluence/connector/callback") +async def confluence_callback( + request: Request, + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +): + """ + Handle Confluence OAuth callback. + + Args: + request: FastAPI request object + code: Authorization code from Atlassian (if user granted access) + error: Error code from Atlassian (if user denied access or error occurred) + state: State parameter containing user/space info + session: Database session + + Returns: + Redirect response to frontend + """ + try: + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Confluence OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=confluence_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=confluence_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e + + user_id = UUID(data["user_id"]) + space_id = data["space_id"] + + # Validate redirect URI (security: ensure it matches configured value) + if not config.CONFLUENCE_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="CONFLUENCE_REDIRECT_URI not configured" + ) + + # Exchange authorization code for access token + token_data = { + "grant_type": "authorization_code", + "client_id": config.ATLASSIAN_CLIENT_ID, + "client_secret": config.ATLASSIAN_CLIENT_SECRET, + "code": code, + "redirect_uri": config.CONFLUENCE_REDIRECT_URI, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + json=token_data, + headers={"Content-Type": "application/json"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get( + "error_description", error_json.get("error", error_detail) + ) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token exchange failed: {error_detail}" + ) + + token_json = token_response.json() + + access_token = token_json.get("access_token") + refresh_token = token_json.get("refresh_token") + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from Atlassian" + ) + + # Get accessible resources to find Confluence cloud ID and site URL + async with httpx.AsyncClient() as client: + resources_response = await client.get( + RESOURCES_URL, + headers={"Authorization": f"Bearer {access_token}"}, + timeout=30.0, + ) + + cloud_id = None + site_url = None + if resources_response.status_code == 200: + resources = resources_response.json() + # Find Confluence resource + for resource in resources: + if resource.get("id") and resource.get("name"): + cloud_id = resource.get("id") + site_url = resource.get("url") + break + + if not cloud_id: + logger.warning( + "Could not determine Confluence cloud ID from accessible resources" + ) + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Encrypt sensitive tokens before storing + token_encryption = get_token_encryption() + + # Store the encrypted tokens and metadata in connector config + connector_config = { + "access_token": token_encryption.encrypt_token(access_token), + "refresh_token": token_encryption.encrypt_token(refresh_token) + if refresh_token + else None, + "token_type": token_json.get("token_type", "Bearer"), + "expires_in": expires_in, + "expires_at": expires_at.isoformat() if expires_at else None, + "scope": token_json.get("scope"), + "cloud_id": cloud_id, + "base_url": site_url, # Store as base_url to match shared schema + # Mark that tokens are encrypted for backward compatibility + "_token_encrypted": True, + } + + # Extract unique identifier from connector credentials + connector_identifier = extract_identifier_from_credentials( + SearchSourceConnectorType.CONFLUENCE_CONNECTOR, connector_config + ) + + # Check for duplicate connector (same Confluence instance already connected) + is_duplicate = await check_duplicate_connector( + session, + SearchSourceConnectorType.CONFLUENCE_CONNECTOR, + space_id, + user_id, + connector_identifier, + ) + if is_duplicate: + logger.warning( + f"Duplicate Confluence connector detected for user {user_id} with instance {connector_identifier}" + ) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=confluence-connector" + ) + + # Generate a unique, user-friendly connector name + connector_name = await generate_unique_connector_name( + session, + SearchSourceConnectorType.CONFLUENCE_CONNECTOR, + space_id, + user_id, + connector_identifier, + ) + # Create new connector + new_connector = SearchSourceConnector( + name=connector_name, + connector_type=SearchSourceConnectorType.CONFLUENCE_CONNECTOR, + is_indexable=True, + config=connector_config, + search_space_id=space_id, + user_id=user_id, + ) + session.add(new_connector) + logger.info( + f"Created new Confluence connector for user {user_id} in space {space_id}" + ) + + try: + await session.commit() + logger.info(f"Successfully saved Confluence connector for user {user_id}") + + # Redirect to the frontend with success params + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=confluence-connector&connectorId={new_connector.id}" + ) + + except ValidationError as e: + await session.rollback() + raise HTTPException( + status_code=422, detail=f"Validation error: {e!s}" + ) from e + except IntegrityError as e: + await session.rollback() + raise HTTPException( + status_code=409, + detail=f"Database integrity error: {e!s}", + ) from e + except Exception as e: + logger.error(f"Failed to create search source connector: {e!s}") + await session.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to create search source connector: {e!s}", + ) from e + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to complete Confluence OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to complete Confluence OAuth: {e!s}" + ) from e + + +async def refresh_confluence_token( + session: AsyncSession, connector: SearchSourceConnector +) -> SearchSourceConnector: + """ + Refresh the Confluence access token for a connector. + + Args: + session: Database session + connector: Confluence connector to refresh + + Returns: + Updated connector object + """ + try: + logger.info(f"Refreshing Confluence token for connector {connector.id}") + + credentials = AtlassianAuthCredentialsBase.from_dict(connector.config) + + # Decrypt tokens if they are encrypted + token_encryption = get_token_encryption() + is_encrypted = connector.config.get("_token_encrypted", False) + + refresh_token = credentials.refresh_token + if is_encrypted and refresh_token: + try: + refresh_token = token_encryption.decrypt_token(refresh_token) + except Exception as e: + logger.error(f"Failed to decrypt refresh token: {e!s}") + raise HTTPException( + status_code=500, detail="Failed to decrypt stored refresh token" + ) from e + + if not refresh_token: + raise HTTPException( + status_code=400, + detail="No refresh token available. Please re-authenticate.", + ) + + # Prepare token refresh data + refresh_data = { + "grant_type": "refresh_token", + "client_id": config.ATLASSIAN_CLIENT_ID, + "client_secret": config.ATLASSIAN_CLIENT_SECRET, + "refresh_token": refresh_token, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + json=refresh_data, + headers={"Content-Type": "application/json"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get( + "error_description", error_json.get("error", error_detail) + ) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token refresh failed: {error_detail}" + ) + + token_json = token_response.json() + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Encrypt new tokens before storing + access_token = token_json.get("access_token") + new_refresh_token = token_json.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=400, + detail="No access token received from Confluence refresh", + ) + + # Update credentials object with encrypted tokens + credentials.access_token = token_encryption.encrypt_token(access_token) + if new_refresh_token: + credentials.refresh_token = token_encryption.encrypt_token( + new_refresh_token + ) + credentials.expires_in = expires_in + credentials.expires_at = expires_at + credentials.scope = token_json.get("scope") + + # Preserve cloud_id and base_url (with backward compatibility for site_url) + if not credentials.cloud_id: + credentials.cloud_id = connector.config.get("cloud_id") + if not credentials.base_url: + # Check both base_url and site_url for backward compatibility + credentials.base_url = connector.config.get( + "base_url" + ) or connector.config.get("site_url") + + # Update connector config with encrypted tokens + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + + logger.info( + f"Successfully refreshed Confluence token for connector {connector.id}" + ) + + return connector + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to refresh Confluence token: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to refresh Confluence token: {e!s}" + ) from e diff --git a/surfsense_backend/app/routes/discord_add_connector_route.py b/surfsense_backend/app/routes/discord_add_connector_route.py new file mode 100644 index 000000000..1d8b40fcf --- /dev/null +++ b/surfsense_backend/app/routes/discord_add_connector_route.py @@ -0,0 +1,533 @@ +""" +Discord Connector OAuth Routes. + +Handles OAuth 2.0 authentication flow for Discord connector. +""" + +import logging +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import ( + SearchSourceConnector, + SearchSourceConnectorType, + User, + get_async_session, +) +from app.schemas.discord_auth_credentials import DiscordAuthCredentialsBase +from app.users import current_active_user +from app.utils.connector_naming import ( + check_duplicate_connector, + extract_identifier_from_credentials, + generate_unique_connector_name, +) +from app.utils.oauth_security import OAuthStateManager, TokenEncryption + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# Discord OAuth endpoints +AUTHORIZATION_URL = "https://discord.com/api/oauth2/authorize" +TOKEN_URL = "https://discord.com/api/oauth2/token" + +# OAuth scopes for Discord (Bot Token) +SCOPES = [ + "bot", # Basic bot scope + "guilds", # Access to guild information + "guilds.members.read", # Read member information +] + +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + +@router.get("/auth/discord/connector/add") +async def connect_discord(space_id: int, user: User = Depends(current_active_user)): + """ + Initiate Discord OAuth flow. + + Args: + space_id: The search space ID + user: Current authenticated user + + Returns: + Authorization URL for redirect + """ + try: + if not space_id: + raise HTTPException(status_code=400, detail="space_id is required") + + if not config.DISCORD_CLIENT_ID: + raise HTTPException(status_code=500, detail="Discord OAuth not configured.") + + if not config.DISCORD_BOT_TOKEN: + raise HTTPException( + status_code=500, + detail="Discord bot token not configured. Please set DISCORD_BOT_TOKEN in backend configuration.", + ) + + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) + + # Build authorization URL + from urllib.parse import urlencode + + auth_params = { + "client_id": config.DISCORD_CLIENT_ID, + "scope": " ".join(SCOPES), # Discord uses space-separated scopes + "redirect_uri": config.DISCORD_REDIRECT_URI, + "response_type": "code", + "state": state_encoded, + } + + auth_url = f"{AUTHORIZATION_URL}?{urlencode(auth_params)}" + + logger.info(f"Generated Discord OAuth URL for user {user.id}, space {space_id}") + return {"auth_url": auth_url} + + except Exception as e: + logger.error(f"Failed to initiate Discord OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to initiate Discord OAuth: {e!s}" + ) from e + + +@router.get("/auth/discord/connector/callback") +async def discord_callback( + request: Request, + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +): + """ + Handle Discord OAuth callback. + + Args: + request: FastAPI request object + code: Authorization code from Discord (if user granted access) + error: Error code from Discord (if user denied access or error occurred) + state: State parameter containing user/space info + session: Database session + + Returns: + Redirect response to frontend + """ + try: + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Discord OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=discord_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=discord_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e + + user_id = UUID(data["user_id"]) + space_id = data["space_id"] + + # Validate redirect URI (security: ensure it matches configured value) + if not config.DISCORD_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="DISCORD_REDIRECT_URI not configured" + ) + + # Exchange authorization code for access token + token_data = { + "client_id": config.DISCORD_CLIENT_ID, + "client_secret": config.DISCORD_CLIENT_SECRET, + "grant_type": "authorization_code", + "code": code, + "redirect_uri": config.DISCORD_REDIRECT_URI, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=token_data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get( + "error_description", error_json.get("error", error_detail) + ) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token exchange failed: {error_detail}" + ) + + token_json = token_response.json() + + # Log OAuth response for debugging (without sensitive data) + logger.info(f"Discord OAuth response received. Keys: {list(token_json.keys())}") + + # Discord OAuth with 'bot' scope returns access_token (user token), not bot token + # The bot token must come from backend config (DISCORD_BOT_TOKEN) + # OAuth is used to authorize bot installation to servers, not to get bot token + if not config.DISCORD_BOT_TOKEN: + raise HTTPException( + status_code=500, + detail="Discord bot token not configured. Please set DISCORD_BOT_TOKEN in backend configuration.", + ) + + # Use the bot token from backend config (not the OAuth access_token) + bot_token = config.DISCORD_BOT_TOKEN + + # Extract OAuth access_token and refresh_token (for reference, not used for bot operations) + oauth_access_token = token_json.get("access_token") + refresh_token = token_json.get("refresh_token") + + # Encrypt sensitive tokens before storing + token_encryption = get_token_encryption() + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + if token_json.get("expires_in"): + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(token_json["expires_in"])) + + # Extract guild info from OAuth response if available + guild_id = None + guild_name = None + if token_json.get("guild"): + guild_id = token_json["guild"].get("id") + guild_name = token_json["guild"].get("name") + + # Store the bot token from config and OAuth metadata + connector_config = { + "bot_token": token_encryption.encrypt_token( + bot_token + ), # Use bot token from config + "oauth_access_token": token_encryption.encrypt_token(oauth_access_token) + if oauth_access_token + else None, # Store OAuth token for reference + "refresh_token": token_encryption.encrypt_token(refresh_token) + if refresh_token + else None, + "token_type": token_json.get("token_type", "Bearer"), + "expires_in": token_json.get("expires_in"), + "expires_at": expires_at.isoformat() if expires_at else None, + "scope": token_json.get("scope"), + "guild_id": guild_id, + "guild_name": guild_name, + # Mark that tokens are encrypted for backward compatibility + "_token_encrypted": True, + } + + # Extract unique identifier from connector credentials + connector_identifier = extract_identifier_from_credentials( + SearchSourceConnectorType.DISCORD_CONNECTOR, connector_config + ) + + # Check for duplicate connector (same server already connected) + is_duplicate = await check_duplicate_connector( + session, + SearchSourceConnectorType.DISCORD_CONNECTOR, + space_id, + user_id, + connector_identifier, + ) + if is_duplicate: + logger.warning( + f"Duplicate Discord connector detected for user {user_id} with server {connector_identifier}" + ) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=discord-connector" + ) + + # Generate a unique, user-friendly connector name + connector_name = await generate_unique_connector_name( + session, + SearchSourceConnectorType.DISCORD_CONNECTOR, + space_id, + user_id, + connector_identifier, + ) + # Create new connector + new_connector = SearchSourceConnector( + name=connector_name, + connector_type=SearchSourceConnectorType.DISCORD_CONNECTOR, + is_indexable=True, + config=connector_config, + search_space_id=space_id, + user_id=user_id, + ) + session.add(new_connector) + logger.info( + f"Created new Discord connector for user {user_id} in space {space_id}" + ) + + try: + await session.commit() + logger.info(f"Successfully saved Discord connector for user {user_id}") + + # Redirect to the frontend with success params + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=discord-connector&connectorId={new_connector.id}" + ) + + except ValidationError as e: + await session.rollback() + raise HTTPException( + status_code=422, detail=f"Validation error: {e!s}" + ) from e + except IntegrityError as e: + await session.rollback() + raise HTTPException( + status_code=409, + detail=f"Database integrity error: {e!s}", + ) from e + except Exception as e: + logger.error(f"Failed to create search source connector: {e!s}") + await session.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to create search source connector: {e!s}", + ) from e + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to complete Discord OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to complete Discord OAuth: {e!s}" + ) from e + + +async def refresh_discord_token( + session: AsyncSession, connector: SearchSourceConnector +) -> SearchSourceConnector: + """ + Refresh the Discord OAuth tokens for a connector. + + Note: Bot tokens from config don't expire, but OAuth access tokens might. + This function refreshes OAuth tokens if needed, but always uses bot token from config. + + Args: + session: Database session + connector: Discord connector to refresh + + Returns: + Updated connector object + """ + try: + logger.info(f"Refreshing Discord OAuth tokens for connector {connector.id}") + + # Bot token always comes from config, not from OAuth + if not config.DISCORD_BOT_TOKEN: + raise HTTPException( + status_code=500, + detail="Discord bot token not configured. Please set DISCORD_BOT_TOKEN in backend configuration.", + ) + + credentials = DiscordAuthCredentialsBase.from_dict(connector.config) + + # Decrypt tokens if they are encrypted + token_encryption = get_token_encryption() + is_encrypted = connector.config.get("_token_encrypted", False) + + refresh_token = credentials.refresh_token + if is_encrypted and refresh_token: + try: + refresh_token = token_encryption.decrypt_token(refresh_token) + except Exception as e: + logger.error(f"Failed to decrypt refresh token: {e!s}") + raise HTTPException( + status_code=500, detail="Failed to decrypt stored refresh token" + ) from e + + # If no refresh token, bot token from config is still valid (bot tokens don't expire) + # Just update the bot token from config in case it was changed + if not refresh_token: + logger.info( + f"No refresh token available for connector {connector.id}. Using bot token from config." + ) + # Update bot token from config (in case it was changed) + credentials.bot_token = token_encryption.encrypt_token( + config.DISCORD_BOT_TOKEN + ) + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + return connector + + # Discord uses oauth2/token for token refresh with grant_type=refresh_token + refresh_data = { + "client_id": config.DISCORD_CLIENT_ID, + "client_secret": config.DISCORD_CLIENT_SECRET, + "grant_type": "refresh_token", + "refresh_token": refresh_token, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=refresh_data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get( + "error_description", error_json.get("error", error_detail) + ) + except Exception: + pass + # If refresh fails, bot token from config is still valid + logger.warning( + f"OAuth token refresh failed for connector {connector.id}: {error_detail}. " + "Using bot token from config." + ) + # Update bot token from config + credentials.bot_token = token_encryption.encrypt_token( + config.DISCORD_BOT_TOKEN + ) + credentials.refresh_token = None # Clear invalid refresh token + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + return connector + + token_json = token_response.json() + + # Extract OAuth access token from refresh response (for reference) + oauth_access_token = token_json.get("access_token") + + # Get new refresh token if provided (Discord may rotate refresh tokens) + new_refresh_token = token_json.get("refresh_token") + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Always use bot token from config (bot tokens don't expire) + credentials.bot_token = token_encryption.encrypt_token(config.DISCORD_BOT_TOKEN) + + # Update OAuth tokens if available + if oauth_access_token: + # Store OAuth access token for reference + connector.config["oauth_access_token"] = token_encryption.encrypt_token( + oauth_access_token + ) + if new_refresh_token: + credentials.refresh_token = token_encryption.encrypt_token( + new_refresh_token + ) + credentials.expires_in = expires_in + credentials.expires_at = expires_at + credentials.scope = token_json.get("scope") + + # Preserve guild info if present + if not credentials.guild_id: + credentials.guild_id = connector.config.get("guild_id") + if not credentials.guild_name: + credentials.guild_name = connector.config.get("guild_name") + if not credentials.bot_user_id: + credentials.bot_user_id = connector.config.get("bot_user_id") + + # Update connector config with encrypted tokens + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + + logger.info( + f"Successfully refreshed Discord OAuth tokens for connector {connector.id}" + ) + + return connector + except HTTPException: + raise + except Exception as e: + logger.error( + f"Failed to refresh Discord tokens for connector {connector.id}: {e!s}", + exc_info=True, + ) + raise HTTPException( + status_code=500, detail=f"Failed to refresh Discord tokens: {e!s}" + ) from e diff --git a/surfsense_backend/app/routes/google_calendar_add_connector_route.py b/surfsense_backend/app/routes/google_calendar_add_connector_route.py index 8bb685450..08e5c2f04 100644 --- a/surfsense_backend/app/routes/google_calendar_add_connector_route.py +++ b/surfsense_backend/app/routes/google_calendar_add_connector_route.py @@ -2,7 +2,6 @@ import os os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" -import base64 import json import logging from uuid import UUID @@ -13,9 +12,9 @@ from google_auth_oauthlib.flow import Flow from pydantic import ValidationError from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select from app.config import config +from app.connectors.google_gmail_connector import fetch_google_user_email from app.db import ( SearchSourceConnector, SearchSourceConnectorType, @@ -23,6 +22,11 @@ from app.db import ( get_async_session, ) from app.users import current_active_user +from app.utils.connector_naming import ( + check_duplicate_connector, + generate_unique_connector_name, +) +from app.utils.oauth_security import OAuthStateManager, TokenEncryption logger = logging.getLogger(__name__) @@ -31,6 +35,30 @@ router = APIRouter() SCOPES = ["https://www.googleapis.com/auth/calendar.readonly"] REDIRECT_URI = config.GOOGLE_CALENDAR_REDIRECT_URI +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + def get_google_flow(): try: @@ -59,16 +87,16 @@ async def connect_calendar(space_id: int, user: User = Depends(current_active_us if not space_id: raise HTTPException(status_code=400, detail="space_id is required") + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + flow = get_google_flow() - # Encode space_id and user_id in state - state_payload = json.dumps( - { - "space_id": space_id, - "user_id": str(user.id), - } - ) - state_encoded = base64.urlsafe_b64encode(state_payload.encode()).decode() + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) auth_url, _ = flow.authorization_url( access_type="offline", @@ -86,42 +114,116 @@ async def connect_calendar(space_id: int, user: User = Depends(current_active_us @router.get("/auth/google/calendar/connector/callback") async def calendar_callback( request: Request, - code: str, - state: str, + code: str | None = None, + error: str | None = None, + state: str | None = None, session: AsyncSession = Depends(get_async_session), ): try: - # Decode and parse the state - decoded_state = base64.urlsafe_b64decode(state.encode()).decode() - data = json.loads(decoded_state) + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Google Calendar OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=google_calendar_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=google_calendar_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e user_id = UUID(data["user_id"]) space_id = data["space_id"] + # Validate redirect URI (security: ensure it matches configured value) + if not config.GOOGLE_CALENDAR_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="GOOGLE_CALENDAR_REDIRECT_URI not configured" + ) + flow = get_google_flow() flow.fetch_token(code=code) creds = flow.credentials creds_dict = json.loads(creds.to_json()) - try: - # Check if a connector with the same type already exists for this search space and user - result = await session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, - ) + # Fetch user email + user_email = fetch_google_user_email(creds) + + # Encrypt sensitive credentials before storing + token_encryption = get_token_encryption() + + # Encrypt sensitive fields: token, refresh_token, client_secret + if creds_dict.get("token"): + creds_dict["token"] = token_encryption.encrypt_token(creds_dict["token"]) + if creds_dict.get("refresh_token"): + creds_dict["refresh_token"] = token_encryption.encrypt_token( + creds_dict["refresh_token"] + ) + if creds_dict.get("client_secret"): + creds_dict["client_secret"] = token_encryption.encrypt_token( + creds_dict["client_secret"] + ) + + # Mark that credentials are encrypted for backward compatibility + creds_dict["_token_encrypted"] = True + + # Check for duplicate connector (same account already connected) + is_duplicate = await check_duplicate_connector( + session, + SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, + space_id, + user_id, + user_email, + ) + if is_duplicate: + logger.warning( + f"Duplicate Google Calendar connector detected for user {user_id} with email {user_email}" + ) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=google-calendar-connector" + ) + + try: + # Generate a unique, user-friendly connector name + connector_name = await generate_unique_connector_name( + session, + SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, + space_id, + user_id, + user_email, ) - existing_connector = result.scalars().first() - if existing_connector: - raise HTTPException( - status_code=409, - detail="A GOOGLE_CALENDAR_CONNECTOR connector already exists in this search space. Each search space can have only one connector of each type per user.", - ) db_connector = SearchSourceConnector( - name="Google Calendar Connector", + name=connector_name, connector_type=SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, config=creds_dict, search_space_id=space_id, @@ -134,7 +236,7 @@ async def calendar_callback( # Redirect to the frontend with success params for indexing config # Using query params to auto-open the popup with config view on new-chat page return RedirectResponse( - f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-calendar-connector" + f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-calendar-connector&connectorId={db_connector.id}" ) except ValidationError as e: await session.rollback() @@ -145,7 +247,7 @@ async def calendar_callback( await session.rollback() raise HTTPException( status_code=409, - detail=f"Integrity error: A connector with this type already exists. {e!s}", + detail=f"Database integrity error: {e!s}", ) from e except HTTPException: await session.rollback() diff --git a/surfsense_backend/app/routes/google_drive_add_connector_route.py b/surfsense_backend/app/routes/google_drive_add_connector_route.py index 52461319b..e15aed762 100644 --- a/surfsense_backend/app/routes/google_drive_add_connector_route.py +++ b/surfsense_backend/app/routes/google_drive_add_connector_route.py @@ -10,7 +10,6 @@ Endpoints: - GET /connectors/{connector_id}/google-drive/folders - List user's folders (for index-time selection) """ -import base64 import json import logging import os @@ -30,6 +29,7 @@ from app.connectors.google_drive import ( get_start_page_token, list_folder_contents, ) +from app.connectors.google_gmail_connector import fetch_google_user_email from app.db import ( SearchSourceConnector, SearchSourceConnectorType, @@ -37,6 +37,11 @@ from app.db import ( get_async_session, ) from app.users import current_active_user +from app.utils.connector_naming import ( + check_duplicate_connector, + generate_unique_connector_name, +) +from app.utils.oauth_security import OAuthStateManager, TokenEncryption # Relax token scope validation for Google OAuth os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" @@ -44,6 +49,31 @@ os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" logger = logging.getLogger(__name__) router = APIRouter() +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + # Google Drive OAuth scopes SCOPES = [ "https://www.googleapis.com/auth/drive.readonly", # Read-only access to Drive @@ -90,16 +120,16 @@ async def connect_drive(space_id: int, user: User = Depends(current_active_user) if not space_id: raise HTTPException(status_code=400, detail="space_id is required") + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + flow = get_google_flow() - # Encode space_id and user_id in state parameter - state_payload = json.dumps( - { - "space_id": space_id, - "user_id": str(user.id), - } - ) - state_encoded = base64.urlsafe_b64encode(state_payload.encode()).decode() + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) # Generate authorization URL auth_url, _ = flow.authorization_url( @@ -124,8 +154,9 @@ async def connect_drive(space_id: int, user: User = Depends(current_active_user) @router.get("/auth/google/drive/connector/callback") async def drive_callback( request: Request, - code: str, - state: str, + code: str | None = None, + error: str | None = None, + state: str | None = None, session: AsyncSession = Depends(get_async_session), ): """ @@ -133,15 +164,53 @@ async def drive_callback( Query params: code: Authorization code from Google + error: OAuth error (if user denied access) state: Encoded state with space_id and user_id Returns: Redirect to frontend success page """ try: - # Decode and parse state - decoded_state = base64.urlsafe_b64decode(state.encode()).decode() - data = json.loads(decoded_state) + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Google Drive OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=google_drive_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=google_drive_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e user_id = UUID(data["user_id"]) space_id = data["space_id"] @@ -150,6 +219,12 @@ async def drive_callback( f"Processing Google Drive callback for user {user_id}, space {space_id}" ) + # Validate redirect URI (security: ensure it matches configured value) + if not config.GOOGLE_DRIVE_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="GOOGLE_DRIVE_REDIRECT_URI not configured" + ) + # Exchange authorization code for tokens flow = get_google_flow() flow.fetch_token(code=code) @@ -157,26 +232,54 @@ async def drive_callback( creds = flow.credentials creds_dict = json.loads(creds.to_json()) - # Check if connector already exists for this space/user - result = await session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR, + # Fetch user email + user_email = fetch_google_user_email(creds) + + # Encrypt sensitive credentials before storing + token_encryption = get_token_encryption() + + # Encrypt sensitive fields: token, refresh_token, client_secret + if creds_dict.get("token"): + creds_dict["token"] = token_encryption.encrypt_token(creds_dict["token"]) + if creds_dict.get("refresh_token"): + creds_dict["refresh_token"] = token_encryption.encrypt_token( + creds_dict["refresh_token"] ) + if creds_dict.get("client_secret"): + creds_dict["client_secret"] = token_encryption.encrypt_token( + creds_dict["client_secret"] + ) + + # Mark that credentials are encrypted for backward compatibility + creds_dict["_token_encrypted"] = True + + # Check for duplicate connector (same account already connected) + is_duplicate = await check_duplicate_connector( + session, + SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR, + space_id, + user_id, + user_email, ) - existing_connector = result.scalars().first() - - if existing_connector: - raise HTTPException( - status_code=409, - detail="A GOOGLE_DRIVE_CONNECTOR already exists in this search space. Each search space can have only one connector of each type per user.", + if is_duplicate: + logger.warning( + f"Duplicate Google Drive connector detected for user {user_id} with email {user_email}" + ) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=google-drive-connector" ) - # Create new connector (NO folder selection here - happens at index time) + # Generate a unique, user-friendly connector name + connector_name = await generate_unique_connector_name( + session, + SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR, + space_id, + user_id, + user_email, + ) + db_connector = SearchSourceConnector( - name="Google Drive Connector", + name=connector_name, connector_type=SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR, config={ **creds_dict, @@ -213,7 +316,7 @@ async def drive_callback( ) return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-drive-connector" + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-drive-connector&connectorId={db_connector.id}" ) except HTTPException: @@ -230,7 +333,7 @@ async def drive_callback( logger.error(f"Database integrity error: {e!s}", exc_info=True) raise HTTPException( status_code=409, - detail="A connector with this configuration already exists.", + detail=f"Database integrity error: {e!s}", ) from e except Exception as e: await session.rollback() diff --git a/surfsense_backend/app/routes/google_gmail_add_connector_route.py b/surfsense_backend/app/routes/google_gmail_add_connector_route.py index 21fcf2c38..19fa019ce 100644 --- a/surfsense_backend/app/routes/google_gmail_add_connector_route.py +++ b/surfsense_backend/app/routes/google_gmail_add_connector_route.py @@ -2,7 +2,6 @@ import os os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" -import base64 import json import logging from uuid import UUID @@ -13,9 +12,9 @@ from google_auth_oauthlib.flow import Flow from pydantic import ValidationError from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select from app.config import config +from app.connectors.google_gmail_connector import fetch_google_user_email from app.db import ( SearchSourceConnector, SearchSourceConnectorType, @@ -23,51 +22,94 @@ from app.db import ( get_async_session, ) from app.users import current_active_user +from app.utils.connector_naming import ( + check_duplicate_connector, + generate_unique_connector_name, +) +from app.utils.oauth_security import OAuthStateManager, TokenEncryption logger = logging.getLogger(__name__) router = APIRouter() +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + def get_google_flow(): """Create and return a Google OAuth flow for Gmail API.""" - flow = Flow.from_client_config( - { - "web": { - "client_id": config.GOOGLE_OAUTH_CLIENT_ID, - "client_secret": config.GOOGLE_OAUTH_CLIENT_SECRET, - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "redirect_uris": [config.GOOGLE_GMAIL_REDIRECT_URI], - } - }, - scopes=[ - "https://www.googleapis.com/auth/gmail.readonly", - "https://www.googleapis.com/auth/userinfo.email", - "https://www.googleapis.com/auth/userinfo.profile", - "openid", - ], - ) - flow.redirect_uri = config.GOOGLE_GMAIL_REDIRECT_URI - return flow + try: + flow = Flow.from_client_config( + { + "web": { + "client_id": config.GOOGLE_OAUTH_CLIENT_ID, + "client_secret": config.GOOGLE_OAUTH_CLIENT_SECRET, + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "redirect_uris": [config.GOOGLE_GMAIL_REDIRECT_URI], + } + }, + scopes=[ + "https://www.googleapis.com/auth/gmail.readonly", + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/userinfo.profile", + "openid", + ], + ) + flow.redirect_uri = config.GOOGLE_GMAIL_REDIRECT_URI + return flow + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Failed to create Google flow: {e!s}" + ) from e @router.get("/auth/google/gmail/connector/add") async def connect_gmail(space_id: int, user: User = Depends(current_active_user)): + """ + Initiate Google Gmail OAuth flow. + + Query params: + space_id: Search space ID to add connector to + + Returns: + JSON with auth_url to redirect user to Google authorization + """ try: if not space_id: raise HTTPException(status_code=400, detail="space_id is required") + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + flow = get_google_flow() - # Encode space_id and user_id in state - state_payload = json.dumps( - { - "space_id": space_id, - "user_id": str(user.id), - } - ) - state_encoded = base64.urlsafe_b64encode(state_payload.encode()).decode() + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) auth_url, _ = flow.authorization_url( access_type="offline", @@ -75,8 +117,13 @@ async def connect_gmail(space_id: int, user: User = Depends(current_active_user) include_granted_scopes="true", state=state_encoded, ) + + logger.info( + f"Initiating Google Gmail OAuth for user {user.id}, space {space_id}" + ) return {"auth_url": auth_url} except Exception as e: + logger.error(f"Failed to initiate Google Gmail OAuth: {e!s}", exc_info=True) raise HTTPException( status_code=500, detail=f"Failed to initiate Google OAuth: {e!s}" ) from e @@ -85,42 +132,129 @@ async def connect_gmail(space_id: int, user: User = Depends(current_active_user) @router.get("/auth/google/gmail/connector/callback") async def gmail_callback( request: Request, - code: str, - state: str, + code: str | None = None, + error: str | None = None, + state: str | None = None, session: AsyncSession = Depends(get_async_session), ): + """ + Handle Google Gmail OAuth callback. + + Args: + request: FastAPI request object + code: Authorization code from Google (if user granted access) + error: Error code from Google (if user denied access or error occurred) + state: State parameter containing user/space info + session: Database session + + Returns: + Redirect response to frontend + """ try: - # Decode and parse the state - decoded_state = base64.urlsafe_b64decode(state.encode()).decode() - data = json.loads(decoded_state) + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Google Gmail OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=google_gmail_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=google_gmail_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e user_id = UUID(data["user_id"]) space_id = data["space_id"] + # Validate redirect URI (security: ensure it matches configured value) + if not config.GOOGLE_GMAIL_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="GOOGLE_GMAIL_REDIRECT_URI not configured" + ) + flow = get_google_flow() flow.fetch_token(code=code) creds = flow.credentials creds_dict = json.loads(creds.to_json()) - try: - # Check if a connector with the same type already exists for this search space and user - result = await session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, - ) + # Fetch user email + user_email = fetch_google_user_email(creds) + + # Encrypt sensitive credentials before storing + token_encryption = get_token_encryption() + + # Encrypt sensitive fields: token, refresh_token, client_secret + if creds_dict.get("token"): + creds_dict["token"] = token_encryption.encrypt_token(creds_dict["token"]) + if creds_dict.get("refresh_token"): + creds_dict["refresh_token"] = token_encryption.encrypt_token( + creds_dict["refresh_token"] + ) + if creds_dict.get("client_secret"): + creds_dict["client_secret"] = token_encryption.encrypt_token( + creds_dict["client_secret"] + ) + + # Mark that credentials are encrypted for backward compatibility + creds_dict["_token_encrypted"] = True + + # Check for duplicate connector (same account already connected) + is_duplicate = await check_duplicate_connector( + session, + SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, + space_id, + user_id, + user_email, + ) + if is_duplicate: + logger.warning( + f"Duplicate Gmail connector detected for user {user_id} with email {user_email}" + ) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=google-gmail-connector" + ) + + try: + # Generate a unique, user-friendly connector name + connector_name = await generate_unique_connector_name( + session, + SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, + space_id, + user_id, + user_email, ) - existing_connector = result.scalars().first() - if existing_connector: - raise HTTPException( - status_code=409, - detail="A GOOGLE_GMAIL_CONNECTOR connector already exists in this search space. Each search space can have only one connector of each type per user.", - ) db_connector = SearchSourceConnector( - name="Google Gmail Connector", + name=connector_name, connector_type=SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, config=creds_dict, search_space_id=space_id, @@ -138,7 +272,7 @@ async def gmail_callback( # Redirect to the frontend with success params for indexing config # Using query params to auto-open the popup with config view on new-chat page return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-gmail-connector" + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-gmail-connector&connectorId={db_connector.id}" ) except IntegrityError as e: @@ -146,7 +280,7 @@ async def gmail_callback( logger.error(f"Database integrity error: {e!s}") raise HTTPException( status_code=409, - detail="A connector with this configuration already exists.", + detail=f"Database integrity error: {e!s}", ) from e except ValidationError as e: await session.rollback() @@ -160,3 +294,6 @@ async def gmail_callback( raise except Exception as e: logger.error(f"Unexpected error in Gmail callback: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to complete Google Gmail OAuth: {e!s}" + ) from e diff --git a/surfsense_backend/app/routes/jira_add_connector_route.py b/surfsense_backend/app/routes/jira_add_connector_route.py new file mode 100644 index 000000000..fb66f4da7 --- /dev/null +++ b/surfsense_backend/app/routes/jira_add_connector_route.py @@ -0,0 +1,510 @@ +""" +Jira Connector OAuth Routes. + +Handles OAuth 2.0 authentication flow for Jira connector. +Uses Atlassian OAuth 2.0 (3LO) with accessible-resources API to discover Jira instances. +""" + +import logging +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import ( + SearchSourceConnector, + SearchSourceConnectorType, + User, + get_async_session, +) +from app.schemas.atlassian_auth_credentials import AtlassianAuthCredentialsBase +from app.users import current_active_user +from app.utils.connector_naming import ( + check_duplicate_connector, + extract_identifier_from_credentials, + generate_unique_connector_name, +) +from app.utils.oauth_security import OAuthStateManager, TokenEncryption + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# Atlassian OAuth endpoints +AUTHORIZATION_URL = "https://auth.atlassian.com/authorize" +TOKEN_URL = "https://auth.atlassian.com/oauth/token" +ACCESSIBLE_RESOURCES_URL = "https://api.atlassian.com/oauth/token/accessible-resources" + +# OAuth scopes for Jira +SCOPES = [ + "read:jira-work", + "read:jira-user", + "offline_access", # Required for refresh tokens +] + +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + +@router.get("/auth/jira/connector/add") +async def connect_jira(space_id: int, user: User = Depends(current_active_user)): + """ + Initiate Jira OAuth flow. + + Args: + space_id: The search space ID + user: Current authenticated user + + Returns: + Authorization URL for redirect + """ + try: + if not space_id: + raise HTTPException(status_code=400, detail="space_id is required") + + if not config.ATLASSIAN_CLIENT_ID: + raise HTTPException( + status_code=500, detail="Atlassian OAuth not configured." + ) + + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) + + # Build authorization URL + from urllib.parse import urlencode + + auth_params = { + "audience": "api.atlassian.com", + "client_id": config.ATLASSIAN_CLIENT_ID, + "scope": " ".join(SCOPES), + "redirect_uri": config.JIRA_REDIRECT_URI, + "state": state_encoded, + "response_type": "code", + "prompt": "consent", # Force consent screen to get refresh token + } + + auth_url = f"{AUTHORIZATION_URL}?{urlencode(auth_params)}" + + logger.info(f"Generated Jira OAuth URL for user {user.id}, space {space_id}") + return {"auth_url": auth_url} + + except Exception as e: + logger.error(f"Failed to initiate Jira OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to initiate Jira OAuth: {e!s}" + ) from e + + +@router.get("/auth/jira/connector/callback") +async def jira_callback( + request: Request, + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +): + """ + Handle Jira OAuth callback. + + Args: + request: FastAPI request object + code: Authorization code from Atlassian (if user granted access) + error: Error code from Atlassian (if user denied access or error occurred) + state: State parameter containing user/space info + session: Database session + + Returns: + Redirect response to frontend + """ + try: + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Jira OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=jira_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=jira_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e + + user_id = UUID(data["user_id"]) + space_id = data["space_id"] + + # Validate redirect URI (security: ensure it matches configured value) + if not config.JIRA_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="JIRA_REDIRECT_URI not configured" + ) + + # Exchange authorization code for access token + token_data = { + "grant_type": "authorization_code", + "client_id": config.ATLASSIAN_CLIENT_ID, + "client_secret": config.ATLASSIAN_CLIENT_SECRET, + "code": code, + "redirect_uri": config.JIRA_REDIRECT_URI, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=token_data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get( + "error_description", error_json.get("error", error_detail) + ) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token exchange failed: {error_detail}" + ) + + token_json = token_response.json() + + # Encrypt sensitive tokens before storing + token_encryption = get_token_encryption() + access_token = token_json.get("access_token") + refresh_token = token_json.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from Atlassian" + ) + + # Fetch accessible resources to get Jira instance information + async with httpx.AsyncClient() as client: + resources_response = await client.get( + ACCESSIBLE_RESOURCES_URL, + headers={"Authorization": f"Bearer {access_token}"}, + timeout=30.0, + ) + + if resources_response.status_code != 200: + error_detail = resources_response.text + logger.error(f"Failed to fetch accessible resources: {error_detail}") + raise HTTPException( + status_code=400, + detail=f"Failed to fetch Jira instances: {error_detail}", + ) + + resources = resources_response.json() + + # Filter for Jira instances (resources with type "jira" or id field) + jira_instances = [ + r for r in resources if r.get("id") and (r.get("name") or r.get("url")) + ] + + if not jira_instances: + raise HTTPException( + status_code=400, + detail="No accessible Jira instances found. Please ensure you have access to at least one Jira instance.", + ) + + # For now, use the first Jira instance + # TODO: Support multiple instances by letting user choose during OAuth + jira_instance = jira_instances[0] + cloud_id = jira_instance["id"] + base_url = jira_instance.get("url") + + # If URL is not provided, construct it from cloud_id + if not base_url: + # Try to extract from name or construct default format + instance_name = jira_instance.get("name", "").lower().replace(" ", "") + if instance_name: + base_url = f"https://{instance_name}.atlassian.net" + else: + # Fallback: use cloud_id directly (though this may not work) + base_url = f"https://{cloud_id}.atlassian.net" + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Store the encrypted access token and refresh token in connector config + connector_config = { + "access_token": token_encryption.encrypt_token(access_token), + "refresh_token": token_encryption.encrypt_token(refresh_token) + if refresh_token + else None, + "token_type": token_json.get("token_type", "Bearer"), + "expires_in": expires_in, + "expires_at": expires_at.isoformat() if expires_at else None, + "scope": token_json.get("scope"), + "cloud_id": cloud_id, + "base_url": base_url.rstrip("/") if base_url else None, + # Mark that tokens are encrypted for backward compatibility + "_token_encrypted": True, + } + + # Extract unique identifier from connector credentials + connector_identifier = extract_identifier_from_credentials( + SearchSourceConnectorType.JIRA_CONNECTOR, connector_config + ) + + # Check for duplicate connector (same Jira instance already connected) + is_duplicate = await check_duplicate_connector( + session, + SearchSourceConnectorType.JIRA_CONNECTOR, + space_id, + user_id, + connector_identifier, + ) + if is_duplicate: + logger.warning( + f"Duplicate Jira connector detected for user {user_id} with instance {connector_identifier}" + ) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=jira-connector" + ) + + # Generate a unique, user-friendly connector name + connector_name = await generate_unique_connector_name( + session, + SearchSourceConnectorType.JIRA_CONNECTOR, + space_id, + user_id, + connector_identifier, + ) + # Create new connector + new_connector = SearchSourceConnector( + name=connector_name, + connector_type=SearchSourceConnectorType.JIRA_CONNECTOR, + is_indexable=True, + config=connector_config, + search_space_id=space_id, + user_id=user_id, + ) + session.add(new_connector) + logger.info( + f"Created new Jira connector for user {user_id} in space {space_id}" + ) + + try: + await session.commit() + logger.info(f"Successfully saved Jira connector for user {user_id}") + + # Redirect to the frontend with success params + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=jira-connector&connectorId={new_connector.id}" + ) + + except ValidationError as e: + await session.rollback() + raise HTTPException( + status_code=422, detail=f"Validation error: {e!s}" + ) from e + except IntegrityError as e: + await session.rollback() + raise HTTPException( + status_code=409, + detail=f"Database integrity error: {e!s}", + ) from e + except Exception as e: + logger.error(f"Failed to create search source connector: {e!s}") + await session.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to create search source connector: {e!s}", + ) from e + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to complete Jira OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to complete Jira OAuth: {e!s}" + ) from e + + +async def refresh_jira_token( + session: AsyncSession, connector: SearchSourceConnector +) -> SearchSourceConnector: + """ + Refresh the Jira access token for a connector. + + Args: + session: Database session + connector: Jira connector to refresh + + Returns: + Updated connector object + """ + try: + logger.info(f"Refreshing Jira token for connector {connector.id}") + + credentials = AtlassianAuthCredentialsBase.from_dict(connector.config) + + # Decrypt tokens if they are encrypted + token_encryption = get_token_encryption() + is_encrypted = connector.config.get("_token_encrypted", False) + + refresh_token = credentials.refresh_token + if is_encrypted and refresh_token: + try: + refresh_token = token_encryption.decrypt_token(refresh_token) + except Exception as e: + logger.error(f"Failed to decrypt refresh token: {e!s}") + raise HTTPException( + status_code=500, detail="Failed to decrypt stored refresh token" + ) from e + + if not refresh_token: + raise HTTPException( + status_code=400, + detail="No refresh token available. Please re-authenticate.", + ) + + # Prepare token refresh data + refresh_data = { + "grant_type": "refresh_token", + "client_id": config.ATLASSIAN_CLIENT_ID, + "client_secret": config.ATLASSIAN_CLIENT_SECRET, + "refresh_token": refresh_token, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=refresh_data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get( + "error_description", error_json.get("error", error_detail) + ) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token refresh failed: {error_detail}" + ) + + token_json = token_response.json() + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Encrypt new tokens before storing + access_token = token_json.get("access_token") + new_refresh_token = token_json.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from Jira refresh" + ) + + # Update credentials object with encrypted tokens + credentials.access_token = token_encryption.encrypt_token(access_token) + if new_refresh_token: + credentials.refresh_token = token_encryption.encrypt_token( + new_refresh_token + ) + credentials.expires_in = expires_in + credentials.expires_at = expires_at + credentials.scope = token_json.get("scope") + + # Preserve cloud_id and base_url + if not credentials.cloud_id: + credentials.cloud_id = connector.config.get("cloud_id") + if not credentials.base_url: + credentials.base_url = connector.config.get("base_url") + + # Update connector config with encrypted tokens + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + + logger.info(f"Successfully refreshed Jira token for connector {connector.id}") + + return connector + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to refresh Jira token: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to refresh Jira token: {e!s}" + ) from e diff --git a/surfsense_backend/app/routes/linear_add_connector_route.py b/surfsense_backend/app/routes/linear_add_connector_route.py new file mode 100644 index 000000000..fc9501bfb --- /dev/null +++ b/surfsense_backend/app/routes/linear_add_connector_route.py @@ -0,0 +1,459 @@ +""" +Linear Connector OAuth Routes. + +Handles OAuth 2.0 authentication flow for Linear connector. +""" + +import logging +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.connectors.linear_connector import fetch_linear_organization_name +from app.db import ( + SearchSourceConnector, + SearchSourceConnectorType, + User, + get_async_session, +) +from app.schemas.linear_auth_credentials import LinearAuthCredentialsBase +from app.users import current_active_user +from app.utils.connector_naming import ( + check_duplicate_connector, + generate_unique_connector_name, +) +from app.utils.oauth_security import OAuthStateManager, TokenEncryption + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# Linear OAuth endpoints +AUTHORIZATION_URL = "https://linear.app/oauth/authorize" +TOKEN_URL = "https://api.linear.app/oauth/token" + +# OAuth scopes for Linear +SCOPES = ["read", "write"] + +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + +def make_basic_auth_header(client_id: str, client_secret: str) -> str: + """Create Basic Auth header for Linear OAuth.""" + import base64 + + credentials = f"{client_id}:{client_secret}".encode() + b64 = base64.b64encode(credentials).decode("ascii") + return f"Basic {b64}" + + +@router.get("/auth/linear/connector/add") +async def connect_linear(space_id: int, user: User = Depends(current_active_user)): + """ + Initiate Linear OAuth flow. + + Args: + space_id: The search space ID + user: Current authenticated user + + Returns: + Authorization URL for redirect + """ + try: + if not space_id: + raise HTTPException(status_code=400, detail="space_id is required") + + if not config.LINEAR_CLIENT_ID: + raise HTTPException(status_code=500, detail="Linear OAuth not configured.") + + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) + + # Build authorization URL + from urllib.parse import urlencode + + auth_params = { + "client_id": config.LINEAR_CLIENT_ID, + "response_type": "code", + "redirect_uri": config.LINEAR_REDIRECT_URI, + "scope": " ".join(SCOPES), + "state": state_encoded, + } + + auth_url = f"{AUTHORIZATION_URL}?{urlencode(auth_params)}" + + logger.info(f"Generated Linear OAuth URL for user {user.id}, space {space_id}") + return {"auth_url": auth_url} + + except Exception as e: + logger.error(f"Failed to initiate Linear OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to initiate Linear OAuth: {e!s}" + ) from e + + +@router.get("/auth/linear/connector/callback") +async def linear_callback( + request: Request, + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +): + """ + Handle Linear OAuth callback. + + Args: + request: FastAPI request object + code: Authorization code from Linear (if user granted access) + error: Error code from Linear (if user denied access or error occurred) + state: State parameter containing user/space info + session: Database session + + Returns: + Redirect response to frontend + """ + try: + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Linear OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=linear_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=linear_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e + + user_id = UUID(data["user_id"]) + space_id = data["space_id"] + + # Validate redirect URI (security: ensure it matches configured value) + if not config.LINEAR_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="LINEAR_REDIRECT_URI not configured" + ) + + # Exchange authorization code for access token + auth_header = make_basic_auth_header( + config.LINEAR_CLIENT_ID, config.LINEAR_CLIENT_SECRET + ) + + token_data = { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": config.LINEAR_REDIRECT_URI, # Use stored value, not from request + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=token_data, + headers={ + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": auth_header, + }, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error_description", error_detail) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token exchange failed: {error_detail}" + ) + + token_json = token_response.json() + + # Encrypt sensitive tokens before storing + token_encryption = get_token_encryption() + access_token = token_json.get("access_token") + refresh_token = token_json.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from Linear" + ) + + # Fetch organization name + org_name = await fetch_linear_organization_name(access_token) + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + if token_json.get("expires_in"): + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(token_json["expires_in"])) + + # Store the encrypted access token and refresh token in connector config + connector_config = { + "access_token": token_encryption.encrypt_token(access_token), + "refresh_token": token_encryption.encrypt_token(refresh_token) + if refresh_token + else None, + "token_type": token_json.get("token_type", "Bearer"), + "expires_in": token_json.get("expires_in"), + "expires_at": expires_at.isoformat() if expires_at else None, + "scope": token_json.get("scope"), + # Mark that tokens are encrypted for backward compatibility + "_token_encrypted": True, + } + + # Check for duplicate connector (same organization already connected) + is_duplicate = await check_duplicate_connector( + session, + SearchSourceConnectorType.LINEAR_CONNECTOR, + space_id, + user_id, + org_name, + ) + if is_duplicate: + logger.warning( + f"Duplicate Linear connector detected for user {user_id} with org {org_name}" + ) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=linear-connector" + ) + + # Generate a unique, user-friendly connector name + connector_name = await generate_unique_connector_name( + session, + SearchSourceConnectorType.LINEAR_CONNECTOR, + space_id, + user_id, + org_name, + ) + # Create new connector + new_connector = SearchSourceConnector( + name=connector_name, + connector_type=SearchSourceConnectorType.LINEAR_CONNECTOR, + is_indexable=True, + config=connector_config, + search_space_id=space_id, + user_id=user_id, + ) + session.add(new_connector) + logger.info( + f"Created new Linear connector for user {user_id} in space {space_id}" + ) + + try: + await session.commit() + logger.info(f"Successfully saved Linear connector for user {user_id}") + + # Redirect to the frontend with success params + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=linear-connector&connectorId={new_connector.id}" + ) + + except ValidationError as e: + await session.rollback() + raise HTTPException( + status_code=422, detail=f"Validation error: {e!s}" + ) from e + except IntegrityError as e: + await session.rollback() + raise HTTPException( + status_code=409, + detail=f"Database integrity error: {e!s}", + ) from e + except Exception as e: + logger.error(f"Failed to create search source connector: {e!s}") + await session.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to create search source connector: {e!s}", + ) from e + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to complete Linear OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to complete Linear OAuth: {e!s}" + ) from e + + +async def refresh_linear_token( + session: AsyncSession, connector: SearchSourceConnector +) -> SearchSourceConnector: + """ + Refresh the Linear access token for a connector. + + Args: + session: Database session + connector: Linear connector to refresh + + Returns: + Updated connector object + """ + try: + logger.info(f"Refreshing Linear token for connector {connector.id}") + + credentials = LinearAuthCredentialsBase.from_dict(connector.config) + + # Decrypt tokens if they are encrypted + token_encryption = get_token_encryption() + is_encrypted = connector.config.get("_token_encrypted", False) + + refresh_token = credentials.refresh_token + if is_encrypted and refresh_token: + try: + refresh_token = token_encryption.decrypt_token(refresh_token) + except Exception as e: + logger.error(f"Failed to decrypt refresh token: {e!s}") + raise HTTPException( + status_code=500, detail="Failed to decrypt stored refresh token" + ) from e + + if not refresh_token: + raise HTTPException( + status_code=400, + detail="No refresh token available. Please re-authenticate.", + ) + + auth_header = make_basic_auth_header( + config.LINEAR_CLIENT_ID, config.LINEAR_CLIENT_SECRET + ) + + # Prepare token refresh data + refresh_data = { + "grant_type": "refresh_token", + "refresh_token": refresh_token, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=refresh_data, + headers={ + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": auth_header, + }, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error_description", error_detail) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token refresh failed: {error_detail}" + ) + + token_json = token_response.json() + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Encrypt new tokens before storing + access_token = token_json.get("access_token") + new_refresh_token = token_json.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from Linear refresh" + ) + + # Update credentials object with encrypted tokens + credentials.access_token = token_encryption.encrypt_token(access_token) + if new_refresh_token: + credentials.refresh_token = token_encryption.encrypt_token( + new_refresh_token + ) + credentials.expires_in = expires_in + credentials.expires_at = expires_at + credentials.scope = token_json.get("scope") + + # Update connector config with encrypted tokens + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + + logger.info(f"Successfully refreshed Linear token for connector {connector.id}") + + return connector + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to refresh Linear token: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to refresh Linear token: {e!s}" + ) from e diff --git a/surfsense_backend/app/routes/notion_add_connector_route.py b/surfsense_backend/app/routes/notion_add_connector_route.py new file mode 100644 index 000000000..aac821793 --- /dev/null +++ b/surfsense_backend/app/routes/notion_add_connector_route.py @@ -0,0 +1,472 @@ +""" +Notion Connector OAuth Routes. + +Handles OAuth 2.0 authentication flow for Notion connector. +""" + +import logging +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import ( + SearchSourceConnector, + SearchSourceConnectorType, + User, + get_async_session, +) +from app.schemas.notion_auth_credentials import NotionAuthCredentialsBase +from app.users import current_active_user +from app.utils.connector_naming import ( + check_duplicate_connector, + extract_identifier_from_credentials, + generate_unique_connector_name, +) +from app.utils.oauth_security import OAuthStateManager, TokenEncryption + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# Notion OAuth endpoints +AUTHORIZATION_URL = "https://api.notion.com/v1/oauth/authorize" +TOKEN_URL = "https://api.notion.com/v1/oauth/token" + +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + +def make_basic_auth_header(client_id: str, client_secret: str) -> str: + """Create Basic Auth header for Notion OAuth.""" + import base64 + + credentials = f"{client_id}:{client_secret}".encode() + b64 = base64.b64encode(credentials).decode("ascii") + return f"Basic {b64}" + + +@router.get("/auth/notion/connector/add") +async def connect_notion(space_id: int, user: User = Depends(current_active_user)): + """ + Initiate Notion OAuth flow. + + Args: + space_id: The search space ID + user: Current authenticated user + + Returns: + Authorization URL for redirect + """ + try: + if not space_id: + raise HTTPException(status_code=400, detail="space_id is required") + + if not config.NOTION_CLIENT_ID: + raise HTTPException(status_code=500, detail="Notion OAuth not configured.") + + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) + + # Build authorization URL + from urllib.parse import urlencode + + auth_params = { + "client_id": config.NOTION_CLIENT_ID, + "response_type": "code", + "owner": "user", # Allows both admins and members to authorize + "redirect_uri": config.NOTION_REDIRECT_URI, + "state": state_encoded, + } + + auth_url = f"{AUTHORIZATION_URL}?{urlencode(auth_params)}" + + logger.info(f"Generated Notion OAuth URL for user {user.id}, space {space_id}") + return {"auth_url": auth_url} + + except Exception as e: + logger.error(f"Failed to initiate Notion OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to initiate Notion OAuth: {e!s}" + ) from e + + +@router.get("/auth/notion/connector/callback") +async def notion_callback( + request: Request, + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +): + """ + Handle Notion OAuth callback. + + Args: + request: FastAPI request object + code: Authorization code from Notion (if user granted access) + error: Error code from Notion (if user denied access or error occurred) + state: State parameter containing user/space info + session: Database session + + Returns: + Redirect response to frontend + """ + try: + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Notion OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=notion_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=notion_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e + + user_id = UUID(data["user_id"]) + space_id = data["space_id"] + + # Validate redirect URI (security: ensure it matches configured value) + # Note: Notion doesn't send redirect_uri in callback, but we validate + # that we're using the configured one in token exchange + if not config.NOTION_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="NOTION_REDIRECT_URI not configured" + ) + + # Exchange authorization code for access token + auth_header = make_basic_auth_header( + config.NOTION_CLIENT_ID, config.NOTION_CLIENT_SECRET + ) + + token_data = { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": config.NOTION_REDIRECT_URI, # Use stored value, not from request + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + json=token_data, + headers={ + "Content-Type": "application/json", + "Authorization": auth_header, + }, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error_description", error_detail) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token exchange failed: {error_detail}" + ) + + token_json = token_response.json() + + # Encrypt sensitive tokens before storing + token_encryption = get_token_encryption() + access_token = token_json.get("access_token") + refresh_token = token_json.get("refresh_token") + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from Notion" + ) + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Notion returns access_token, refresh_token (if available), and workspace information + # Store the encrypted tokens and workspace info in connector config + connector_config = { + "access_token": token_encryption.encrypt_token(access_token), + "refresh_token": token_encryption.encrypt_token(refresh_token) + if refresh_token + else None, + "expires_in": expires_in, + "expires_at": expires_at.isoformat() if expires_at else None, + "workspace_id": token_json.get("workspace_id"), + "workspace_name": token_json.get("workspace_name"), + "workspace_icon": token_json.get("workspace_icon"), + "bot_id": token_json.get("bot_id"), + # Mark that token is encrypted for backward compatibility + "_token_encrypted": True, + } + + # Extract unique identifier from connector credentials + connector_identifier = extract_identifier_from_credentials( + SearchSourceConnectorType.NOTION_CONNECTOR, connector_config + ) + + # Check for duplicate connector (same workspace already connected) + is_duplicate = await check_duplicate_connector( + session, + SearchSourceConnectorType.NOTION_CONNECTOR, + space_id, + user_id, + connector_identifier, + ) + if is_duplicate: + logger.warning( + f"Duplicate Notion connector detected for user {user_id} with workspace {connector_identifier}" + ) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=notion-connector" + ) + + # Generate a unique, user-friendly connector name + connector_name = await generate_unique_connector_name( + session, + SearchSourceConnectorType.NOTION_CONNECTOR, + space_id, + user_id, + connector_identifier, + ) + # Create new connector + new_connector = SearchSourceConnector( + name=connector_name, + connector_type=SearchSourceConnectorType.NOTION_CONNECTOR, + is_indexable=True, + config=connector_config, + search_space_id=space_id, + user_id=user_id, + ) + session.add(new_connector) + logger.info( + f"Created new Notion connector for user {user_id} in space {space_id}" + ) + + try: + await session.commit() + logger.info(f"Successfully saved Notion connector for user {user_id}") + + # Redirect to the frontend with success params + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=notion-connector&connectorId={new_connector.id}" + ) + + except ValidationError as e: + await session.rollback() + raise HTTPException( + status_code=422, detail=f"Validation error: {e!s}" + ) from e + except IntegrityError as e: + await session.rollback() + raise HTTPException( + status_code=409, + detail=f"Database integrity error: {e!s}", + ) from e + except Exception as e: + logger.error(f"Failed to create search source connector: {e!s}") + await session.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to create search source connector: {e!s}", + ) from e + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to complete Notion OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to complete Notion OAuth: {e!s}" + ) from e + + +async def refresh_notion_token( + session: AsyncSession, connector: SearchSourceConnector +) -> SearchSourceConnector: + """ + Refresh the Notion access token for a connector. + + Args: + session: Database session + connector: Notion connector to refresh + + Returns: + Updated connector object + """ + try: + logger.info(f"Refreshing Notion token for connector {connector.id}") + + credentials = NotionAuthCredentialsBase.from_dict(connector.config) + + # Decrypt tokens if they are encrypted + token_encryption = get_token_encryption() + is_encrypted = connector.config.get("_token_encrypted", False) + + refresh_token = credentials.refresh_token + if is_encrypted and refresh_token: + try: + refresh_token = token_encryption.decrypt_token(refresh_token) + except Exception as e: + logger.error(f"Failed to decrypt refresh token: {e!s}") + raise HTTPException( + status_code=500, detail="Failed to decrypt stored refresh token" + ) from e + + if not refresh_token: + raise HTTPException( + status_code=400, + detail="No refresh token available. Please re-authenticate.", + ) + + auth_header = make_basic_auth_header( + config.NOTION_CLIENT_ID, config.NOTION_CLIENT_SECRET + ) + + # Prepare token refresh data + refresh_data = { + "grant_type": "refresh_token", + "refresh_token": refresh_token, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + json=refresh_data, + headers={ + "Content-Type": "application/json", + "Authorization": auth_header, + }, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error_description", error_detail) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token refresh failed: {error_detail}" + ) + + token_json = token_response.json() + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Encrypt new tokens before storing + access_token = token_json.get("access_token") + new_refresh_token = token_json.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from Notion refresh" + ) + + # Update credentials object with encrypted tokens + credentials.access_token = token_encryption.encrypt_token(access_token) + if new_refresh_token: + credentials.refresh_token = token_encryption.encrypt_token( + new_refresh_token + ) + credentials.expires_in = expires_in + credentials.expires_at = expires_at + + # Preserve workspace info + if not credentials.workspace_id: + credentials.workspace_id = connector.config.get("workspace_id") + if not credentials.workspace_name: + credentials.workspace_name = connector.config.get("workspace_name") + if not credentials.workspace_icon: + credentials.workspace_icon = connector.config.get("workspace_icon") + if not credentials.bot_id: + credentials.bot_id = connector.config.get("bot_id") + + # Update connector config with encrypted tokens + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + + logger.info(f"Successfully refreshed Notion token for connector {connector.id}") + + return connector + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to refresh Notion token: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to refresh Notion token: {e!s}" + ) from e diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index d6fdedd7c..58a50a6f8 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -7,7 +7,8 @@ PUT /search-source-connectors/{connector_id} - Update a specific connector DELETE /search-source-connectors/{connector_id} - Delete a specific connector POST /search-source-connectors/{connector_id}/index - Index content from a connector to a search space -Note: Each search space can have only one connector of each type per user (based on search_space_id, user_id, and connector_type). +Note: OAuth connectors (Gmail, Drive, Slack, etc.) support multiple accounts per search space. +Non-OAuth connectors (BookStack, GitHub, etc.) are limited to one per search space. """ import logging @@ -125,6 +126,7 @@ async def create_search_source_connector( ) # Check if a connector with the same type already exists for this search space + # (for non-OAuth connectors that don't support multiple accounts) result = await session.execute( select(SearchSourceConnector).filter( SearchSourceConnector.search_space_id == search_space_id, diff --git a/surfsense_backend/app/routes/slack_add_connector_route.py b/surfsense_backend/app/routes/slack_add_connector_route.py new file mode 100644 index 000000000..62d2ccaaa --- /dev/null +++ b/surfsense_backend/app/routes/slack_add_connector_route.py @@ -0,0 +1,492 @@ +""" +Slack Connector OAuth Routes. + +Handles OAuth 2.0 authentication flow for Slack connector. +""" + +import logging +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import ( + SearchSourceConnector, + SearchSourceConnectorType, + User, + get_async_session, +) +from app.schemas.slack_auth_credentials import SlackAuthCredentialsBase +from app.users import current_active_user +from app.utils.connector_naming import ( + check_duplicate_connector, + extract_identifier_from_credentials, + generate_unique_connector_name, +) +from app.utils.oauth_security import OAuthStateManager, TokenEncryption + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# Slack OAuth endpoints +AUTHORIZATION_URL = "https://slack.com/oauth/v2/authorize" +TOKEN_URL = "https://slack.com/api/oauth.v2.access" + +# OAuth scopes for Slack (Bot Token) +SCOPES = [ + "channels:history", # Read messages in public channels + "channels:read", # View basic information about public channels + "groups:history", # Read messages in private channels + "groups:read", # View basic information about private channels + "im:history", # Read messages in direct messages + "mpim:history", # Read messages in group direct messages + "users:read", # Read user information +] + +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + +@router.get("/auth/slack/connector/add") +async def connect_slack(space_id: int, user: User = Depends(current_active_user)): + """ + Initiate Slack OAuth flow. + + Args: + space_id: The search space ID + user: Current authenticated user + + Returns: + Authorization URL for redirect + """ + try: + if not space_id: + raise HTTPException(status_code=400, detail="space_id is required") + + if not config.SLACK_CLIENT_ID: + raise HTTPException(status_code=500, detail="Slack OAuth not configured.") + + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) + + # Build authorization URL + from urllib.parse import urlencode + + auth_params = { + "client_id": config.SLACK_CLIENT_ID, + "scope": ",".join(SCOPES), + "redirect_uri": config.SLACK_REDIRECT_URI, + "state": state_encoded, + } + + auth_url = f"{AUTHORIZATION_URL}?{urlencode(auth_params)}" + + logger.info(f"Generated Slack OAuth URL for user {user.id}, space {space_id}") + return {"auth_url": auth_url} + + except Exception as e: + logger.error(f"Failed to initiate Slack OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to initiate Slack OAuth: {e!s}" + ) from e + + +@router.get("/auth/slack/connector/callback") +async def slack_callback( + request: Request, + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +): + """ + Handle Slack OAuth callback. + + Args: + request: FastAPI request object + code: Authorization code from Slack (if user granted access) + error: Error code from Slack (if user denied access or error occurred) + state: State parameter containing user/space info + session: Database session + + Returns: + Redirect response to frontend + """ + try: + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Slack OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=slack_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=slack_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e + + user_id = UUID(data["user_id"]) + space_id = data["space_id"] + + # Validate redirect URI (security: ensure it matches configured value) + if not config.SLACK_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="SLACK_REDIRECT_URI not configured" + ) + + # Exchange authorization code for access token + token_data = { + "client_id": config.SLACK_CLIENT_ID, + "client_secret": config.SLACK_CLIENT_SECRET, + "code": code, + "redirect_uri": config.SLACK_REDIRECT_URI, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=token_data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error", error_detail) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token exchange failed: {error_detail}" + ) + + token_json = token_response.json() + + # Slack OAuth v2 returns success status in the JSON + if not token_json.get("ok", False): + error_msg = token_json.get("error", "Unknown error") + raise HTTPException( + status_code=400, detail=f"Slack OAuth error: {error_msg}" + ) + + # Extract bot token from Slack response + # Slack OAuth v2 returns: { "ok": true, "access_token": "...", "bot": { "bot_user_id": "...", "bot_access_token": "xoxb-..." }, "refresh_token": "...", ... } + bot_token = None + if token_json.get("bot") and token_json["bot"].get("bot_access_token"): + bot_token = token_json["bot"]["bot_access_token"] + elif token_json.get("access_token"): + # Fallback to access_token if bot token not available + bot_token = token_json["access_token"] + else: + raise HTTPException( + status_code=400, detail="No bot token received from Slack" + ) + + # Extract refresh token if available (for token rotation) + refresh_token = token_json.get("refresh_token") + + # Encrypt sensitive tokens before storing + token_encryption = get_token_encryption() + + # Calculate expiration time (UTC, tz-aware) + # Slack tokens don't expire by default, but we'll store expiration info if provided + expires_at = None + if token_json.get("expires_in"): + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(token_json["expires_in"])) + + # Store the encrypted bot token and refresh token in connector config + connector_config = { + "bot_token": token_encryption.encrypt_token(bot_token), + "refresh_token": token_encryption.encrypt_token(refresh_token) + if refresh_token + else None, + "bot_user_id": token_json.get("bot", {}).get("bot_user_id"), + "team_id": token_json.get("team", {}).get("id"), + "team_name": token_json.get("team", {}).get("name"), + "token_type": token_json.get("token_type", "Bearer"), + "expires_in": token_json.get("expires_in"), + "expires_at": expires_at.isoformat() if expires_at else None, + "scope": token_json.get("scope"), + # Mark that tokens are encrypted for backward compatibility + "_token_encrypted": True, + } + + # Extract unique identifier from connector credentials + connector_identifier = extract_identifier_from_credentials( + SearchSourceConnectorType.SLACK_CONNECTOR, connector_config + ) + + # Check for duplicate connector (same workspace already connected) + is_duplicate = await check_duplicate_connector( + session, + SearchSourceConnectorType.SLACK_CONNECTOR, + space_id, + user_id, + connector_identifier, + ) + if is_duplicate: + logger.warning( + f"Duplicate Slack connector detected for user {user_id} with workspace {connector_identifier}" + ) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=slack-connector" + ) + + # Generate a unique, user-friendly connector name + connector_name = await generate_unique_connector_name( + session, + SearchSourceConnectorType.SLACK_CONNECTOR, + space_id, + user_id, + connector_identifier, + ) + + # Create new connector + new_connector = SearchSourceConnector( + name=connector_name, + connector_type=SearchSourceConnectorType.SLACK_CONNECTOR, + is_indexable=True, + config=connector_config, + search_space_id=space_id, + user_id=user_id, + ) + session.add(new_connector) + logger.info( + f"Created new Slack connector for user {user_id} in space {space_id}" + ) + + try: + await session.commit() + logger.info(f"Successfully saved Slack connector for user {user_id}") + + # Redirect to the frontend with success params + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=slack-connector&connectorId={new_connector.id}" + ) + + except ValidationError as e: + await session.rollback() + raise HTTPException( + status_code=422, detail=f"Validation error: {e!s}" + ) from e + except IntegrityError as e: + await session.rollback() + raise HTTPException( + status_code=409, + detail=f"Database integrity error: {e!s}", + ) from e + except Exception as e: + logger.error(f"Failed to create search source connector: {e!s}") + await session.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to create search source connector: {e!s}", + ) from e + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to complete Slack OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to complete Slack OAuth: {e!s}" + ) from e + + +async def refresh_slack_token( + session: AsyncSession, connector: SearchSourceConnector +) -> SearchSourceConnector: + """ + Refresh the Slack bot token for a connector. + + Args: + session: Database session + connector: Slack connector to refresh + + Returns: + Updated connector object + """ + try: + logger.info(f"Refreshing Slack token for connector {connector.id}") + + credentials = SlackAuthCredentialsBase.from_dict(connector.config) + + # Decrypt tokens if they are encrypted + token_encryption = get_token_encryption() + is_encrypted = connector.config.get("_token_encrypted", False) + + refresh_token = credentials.refresh_token + if is_encrypted and refresh_token: + try: + refresh_token = token_encryption.decrypt_token(refresh_token) + except Exception as e: + logger.error(f"Failed to decrypt refresh token: {e!s}") + raise HTTPException( + status_code=500, detail="Failed to decrypt stored refresh token" + ) from e + + if not refresh_token: + raise HTTPException( + status_code=400, + detail="No refresh token available. Please re-authenticate.", + ) + + # Slack uses oauth.v2.access for token refresh with grant_type=refresh_token + refresh_data = { + "client_id": config.SLACK_CLIENT_ID, + "client_secret": config.SLACK_CLIENT_SECRET, + "grant_type": "refresh_token", + "refresh_token": refresh_token, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=refresh_data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error", error_detail) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token refresh failed: {error_detail}" + ) + + token_json = token_response.json() + + # Slack OAuth v2 returns success status in the JSON + if not token_json.get("ok", False): + error_msg = token_json.get("error", "Unknown error") + raise HTTPException( + status_code=400, detail=f"Slack OAuth refresh error: {error_msg}" + ) + + # Extract bot token from refresh response + bot_token = None + if token_json.get("bot") and token_json["bot"].get("bot_access_token"): + bot_token = token_json["bot"]["bot_access_token"] + elif token_json.get("access_token"): + bot_token = token_json["access_token"] + else: + raise HTTPException( + status_code=400, detail="No bot token received from Slack refresh" + ) + + # Get new refresh token if provided (Slack may rotate refresh tokens) + new_refresh_token = token_json.get("refresh_token") + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Update credentials object with encrypted tokens + credentials.bot_token = token_encryption.encrypt_token(bot_token) + if new_refresh_token: + credentials.refresh_token = token_encryption.encrypt_token( + new_refresh_token + ) + credentials.expires_in = expires_in + credentials.expires_at = expires_at + credentials.scope = token_json.get("scope") + + # Preserve team info + if not credentials.team_id: + credentials.team_id = connector.config.get("team_id") + if not credentials.team_name: + credentials.team_name = connector.config.get("team_name") + if not credentials.bot_user_id: + credentials.bot_user_id = connector.config.get("bot_user_id") + + # Update connector config with encrypted tokens + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + + logger.info(f"Successfully refreshed Slack token for connector {connector.id}") + + return connector + except HTTPException: + raise + except Exception as e: + logger.error( + f"Failed to refresh Slack token for connector {connector.id}: {e!s}", + exc_info=True, + ) + raise HTTPException( + status_code=500, detail=f"Failed to refresh Slack token: {e!s}" + ) from e diff --git a/surfsense_backend/app/schemas/atlassian_auth_credentials.py b/surfsense_backend/app/schemas/atlassian_auth_credentials.py new file mode 100644 index 000000000..cbb4772e6 --- /dev/null +++ b/surfsense_backend/app/schemas/atlassian_auth_credentials.py @@ -0,0 +1,86 @@ +""" +Atlassian OAuth 2.0 Authentication Credentials Schema. + +Shared schema for both Jira and Confluence OAuth credentials. +Both products use the same Atlassian OAuth 2.0 (3LO) flow and token structure. +""" + +from datetime import UTC, datetime + +from pydantic import BaseModel, field_validator + + +class AtlassianAuthCredentialsBase(BaseModel): + """ + Base model for Atlassian OAuth 2.0 credentials. + + Used for both Jira and Confluence connectors since they share + the same Atlassian OAuth infrastructure and token structure. + """ + + access_token: str + refresh_token: str | None = None + token_type: str = "Bearer" + expires_in: int | None = None + expires_at: datetime | None = None + scope: str | None = None + cloud_id: str | None = None + base_url: str | None = None + + @property + def is_expired(self) -> bool: + """Check if the credentials have expired.""" + if self.expires_at is None: + return False + return self.expires_at <= datetime.now(UTC) + + @property + def is_refreshable(self) -> bool: + """Check if the credentials can be refreshed.""" + return self.refresh_token is not None + + def to_dict(self) -> dict: + """Convert credentials to dictionary for storage.""" + return { + "access_token": self.access_token, + "refresh_token": self.refresh_token, + "token_type": self.token_type, + "expires_in": self.expires_in, + "expires_at": self.expires_at.isoformat() if self.expires_at else None, + "scope": self.scope, + "cloud_id": self.cloud_id, + "base_url": self.base_url, + } + + @classmethod + def from_dict(cls, data: dict) -> "AtlassianAuthCredentialsBase": + """Create credentials from dictionary.""" + expires_at = None + if data.get("expires_at"): + expires_at = datetime.fromisoformat(data["expires_at"]) + + return cls( + access_token=data["access_token"], + refresh_token=data.get("refresh_token"), + token_type=data.get("token_type", "Bearer"), + expires_in=data.get("expires_in"), + expires_at=expires_at, + scope=data.get("scope"), + cloud_id=data.get("cloud_id"), + base_url=data.get("base_url"), + ) + + @field_validator("expires_at", mode="before") + @classmethod + def ensure_aware_utc(cls, v): + # Strings like "2025-08-26T14:46:57.367184" + if isinstance(v, str): + # add +00:00 if missing tz info + if v.endswith("Z"): + return datetime.fromisoformat(v.replace("Z", "+00:00")) + dt = datetime.fromisoformat(v) + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + # datetime objects + if isinstance(v, datetime): + return v if v.tzinfo else v.replace(tzinfo=UTC) + return v diff --git a/surfsense_backend/app/schemas/clickup_auth_credentials.py b/surfsense_backend/app/schemas/clickup_auth_credentials.py new file mode 100644 index 000000000..d116ca92b --- /dev/null +++ b/surfsense_backend/app/schemas/clickup_auth_credentials.py @@ -0,0 +1,85 @@ +from datetime import UTC, datetime + +from pydantic import BaseModel, field_validator + + +class ClickUpAuthCredentialsBase(BaseModel): + access_token: str + refresh_token: str | None = None + expires_in: int | None = None + expires_at: datetime | None = None + user_id: str | None = None + user_email: str | None = None + user_name: str | None = None + workspace_id: str | None = None + workspace_name: str | None = None + + @property + def is_expired(self) -> bool: + """Check if the credentials have expired.""" + if self.expires_at is None: + return False # Long-lived token, treat as not expired + return self.expires_at <= datetime.now(UTC) + + @property + def is_refreshable(self) -> bool: + """Check if the credentials can be refreshed.""" + return self.refresh_token is not None + + def to_dict(self) -> dict: + """Convert credentials to dictionary for storage.""" + return { + "access_token": self.access_token, + "refresh_token": self.refresh_token, + "expires_in": self.expires_in, + "expires_at": self.expires_at.isoformat() if self.expires_at else None, + "user_id": self.user_id, + "user_email": self.user_email, + "user_name": self.user_name, + "workspace_id": self.workspace_id, + "workspace_name": self.workspace_name, + } + + @classmethod + def from_dict(cls, data: dict) -> "ClickUpAuthCredentialsBase": + """Create credentials from dictionary.""" + expires_at = None + if data.get("expires_at"): + expires_at = datetime.fromisoformat(data["expires_at"]) + + # Convert user_id to string if it's an integer (for backward compatibility) + user_id = data.get("user_id") + if user_id is not None and not isinstance(user_id, str): + user_id = str(user_id) + + # Convert workspace_id to string if it's an integer (for backward compatibility) + workspace_id = data.get("workspace_id") + if workspace_id is not None and not isinstance(workspace_id, str): + workspace_id = str(workspace_id) + + return cls( + access_token=data.get("access_token", ""), + refresh_token=data.get("refresh_token"), + expires_in=data.get("expires_in"), + expires_at=expires_at, + user_id=user_id, + user_email=data.get("user_email"), + user_name=data.get("user_name"), + workspace_id=workspace_id, + workspace_name=data.get("workspace_name"), + ) + + @field_validator("expires_at", mode="before") + @classmethod + def ensure_aware_utc(cls, v): + # Strings like "2025-08-26T14:46:57.367184" + if isinstance(v, str): + # add +00:00 if missing tz info + if v.endswith("Z"): + return datetime.fromisoformat(v.replace("Z", "+00:00")) + dt = datetime.fromisoformat(v) + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + # datetime objects + if isinstance(v, datetime): + return v if v.tzinfo else v.replace(tzinfo=UTC) + return v diff --git a/surfsense_backend/app/schemas/discord_auth_credentials.py b/surfsense_backend/app/schemas/discord_auth_credentials.py new file mode 100644 index 000000000..7ea4ee55c --- /dev/null +++ b/surfsense_backend/app/schemas/discord_auth_credentials.py @@ -0,0 +1,75 @@ +from datetime import UTC, datetime + +from pydantic import BaseModel, field_validator + + +class DiscordAuthCredentialsBase(BaseModel): + bot_token: str + refresh_token: str | None = None + token_type: str = "Bearer" + expires_in: int | None = None + expires_at: datetime | None = None + scope: str | None = None + bot_user_id: str | None = None + guild_id: str | None = None + guild_name: str | None = None + + @property + def is_expired(self) -> bool: + """Check if the credentials have expired.""" + if self.expires_at is None: + return False # Long-lived token, treat as not expired + return self.expires_at <= datetime.now(UTC) + + @property + def is_refreshable(self) -> bool: + """Check if the credentials can be refreshed.""" + return self.refresh_token is not None + + def to_dict(self) -> dict: + """Convert credentials to dictionary for storage.""" + return { + "bot_token": self.bot_token, + "refresh_token": self.refresh_token, + "token_type": self.token_type, + "expires_in": self.expires_in, + "expires_at": self.expires_at.isoformat() if self.expires_at else None, + "scope": self.scope, + "bot_user_id": self.bot_user_id, + "guild_id": self.guild_id, + "guild_name": self.guild_name, + } + + @classmethod + def from_dict(cls, data: dict) -> "DiscordAuthCredentialsBase": + """Create credentials from dictionary.""" + expires_at = None + if data.get("expires_at"): + expires_at = datetime.fromisoformat(data["expires_at"]) + + return cls( + bot_token=data.get("bot_token", ""), + refresh_token=data.get("refresh_token"), + token_type=data.get("token_type", "Bearer"), + expires_in=data.get("expires_in"), + expires_at=expires_at, + scope=data.get("scope"), + bot_user_id=data.get("bot_user_id"), + guild_id=data.get("guild_id"), + guild_name=data.get("guild_name"), + ) + + @field_validator("expires_at", mode="before") + @classmethod + def ensure_aware_utc(cls, v): + # Strings like "2025-08-26T14:46:57.367184" + if isinstance(v, str): + # add +00:00 if missing tz info + if v.endswith("Z"): + return datetime.fromisoformat(v.replace("Z", "+00:00")) + dt = datetime.fromisoformat(v) + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + # datetime objects + if isinstance(v, datetime): + return v if v.tzinfo else v.replace(tzinfo=UTC) + return v diff --git a/surfsense_backend/app/schemas/linear_auth_credentials.py b/surfsense_backend/app/schemas/linear_auth_credentials.py new file mode 100644 index 000000000..99e8d9111 --- /dev/null +++ b/surfsense_backend/app/schemas/linear_auth_credentials.py @@ -0,0 +1,66 @@ +from datetime import UTC, datetime + +from pydantic import BaseModel, field_validator + + +class LinearAuthCredentialsBase(BaseModel): + access_token: str + refresh_token: str | None = None + token_type: str = "Bearer" + expires_in: int | None = None + expires_at: datetime | None = None + scope: str | None = None + + @property + def is_expired(self) -> bool: + """Check if the credentials have expired.""" + if self.expires_at is None: + return False + return self.expires_at <= datetime.now(UTC) + + @property + def is_refreshable(self) -> bool: + """Check if the credentials can be refreshed.""" + return self.refresh_token is not None + + def to_dict(self) -> dict: + """Convert credentials to dictionary for storage.""" + return { + "access_token": self.access_token, + "refresh_token": self.refresh_token, + "token_type": self.token_type, + "expires_in": self.expires_in, + "expires_at": self.expires_at.isoformat() if self.expires_at else None, + "scope": self.scope, + } + + @classmethod + def from_dict(cls, data: dict) -> "LinearAuthCredentialsBase": + """Create credentials from dictionary.""" + expires_at = None + if data.get("expires_at"): + expires_at = datetime.fromisoformat(data["expires_at"]) + + return cls( + access_token=data["access_token"], + refresh_token=data.get("refresh_token"), + token_type=data.get("token_type", "Bearer"), + expires_in=data.get("expires_in"), + expires_at=expires_at, + scope=data.get("scope"), + ) + + @field_validator("expires_at", mode="before") + @classmethod + def ensure_aware_utc(cls, v): + # Strings like "2025-08-26T14:46:57.367184" + if isinstance(v, str): + # add +00:00 if missing tz info + if v.endswith("Z"): + return datetime.fromisoformat(v.replace("Z", "+00:00")) + dt = datetime.fromisoformat(v) + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + # datetime objects + if isinstance(v, datetime): + return v if v.tzinfo else v.replace(tzinfo=UTC) + return v diff --git a/surfsense_backend/app/schemas/notion_auth_credentials.py b/surfsense_backend/app/schemas/notion_auth_credentials.py new file mode 100644 index 000000000..e66afb903 --- /dev/null +++ b/surfsense_backend/app/schemas/notion_auth_credentials.py @@ -0,0 +1,72 @@ +from datetime import UTC, datetime + +from pydantic import BaseModel, field_validator + + +class NotionAuthCredentialsBase(BaseModel): + access_token: str + refresh_token: str | None = None + expires_in: int | None = None + expires_at: datetime | None = None + workspace_id: str | None = None + workspace_name: str | None = None + workspace_icon: str | None = None + bot_id: str | None = None + + @property + def is_expired(self) -> bool: + """Check if the credentials have expired.""" + if self.expires_at is None: + return False # Long-lived token, treat as not expired + return self.expires_at <= datetime.now(UTC) + + @property + def is_refreshable(self) -> bool: + """Check if the credentials can be refreshed.""" + return self.refresh_token is not None + + def to_dict(self) -> dict: + """Convert credentials to dictionary for storage.""" + return { + "access_token": self.access_token, + "refresh_token": self.refresh_token, + "expires_in": self.expires_in, + "expires_at": self.expires_at.isoformat() if self.expires_at else None, + "workspace_id": self.workspace_id, + "workspace_name": self.workspace_name, + "workspace_icon": self.workspace_icon, + "bot_id": self.bot_id, + } + + @classmethod + def from_dict(cls, data: dict) -> "NotionAuthCredentialsBase": + """Create credentials from dictionary.""" + expires_at = None + if data.get("expires_at"): + expires_at = datetime.fromisoformat(data["expires_at"]) + + return cls( + access_token=data["access_token"], + refresh_token=data.get("refresh_token"), + expires_in=data.get("expires_in"), + expires_at=expires_at, + workspace_id=data.get("workspace_id"), + workspace_name=data.get("workspace_name"), + workspace_icon=data.get("workspace_icon"), + bot_id=data.get("bot_id"), + ) + + @field_validator("expires_at", mode="before") + @classmethod + def ensure_aware_utc(cls, v): + # Strings like "2025-08-26T14:46:57.367184" + if isinstance(v, str): + # add +00:00 if missing tz info + if v.endswith("Z"): + return datetime.fromisoformat(v.replace("Z", "+00:00")) + dt = datetime.fromisoformat(v) + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + # datetime objects + if isinstance(v, datetime): + return v if v.tzinfo else v.replace(tzinfo=UTC) + return v diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index 1e8a7a38d..dbe4dce1f 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -30,7 +30,12 @@ class SearchSourceConnectorBase(BaseModel): @model_validator(mode="after") def validate_periodic_indexing(self): - """Validate that periodic indexing configuration is consistent.""" + """Validate that periodic indexing configuration is consistent. + + Supported frequencies: Any positive integer (in minutes). + Common values: 5, 15, 60 (1 hour), 360 (6 hours), 720 (12 hours), 1440 (daily), etc. + The schedule checker will handle any frequency >= 1 minute. + """ if self.periodic_indexing_enabled: if not self.is_indexable: raise ValueError( diff --git a/surfsense_backend/app/schemas/slack_auth_credentials.py b/surfsense_backend/app/schemas/slack_auth_credentials.py new file mode 100644 index 000000000..5148a0985 --- /dev/null +++ b/surfsense_backend/app/schemas/slack_auth_credentials.py @@ -0,0 +1,75 @@ +from datetime import UTC, datetime + +from pydantic import BaseModel, field_validator + + +class SlackAuthCredentialsBase(BaseModel): + bot_token: str + refresh_token: str | None = None + token_type: str = "Bearer" + expires_in: int | None = None + expires_at: datetime | None = None + scope: str | None = None + bot_user_id: str | None = None + team_id: str | None = None + team_name: str | None = None + + @property + def is_expired(self) -> bool: + """Check if the credentials have expired.""" + if self.expires_at is None: + return False # Long-lived token, treat as not expired + return self.expires_at <= datetime.now(UTC) + + @property + def is_refreshable(self) -> bool: + """Check if the credentials can be refreshed.""" + return self.refresh_token is not None + + def to_dict(self) -> dict: + """Convert credentials to dictionary for storage.""" + return { + "bot_token": self.bot_token, + "refresh_token": self.refresh_token, + "token_type": self.token_type, + "expires_in": self.expires_in, + "expires_at": self.expires_at.isoformat() if self.expires_at else None, + "scope": self.scope, + "bot_user_id": self.bot_user_id, + "team_id": self.team_id, + "team_name": self.team_name, + } + + @classmethod + def from_dict(cls, data: dict) -> "SlackAuthCredentialsBase": + """Create credentials from dictionary.""" + expires_at = None + if data.get("expires_at"): + expires_at = datetime.fromisoformat(data["expires_at"]) + + return cls( + bot_token=data.get("bot_token", ""), + refresh_token=data.get("refresh_token"), + token_type=data.get("token_type", "Bearer"), + expires_in=data.get("expires_in"), + expires_at=expires_at, + scope=data.get("scope"), + bot_user_id=data.get("bot_user_id"), + team_id=data.get("team_id"), + team_name=data.get("team_name"), + ) + + @field_validator("expires_at", mode="before") + @classmethod + def ensure_aware_utc(cls, v): + # Strings like "2025-08-26T14:46:57.367184" + if isinstance(v, str): + # add +00:00 if missing tz info + if v.endswith("Z"): + return datetime.fromisoformat(v.replace("Z", "+00:00")) + dt = datetime.fromisoformat(v) + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + # datetime objects + if isinstance(v, datetime): + return v if v.tzinfo else v.replace(tzinfo=UTC) + return v diff --git a/surfsense_backend/app/services/docling_service.py b/surfsense_backend/app/services/docling_service.py index a61148c6d..82eaf7f74 100644 --- a/surfsense_backend/app/services/docling_service.py +++ b/surfsense_backend/app/services/docling_service.py @@ -128,42 +128,6 @@ class DoclingService: logger.error(f"❌ Docling initialization failed: {e}") raise RuntimeError(f"Docling initialization failed: {e}") from e - def _configure_easyocr_local_models(self): - """Configure EasyOCR to use pre-downloaded local models.""" - try: - import os - - import easyocr - - # Set SSL environment for EasyOCR downloads - os.environ["CURL_CA_BUNDLE"] = "" - os.environ["REQUESTS_CA_BUNDLE"] = "" - - # Try to use local models first, fallback to download if needed - try: - reader = easyocr.Reader( - ["en"], - download_enabled=False, - model_storage_directory="/root/.EasyOCR/model", - ) - logger.info("✅ EasyOCR configured for local models") - return reader - except Exception: - # If local models fail, allow download with SSL bypass - logger.info( - "🔄 Local models failed, attempting download with SSL bypass..." - ) - reader = easyocr.Reader( - ["en"], - download_enabled=True, - model_storage_directory="/root/.EasyOCR/model", - ) - logger.info("✅ EasyOCR configured with downloaded models") - return reader - except Exception as e: - logger.warning(f"⚠️ EasyOCR configuration failed: {e}") - return None - async def process_document( self, file_path: str, filename: str | None = None ) -> dict[str, Any]: diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index 68dd167b5..33f073d61 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -342,40 +342,7 @@ async def get_document_summary_llm( ) -# Backward-compatible aliases (deprecated - will be removed in future versions) -async def get_user_llm_instance( - session: AsyncSession, user_id: str, search_space_id: int, role: str -) -> ChatLiteLLM | None: - """ - Deprecated: Use get_search_space_llm_instance instead. - LLM preferences are now stored at the search space level, not per-user. - """ - return await get_search_space_llm_instance(session, search_space_id, role) - - -# Legacy aliases for backward compatibility -async def get_long_context_llm( - session: AsyncSession, search_space_id: int -) -> ChatLiteLLM | None: - """Deprecated: Use get_document_summary_llm instead.""" - return await get_document_summary_llm(session, search_space_id) - - -async def get_fast_llm( - session: AsyncSession, search_space_id: int -) -> ChatLiteLLM | None: - """Deprecated: Use get_agent_llm instead.""" - return await get_agent_llm(session, search_space_id) - - -async def get_strategic_llm( - session: AsyncSession, search_space_id: int -) -> ChatLiteLLM | None: - """Deprecated: Use get_document_summary_llm instead.""" - return await get_document_summary_llm(session, search_space_id) - - -# User-based legacy aliases (LLM preferences are now per-search-space, not per-user) +# Backward-compatible alias (LLM preferences are now per-search-space, not per-user) async def get_user_long_context_llm( session: AsyncSession, user_id: str, search_space_id: int ) -> ChatLiteLLM | None: @@ -384,23 +351,3 @@ async def get_user_long_context_llm( The user_id parameter is ignored as LLM preferences are now per-search-space. """ return await get_document_summary_llm(session, search_space_id) - - -async def get_user_fast_llm( - session: AsyncSession, user_id: str, search_space_id: int -) -> ChatLiteLLM | None: - """ - Deprecated: Use get_agent_llm instead. - The user_id parameter is ignored as LLM preferences are now per-search-space. - """ - return await get_agent_llm(session, search_space_id) - - -async def get_user_strategic_llm( - session: AsyncSession, user_id: str, search_space_id: int -) -> ChatLiteLLM | None: - """ - Deprecated: Use get_document_summary_llm instead. - The user_id parameter is ignored as LLM preferences are now per-search-space. - """ - return await get_document_summary_llm(session, search_space_id) diff --git a/surfsense_backend/app/services/query_service.py b/surfsense_backend/app/services/query_service.py deleted file mode 100644 index 863ff58a4..000000000 --- a/surfsense_backend/app/services/query_service.py +++ /dev/null @@ -1,114 +0,0 @@ -import datetime -from typing import Any - -from langchain_core.messages import AIMessage, HumanMessage, SystemMessage -from sqlalchemy.ext.asyncio import AsyncSession - -from app.services.llm_service import get_document_summary_llm - - -class QueryService: - """ - Service for query-related operations, including reformulation and processing. - """ - - @staticmethod - async def reformulate_query_with_chat_history( - user_query: str, - session: AsyncSession, - search_space_id: int, - chat_history_str: str | None = None, - ) -> str: - """ - Reformulate the user query using the search space's document summary LLM to make it more - effective for information retrieval and research purposes. - - Args: - user_query: The original user query - session: Database session for accessing LLM configs - search_space_id: Search Space ID to get LLM preferences - chat_history_str: Optional chat history string - - Returns: - str: The reformulated query - """ - if not user_query or not user_query.strip(): - return user_query - - try: - # Get the search space's document summary LLM instance - llm = await get_document_summary_llm(session, search_space_id) - if not llm: - print( - f"Warning: No document summary LLM configured for search space {search_space_id}. Using original query." - ) - return user_query - - # Create system message with instructions - system_message = SystemMessage( - content=f""" - Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} - You are a highly skilled AI assistant specializing in query optimization for advanced research. - Your primary objective is to transform a user's initial query into a highly effective search query. - This reformulated query will be used to retrieve information from diverse data sources. - - **Chat History Context:** - {chat_history_str if chat_history_str else "No prior conversation history is available."} - If chat history is provided, analyze it to understand the user's evolving information needs and the broader context of their request. Use this understanding to refine the current query, ensuring it builds upon or clarifies previous interactions. - - **Query Reformulation Guidelines:** - Your reformulated query should: - 1. **Enhance Specificity and Detail:** Add precision to narrow the search focus effectively, making the query less ambiguous and more targeted. - 2. **Resolve Ambiguities:** Identify and clarify vague terms or phrases. If a term has multiple meanings, orient the query towards the most likely one given the context. - 3. **Expand Key Concepts:** Incorporate relevant synonyms, related terms, and alternative phrasings for core concepts. This helps capture a wider range of relevant documents. - 4. **Deconstruct Complex Questions:** If the original query is multifaceted, break it down into its core searchable components or rephrase it to address each aspect clearly. The final output must still be a single, coherent query string. - 5. **Optimize for Comprehensiveness:** Ensure the query is structured to uncover all essential facets of the original request, aiming for thorough information retrieval suitable for research. - 6. **Maintain User Intent:** The reformulated query must stay true to the original intent of the user's query. Do not introduce new topics or shift the focus significantly. - - **Crucial Constraints:** - * **Conciseness and Effectiveness:** While aiming for comprehensiveness, the reformulated query MUST be as concise as possible. Eliminate all unnecessary verbosity. Focus on essential keywords, entities, and concepts that directly contribute to effective retrieval. - * **Single, Direct Output:** Return ONLY the reformulated query itself. Do NOT include any explanations, introductory phrases (e.g., "Reformulated query:", "Here is the optimized query:"), or any other surrounding text or markdown formatting. - - Your output should be a single, optimized query string, ready for immediate use in a search system. - """ - ) - - # Create human message with the user query - human_message = HumanMessage( - content=f"Reformulate this query for better research results: {user_query}" - ) - - # Get the response from the LLM - response = await llm.agenerate(messages=[[system_message, human_message]]) - - # Extract the reformulated query from the response - reformulated_query = response.generations[0][0].text.strip() - - # Return the original query if the reformulation is empty - if not reformulated_query: - return user_query - - return reformulated_query - - except Exception as e: - # Log the error and return the original query - print(f"Error reformulating query: {e}") - return user_query - - @staticmethod - async def langchain_chat_history_to_str(chat_history: list[Any]) -> str: - """ - Convert a list of chat history messages to a string. - """ - chat_history_str = "\n" - - for chat_message in chat_history: - if isinstance(chat_message, HumanMessage): - chat_history_str += f"{chat_message.content}\n" - elif isinstance(chat_message, AIMessage): - chat_history_str += f"{chat_message.content}\n" - elif isinstance(chat_message, SystemMessage): - chat_history_str += f"{chat_message.content}\n" - - chat_history_str += "" - return chat_history_str diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py index 69b75e5c4..3b87c33f1 100644 --- a/surfsense_backend/app/tasks/chat/stream_new_chat.py +++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py @@ -270,7 +270,8 @@ async def stream_new_chat( # Track if we just finished a tool (text flows silently after tools) just_finished_tool: bool = False # Track write_todos calls to show "Creating plan" vs "Updating plan" - write_todos_call_count: int = 0 + # Disabled for now + # write_todos_call_count: int = 0 def next_thinking_step_id() -> str: nonlocal thinking_step_counter @@ -479,60 +480,60 @@ async def stream_new_chat( status="in_progress", items=last_active_step_items, ) - elif tool_name == "write_todos": - # Track write_todos calls for better messaging - write_todos_call_count += 1 - todos = ( - tool_input.get("todos", []) - if isinstance(tool_input, dict) - else [] - ) - todo_count = len(todos) if isinstance(todos, list) else 0 + # elif tool_name == "write_todos": # Disabled for now + # # Track write_todos calls for better messaging + # write_todos_call_count += 1 + # todos = ( + # tool_input.get("todos", []) + # if isinstance(tool_input, dict) + # else [] + # ) + # todo_count = len(todos) if isinstance(todos, list) else 0 - if write_todos_call_count == 1: - # First call - creating the plan - last_active_step_title = "Creating plan" - last_active_step_items = [f"Defining {todo_count} tasks..."] - else: - # Subsequent calls - updating the plan - # Try to provide context about what's being updated - in_progress_count = ( - sum( - 1 - for t in todos - if isinstance(t, dict) - and t.get("status") == "in_progress" - ) - if isinstance(todos, list) - else 0 - ) - completed_count = ( - sum( - 1 - for t in todos - if isinstance(t, dict) - and t.get("status") == "completed" - ) - if isinstance(todos, list) - else 0 - ) + # if write_todos_call_count == 1: + # # First call - creating the plan + # last_active_step_title = "Creating plan" + # last_active_step_items = [f"Defining {todo_count} tasks..."] + # else: + # # Subsequent calls - updating the plan + # # Try to provide context about what's being updated + # in_progress_count = ( + # sum( + # 1 + # for t in todos + # if isinstance(t, dict) + # and t.get("status") == "in_progress" + # ) + # if isinstance(todos, list) + # else 0 + # ) + # completed_count = ( + # sum( + # 1 + # for t in todos + # if isinstance(t, dict) + # and t.get("status") == "completed" + # ) + # if isinstance(todos, list) + # else 0 + # ) - last_active_step_title = "Updating progress" - last_active_step_items = ( - [ - f"Progress: {completed_count}/{todo_count} completed", - f"In progress: {in_progress_count} tasks", - ] - if completed_count > 0 - else [f"Working on {todo_count} tasks"] - ) + # last_active_step_title = "Updating progress" + # last_active_step_items = ( + # [ + # f"Progress: {completed_count}/{todo_count} completed", + # f"In progress: {in_progress_count} tasks", + # ] + # if completed_count > 0 + # else [f"Working on {todo_count} tasks"] + # ) - yield streaming_service.format_thinking_step( - step_id=tool_step_id, - title=last_active_step_title, - status="in_progress", - items=last_active_step_items, - ) + # yield streaming_service.format_thinking_step( + # step_id=tool_step_id, + # title=last_active_step_title, + # status="in_progress", + # items=last_active_step_items, + # ) elif tool_name == "generate_podcast": podcast_title = ( tool_input.get("podcast_title", "SurfSense Podcast") @@ -596,10 +597,12 @@ async def stream_new_chat( raw_output = event.get("data", {}).get("output", "") # Handle deepagents' write_todos Command object specially - if tool_name == "write_todos" and hasattr(raw_output, "update"): - # deepagents returns a Command object - extract todos directly - tool_output = extract_todos_from_deepagents(raw_output) - elif hasattr(raw_output, "content"): + # Disabled for now + # if tool_name == "write_todos" and hasattr(raw_output, "update"): + # # deepagents returns a Command object - extract todos directly + # tool_output = extract_todos_from_deepagents(raw_output) + # elif hasattr(raw_output, "content"): + if hasattr(raw_output, "content"): # It's a ToolMessage object - extract the content content = raw_output.content # If content is a string that looks like JSON, try to parse it @@ -758,63 +761,63 @@ async def stream_new_chat( status="completed", items=completed_items, ) - elif tool_name == "write_todos": - # Build completion items for planning/updating - if isinstance(tool_output, dict): - todos = tool_output.get("todos", []) - todo_count = len(todos) if isinstance(todos, list) else 0 - completed_count = ( - sum( - 1 - for t in todos - if isinstance(t, dict) - and t.get("status") == "completed" - ) - if isinstance(todos, list) - else 0 - ) - in_progress_count = ( - sum( - 1 - for t in todos - if isinstance(t, dict) - and t.get("status") == "in_progress" - ) - if isinstance(todos, list) - else 0 - ) + # elif tool_name == "write_todos": # Disabled for now + # # Build completion items for planning/updating + # if isinstance(tool_output, dict): + # todos = tool_output.get("todos", []) + # todo_count = len(todos) if isinstance(todos, list) else 0 + # completed_count = ( + # sum( + # 1 + # for t in todos + # if isinstance(t, dict) + # and t.get("status") == "completed" + # ) + # if isinstance(todos, list) + # else 0 + # ) + # in_progress_count = ( + # sum( + # 1 + # for t in todos + # if isinstance(t, dict) + # and t.get("status") == "in_progress" + # ) + # if isinstance(todos, list) + # else 0 + # ) - # Use context-aware completion message - if last_active_step_title == "Creating plan": - completed_items = [f"Created {todo_count} tasks"] - else: - # Updating progress - show stats - completed_items = [ - f"Progress: {completed_count}/{todo_count} completed", - ] - if in_progress_count > 0: - # Find the currently in-progress task name - in_progress_task = next( - ( - t.get("content", "")[:40] - for t in todos - if isinstance(t, dict) - and t.get("status") == "in_progress" - ), - None, - ) - if in_progress_task: - completed_items.append( - f"Current: {in_progress_task}..." - ) - else: - completed_items = ["Plan updated"] - yield streaming_service.format_thinking_step( - step_id=original_step_id, - title=last_active_step_title, - status="completed", - items=completed_items, - ) + # # Use context-aware completion message + # if last_active_step_title == "Creating plan": + # completed_items = [f"Created {todo_count} tasks"] + # else: + # # Updating progress - show stats + # completed_items = [ + # f"Progress: {completed_count}/{todo_count} completed", + # ] + # if in_progress_count > 0: + # # Find the currently in-progress task name + # in_progress_task = next( + # ( + # t.get("content", "")[:40] + # for t in todos + # if isinstance(t, dict) + # and t.get("status") == "in_progress" + # ), + # None, + # ) + # if in_progress_task: + # completed_items.append( + # f"Current: {in_progress_task}..." + # ) + # else: + # completed_items = ["Plan updated"] + # yield streaming_service.format_thinking_step( + # step_id=original_step_id, + # title=last_active_step_title, + # status="completed", + # items=completed_items, + # ) elif tool_name == "ls": # Build completion items showing file names found if isinstance(tool_output, dict): @@ -992,27 +995,27 @@ async def stream_new_chat( yield streaming_service.format_terminal_info( "Knowledge base search completed", "success" ) - elif tool_name == "write_todos": - # Stream the full write_todos result so frontend can render the Plan component - yield streaming_service.format_tool_output_available( - tool_call_id, - tool_output - if isinstance(tool_output, dict) - else {"result": tool_output}, - ) - # Send terminal message with plan info - if isinstance(tool_output, dict): - todos = tool_output.get("todos", []) - todo_count = len(todos) if isinstance(todos, list) else 0 - yield streaming_service.format_terminal_info( - f"Plan created ({todo_count} tasks)", - "success", - ) - else: - yield streaming_service.format_terminal_info( - "Plan created", - "success", - ) + # elif tool_name == "write_todos": # Disabled for now + # # Stream the full write_todos result so frontend can render the Plan component + # yield streaming_service.format_tool_output_available( + # tool_call_id, + # tool_output + # if isinstance(tool_output, dict) + # else {"result": tool_output}, + # ) + # # Send terminal message with plan info + # if isinstance(tool_output, dict): + # todos = tool_output.get("todos", []) + # todo_count = len(todos) if isinstance(todos, list) else 0 + # yield streaming_service.format_terminal_info( + # f"Plan created ({todo_count} tasks)", + # "success", + # ) + # else: + # yield streaming_service.format_terminal_info( + # "Plan created", + # "success", + # ) else: # Default handling for other tools yield streaming_service.format_tool_output_available( diff --git a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py index cea2a0529..4d5a33b79 100644 --- a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py @@ -6,10 +6,8 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession from app.config import config -from app.connectors.airtable_connector import AirtableConnector +from app.connectors.airtable_history import AirtableHistoryConnector from app.db import Document, DocumentType, SearchSourceConnectorType -from app.routes.airtable_add_connector_route import refresh_airtable_token -from app.schemas.airtable_auth_credentials import AirtableAuthCredentialsBase from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( @@ -84,31 +82,11 @@ async def index_airtable_records( ) return 0, f"Connector with ID {connector_id} not found" - # Create credentials from connector config - config_data = connector.config - try: - credentials = AirtableAuthCredentialsBase.from_dict(config_data) - except Exception as e: - await task_logger.log_task_failure( - log_entry, - f"Invalid Airtable credentials in connector {connector_id}", - str(e), - {"error_type": "InvalidCredentials"}, - ) - return 0, f"Invalid Airtable credentials: {e!s}" - - # Check if credentials are expired - if credentials.is_expired: - await task_logger.log_task_failure( - log_entry, - f"Airtable credentials expired for connector {connector_id}", - "Credentials expired", - {"error_type": "ExpiredCredentials"}, - ) - - connector = await refresh_airtable_token(session, connector) - - # return 0, "Airtable credentials have expired. Please re-authenticate." + # Normalize "undefined" strings to None (from frontend) + if start_date == "undefined" or start_date == "": + start_date = None + if end_date == "undefined" or end_date == "": + end_date = None # Calculate date range for indexing start_date_str, end_date_str = calculate_date_range( @@ -120,8 +98,9 @@ async def index_airtable_records( f"from {start_date_str} to {end_date_str}" ) - # Initialize Airtable connector - airtable_connector = AirtableConnector(credentials) + # Initialize Airtable history connector with auto-refresh capability + airtable_history = AirtableHistoryConnector(session, connector_id) + airtable_connector = await airtable_history._get_connector() total_processed = 0 try: @@ -413,47 +392,56 @@ async def index_airtable_records( documents_skipped += 1 continue # Skip this message and continue with others - # Update the last_indexed_at timestamp for the connector only if requested - total_processed = documents_indexed - if total_processed > 0: - await update_connector_last_indexed( - session, connector, update_last_indexed - ) + # Accumulate total processed across all tables + total_processed += documents_indexed # Final commit for any remaining documents not yet committed in batches - logger.info( - f"Final commit: Total {documents_indexed} Airtable records processed" - ) - await session.commit() - logger.info( - "Successfully committed all Airtable document changes to database" - ) + if documents_indexed > 0: + logger.info( + f"Final commit for table {table_name}: {documents_indexed} Airtable records processed" + ) + await session.commit() + logger.info( + f"Successfully committed all Airtable document changes for table {table_name}" + ) - # Log success - await task_logger.log_task_success( - log_entry, - f"Successfully completed Airtable indexing for connector {connector_id}", - { - "events_processed": total_processed, - "documents_indexed": documents_indexed, - "documents_skipped": documents_skipped, - "skipped_messages_count": len(skipped_messages), - }, - ) + # Update the last_indexed_at timestamp for the connector only if requested + # (after all tables in all bases are processed) + if total_processed > 0: + await update_connector_last_indexed( + session, connector, update_last_indexed + ) - logger.info( - f"Airtable indexing completed: {documents_indexed} new records, {documents_skipped} skipped" - ) - return ( - total_processed, - None, - ) # Return None as the error message to indicate success + # Log success after processing all bases and tables + await task_logger.log_task_success( + log_entry, + f"Successfully completed Airtable indexing for connector {connector_id}", + { + "events_processed": total_processed, + "documents_indexed": total_processed, + }, + ) + + logger.info( + f"Airtable indexing completed: {total_processed} total records processed" + ) + return ( + total_processed, + None, + ) # Return None as the error message to indicate success except Exception as e: logger.error( f"Fetching Airtable bases for connector {connector_id} failed: {e!s}", exc_info=True, ) + await task_logger.log_task_failure( + log_entry, + f"Failed to fetch Airtable bases for connector {connector_id}", + str(e), + {"error_type": type(e).__name__}, + ) + return 0, f"Failed to fetch Airtable bases: {e!s}" except SQLAlchemyError as db_error: await session.rollback() diff --git a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py index b4a349163..e459584f8 100644 --- a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py @@ -2,13 +2,14 @@ ClickUp connector indexer. """ +import contextlib from datetime import datetime from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession from app.config import config -from app.connectors.clickup_connector import ClickUpConnector +from app.connectors.clickup_history import ClickUpHistoryConnector from app.db import Document, DocumentType, SearchSourceConnectorType from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService @@ -82,26 +83,30 @@ async def index_clickup_tasks( ) return 0, error_msg - # Extract ClickUp configuration - clickup_api_token = connector.config.get("CLICKUP_API_TOKEN") + # Check if using OAuth (has access_token in config) or legacy (has CLICKUP_API_TOKEN) + has_oauth = connector.config.get("access_token") is not None + has_legacy = connector.config.get("CLICKUP_API_TOKEN") is not None - if not clickup_api_token: - error_msg = "ClickUp API token not found in connector configuration" + if not has_oauth and not has_legacy: + error_msg = "ClickUp credentials not found in connector configuration (neither OAuth nor API token)" await task_logger.log_task_failure( log_entry, - f"ClickUp API token not found in connector config for connector {connector_id}", - "Missing ClickUp token", - {"error_type": "MissingToken"}, + f"ClickUp credentials not found in connector config for connector {connector_id}", + "Missing ClickUp credentials", + {"error_type": "MissingCredentials"}, ) return 0, error_msg await task_logger.log_task_progress( log_entry, - f"Initializing ClickUp client for connector {connector_id}", + f"Initializing ClickUp client for connector {connector_id} ({'OAuth' if has_oauth else 'API Token'})", {"stage": "client_initialization"}, ) - clickup_client = ClickUpConnector(api_token=clickup_api_token) + # Use history connector which supports both OAuth and legacy API tokens + clickup_client = ClickUpHistoryConnector( + session=session, connector_id=connector_id + ) # Get authorized workspaces await task_logger.log_task_progress( @@ -110,7 +115,7 @@ async def index_clickup_tasks( {"stage": "workspace_fetching"}, ) - workspaces_response = clickup_client.get_authorized_workspaces() + workspaces_response = await clickup_client.get_authorized_workspaces() workspaces = workspaces_response.get("teams", []) if not workspaces: @@ -141,7 +146,7 @@ async def index_clickup_tasks( # Fetch tasks for date range if provided if start_date and end_date: - tasks, error = clickup_client.get_tasks_in_date_range( + tasks, error = await clickup_client.get_tasks_in_date_range( workspace_id=workspace_id, start_date=start_date, end_date=end_date, @@ -153,7 +158,7 @@ async def index_clickup_tasks( ) continue else: - tasks = clickup_client.get_workspace_tasks( + tasks = await clickup_client.get_workspace_tasks( workspace_id=workspace_id, include_closed=True ) @@ -393,10 +398,21 @@ async def index_clickup_tasks( logger.info( f"clickup indexing completed: {documents_indexed} new tasks, {documents_skipped} skipped" ) + + # Close client connection + try: + await clickup_client.close() + except Exception as e: + logger.warning(f"Error closing ClickUp client: {e!s}") + return total_processed, None except SQLAlchemyError as db_error: await session.rollback() + # Clean up the connector in case of error + if "clickup_client" in locals(): + with contextlib.suppress(Exception): + await clickup_client.close() await task_logger.log_task_failure( log_entry, f"Database error during ClickUp indexing for connector {connector_id}", @@ -407,6 +423,10 @@ async def index_clickup_tasks( return 0, f"Database error: {db_error!s}" except Exception as e: await session.rollback() + # Clean up the connector in case of error + if "clickup_client" in locals(): + with contextlib.suppress(Exception): + await clickup_client.close() await task_logger.log_task_failure( log_entry, f"Failed to index ClickUp tasks for connector {connector_id}", diff --git a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py index d5e68fb8f..7289b0ccd 100644 --- a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py @@ -2,13 +2,14 @@ Confluence connector indexer. """ +import contextlib from datetime import datetime from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession from app.config import config -from app.connectors.confluence_connector import ConfluenceConnector +from app.connectors.confluence_history import ConfluenceHistoryConnector from app.db import Document, DocumentType, SearchSourceConnectorType from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService @@ -83,31 +84,18 @@ async def index_confluence_pages( ) return 0, f"Connector with ID {connector_id} not found" - # Get the Confluence credentials from the connector config - confluence_email = connector.config.get("CONFLUENCE_EMAIL") - confluence_api_token = connector.config.get("CONFLUENCE_API_TOKEN") - confluence_base_url = connector.config.get("CONFLUENCE_BASE_URL") - - if not confluence_email or not confluence_api_token or not confluence_base_url: - await task_logger.log_task_failure( - log_entry, - f"Confluence credentials not found in connector config for connector {connector_id}", - "Missing Confluence credentials", - {"error_type": "MissingCredentials"}, - ) - return 0, "Confluence credentials not found in connector config" - - # Initialize Confluence client + # Initialize Confluence OAuth client await task_logger.log_task_progress( log_entry, - f"Initializing Confluence client for connector {connector_id}", + f"Initializing Confluence OAuth client for connector {connector_id}", {"stage": "client_initialization"}, ) - confluence_client = ConfluenceConnector( - base_url=confluence_base_url, - email=confluence_email, - api_token=confluence_api_token, + confluence_client: ConfluenceHistoryConnector | None = ( + ConfluenceHistoryConnector( + session=session, + connector_id=connector_id, + ) ) # Calculate date range @@ -127,7 +115,7 @@ async def index_confluence_pages( # Get pages within date range try: - pages, error = confluence_client.get_pages_by_date_range( + pages, error = await confluence_client.get_pages_by_date_range( start_date=start_date_str, end_date=end_date_str, include_comments=True ) @@ -153,6 +141,10 @@ async def index_confluence_pages( f"No Confluence pages found in date range {start_date_str} to {end_date_str}", {"pages_found": 0}, ) + # Close client before returning + if confluence_client: + with contextlib.suppress(Exception): + await confluence_client.close() return 0, None else: await task_logger.log_task_failure( @@ -161,12 +153,20 @@ async def index_confluence_pages( "API Error", {"error_type": "APIError"}, ) + # Close client on error + if confluence_client: + with contextlib.suppress(Exception): + await confluence_client.close() return 0, f"Failed to get Confluence pages: {error}" logger.info(f"Retrieved {len(pages)} pages from Confluence API") except Exception as e: logger.error(f"Error fetching Confluence pages: {e!s}", exc_info=True) + # Close client on error + if confluence_client: + with contextlib.suppress(Exception): + await confluence_client.close() return 0, f"Error fetching Confluence pages: {e!s}" # Process and index each page @@ -418,6 +418,11 @@ async def index_confluence_pages( logger.info( f"Confluence indexing completed: {documents_indexed} new pages, {documents_skipped} skipped" ) + + # Close the client connection + if confluence_client: + await confluence_client.close() + return ( total_processed, None, @@ -425,6 +430,10 @@ async def index_confluence_pages( except SQLAlchemyError as db_error: await session.rollback() + # Close client if it exists + if confluence_client: + with contextlib.suppress(Exception): + await confluence_client.close() await task_logger.log_task_failure( log_entry, f"Database error during Confluence indexing for connector {connector_id}", @@ -435,6 +444,10 @@ async def index_confluence_pages( return 0, f"Database error: {db_error!s}" except Exception as e: await session.rollback() + # Close client if it exists + if confluence_client: + with contextlib.suppress(Exception): + await confluence_client.close() await task_logger.log_task_failure( log_entry, f"Failed to index Confluence pages for connector {connector_id}", diff --git a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py index 9391be788..110732831 100644 --- a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py @@ -8,19 +8,18 @@ from datetime import UTC, datetime, timedelta from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession +from app.config import config from app.connectors.discord_connector import DiscordConnector from app.db import Document, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) from .base import ( - build_document_metadata_string, + build_document_metadata_markdown, check_document_by_unique_identifier, get_connector_by_id, get_current_timestamp, @@ -69,6 +68,14 @@ async def index_discord_messages( ) try: + # Normalize date parameters - handle 'undefined' strings from frontend + if start_date and ( + start_date.lower() == "undefined" or start_date.strip() == "" + ): + start_date = None + if end_date and (end_date.lower() == "undefined" or end_date.strip() == ""): + end_date = None + # Get the connector await task_logger.log_task_progress( log_entry, @@ -92,27 +99,55 @@ async def index_discord_messages( f"Connector with ID {connector_id} not found or is not a Discord connector", ) - # Get the Discord token from the connector config - discord_token = connector.config.get("DISCORD_BOT_TOKEN") - if not discord_token: - await task_logger.log_task_failure( - log_entry, - f"Discord token not found in connector config for connector {connector_id}", - "Missing Discord token", - {"error_type": "MissingToken"}, - ) - return 0, "Discord token not found in connector config" - logger.info(f"Starting Discord indexing for connector {connector_id}") - # Initialize Discord client + # Initialize Discord client with OAuth credentials support await task_logger.log_task_progress( log_entry, f"Initializing Discord client for connector {connector_id}", {"stage": "client_initialization"}, ) - discord_client = DiscordConnector(token=discord_token) + # Check if using OAuth (has bot_token in config) or legacy (has DISCORD_BOT_TOKEN) + has_oauth = connector.config.get("bot_token") is not None + has_legacy = connector.config.get("DISCORD_BOT_TOKEN") is not None + + if has_oauth: + # Use OAuth credentials with auto-refresh + discord_client = DiscordConnector( + session=session, connector_id=connector_id + ) + elif has_legacy: + # Backward compatibility: use legacy token format + discord_token = connector.config.get("DISCORD_BOT_TOKEN") + + # Decrypt token if it's encrypted (legacy tokens might be encrypted) + token_encrypted = connector.config.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY and discord_token: + try: + from app.utils.oauth_security import TokenEncryption + + token_encryption = TokenEncryption(config.SECRET_KEY) + discord_token = token_encryption.decrypt_token(discord_token) + logger.info( + f"Decrypted legacy Discord token for connector {connector_id}" + ) + except Exception as e: + logger.warning( + f"Failed to decrypt legacy Discord token for connector {connector_id}: {e!s}. " + "Trying to use token as-is (might be unencrypted)." + ) + # Continue with token as-is - might be unencrypted legacy token + + discord_client = DiscordConnector(token=discord_token) + else: + await task_logger.log_task_failure( + log_entry, + f"Discord credentials not found in connector config for connector {connector_id}", + "Missing Discord credentials", + {"error_type": "MissingCredentials"}, + ) + return 0, "Discord credentials not found in connector config" # Calculate date range if start_date is None or end_date is None: @@ -135,32 +170,71 @@ async def index_discord_messages( if start_date is None: start_date_iso = calculated_start_date.isoformat() else: - # Convert YYYY-MM-DD to ISO format + # Validate and convert YYYY-MM-DD to ISO format + try: + start_date_iso = ( + datetime.strptime(start_date, "%Y-%m-%d") + .replace(tzinfo=UTC) + .isoformat() + ) + except ValueError as e: + logger.warning( + f"Invalid start_date format '{start_date}', using calculated start date: {e!s}" + ) + start_date_iso = calculated_start_date.isoformat() + + if end_date is None: + end_date_iso = calculated_end_date.isoformat() + else: + # Validate and convert YYYY-MM-DD to ISO format + try: + end_date_iso = ( + datetime.strptime(end_date, "%Y-%m-%d") + .replace(tzinfo=UTC) + .isoformat() + ) + except ValueError as e: + logger.warning( + f"Invalid end_date format '{end_date}', using calculated end date: {e!s}" + ) + end_date_iso = calculated_end_date.isoformat() + else: + # Convert provided dates to ISO format for Discord API + try: start_date_iso = ( datetime.strptime(start_date, "%Y-%m-%d") .replace(tzinfo=UTC) .isoformat() ) + except ValueError as e: + await task_logger.log_task_failure( + log_entry, + f"Invalid start_date format: {start_date}", + f"Date parsing error: {e!s}", + {"error_type": "InvalidDateFormat", "start_date": start_date}, + ) + return ( + 0, + f"Invalid start_date format: {start_date}. Expected YYYY-MM-DD format.", + ) - if end_date is None: - end_date_iso = calculated_end_date.isoformat() - else: - # Convert YYYY-MM-DD to ISO format + try: end_date_iso = ( datetime.strptime(end_date, "%Y-%m-%d") .replace(tzinfo=UTC) .isoformat() ) - else: - # Convert provided dates to ISO format for Discord API - start_date_iso = ( - datetime.strptime(start_date, "%Y-%m-%d") - .replace(tzinfo=UTC) - .isoformat() - ) - end_date_iso = ( - datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC).isoformat() - ) + except ValueError as e: + await task_logger.log_task_failure( + log_entry, + f"Invalid end_date format: {end_date}", + f"Date parsing error: {e!s}", + {"error_type": "InvalidDateFormat", "end_date": end_date}, + ) + return ( + 0, + f"Invalid end_date format: {end_date}. Expected YYYY-MM-DD format.", + ) logger.info( f"Indexing Discord messages from {start_date_iso} to {end_date_iso}" @@ -271,207 +345,163 @@ async def index_discord_messages( documents_skipped += 1 continue - # Convert messages to markdown format - channel_content = ( - f"# Discord Channel: {guild_name} / {channel_name}\n\n" - ) + # Process each message as an individual document (like Slack) for msg in formatted_messages: - user_name = msg.get("author_name", "Unknown User") - timestamp = msg.get("created_at", "Unknown Time") - text = msg.get("content", "") - channel_content += ( - f"## {user_name} ({timestamp})\n\n{text}\n\n---\n\n" + msg_id = msg.get("id", "") + msg_user_name = msg.get("author_name", "Unknown User") + msg_timestamp = msg.get("created_at", "Unknown Time") + msg_text = msg.get("content", "") + + # Format document metadata (similar to Slack) + metadata_sections = [ + ( + "METADATA", + [ + f"GUILD_NAME: {guild_name}", + f"GUILD_ID: {guild_id}", + f"CHANNEL_NAME: {channel_name}", + f"CHANNEL_ID: {channel_id}", + f"MESSAGE_TIMESTAMP: {msg_timestamp}", + f"MESSAGE_USER_NAME: {msg_user_name}", + ], + ), + ( + "CONTENT", + [ + "FORMAT: markdown", + "TEXT_START", + msg_text, + "TEXT_END", + ], + ), + ] + + # Build the document string + combined_document_string = build_document_metadata_markdown( + metadata_sections ) - # Metadata sections - metadata_sections = [ - ( - "METADATA", - [ - f"GUILD_NAME: {guild_name}", - f"GUILD_ID: {guild_id}", - f"CHANNEL_NAME: {channel_name}", - f"CHANNEL_ID: {channel_id}", - f"MESSAGE_COUNT: {len(formatted_messages)}", - ], - ), - ( - "CONTENT", - [ - "FORMAT: markdown", - "TEXT_START", - channel_content, - "TEXT_END", - ], - ), - ] + # Generate unique identifier hash for this Discord message + unique_identifier = f"{channel_id}_{msg_id}" + unique_identifier_hash = generate_unique_identifier_hash( + DocumentType.DISCORD_CONNECTOR, + unique_identifier, + search_space_id, + ) - combined_document_string = build_document_metadata_string( - metadata_sections - ) + # Generate content hash + content_hash = generate_content_hash( + combined_document_string, search_space_id + ) - # Generate unique identifier hash for this Discord channel - unique_identifier_hash = generate_unique_identifier_hash( - DocumentType.DISCORD_CONNECTOR, channel_id, search_space_id - ) - - # Generate content hash - content_hash = generate_content_hash( - combined_document_string, search_space_id - ) - - # Check if document with this unique identifier already exists - existing_document = await check_document_by_unique_identifier( - session, unique_identifier_hash - ) - - if existing_document: - # Document exists - check if content has changed - if existing_document.content_hash == content_hash: - logger.info( - f"Document for Discord channel {guild_name}#{channel_name} unchanged. Skipping." - ) - documents_skipped += 1 - continue - else: - # Content has changed - update the existing document - logger.info( - f"Content changed for Discord channel {guild_name}#{channel_name}. Updating document." + # Check if document with this unique identifier already exists + existing_document = ( + await check_document_by_unique_identifier( + session, unique_identifier_hash ) + ) - # Get user's long context LLM - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) - if not user_llm: - logger.error( - f"No long context LLM configured for user {user_id}" - ) - skipped_channels.append( - f"{guild_name}#{channel_name} (no LLM configured)" + if existing_document: + # Document exists - check if content has changed + if existing_document.content_hash == content_hash: + logger.info( + f"Document for Discord message {msg_id} in {guild_name}#{channel_name} unchanged. Skipping." ) documents_skipped += 1 continue + else: + # Content has changed - update the existing document + logger.info( + f"Content changed for Discord message {msg_id} in {guild_name}#{channel_name}. Updating document." + ) - # Generate summary with metadata - document_metadata = { - "guild_name": guild_name, - "channel_name": channel_name, - "message_count": len(formatted_messages), - "document_type": "Discord Channel Messages", - "connector_type": "Discord", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - combined_document_string, - user_llm, - document_metadata, - ) + # Update chunks and embedding + chunks = await create_document_chunks( + combined_document_string + ) + doc_embedding = ( + config.embedding_model_instance.embed( + combined_document_string + ) + ) - # Chunks from channel content - chunks = await create_document_chunks(channel_content) + # Update existing document + existing_document.content = combined_document_string + existing_document.content_hash = content_hash + existing_document.embedding = doc_embedding + existing_document.document_metadata = { + "guild_name": guild_name, + "guild_id": guild_id, + "channel_name": channel_name, + "channel_id": channel_id, + "message_id": msg_id, + "message_timestamp": msg_timestamp, + "message_user_name": msg_user_name, + "indexed_at": datetime.now(UTC).strftime( + "%Y-%m-%d %H:%M:%S" + ), + } - # Update existing document - existing_document.title = ( - f"Discord - {guild_name}#{channel_name}" - ) - existing_document.content = summary_content - existing_document.content_hash = content_hash - existing_document.embedding = summary_embedding - existing_document.document_metadata = { + # Delete old chunks and add new ones + existing_document.chunks = chunks + existing_document.updated_at = ( + get_current_timestamp() + ) + + documents_indexed += 1 + logger.info( + f"Successfully updated Discord message {msg_id}" + ) + continue + + # Document doesn't exist - create new one + # Process chunks + chunks = await create_document_chunks( + combined_document_string + ) + doc_embedding = config.embedding_model_instance.embed( + combined_document_string + ) + + # Create and store new document + document = Document( + search_space_id=search_space_id, + title=f"Discord - {guild_name}#{channel_name}", + document_type=DocumentType.DISCORD_CONNECTOR, + document_metadata={ "guild_name": guild_name, "guild_id": guild_id, "channel_name": channel_name, "channel_id": channel_id, - "message_count": len(formatted_messages), - "start_date": start_date_iso, - "end_date": end_date_iso, + "message_id": msg_id, + "message_timestamp": msg_timestamp, + "message_user_name": msg_user_name, "indexed_at": datetime.now(UTC).strftime( "%Y-%m-%d %H:%M:%S" ), - } - existing_document.chunks = chunks - existing_document.updated_at = get_current_timestamp() + }, + content=combined_document_string, + embedding=doc_embedding, + chunks=chunks, + content_hash=content_hash, + unique_identifier_hash=unique_identifier_hash, + updated_at=get_current_timestamp(), + ) - documents_indexed += 1 + session.add(document) + documents_indexed += 1 + + # Batch commit every 10 documents + if documents_indexed % 10 == 0: logger.info( - f"Successfully updated Discord channel {guild_name}#{channel_name}" + f"Committing batch: {documents_indexed} Discord messages processed so far" ) - continue + await session.commit() - # Document doesn't exist - create new one - # Get user's long context LLM - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) - if not user_llm: - logger.error( - f"No long context LLM configured for user {user_id}" - ) - skipped_channels.append( - f"{guild_name}#{channel_name} (no LLM configured)" - ) - documents_skipped += 1 - continue - - # Generate summary with metadata - document_metadata = { - "guild_name": guild_name, - "channel_name": channel_name, - "message_count": len(formatted_messages), - "document_type": "Discord Channel Messages", - "connector_type": "Discord", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - combined_document_string, user_llm, document_metadata - ) - - # Chunks from channel content - chunks = await create_document_chunks(channel_content) - - # Create and store new document - document = Document( - search_space_id=search_space_id, - title=f"Discord - {guild_name}#{channel_name}", - document_type=DocumentType.DISCORD_CONNECTOR, - document_metadata={ - "guild_name": guild_name, - "guild_id": guild_id, - "channel_name": channel_name, - "channel_id": channel_id, - "message_count": len(formatted_messages), - "start_date": start_date_iso, - "end_date": end_date_iso, - "indexed_at": datetime.now(UTC).strftime( - "%Y-%m-%d %H:%M:%S" - ), - }, - content=summary_content, - content_hash=content_hash, - unique_identifier_hash=unique_identifier_hash, - embedding=summary_embedding, - chunks=chunks, - updated_at=get_current_timestamp(), - ) - - session.add(document) - documents_indexed += 1 logger.info( - f"Successfully indexed new channel {guild_name}#{channel_name} with {len(formatted_messages)} messages" + f"Successfully indexed channel {guild_name}#{channel_name} with {len(formatted_messages)} messages" ) - # Batch commit every 10 documents - if documents_indexed % 10 == 0: - logger.info( - f"Committing batch: {documents_indexed} Discord channels processed so far" - ) - await session.commit() - except Exception as e: logger.error( f"Error processing guild {guild_name}: {e!s}", exc_info=True @@ -488,7 +518,7 @@ async def index_discord_messages( # Final commit for any remaining documents not yet committed in batches logger.info( - f"Final commit: Total {documents_indexed} Discord channels processed" + f"Final commit: Total {documents_indexed} Discord messages processed" ) await session.commit() @@ -496,18 +526,18 @@ async def index_discord_messages( result_message = None if skipped_channels: result_message = ( - f"Processed {documents_indexed} channels. Skipped {len(skipped_channels)} channels: " + f"Processed {documents_indexed} messages. Skipped {len(skipped_channels)} channels: " + ", ".join(skipped_channels) ) else: - result_message = f"Processed {documents_indexed} channels." + result_message = f"Processed {documents_indexed} messages." # Log success await task_logger.log_task_success( log_entry, f"Successfully completed Discord indexing for connector {connector_id}", { - "channels_processed": documents_indexed, + "messages_processed": documents_indexed, "documents_indexed": documents_indexed, "documents_skipped": documents_skipped, "skipped_channels_count": len(skipped_channels), @@ -517,7 +547,7 @@ async def index_discord_messages( ) logger.info( - f"Discord indexing completed: {documents_indexed} new channels, {documents_skipped} skipped" + f"Discord indexing completed: {documents_indexed} new messages, {documents_skipped} skipped" ) return documents_indexed, result_message diff --git a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py index a5d2bc73a..499f01d66 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py @@ -8,7 +8,6 @@ from google.oauth2.credentials import Credentials from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession -from app.config import config from app.connectors.google_calendar_connector import GoogleCalendarConnector from app.db import Document, DocumentType, SearchSourceConnectorType from app.services.llm_service import get_user_long_context_llm @@ -84,15 +83,52 @@ async def index_google_calendar_events( return 0, f"Connector with ID {connector_id} not found" # Get the Google Calendar credentials from the connector config - exp = connector.config.get("expiry").replace("Z", "") + config_data = connector.config + + # Decrypt sensitive credentials if encrypted (for backward compatibility) + from app.config import config + from app.utils.oauth_security import TokenEncryption + + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt sensitive fields + if config_data.get("token"): + config_data["token"] = token_encryption.decrypt_token( + config_data["token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + if config_data.get("client_secret"): + config_data["client_secret"] = token_encryption.decrypt_token( + config_data["client_secret"] + ) + + logger.info( + f"Decrypted Google Calendar credentials for connector {connector_id}" + ) + except Exception as e: + await task_logger.log_task_failure( + log_entry, + f"Failed to decrypt Google Calendar credentials for connector {connector_id}: {e!s}", + "Credential decryption failed", + {"error_type": "CredentialDecryptionError"}, + ) + return 0, f"Failed to decrypt Google Calendar credentials: {e!s}" + + exp = config_data.get("expiry", "").replace("Z", "") credentials = Credentials( - token=connector.config.get("token"), - refresh_token=connector.config.get("refresh_token"), - token_uri=connector.config.get("token_uri"), - client_id=connector.config.get("client_id"), - client_secret=connector.config.get("client_secret"), - scopes=connector.config.get("scopes"), - expiry=datetime.fromisoformat(exp), + token=config_data.get("token"), + refresh_token=config_data.get("refresh_token"), + token_uri=config_data.get("token_uri"), + client_id=config_data.get("client_id"), + client_secret=config_data.get("client_secret"), + scopes=config_data.get("scopes"), + expiry=datetime.fromisoformat(exp) if exp else None, ) if ( @@ -122,6 +158,12 @@ async def index_google_calendar_events( connector_id=connector_id, ) + # Handle 'undefined' string from frontend (treat as None) + if start_date == "undefined" or start_date == "": + start_date = None + if end_date == "undefined" or end_date == "": + end_date = None + # Calculate date range if start_date is None or end_date is None: # Fall back to calculating dates based on last_indexed_at diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index 343d44072..9eeb46fc8 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -5,6 +5,7 @@ import logging from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession +from app.config import config from app.connectors.google_drive import ( GoogleDriveClient, categorize_change, @@ -87,6 +88,26 @@ async def index_google_drive_files( {"stage": "client_initialization"}, ) + # Check if credentials are encrypted (only when explicitly marked) + token_encrypted = connector.config.get("_token_encrypted", False) + if token_encrypted: + # Credentials are explicitly marked as encrypted, will be decrypted during client initialization + if not config.SECRET_KEY: + await task_logger.log_task_failure( + log_entry, + f"SECRET_KEY not configured but credentials are marked as encrypted for connector {connector_id}", + "Missing SECRET_KEY for token decryption", + {"error_type": "MissingSecretKey"}, + ) + return ( + 0, + "SECRET_KEY not configured but credentials are marked as encrypted", + ) + logger.info( + f"Google Drive credentials are encrypted for connector {connector_id}, will decrypt during client initialization" + ) + # If _token_encrypted is False or not set, treat credentials as plaintext + drive_client = GoogleDriveClient(session, connector_id) if not folder_id: @@ -249,6 +270,26 @@ async def index_google_drive_single_file( {"stage": "client_initialization"}, ) + # Check if credentials are encrypted (only when explicitly marked) + token_encrypted = connector.config.get("_token_encrypted", False) + if token_encrypted: + # Credentials are explicitly marked as encrypted, will be decrypted during client initialization + if not config.SECRET_KEY: + await task_logger.log_task_failure( + log_entry, + f"SECRET_KEY not configured but credentials are marked as encrypted for connector {connector_id}", + "Missing SECRET_KEY for token decryption", + {"error_type": "MissingSecretKey"}, + ) + return ( + 0, + "SECRET_KEY not configured but credentials are marked as encrypted", + ) + logger.info( + f"Google Drive credentials are encrypted for connector {connector_id}, will decrypt during client initialization" + ) + # If _token_encrypted is False or not set, treat credentials as plaintext + drive_client = GoogleDriveClient(session, connector_id) # Fetch the file metadata diff --git a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py index d350411e1..e10297057 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py @@ -8,7 +8,6 @@ from google.oauth2.credentials import Credentials from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession -from app.config import config from app.connectors.google_gmail_connector import GoogleGmailConnector from app.db import ( Document, @@ -88,9 +87,47 @@ async def index_google_gmail_messages( ) return 0, error_msg - # Create credentials from connector config + # Get the Google Gmail credentials from the connector config config_data = connector.config - exp = config_data.get("expiry").replace("Z", "") + + # Decrypt sensitive credentials if encrypted (for backward compatibility) + from app.config import config + from app.utils.oauth_security import TokenEncryption + + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt sensitive fields + if config_data.get("token"): + config_data["token"] = token_encryption.decrypt_token( + config_data["token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + if config_data.get("client_secret"): + config_data["client_secret"] = token_encryption.decrypt_token( + config_data["client_secret"] + ) + + logger.info( + f"Decrypted Google Gmail credentials for connector {connector_id}" + ) + except Exception as e: + await task_logger.log_task_failure( + log_entry, + f"Failed to decrypt Google Gmail credentials for connector {connector_id}: {e!s}", + "Credential decryption failed", + {"error_type": "CredentialDecryptionError"}, + ) + return 0, f"Failed to decrypt Google Gmail credentials: {e!s}" + + exp = config_data.get("expiry", "") + if exp: + exp = exp.replace("Z", "") credentials = Credentials( token=config_data.get("token"), refresh_token=config_data.get("refresh_token"), @@ -98,7 +135,7 @@ async def index_google_gmail_messages( client_id=config_data.get("client_id"), client_secret=config_data.get("client_secret"), scopes=config_data.get("scopes", []), - expiry=datetime.fromisoformat(exp), + expiry=datetime.fromisoformat(exp) if exp else None, ) if ( diff --git a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py index 8c56b10ab..fdbeb93b0 100644 --- a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py @@ -2,13 +2,14 @@ Jira connector indexer. """ +import contextlib from datetime import datetime from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession from app.config import config -from app.connectors.jira_connector import JiraConnector +from app.connectors.jira_history import JiraHistoryConnector from app.db import Document, DocumentType, SearchSourceConnectorType from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService @@ -83,32 +84,27 @@ async def index_jira_issues( ) return 0, f"Connector with ID {connector_id} not found" - # Get the Jira credentials from the connector config - jira_email = connector.config.get("JIRA_EMAIL") - jira_api_token = connector.config.get("JIRA_API_TOKEN") - jira_base_url = connector.config.get("JIRA_BASE_URL") - - if not jira_email or not jira_api_token or not jira_base_url: - await task_logger.log_task_failure( - log_entry, - f"Jira credentials not found in connector config for connector {connector_id}", - "Missing Jira credentials", - {"error_type": "MissingCredentials"}, - ) - return 0, "Jira credentials not found in connector config" - - # Initialize Jira client + # Initialize Jira client with internal refresh capability + # Token refresh will happen automatically when needed await task_logger.log_task_progress( log_entry, f"Initializing Jira client for connector {connector_id}", {"stage": "client_initialization"}, ) - jira_client = JiraConnector( - base_url=jira_base_url, email=jira_email, api_token=jira_api_token - ) + logger.info(f"Initializing Jira client for connector {connector_id}") + + # Create connector with session and connector_id for internal refresh + # Token refresh will happen automatically when needed + jira_client = JiraHistoryConnector(session=session, connector_id=connector_id) # Calculate date range + # Handle "undefined" strings from frontend + if start_date == "undefined" or start_date == "": + start_date = None + if end_date == "undefined" or end_date == "": + end_date = None + start_date_str, end_date_str = calculate_date_range( connector, start_date, end_date, default_days_back=365 ) @@ -125,7 +121,7 @@ async def index_jira_issues( # Get issues within date range try: - issues, error = jira_client.get_issues_by_date_range( + issues, error = await jira_client.get_issues_by_date_range( start_date=start_date_str, end_date=end_date_str, include_comments=True ) @@ -398,6 +394,10 @@ async def index_jira_issues( logger.info( f"JIRA indexing completed: {documents_indexed} new issues, {documents_skipped} skipped" ) + + # Clean up the connector + await jira_client.close() + return ( total_processed, None, @@ -412,6 +412,10 @@ async def index_jira_issues( {"error_type": "SQLAlchemyError"}, ) logger.error(f"Database error: {db_error!s}", exc_info=True) + # Clean up the connector in case of error + if "jira_client" in locals(): + with contextlib.suppress(Exception): + await jira_client.close() return 0, f"Database error: {db_error!s}" except Exception as e: await session.rollback() @@ -422,4 +426,8 @@ async def index_jira_issues( {"error_type": type(e).__name__}, ) logger.error(f"Failed to index JIRA issues: {e!s}", exc_info=True) + # Clean up the connector in case of error + if "jira_client" in locals(): + with contextlib.suppress(Exception): + await jira_client.close() return 0, f"Failed to index JIRA issues: {e!s}" diff --git a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py index afc9ffd3b..f1bfd42e8 100644 --- a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py @@ -92,25 +92,34 @@ async def index_linear_issues( f"Connector with ID {connector_id} not found or is not a Linear connector", ) - # Get the Linear token from the connector config - linear_token = connector.config.get("LINEAR_API_KEY") - if not linear_token: + # Check if access_token exists (support both new OAuth format and old API key format) + if not connector.config.get("access_token") and not connector.config.get( + "LINEAR_API_KEY" + ): await task_logger.log_task_failure( log_entry, - f"Linear API token not found in connector config for connector {connector_id}", - "Missing Linear token", + f"Linear access token not found in connector config for connector {connector_id}", + "Missing Linear access token", {"error_type": "MissingToken"}, ) - return 0, "Linear API token not found in connector config" + return 0, "Linear access token not found in connector config" - # Initialize Linear client + # Initialize Linear client with internal refresh capability await task_logger.log_task_progress( log_entry, f"Initializing Linear client for connector {connector_id}", {"stage": "client_initialization"}, ) - linear_client = LinearConnector(token=linear_token) + # Create connector with session and connector_id for internal refresh + # Token refresh will happen automatically when needed + linear_client = LinearConnector(session=session, connector_id=connector_id) + + # Handle 'undefined' string from frontend (treat as None) + if start_date == "undefined" or start_date == "": + start_date = None + if end_date == "undefined" or end_date == "": + end_date = None # Calculate date range start_date_str, end_date_str = calculate_date_range( @@ -131,7 +140,7 @@ async def index_linear_issues( # Get issues within date range try: - issues, error = linear_client.get_issues_by_date_range( + issues, error = await linear_client.get_issues_by_date_range( start_date=start_date_str, end_date=end_date_str, include_comments=True ) diff --git a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py index 332d3e39d..13923269d 100644 --- a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py @@ -2,7 +2,7 @@ Notion connector indexer. """ -from datetime import datetime, timedelta +from datetime import datetime from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession @@ -20,6 +20,7 @@ from app.utils.document_converters import ( from .base import ( build_document_metadata_string, + calculate_date_range, check_document_by_unique_identifier, get_connector_by_id, get_current_timestamp, @@ -91,18 +92,19 @@ async def index_notion_pages( f"Connector with ID {connector_id} not found or is not a Notion connector", ) - # Get the Notion token from the connector config - notion_token = connector.config.get("NOTION_INTEGRATION_TOKEN") - if not notion_token: + # Check if access_token exists (support both new OAuth format and old integration token format) + if not connector.config.get("access_token") and not connector.config.get( + "NOTION_INTEGRATION_TOKEN" + ): await task_logger.log_task_failure( log_entry, - f"Notion integration token not found in connector config for connector {connector_id}", - "Missing Notion token", + f"Notion access token not found in connector config for connector {connector_id}", + "Missing Notion access token", {"error_type": "MissingToken"}, ) - return 0, "Notion integration token not found in connector config" + return 0, "Notion access token not found in connector config" - # Initialize Notion client + # Initialize Notion client with internal refresh capability await task_logger.log_task_progress( log_entry, f"Initializing Notion client for connector {connector_id}", @@ -111,40 +113,30 @@ async def index_notion_pages( logger.info(f"Initializing Notion client for connector {connector_id}") - # Calculate date range - if start_date is None or end_date is None: - # Fall back to calculating dates - calculated_end_date = datetime.now() - calculated_start_date = calculated_end_date - timedelta( - days=365 - ) # Check for last 1 year of pages + # Handle 'undefined' string from frontend (treat as None) + if start_date == "undefined" or start_date == "": + start_date = None + if end_date == "undefined" or end_date == "": + end_date = None - # Use calculated dates if not provided - if start_date is None: - start_date_iso = calculated_start_date.strftime("%Y-%m-%dT%H:%M:%SZ") - else: - # Convert YYYY-MM-DD to ISO format - start_date_iso = datetime.strptime(start_date, "%Y-%m-%d").strftime( - "%Y-%m-%dT%H:%M:%SZ" - ) + # Calculate date range using the shared utility function + start_date_str, end_date_str = calculate_date_range( + connector, start_date, end_date, default_days_back=365 + ) - if end_date is None: - end_date_iso = calculated_end_date.strftime("%Y-%m-%dT%H:%M:%SZ") - else: - # Convert YYYY-MM-DD to ISO format - end_date_iso = datetime.strptime(end_date, "%Y-%m-%d").strftime( - "%Y-%m-%dT%H:%M:%SZ" - ) - else: - # Convert provided dates to ISO format for Notion API - start_date_iso = datetime.strptime(start_date, "%Y-%m-%d").strftime( - "%Y-%m-%dT%H:%M:%SZ" - ) - end_date_iso = datetime.strptime(end_date, "%Y-%m-%d").strftime( - "%Y-%m-%dT%H:%M:%SZ" - ) + # Convert YYYY-MM-DD to ISO format for Notion API + start_date_iso = datetime.strptime(start_date_str, "%Y-%m-%d").strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + end_date_iso = datetime.strptime(end_date_str, "%Y-%m-%d").strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) - notion_client = NotionHistoryConnector(token=notion_token) + # Create connector with session and connector_id for internal refresh + # Token refresh will happen automatically when needed + notion_client = NotionHistoryConnector( + session=session, connector_id=connector_id + ) logger.info(f"Fetching Notion pages from {start_date_iso} to {end_date_iso}") diff --git a/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py index 5119aba2e..dad64ad27 100644 --- a/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py @@ -92,25 +92,24 @@ async def index_slack_messages( f"Connector with ID {connector_id} not found or is not a Slack connector", ) - # Get the Slack token from the connector config - slack_token = connector.config.get("SLACK_BOT_TOKEN") - if not slack_token: - await task_logger.log_task_failure( - log_entry, - f"Slack token not found in connector config for connector {connector_id}", - "Missing Slack token", - {"error_type": "MissingToken"}, - ) - return 0, "Slack token not found in connector config" + # Note: Token handling is now done automatically by SlackHistory + # with auto-refresh support. We just need to pass session and connector_id. - # Initialize Slack client + # Initialize Slack client with auto-refresh support await task_logger.log_task_progress( log_entry, f"Initializing Slack client for connector {connector_id}", {"stage": "client_initialization"}, ) - slack_client = SlackHistory(token=slack_token) + # Use the new pattern with session and connector_id for auto-refresh + slack_client = SlackHistory(session=session, connector_id=connector_id) + + # Handle 'undefined' string from frontend (treat as None) + if start_date == "undefined" or start_date == "": + start_date = None + if end_date == "undefined" or end_date == "": + end_date = None # Calculate date range await task_logger.log_task_progress( @@ -141,7 +140,7 @@ async def index_slack_messages( # Get all channels try: - channels = slack_client.get_all_channels() + channels = await slack_client.get_all_channels() except Exception as e: await task_logger.log_task_failure( log_entry, @@ -190,7 +189,7 @@ async def index_slack_messages( continue # Get messages for this channel - messages, error = slack_client.get_history_by_date_range( + messages, error = await slack_client.get_history_by_date_range( channel_id=channel_id, start_date=start_date_str, end_date=end_date_str, @@ -223,7 +222,7 @@ async def index_slack_messages( ]: continue - formatted_msg = slack_client.format_message( + formatted_msg = await slack_client.format_message( msg, include_user_info=True ) formatted_messages.append(formatted_msg) diff --git a/surfsense_backend/app/utils/connector_naming.py b/surfsense_backend/app/utils/connector_naming.py new file mode 100644 index 000000000..f9f1fdd21 --- /dev/null +++ b/surfsense_backend/app/utils/connector_naming.py @@ -0,0 +1,189 @@ +""" +Connector Naming Utilities. + +Provides functions for generating unique, user-friendly connector names. +""" + +from typing import Any +from urllib.parse import urlparse +from uuid import UUID + +from sqlalchemy import func +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.db import SearchSourceConnector, SearchSourceConnectorType + +# Friendly display names for connector types +BASE_NAME_FOR_TYPE = { + SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR: "Gmail", + SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR: "Google Drive", + SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR: "Google Calendar", + SearchSourceConnectorType.SLACK_CONNECTOR: "Slack", + SearchSourceConnectorType.NOTION_CONNECTOR: "Notion", + SearchSourceConnectorType.LINEAR_CONNECTOR: "Linear", + SearchSourceConnectorType.JIRA_CONNECTOR: "Jira", + SearchSourceConnectorType.DISCORD_CONNECTOR: "Discord", + SearchSourceConnectorType.CONFLUENCE_CONNECTOR: "Confluence", + SearchSourceConnectorType.AIRTABLE_CONNECTOR: "Airtable", +} + + +def get_base_name_for_type(connector_type: SearchSourceConnectorType) -> str: + """Get a friendly display name for a connector type.""" + return BASE_NAME_FOR_TYPE.get( + connector_type, connector_type.replace("_", " ").title() + ) + + +def extract_identifier_from_credentials( + connector_type: SearchSourceConnectorType, + credentials: dict[str, Any], +) -> str | None: + """ + Extract a unique identifier from connector credentials. + + Args: + connector_type: The type of connector + credentials: The connector credentials dict + + Returns: + Identifier string (workspace name, email, etc.) or None + """ + if connector_type == SearchSourceConnectorType.SLACK_CONNECTOR: + return credentials.get("team_name") + + if connector_type == SearchSourceConnectorType.NOTION_CONNECTOR: + return credentials.get("workspace_name") + + if connector_type == SearchSourceConnectorType.DISCORD_CONNECTOR: + return credentials.get("guild_name") + + if connector_type in ( + SearchSourceConnectorType.JIRA_CONNECTOR, + SearchSourceConnectorType.CONFLUENCE_CONNECTOR, + ): + base_url = credentials.get("base_url", "") + if base_url: + try: + parsed = urlparse(base_url) + hostname = parsed.netloc or parsed.path + if ".atlassian.net" in hostname: + return hostname.replace(".atlassian.net", "") + return hostname + except Exception: + pass + return None + + # Google, Linear, Airtable require API calls - return None + return None + + +def generate_connector_name_with_identifier( + connector_type: SearchSourceConnectorType, + identifier: str | None, +) -> str: + """ + Generate a connector name with an identifier. + + Args: + connector_type: The type of connector + identifier: User identifier (email, workspace name, etc.) + + Returns: + Name like "Gmail - john@example.com" or just "Gmail" if no identifier + """ + base = get_base_name_for_type(connector_type) + if identifier: + return f"{base} - {identifier}" + return base + + +async def count_connectors_of_type( + session: AsyncSession, + connector_type: SearchSourceConnectorType, + search_space_id: int, + user_id: UUID, +) -> int: + """Count existing connectors of a type for a user in a search space.""" + result = await session.execute( + select(func.count(SearchSourceConnector.id)).where( + SearchSourceConnector.connector_type == connector_type, + SearchSourceConnector.search_space_id == search_space_id, + SearchSourceConnector.user_id == user_id, + ) + ) + return result.scalar() or 0 + + +async def check_duplicate_connector( + session: AsyncSession, + connector_type: SearchSourceConnectorType, + search_space_id: int, + user_id: UUID, + identifier: str | None, +) -> bool: + """ + Check if a connector with the same identifier already exists. + + Args: + session: Database session + connector_type: The type of connector + search_space_id: The search space ID + user_id: The user ID + identifier: User identifier (email, workspace name, etc.) + + Returns: + True if a duplicate exists, False otherwise + """ + if not identifier: + return False + + expected_name = f"{get_base_name_for_type(connector_type)} - {identifier}" + result = await session.execute( + select(func.count(SearchSourceConnector.id)).where( + SearchSourceConnector.connector_type == connector_type, + SearchSourceConnector.search_space_id == search_space_id, + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.name == expected_name, + ) + ) + return (result.scalar() or 0) > 0 + + +async def generate_unique_connector_name( + session: AsyncSession, + connector_type: SearchSourceConnectorType, + search_space_id: int, + user_id: UUID, + identifier: str | None = None, +) -> str: + """ + Generate a unique connector name. + + If an identifier is provided (email, workspace name, etc.), uses it with base name. + Otherwise, falls back to counting existing connectors for uniqueness. + + Args: + session: Database session + connector_type: The type of connector + search_space_id: The search space ID + user_id: The user ID + identifier: Optional user identifier (email, workspace name, etc.) + + Returns: + Unique name like "Gmail - john@example.com" or "Gmail (2)" + """ + base = get_base_name_for_type(connector_type) + + if identifier: + return f"{base} - {identifier}" + + # Fallback: use counter for uniqueness + count = await count_connectors_of_type( + session, connector_type, search_space_id, user_id + ) + + if count == 0: + return base + return f"{base} ({count + 1})" diff --git a/surfsense_backend/app/utils/document_converters.py b/surfsense_backend/app/utils/document_converters.py index 9883a74ed..279b1dbf6 100644 --- a/surfsense_backend/app/utils/document_converters.py +++ b/surfsense_backend/app/utils/document_converters.py @@ -222,88 +222,6 @@ async def convert_document_to_markdown(elements): return "".join(markdown_parts) -def convert_chunks_to_langchain_documents(chunks): - """ - Convert chunks from hybrid search results to LangChain Document objects. - - Args: - chunks: List of chunk dictionaries from hybrid search results - - Returns: - List of LangChain Document objects - """ - try: - from langchain_core.documents import Document as LangChainDocument - except ImportError: - raise ImportError( - "LangChain is not installed. Please install it with `pip install langchain langchain-core`" - ) from None - - langchain_docs = [] - - for chunk in chunks: - # Extract content from the chunk - content = chunk.get("content", "") - - # Create metadata dictionary - metadata = { - "chunk_id": chunk.get("chunk_id"), - "score": chunk.get("score"), - "rank": chunk.get("rank") if "rank" in chunk else None, - } - - # Add document information to metadata - if "document" in chunk: - doc = chunk["document"] - metadata.update( - { - "document_id": doc.get("id"), - "document_title": doc.get("title"), - "document_type": doc.get("document_type"), - } - ) - - # Add document metadata if available - if "metadata" in doc: - # Prefix document metadata keys to avoid conflicts - doc_metadata = { - f"doc_meta_{k}": v for k, v in doc.get("metadata", {}).items() - } - metadata.update(doc_metadata) - - # Add source URL if available in metadata - if "url" in doc.get("metadata", {}): - metadata["source"] = doc["metadata"]["url"] - elif "sourceURL" in doc.get("metadata", {}): - metadata["source"] = doc["metadata"]["sourceURL"] - - # Ensure source_id is set for citation purposes - # Use document_id as the source_id if available - if "document_id" in metadata: - metadata["source_id"] = metadata["document_id"] - - # Update content for citation mode - format as XML with explicit source_id - new_content = f""" - - - {metadata.get("source_id", metadata.get("document_id", "unknown"))} - - - - {content} - - - - """ - - # Create LangChain Document - langchain_doc = LangChainDocument(page_content=new_content, metadata=metadata) - - langchain_docs.append(langchain_doc) - - return langchain_docs - - def generate_content_hash(content: str, search_space_id: int) -> str: """Generate SHA-256 hash for the given content combined with search space ID.""" combined_data = f"{search_space_id}:{content}" diff --git a/surfsense_backend/app/utils/oauth_security.py b/surfsense_backend/app/utils/oauth_security.py new file mode 100644 index 000000000..5135cdef4 --- /dev/null +++ b/surfsense_backend/app/utils/oauth_security.py @@ -0,0 +1,210 @@ +""" +OAuth Security Utilities. + +Provides secure state parameter generation/validation and token encryption +for OAuth 2.0 flows. +""" + +import base64 +import hashlib +import hmac +import json +import logging +import time +from uuid import UUID + +from cryptography.fernet import Fernet +from fastapi import HTTPException + +logger = logging.getLogger(__name__) + + +class OAuthStateManager: + """Manages secure OAuth state parameters with HMAC signatures.""" + + def __init__(self, secret_key: str, max_age_seconds: int = 600): + """ + Initialize OAuth state manager. + + Args: + secret_key: Secret key for HMAC signing (should be SECRET_KEY from config) + max_age_seconds: Maximum age of state parameter in seconds (default 10 minutes) + """ + if not secret_key: + raise ValueError("secret_key is required for OAuth state management") + self.secret_key = secret_key + self.max_age_seconds = max_age_seconds + + def generate_secure_state( + self, space_id: int, user_id: UUID, **extra_fields + ) -> str: + """ + Generate cryptographically signed state parameter. + + Args: + space_id: The search space ID + user_id: The user ID + **extra_fields: Additional fields to include in state (e.g., code_verifier for PKCE) + + Returns: + Base64-encoded state parameter with HMAC signature + """ + timestamp = int(time.time()) + state_payload = { + "space_id": space_id, + "user_id": str(user_id), + "timestamp": timestamp, + } + + # Add any extra fields (e.g., code_verifier for PKCE) + state_payload.update(extra_fields) + + # Create signature + payload_str = json.dumps(state_payload, sort_keys=True) + signature = hmac.new( + self.secret_key.encode(), + payload_str.encode(), + hashlib.sha256, + ).hexdigest() + + # Include signature in state + state_payload["signature"] = signature + state_encoded = base64.urlsafe_b64encode( + json.dumps(state_payload).encode() + ).decode() + + return state_encoded + + def validate_state(self, state: str) -> dict: + """ + Validate and decode state parameter with signature verification. + + Args: + state: The state parameter from OAuth callback + + Returns: + Decoded state data (space_id, user_id, timestamp) + + Raises: + HTTPException: If state is invalid, expired, or tampered with + """ + try: + decoded = base64.urlsafe_b64decode(state.encode()).decode() + data = json.loads(decoded) + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state format: {e!s}" + ) from e + + # Verify signature exists + signature = data.pop("signature", None) + if not signature: + raise HTTPException(status_code=400, detail="Missing state signature") + + # Verify signature + payload_str = json.dumps(data, sort_keys=True) + expected_signature = hmac.new( + self.secret_key.encode(), + payload_str.encode(), + hashlib.sha256, + ).hexdigest() + + if not hmac.compare_digest(signature, expected_signature): + raise HTTPException( + status_code=400, detail="Invalid state signature - possible tampering" + ) + + # Verify timestamp (prevent replay attacks) + timestamp = data.get("timestamp", 0) + current_time = time.time() + age = current_time - timestamp + + if age < 0: + raise HTTPException(status_code=400, detail="Invalid state timestamp") + + if age > self.max_age_seconds: + raise HTTPException( + status_code=400, + detail="State parameter expired. Please try again.", + ) + + return data + + +class TokenEncryption: + """Encrypt/decrypt sensitive OAuth tokens for storage.""" + + def __init__(self, secret_key: str): + """ + Initialize token encryption. + + Args: + secret_key: Secret key for encryption (should be SECRET_KEY from config) + """ + if not secret_key: + raise ValueError("secret_key is required for token encryption") + # Derive Fernet key from secret using SHA256 + # Note: In production, consider using HKDF for key derivation + key = base64.urlsafe_b64encode(hashlib.sha256(secret_key.encode()).digest()) + try: + self.cipher = Fernet(key) + except Exception as e: + raise ValueError(f"Failed to initialize encryption cipher: {e!s}") from e + + def encrypt_token(self, token: str) -> str: + """ + Encrypt a token for storage. + + Args: + token: Plaintext token to encrypt + + Returns: + Encrypted token string + """ + if not token: + return token + try: + return self.cipher.encrypt(token.encode()).decode() + except Exception as e: + logger.error(f"Failed to encrypt token: {e!s}") + raise ValueError(f"Token encryption failed: {e!s}") from e + + def decrypt_token(self, encrypted_token: str) -> str: + """ + Decrypt a stored token. + + Args: + encrypted_token: Encrypted token string + + Returns: + Decrypted plaintext token + """ + if not encrypted_token: + return encrypted_token + try: + return self.cipher.decrypt(encrypted_token.encode()).decode() + except Exception as e: + logger.error(f"Failed to decrypt token: {e!s}") + raise ValueError(f"Token decryption failed: {e!s}") from e + + def is_encrypted(self, token: str) -> bool: + """ + Check if a token appears to be encrypted. + + Args: + token: Token string to check + + Returns: + True if token appears encrypted, False otherwise + """ + if not token: + return False + # Encrypted tokens are base64-encoded and have specific format + # This is a heuristic check - encrypted tokens are longer and base64-like + try: + # Try to decode as base64 + base64.urlsafe_b64decode(token.encode()) + # If it's base64 and reasonably long, likely encrypted + return len(token) > 20 + except Exception: + return False diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index 6b69fb3e1..54e681518 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -513,11 +513,22 @@ def validate_connector_config( ], "validators": {}, }, - "SLACK_CONNECTOR": {"required": ["SLACK_BOT_TOKEN"], "validators": {}}, - "NOTION_CONNECTOR": { - "required": ["NOTION_INTEGRATION_TOKEN"], - "validators": {}, - }, + # "SLACK_CONNECTOR": { + # "required": [], # OAuth uses bot_token (encrypted), legacy uses SLACK_BOT_TOKEN + # "optional": [ + # "bot_token", + # "SLACK_BOT_TOKEN", + # "bot_user_id", + # "team_id", + # "team_name", + # "token_type", + # "expires_in", + # "expires_at", + # "scope", + # "_token_encrypted", + # ], + # "validators": {}, + # }, "GITHUB_CONNECTOR": { "required": ["GITHUB_PAT", "repo_full_names"], "validators": { @@ -526,31 +537,21 @@ def validate_connector_config( ) }, }, - "LINEAR_CONNECTOR": {"required": ["LINEAR_API_KEY"], "validators": {}}, - "DISCORD_CONNECTOR": {"required": ["DISCORD_BOT_TOKEN"], "validators": {}}, - "JIRA_CONNECTOR": { - "required": ["JIRA_EMAIL", "JIRA_API_TOKEN", "JIRA_BASE_URL"], - "validators": { - "JIRA_EMAIL": lambda: validate_email_field("JIRA_EMAIL", "JIRA"), - "JIRA_BASE_URL": lambda: validate_url_field("JIRA_BASE_URL", "JIRA"), - }, - }, - "CONFLUENCE_CONNECTOR": { - "required": [ - "CONFLUENCE_BASE_URL", - "CONFLUENCE_EMAIL", - "CONFLUENCE_API_TOKEN", - ], - "validators": { - "CONFLUENCE_EMAIL": lambda: validate_email_field( - "CONFLUENCE_EMAIL", "Confluence" - ), - "CONFLUENCE_BASE_URL": lambda: validate_url_field( - "CONFLUENCE_BASE_URL", "Confluence" - ), - }, - }, - "CLICKUP_CONNECTOR": {"required": ["CLICKUP_API_TOKEN"], "validators": {}}, + # "DISCORD_CONNECTOR": {"required": ["DISCORD_BOT_TOKEN"], "validators": {}}, + # "JIRA_CONNECTOR": { + # "required": ["JIRA_EMAIL", "JIRA_API_TOKEN", "JIRA_BASE_URL"], + # "validators": { + # "JIRA_EMAIL": lambda: validate_email_field("JIRA_EMAIL", "JIRA"), + # "JIRA_BASE_URL": lambda: validate_url_field("JIRA_BASE_URL", "JIRA"), + # }, + # }, + # "CONFLUENCE_CONNECTOR": { + # "required": [ + # "access_token", + # ], + # "validators": {}, + # }, + # "CLICKUP_CONNECTOR": {"required": ["CLICKUP_API_TOKEN"], "validators": {}}, # "GOOGLE_CALENDAR_CONNECTOR": { # "required": ["token", "refresh_token", "token_uri", "client_id", "expiry", "scopes", "client_secret"], # "validators": {}, diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index ba1d69939..e3e7583f8 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -1,8 +1,7 @@ [project] name = "surf-new-backend" -version = "0.0.10" +version = "0.0.11" description = "SurfSense Backend" -readme = "README.md" requires-python = ">=3.12" dependencies = [ "alembic>=1.13.0", @@ -153,3 +152,11 @@ line-ending = "auto" known-first-party = ["app"] force-single-line = false combine-as-imports = true + +[tool.setuptools.packages.find] +where = ["."] +include = ["app*", "alembic*"] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index a6ef20cca..8ec09ddd9 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -6409,8 +6409,8 @@ wheels = [ [[package]] name = "surf-new-backend" -version = "0.0.10" -source = { virtual = "." } +version = "0.0.11" +source = { editable = "." } dependencies = [ { name = "alembic" }, { name = "asyncpg" }, diff --git a/surfsense_browser_extension/package.json b/surfsense_browser_extension/package.json index d7edcc95b..b225bc206 100644 --- a/surfsense_browser_extension/package.json +++ b/surfsense_browser_extension/package.json @@ -1,7 +1,7 @@ { "name": "surfsense_browser_extension", "displayName": "Surfsense Browser Extension", - "version": "0.0.10", + "version": "0.0.11", "description": "Extension to collect Browsing History for SurfSense.", "author": "https://github.com/MODSetter", "engines": { diff --git a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx index 6e61ff7ac..c78cc7762 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx @@ -17,6 +17,7 @@ import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-quer import { DocumentUploadDialogProvider } from "@/components/assistant-ui/document-upload-popup"; import { DashboardBreadcrumb } from "@/components/dashboard-breadcrumb"; import { LanguageSwitcher } from "@/components/LanguageSwitcher"; +import { OnboardingTour } from "@/components/onboarding-tour"; import { AppSidebarProvider } from "@/components/sidebar/AppSidebarProvider"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Separator } from "@/components/ui/separator"; @@ -242,6 +243,7 @@ export function DashboardClientLayout({ return ( + {/* Use AppSidebarProvider which fetches user, search space, and recent chats */} { setIsDeleting(true); try { @@ -65,28 +72,30 @@ export function RowActions({
{/* Desktop Actions */}
- - - - - - - -

Edit Document

-
-
+ + + + +

Edit Document

+
+ + )} @@ -146,10 +155,12 @@ export function RowActions({ - - - Edit - + {isEditable && ( + + + Edit + + )} setIsMetadataOpen(true)}> Metadata diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 35a096497..b1abd647f 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -20,7 +20,7 @@ import { } from "@/atoms/chat/mentioned-documents.atom"; import { clearPlanOwnerRegistry, - extractWriteTodosFromContent, + // extractWriteTodosFromContent, hydratePlanStateAtom, } from "@/atoms/chat/plan-state.atom"; import { Thread } from "@/components/assistant-ui/thread"; @@ -30,7 +30,7 @@ import { DisplayImageToolUI } from "@/components/tool-ui/display-image"; import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast"; import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview"; import { ScrapeWebpageToolUI } from "@/components/tool-ui/scrape-webpage"; -import { WriteTodosToolUI } from "@/components/tool-ui/write-todos"; +// import { WriteTodosToolUI } from "@/components/tool-ui/write-todos"; import { getBearerToken } from "@/lib/auth-utils"; import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter"; import { @@ -199,7 +199,7 @@ const TOOLS_WITH_UI = new Set([ "link_preview", "display_image", "scrape_webpage", - "write_todos", + // "write_todos", // Disabled for now ]); /** @@ -291,10 +291,11 @@ export default function NewChatPage() { restoredThinkingSteps.set(`msg-${msg.id}`, steps); } // Hydrate write_todos plan state from persisted tool calls - const writeTodosCalls = extractWriteTodosFromContent(msg.content); - for (const todoData of writeTodosCalls) { - hydratePlanState(todoData); - } + // Disabled for now + // const writeTodosCalls = extractWriteTodosFromContent(msg.content); + // for (const todoData of writeTodosCalls) { + // hydratePlanState(todoData); + // } } if (msg.role === "user") { const docs = extractMentionedDocuments(msg.content); @@ -911,7 +912,7 @@ export default function NewChatPage() { - + {/* Disabled for now */}
-
{children}
); diff --git a/surfsense_web/app/docs/[[...slug]]/page.tsx b/surfsense_web/app/docs/[[...slug]]/page.tsx index 1280dfb25..204148bf9 100644 --- a/surfsense_web/app/docs/[[...slug]]/page.tsx +++ b/surfsense_web/app/docs/[[...slug]]/page.tsx @@ -11,7 +11,17 @@ export default async function Page(props: { params: Promise<{ slug?: string[] }> const MDX = page.data.body; return ( - + {page.data.title} {page.data.description} diff --git a/surfsense_web/atoms/announcement.atom.ts b/surfsense_web/atoms/announcement.atom.ts deleted file mode 100644 index 31e032978..000000000 --- a/surfsense_web/atoms/announcement.atom.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { atomWithStorage } from "jotai/utils"; - -// Atom to track whether the announcement banner has been dismissed -// Persists to localStorage automatically -export const announcementDismissedAtom = atomWithStorage("surfsense_announcement_dismissed", false); diff --git a/surfsense_web/components/announcement-banner.tsx b/surfsense_web/components/announcement-banner.tsx deleted file mode 100644 index 537aa6da7..000000000 --- a/surfsense_web/components/announcement-banner.tsx +++ /dev/null @@ -1,47 +0,0 @@ -"use client"; - -import { useAtom } from "jotai"; -import { ExternalLink, Info, X } from "lucide-react"; -import { announcementDismissedAtom } from "@/atoms/announcement.atom"; -import { Button } from "@/components/ui/button"; - -export function AnnouncementBanner() { - const [isDismissed, setIsDismissed] = useAtom(announcementDismissedAtom); - - const handleDismiss = () => { - setIsDismissed(true); - }; - - if (isDismissed) return null; - - return ( -
-
-
- -

- SurfSense is a work in progress.{" "} - - Report issues on GitHub - - -

- -
-
-
- ); -} diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index a955e3972..43307db24 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -19,9 +19,11 @@ import { ConnectorDialogHeader } from "./connector-popup/components/connector-di import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view"; import { ConnectorEditView } from "./connector-popup/connector-configs/views/connector-edit-view"; import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view"; +import { OAUTH_CONNECTORS } from "./connector-popup/constants/connector-constants"; import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog"; import { ActiveConnectorsTab } from "./connector-popup/tabs/active-connectors-tab"; import { AllConnectorsTab } from "./connector-popup/tabs/all-connectors-tab"; +import { ConnectorAccountsListView } from "./connector-popup/views/connector-accounts-list-view"; import { YouTubeCrawlerView } from "./connector-popup/views/youtube-crawler-view"; export const ConnectorIndicator: FC = () => { @@ -60,6 +62,7 @@ export const ConnectorIndicator: FC = () => { periodicEnabled, frequencyMinutes, allConnectors, + viewingAccountsType, setSearchQuery, setStartDate, setEndDate, @@ -81,6 +84,8 @@ export const ConnectorIndicator: FC = () => { handleBackFromEdit, handleBackFromConnect, handleBackFromYouTube, + handleViewAccountsList, + handleBackFromAccountsList, handleQuickIndexConnector, connectorConfig, setConnectorConfig, @@ -162,6 +167,7 @@ export const ConnectorIndicator: FC = () => { return ( { )} - + {/* YouTube Crawler View - shown when adding YouTube videos */} {isYouTubeView && searchSpaceId ? ( + ) : viewingAccountsType ? ( + { + const oauthConnector = OAUTH_CONNECTORS.find( + (c) => c.connectorType === viewingAccountsType.connectorType + ); + if (oauthConnector) { + handleConnectOAuth(oauthConnector); + } + }} + isConnecting={connectingId !== null} + /> ) : connectingConnectorType ? ( { {/* Content */}
-
+
{ onCreateWebcrawler={handleCreateWebcrawler} onCreateYouTubeCrawler={handleCreateYouTubeCrawler} onManage={handleStartEdit} + onViewAccountsList={handleViewAccountsList} /> @@ -302,6 +328,7 @@ export const ConnectorIndicator: FC = () => { searchSpaceId={searchSpaceId} onTabChange={handleTabChange} onManage={handleStartEdit} + onViewAccountsList={handleViewAccountsList} />
diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx index 8cc36fd8e..e8fe6da33 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx @@ -1,7 +1,7 @@ "use client"; import { IconBrandYoutube } from "@tabler/icons-react"; -import { format } from "date-fns"; +import { differenceInDays, differenceInMinutes, format, isToday, isYesterday } from "date-fns"; import { FileText, Loader2 } from "lucide-react"; import type { FC } from "react"; import { Button } from "@/components/ui/button"; @@ -17,6 +17,7 @@ interface ConnectorCardProps { isConnected?: boolean; isConnecting?: boolean; documentCount?: number; + accountCount?: number; lastIndexedAt?: string | null; isIndexing?: boolean; activeTask?: LogActiveTask; @@ -49,6 +50,45 @@ function formatDocumentCount(count: number | undefined): string { return `${m.replace(/\.0$/, "")}M docs`; } +/** + * Format last indexed date with contextual messages + * Examples: "Just now", "10 minutes ago", "Today at 2:30 PM", "Yesterday at 3:45 PM", "3 days ago", "Jan 15, 2026" + */ +function formatLastIndexedDate(dateString: string): string { + const date = new Date(dateString); + const now = new Date(); + const minutesAgo = differenceInMinutes(now, date); + const daysAgo = differenceInDays(now, date); + + // Just now (within last minute) + if (minutesAgo < 1) { + return "Just now"; + } + + // X minutes ago (less than 1 hour) + if (minutesAgo < 60) { + return `${minutesAgo} ${minutesAgo === 1 ? "minute" : "minutes"} ago`; + } + + // Today at [time] + if (isToday(date)) { + return `Today at ${format(date, "h:mm a")}`; + } + + // Yesterday at [time] + if (isYesterday(date)) { + return `Yesterday at ${format(date, "h:mm a")}`; + } + + // X days ago (less than 7 days) + if (daysAgo < 7) { + return `${daysAgo} ${daysAgo === 1 ? "day" : "days"} ago`; + } + + // Full date for older entries + return format(date, "MMM d, yyyy"); +} + export const ConnectorCard: FC = ({ id, title, @@ -57,6 +97,7 @@ export const ConnectorCard: FC = ({ isConnected = false, isConnecting = false, documentCount, + accountCount, lastIndexedAt, isIndexing = false, activeTask, @@ -86,13 +127,13 @@ export const ConnectorCard: FC = ({ // Show last indexed date for connected connectors if (lastIndexedAt) { return ( - - Last indexed: {format(new Date(lastIndexedAt), "MMM d, yyyy")} + + Last indexed: {formatLastIndexedDate(lastIndexedAt)} ); } // Fallback for connected but never indexed - return Never indexed; + return Never indexed; } return description; @@ -100,7 +141,7 @@ export const ConnectorCard: FC = ({ return (
-
+
{connectorType ? ( getConnectorIcon(connectorType, "size-6") ) : id === "youtube-crawler" ? ( @@ -111,12 +152,20 @@ export const ConnectorCard: FC = ({
- {title} + {title}
-
{getStatusContent()}
+
{getStatusContent()}
{isConnected && documentCount !== undefined && ( -

- {formatDocumentCount(documentCount)} +

+ {formatDocumentCount(documentCount)} + {accountCount !== undefined && accountCount > 0 && ( + <> + + + {accountCount} {accountCount === 1 ? "Account" : "Accounts"} + + + )}

)}
@@ -124,18 +173,16 @@ export const ConnectorCard: FC = ({ size="sm" variant={isConnected ? "secondary" : "default"} className={cn( - "h-8 text-[11px] px-3 rounded-lg flex-shrink-0 font-medium", + "h-8 text-[11px] px-3 rounded-lg shrink-0 font-medium", isConnected && "bg-white text-slate-700 hover:bg-slate-50 border-0 shadow-xs dark:bg-secondary dark:text-secondary-foreground dark:hover:bg-secondary/80", !isConnected && "shadow-xs" )} onClick={isConnected ? onManage : onConnect} - disabled={isConnecting || isIndexing} + disabled={isConnecting} > {isConnecting ? ( - ) : isIndexing ? ( - "Syncing..." ) : isConnected ? ( "Manage" ) : id === "youtube-crawler" ? ( diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx index a18c79a1f..34e1ae2e9 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx @@ -24,20 +24,20 @@ export const ConnectorDialogHeader: FC = ({ return (
- + Connectors - + Search across all your apps and data in one place. -
+
= ({
- + = ({
diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/confluence-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/confluence-config.tsx index c3a233406..59fa89554 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/confluence-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/confluence-config.tsx @@ -1,6 +1,6 @@ "use client"; -import { KeyRound } from "lucide-react"; +import { Info, KeyRound } from "lucide-react"; import type { FC } from "react"; import { useEffect, useState } from "react"; import { Input } from "@/components/ui/input"; @@ -16,6 +16,9 @@ export const ConfluenceConfig: FC = ({ onConfigChange, onNameChange, }) => { + // Check if this is an OAuth connector (has access_token or _token_encrypted flag) + const isOAuth = !!(connector.config?.access_token || connector.config?._token_encrypted); + const [baseUrl, setBaseUrl] = useState( (connector.config?.CONFLUENCE_BASE_URL as string) || "" ); @@ -25,16 +28,18 @@ export const ConfluenceConfig: FC = ({ ); const [name, setName] = useState(connector.name || ""); - // Update values when connector changes + // Update values when connector changes (only for legacy connectors) useEffect(() => { - const url = (connector.config?.CONFLUENCE_BASE_URL as string) || ""; - const emailVal = (connector.config?.CONFLUENCE_EMAIL as string) || ""; - const token = (connector.config?.CONFLUENCE_API_TOKEN as string) || ""; - setBaseUrl(url); - setEmail(emailVal); - setApiToken(token); + if (!isOAuth) { + const url = (connector.config?.CONFLUENCE_BASE_URL as string) || ""; + const emailVal = (connector.config?.CONFLUENCE_EMAIL as string) || ""; + const token = (connector.config?.CONFLUENCE_API_TOKEN as string) || ""; + setBaseUrl(url); + setEmail(emailVal); + setApiToken(token); + } setName(connector.name || ""); - }, [connector.config, connector.name]); + }, [connector.config, connector.name, isOAuth]); const handleBaseUrlChange = (value: string) => { setBaseUrl(value); @@ -73,6 +78,35 @@ export const ConfluenceConfig: FC = ({ } }; + // For OAuth connectors, show simple info message + if (isOAuth) { + const siteUrl = + (connector.config?.base_url as string) || (connector.config?.site_url as string) || "Unknown"; + return ( +
+ {/* OAuth Info */} +
+
+ +
+
+

Connected via OAuth

+

+ This connector is authenticated using OAuth 2.0. Your Confluence instance is: +

+

+ {siteUrl} +

+

+ To update your connection, reconnect this connector. +

+
+
+
+ ); + } + + // For legacy API token connectors, show the form return (
{/* Connector Name */} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/discord-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/discord-config.tsx index 377987637..dd4c89c8e 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/discord-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/discord-config.tsx @@ -1,88 +1,26 @@ "use client"; -import { KeyRound } from "lucide-react"; +import { Info } from "lucide-react"; import type { FC } from "react"; -import { useEffect, useState } from "react"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; import type { ConnectorConfigProps } from "../index"; export interface DiscordConfigProps extends ConnectorConfigProps { onNameChange?: (name: string) => void; } -export const DiscordConfig: FC = ({ - connector, - onConfigChange, - onNameChange, -}) => { - const [botToken, setBotToken] = useState( - (connector.config?.DISCORD_BOT_TOKEN as string) || "" - ); - const [name, setName] = useState(connector.name || ""); - - // Update bot token and name when connector changes - useEffect(() => { - const token = (connector.config?.DISCORD_BOT_TOKEN as string) || ""; - setBotToken(token); - setName(connector.name || ""); - }, [connector.config, connector.name]); - - const handleBotTokenChange = (value: string) => { - setBotToken(value); - if (onConfigChange) { - onConfigChange({ - ...connector.config, - DISCORD_BOT_TOKEN: value, - }); - } - }; - - const handleNameChange = (value: string) => { - setName(value); - if (onNameChange) { - onNameChange(value); - } - }; - +export const DiscordConfig: FC = () => { return (
- {/* Connector Name */} -
-
- - handleNameChange(e.target.value)} - placeholder="My Discord Connector" - className="border-slate-400/20 focus-visible:border-slate-400/40" - /> -

- A friendly name to identify this connector. -

+
+
+
-
- - {/* Configuration */} -
-
-

Configuration

-
- -
- - handleBotTokenChange(e.target.value)} - placeholder="Your Bot Token" - className="border-slate-400/20 focus-visible:border-slate-400/40" - /> -

- Update your Discord Bot Token if needed. +

+

Add Bot to Servers

+

+ Before indexing, make sure the Discord bot has been added to the servers (guilds) you + want to index. The bot can only access messages from servers it's been added to. Use the + OAuth authorization flow to add the bot to your servers.

diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx index 3ef16bdb4..dcc83c2d6 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx @@ -1,6 +1,6 @@ "use client"; -import { KeyRound } from "lucide-react"; +import { Info, KeyRound } from "lucide-react"; import type { FC } from "react"; import { useEffect, useState } from "react"; import { Input } from "@/components/ui/input"; @@ -12,6 +12,9 @@ export interface JiraConfigProps extends ConnectorConfigProps { } export const JiraConfig: FC = ({ connector, onConfigChange, onNameChange }) => { + // Check if this is an OAuth connector (has access_token or _token_encrypted flag) + const isOAuth = !!(connector.config?.access_token || connector.config?._token_encrypted); + const [baseUrl, setBaseUrl] = useState((connector.config?.JIRA_BASE_URL as string) || ""); const [email, setEmail] = useState((connector.config?.JIRA_EMAIL as string) || ""); const [apiToken, setApiToken] = useState( @@ -19,16 +22,18 @@ export const JiraConfig: FC = ({ connector, onConfigChange, onN ); const [name, setName] = useState(connector.name || ""); - // Update values when connector changes + // Update values when connector changes (only for legacy connectors) useEffect(() => { - const url = (connector.config?.JIRA_BASE_URL as string) || ""; - const emailVal = (connector.config?.JIRA_EMAIL as string) || ""; - const token = (connector.config?.JIRA_API_TOKEN as string) || ""; - setBaseUrl(url); - setEmail(emailVal); - setApiToken(token); + if (!isOAuth) { + const url = (connector.config?.JIRA_BASE_URL as string) || ""; + const emailVal = (connector.config?.JIRA_EMAIL as string) || ""; + const token = (connector.config?.JIRA_API_TOKEN as string) || ""; + setBaseUrl(url); + setEmail(emailVal); + setApiToken(token); + } setName(connector.name || ""); - }, [connector.config, connector.name]); + }, [connector.config, connector.name, isOAuth]); const handleBaseUrlChange = (value: string) => { setBaseUrl(value); @@ -67,6 +72,34 @@ export const JiraConfig: FC = ({ connector, onConfigChange, onN } }; + // For OAuth connectors, show simple info message + if (isOAuth) { + const baseUrl = (connector.config?.base_url as string) || "Unknown"; + return ( +
+ {/* OAuth Info */} +
+
+ +
+
+

Connected via OAuth

+

+ This connector is authenticated using OAuth 2.0. Your Jira instance is: +

+

+ {baseUrl} +

+

+ To update your connection, reconnect this connector. +

+
+
+
+ ); + } + + // For legacy API token connectors, show the form return (
{/* Connector Name */} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/linear-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/linear-config.tsx deleted file mode 100644 index f5c0534df..000000000 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/linear-config.tsx +++ /dev/null @@ -1,89 +0,0 @@ -"use client"; - -import { KeyRound } from "lucide-react"; -import type { FC } from "react"; -import { useEffect, useState } from "react"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import type { ConnectorConfigProps } from "../index"; - -export interface LinearConfigProps extends ConnectorConfigProps { - onNameChange?: (name: string) => void; -} - -export const LinearConfig: FC = ({ - connector, - onConfigChange, - onNameChange, -}) => { - const [apiKey, setApiKey] = useState((connector.config?.LINEAR_API_KEY as string) || ""); - const [name, setName] = useState(connector.name || ""); - - // Update API key and name when connector changes - useEffect(() => { - const key = (connector.config?.LINEAR_API_KEY as string) || ""; - setApiKey(key); - setName(connector.name || ""); - }, [connector.config, connector.name]); - - const handleApiKeyChange = (value: string) => { - setApiKey(value); - if (onConfigChange) { - onConfigChange({ - ...connector.config, - LINEAR_API_KEY: value, - }); - } - }; - - const handleNameChange = (value: string) => { - setName(value); - if (onNameChange) { - onNameChange(value); - } - }; - - return ( -
- {/* Connector Name */} -
-
- - handleNameChange(e.target.value)} - placeholder="My Linear Connector" - className="border-slate-400/20 focus-visible:border-slate-400/40" - /> -

- A friendly name to identify this connector. -

-
-
- - {/* Configuration */} -
-
-

Configuration

-
- -
- - handleApiKeyChange(e.target.value)} - placeholder="Begins with lin_api_..." - className="border-slate-400/20 focus-visible:border-slate-400/40" - /> -

- Update your Linear API Key if needed. -

-
-
-
- ); -}; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/notion-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/notion-config.tsx deleted file mode 100644 index 72175c31b..000000000 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/notion-config.tsx +++ /dev/null @@ -1,91 +0,0 @@ -"use client"; - -import { KeyRound } from "lucide-react"; -import type { FC } from "react"; -import { useEffect, useState } from "react"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import type { ConnectorConfigProps } from "../index"; - -export interface NotionConfigProps extends ConnectorConfigProps { - onNameChange?: (name: string) => void; -} - -export const NotionConfig: FC = ({ - connector, - onConfigChange, - onNameChange, -}) => { - const [integrationToken, setIntegrationToken] = useState( - (connector.config?.NOTION_INTEGRATION_TOKEN as string) || "" - ); - const [name, setName] = useState(connector.name || ""); - - // Update integration token and name when connector changes - useEffect(() => { - const token = (connector.config?.NOTION_INTEGRATION_TOKEN as string) || ""; - setIntegrationToken(token); - setName(connector.name || ""); - }, [connector.config, connector.name]); - - const handleIntegrationTokenChange = (value: string) => { - setIntegrationToken(value); - if (onConfigChange) { - onConfigChange({ - ...connector.config, - NOTION_INTEGRATION_TOKEN: value, - }); - } - }; - - const handleNameChange = (value: string) => { - setName(value); - if (onNameChange) { - onNameChange(value); - } - }; - - return ( -
- {/* Connector Name */} -
-
- - handleNameChange(e.target.value)} - placeholder="My Notion Connector" - className="border-slate-400/20 focus-visible:border-slate-400/40" - /> -

- A friendly name to identify this connector. -

-
-
- - {/* Configuration */} -
-
-

Configuration

-
- -
- - handleIntegrationTokenChange(e.target.value)} - placeholder="Begins with secret_..." - className="border-slate-400/20 focus-visible:border-slate-400/40" - /> -

- Update your Notion Integration Token if needed. -

-
-
-
- ); -}; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/slack-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/slack-config.tsx index 73ae6a4f3..58293c4de 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/slack-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/slack-config.tsx @@ -1,84 +1,27 @@ "use client"; -import { KeyRound } from "lucide-react"; +import { Info } from "lucide-react"; import type { FC } from "react"; -import { useEffect, useState } from "react"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; import type { ConnectorConfigProps } from "../index"; export interface SlackConfigProps extends ConnectorConfigProps { onNameChange?: (name: string) => void; } -export const SlackConfig: FC = ({ connector, onConfigChange, onNameChange }) => { - const [botToken, setBotToken] = useState( - (connector.config?.SLACK_BOT_TOKEN as string) || "" - ); - const [name, setName] = useState(connector.name || ""); - - // Update bot token and name when connector changes - useEffect(() => { - const token = (connector.config?.SLACK_BOT_TOKEN as string) || ""; - setBotToken(token); - setName(connector.name || ""); - }, [connector.config, connector.name]); - - const handleBotTokenChange = (value: string) => { - setBotToken(value); - if (onConfigChange) { - onConfigChange({ - ...connector.config, - SLACK_BOT_TOKEN: value, - }); - } - }; - - const handleNameChange = (value: string) => { - setName(value); - if (onNameChange) { - onNameChange(value); - } - }; - +export const SlackConfig: FC = () => { return (
- {/* Connector Name */} -
-
- - handleNameChange(e.target.value)} - placeholder="My Slack Connector" - className="border-slate-400/20 focus-visible:border-slate-400/40" - /> -

- A friendly name to identify this connector. -

+
+
+
-
- - {/* Configuration */} -
-
-

Configuration

-
- -
- - handleBotTokenChange(e.target.value)} - placeholder="Begins with xoxb-..." - className="border-slate-400/20 focus-visible:border-slate-400/40" - /> -

- Update your Bot User OAuth Token if needed. +

+

Add Bot to Channels

+

+ Before indexing, add the SurfSense bot to each channel you want to index. The bot can + only access messages from channels it's been added to. Type{" "} + /invite @SurfSense in + any channel to add it.

diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx index c31a4645a..2575b3a69 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx @@ -12,10 +12,8 @@ import { ElasticsearchConfig } from "./components/elasticsearch-config"; import { GithubConfig } from "./components/github-config"; import { GoogleDriveConfig } from "./components/google-drive-config"; import { JiraConfig } from "./components/jira-config"; -import { LinearConfig } from "./components/linear-config"; import { LinkupApiConfig } from "./components/linkup-api-config"; import { LumaConfig } from "./components/luma-config"; -import { NotionConfig } from "./components/notion-config"; import { SearxngConfig } from "./components/searxng-config"; import { SlackConfig } from "./components/slack-config"; import { TavilyApiConfig } from "./components/tavily-api-config"; @@ -46,8 +44,6 @@ export function getConnectorConfigComponent( return LinkupApiConfig; case "BAIDU_SEARCH_API": return BaiduSearchApiConfig; - case "LINEAR_CONNECTOR": - return LinearConfig; case "WEBCRAWLER_CONNECTOR": return WebcrawlerConfig; case "ELASTICSEARCH_CONNECTOR": @@ -56,8 +52,6 @@ export function getConnectorConfigComponent( return SlackConfig; case "DISCORD_CONNECTOR": return DiscordConfig; - case "NOTION_CONNECTOR": - return NotionConfig; case "CONFLUENCE_CONNECTOR": return ConfluenceConfig; case "BOOKSTACK_CONNECTOR": @@ -72,7 +66,7 @@ export function getConnectorConfigComponent( return LumaConfig; case "CIRCLEBACK_CONNECTOR": return CirclebackConfig; - // OAuth connectors (Gmail, Calendar, Airtable) and others don't need special config UI + // OAuth connectors (Gmail, Calendar, Airtable, Notion) and others don't need special config UI default: return null; } diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx index dfd91fe8b..e3941367b 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx @@ -51,16 +51,9 @@ export const ConnectorConnectView: FC = ({ SEARXNG_API: "searxng-connect-form", LINKUP_API: "linkup-api-connect-form", BAIDU_SEARCH_API: "baidu-search-api-connect-form", - LINEAR_CONNECTOR: "linear-connect-form", ELASTICSEARCH_CONNECTOR: "elasticsearch-connect-form", - SLACK_CONNECTOR: "slack-connect-form", - DISCORD_CONNECTOR: "discord-connect-form", - NOTION_CONNECTOR: "notion-connect-form", - CONFLUENCE_CONNECTOR: "confluence-connect-form", BOOKSTACK_CONNECTOR: "bookstack-connect-form", GITHUB_CONNECTOR: "github-connect-form", - JIRA_CONNECTOR: "jira-connect-form", - CLICKUP_CONNECTOR: "clickup-connect-form", LUMA_CONNECTOR: "luma-connect-form", CIRCLEBACK_CONNECTOR: "circleback-connect-form", }; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx index 6d43e6ffc..bdfe9af77 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx @@ -59,6 +59,7 @@ export const ConnectorEditView: FC = ({ const [isScrolled, setIsScrolled] = useState(false); const [hasMoreContent, setHasMoreContent] = useState(false); const [showDisconnectConfirm, setShowDisconnectConfirm] = useState(false); + const [isQuickIndexing, setIsQuickIndexing] = useState(false); const scrollContainerRef = useRef(null); const checkScrollState = useCallback(() => { @@ -94,6 +95,13 @@ export const ConnectorEditView: FC = ({ }; }, [checkScrollState]); + // Reset local quick indexing state when indexing completes + useEffect(() => { + if (!isIndexing) { + setIsQuickIndexing(false); + } + }, [isIndexing]); + const handleDisconnectClick = () => { setShowDisconnectConfirm(true); }; @@ -107,6 +115,13 @@ export const ConnectorEditView: FC = ({ setShowDisconnectConfirm(false); }; + const handleQuickIndex = useCallback(() => { + if (onQuickIndex) { + setIsQuickIndexing(true); + onQuickIndex(); + } + }, [onQuickIndex]); + return (
{/* Fixed Header */} @@ -128,12 +143,14 @@ export const ConnectorEditView: FC = ({ {/* Connector header */}
-
-
+
+
{getConnectorIcon(connector.connector_type, "size-7")}
-

{connector.name}

+

+ {connector.name} +

Manage your connector settings and sync configuration

@@ -146,11 +163,11 @@ export const ConnectorEditView: FC = ({ + {/* Back button - only show if not from OAuth */} + {!isFromOAuth && ( + + )} {/* Success header */} -
+
-

- {config.connectorTitle} Connected! -

+
+ + {getConnectorTypeDisplay(connector?.connector_type || "")} Connected ! + {" "} + + {getConnectorDisplayName(connector?.name || "")} + +

Configure when to start syncing your data

@@ -187,15 +202,7 @@ export const IndexingConfigurationView: FC = ({
{/* Fixed Footer - Action buttons */} -
- +
+
+ ); + })} + + {/* Non-OAuth Connectors - Individual Cards */} + {filteredNonOAuthConnectors.map((connector) => { const isIndexing = indexingConnectorIds.has(connector.id); const activeTask = logsSummary?.active_tasks?.find( (task: LogActiveTask) => task.connector_id === connector.id @@ -125,7 +277,7 @@ export const ActiveConnectorsTab: FC = ({ >
= ({ )}

) : ( -

+

{connector.last_indexed_at - ? `Last indexed: ${format(new Date(connector.last_indexed_at), "MMM d, yyyy")}` + ? `Last indexed: ${formatLastIndexedDate(connector.last_indexed_at)}` : "Never indexed"}

)} -

+

{formatDocumentCount(documentCount)}

); diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index bdec4dcb2..6129b49b7 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -1,12 +1,27 @@ "use client"; +import { Plus } from "lucide-react"; import type { FC } from "react"; +import { Button } from "@/components/ui/button"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types"; import { ConnectorCard } from "../components/connector-card"; import { CRAWLERS, OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants"; import { getDocumentCountForConnector } from "../utils/connector-document-mapping"; +/** + * Extract the display name from a full connector name. + * Full names are in format "Base Name - identifier" (e.g., "Gmail - john@example.com"). + * Returns just the identifier (e.g : john@example.com). + */ +export function getConnectorDisplayName(fullName: string): string { + const separatorIndex = fullName.indexOf(" - "); + if (separatorIndex !== -1) { + return fullName.substring(separatorIndex + 3); + } + return fullName; +} + interface AllConnectorsTabProps { searchQuery: string; searchSpaceId: string; @@ -21,6 +36,7 @@ interface AllConnectorsTabProps { onCreateWebcrawler?: () => void; onCreateYouTubeCrawler?: () => void; onManage?: (connector: SearchSourceConnector) => void; + onViewAccountsList?: (connectorType: string, connectorTitle: string) => void; } export const AllConnectorsTab: FC = ({ @@ -37,6 +53,7 @@ export const AllConnectorsTab: FC = ({ onCreateWebcrawler, onCreateYouTubeCrawler, onManage, + onViewAccountsList, }) => { // Helper to find active task for a connector const getActiveTaskForConnector = (connectorId: number): LogActiveTask | undefined => { @@ -77,22 +94,39 @@ export const AllConnectorsTab: FC = ({ {filteredOAuth.map((connector) => { const isConnected = connectedTypes.has(connector.connectorType); const isConnecting = connectingId === connector.id; - // Find the actual connector object if connected - const actualConnector = + + // Find all connectors of this type + const typeConnectors = isConnected && allConnectors - ? allConnectors.find( + ? allConnectors.filter( (c: SearchSourceConnector) => c.connector_type === connector.connectorType ) - : undefined; + : []; + + // Get the most recent last_indexed_at across all accounts + const mostRecentLastIndexed = typeConnectors.reduce( + (latest, c) => { + if (!c.last_indexed_at) return latest; + if (!latest) return c.last_indexed_at; + return new Date(c.last_indexed_at) > new Date(latest) + ? c.last_indexed_at + : latest; + }, + undefined + ); const documentCount = getDocumentCountForConnector( connector.connectorType, documentTypeCounts ); - const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id); - const activeTask = actualConnector - ? getActiveTaskForConnector(actualConnector.id) - : undefined; + + // Check if any account is currently indexing + const isIndexing = typeConnectors.some((c) => indexingConnectorIds?.has(c.id)); + + // Get active task from any indexing account + const activeTask = typeConnectors + .map((c) => getActiveTaskForConnector(c.id)) + .find((task) => task !== undefined); return ( = ({ isConnected={isConnected} isConnecting={isConnecting} documentCount={documentCount} - lastIndexedAt={actualConnector?.last_indexed_at} + accountCount={typeConnectors.length} + lastIndexedAt={mostRecentLastIndexed} isIndexing={isIndexing} activeTask={activeTask} onConnect={() => onConnectOAuth(connector)} onManage={ - actualConnector && onManage ? () => onManage(actualConnector) : undefined + isConnected && onViewAccountsList + ? () => onViewAccountsList(connector.connectorType, connector.title) + : undefined } /> ); diff --git a/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx new file mode 100644 index 000000000..e45f24d11 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx @@ -0,0 +1,189 @@ +"use client"; + +import { differenceInDays, differenceInMinutes, format, isToday, isYesterday } from "date-fns"; +import { ArrowLeft, Loader2, Plus } from "lucide-react"; +import type { FC } from "react"; +import { Button } from "@/components/ui/button"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types"; +import { cn } from "@/lib/utils"; +import { getConnectorDisplayName } from "../tabs/all-connectors-tab"; + +interface ConnectorAccountsListViewProps { + connectorType: string; + connectorTitle: string; + connectors: SearchSourceConnector[]; + indexingConnectorIds: Set; + logsSummary: LogSummary | undefined; + onBack: () => void; + onManage: (connector: SearchSourceConnector) => void; + onAddAccount: () => void; + isConnecting?: boolean; +} + +/** + * Format last indexed date with contextual messages + */ +function formatLastIndexedDate(dateString: string): string { + const date = new Date(dateString); + const now = new Date(); + const minutesAgo = differenceInMinutes(now, date); + const daysAgo = differenceInDays(now, date); + + if (minutesAgo < 1) { + return "Just now"; + } + + if (minutesAgo < 60) { + return `${minutesAgo} ${minutesAgo === 1 ? "minute" : "minutes"} ago`; + } + + if (isToday(date)) { + return `Today at ${format(date, "h:mm a")}`; + } + + if (isYesterday(date)) { + return `Yesterday at ${format(date, "h:mm a")}`; + } + + if (daysAgo < 7) { + return `${daysAgo} ${daysAgo === 1 ? "day" : "days"} ago`; + } + + return format(date, "MMM d, yyyy"); +} + +export const ConnectorAccountsListView: FC = ({ + connectorType, + connectorTitle, + connectors, + indexingConnectorIds, + logsSummary, + onBack, + onManage, + onAddAccount, + isConnecting = false, +}) => { + // Filter connectors to only show those of this type + const typeConnectors = connectors.filter((c) => c.connector_type === connectorType); + + return ( +
+ {/* Header */} +
+
+
+ +
+
+ {getConnectorIcon(connectorType, "size-5")} +
+
+

{connectorTitle} Accounts

+

+ {typeConnectors.length} connected account{typeConnectors.length !== 1 ? "s" : ""} +

+
+
+
+ {/* Add Account Button with dashed border */} + +
+
+ + {/* Content */} +
+ {/* Connected Accounts Grid */} +
+ {typeConnectors.map((connector) => { + const isIndexing = indexingConnectorIds.has(connector.id); + const activeTask = logsSummary?.active_tasks?.find( + (task: LogActiveTask) => task.connector_id === connector.id + ); + + return ( +
+
+ {getConnectorIcon(connector.connector_type, "size-6")} +
+
+

+ {getConnectorDisplayName(connector.name)} +

+ {isIndexing ? ( +

+ + Indexing... + {activeTask?.message && ( + + • {activeTask.message} + + )} +

+ ) : ( +

+ {connector.last_indexed_at + ? `Last indexed: ${formatLastIndexedDate(connector.last_indexed_at)}` + : "Never indexed"} +

+ )} +
+ +
+ ); + })} +
+
+
+ ); +}; diff --git a/surfsense_web/components/assistant-ui/document-upload-popup.tsx b/surfsense_web/components/assistant-ui/document-upload-popup.tsx index d1fa208d2..6ac1ec979 100644 --- a/surfsense_web/components/assistant-ui/document-upload-popup.tsx +++ b/surfsense_web/components/assistant-ui/document-upload-popup.tsx @@ -1,6 +1,7 @@ "use client"; import { useAtomValue } from "jotai"; +import { Upload } from "lucide-react"; import { useRouter } from "next/navigation"; import { createContext, @@ -85,6 +86,7 @@ const DocumentUploadPopupContent: FC<{ }> = ({ isOpen, onOpenChange }) => { const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); const router = useRouter(); + const [isAccordionExpanded, setIsAccordionExpanded] = useState(false); if (!searchSpaceId) return null; @@ -95,16 +97,40 @@ const DocumentUploadPopupContent: FC<{ return ( - + Upload Document -
-
-
- + + {/* Fixed Header */} +
+ {/* Upload header */} +
+
+ +
+
+

Upload Documents

+

+ Upload and sync your documents to your search space +

- {/* Bottom fade shadow */} -
+
+ + {/* Scrollable Content */} +
+
+
+ +
+
+ {/* Bottom fade shadow - only show when scrolling */} + {isAccordionExpanded && ( +
+ )}
diff --git a/surfsense_web/components/copy-button.tsx b/surfsense_web/components/copy-button.tsx deleted file mode 100644 index c1a752997..000000000 --- a/surfsense_web/components/copy-button.tsx +++ /dev/null @@ -1,38 +0,0 @@ -"use client"; -import { Copy, CopyCheck } from "lucide-react"; -import type { RefObject } from "react"; -import { useEffect, useRef, useState } from "react"; -import { Button } from "./ui/button"; - -export default function CopyButton({ ref }: { ref: RefObject }) { - const [copy, setCopy] = useState(false); - const timeoutRef = useRef(null); - - useEffect(() => { - return () => { - if (timeoutRef.current) { - clearTimeout(timeoutRef.current); - } - }; - }, []); - - const handleClick = () => { - if (ref.current) { - const text = ref.current.innerText; - navigator.clipboard.writeText(text); - - setCopy(true); - timeoutRef.current = setTimeout(() => { - setCopy(false); - }, 2000); - } - }; - - return ( -
- -
- ); -} diff --git a/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx b/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx deleted file mode 100644 index 4b9965632..000000000 --- a/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx +++ /dev/null @@ -1,22 +0,0 @@ -"use client"; - -import { Card, CardContent, CardHeader } from "@/components/ui/card"; -import { Skeleton } from "@/components/ui/skeleton"; - -export function EditConnectorLoadingSkeleton() { - return ( -
- - - - - - - - - - - -
- ); -} diff --git a/surfsense_web/components/editConnector/EditConnectorNameForm.tsx b/surfsense_web/components/editConnector/EditConnectorNameForm.tsx deleted file mode 100644 index 0dae174db..000000000 --- a/surfsense_web/components/editConnector/EditConnectorNameForm.tsx +++ /dev/null @@ -1,28 +0,0 @@ -"use client"; - -import type { Control } from "react-hook-form"; -import { FormControl, FormField, FormItem, FormLabel, FormMessage } from "@/components/ui/form"; -import { Input } from "@/components/ui/input"; - -// Assuming EditConnectorFormValues is defined elsewhere or passed as generic -interface EditConnectorNameFormProps { - control: Control; // Use Control if type is available -} - -export function EditConnectorNameForm({ control }: EditConnectorNameFormProps) { - return ( - ( - - Connector Name - - - - - - )} - /> - ); -} diff --git a/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx b/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx deleted file mode 100644 index aa3eb1404..000000000 --- a/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx +++ /dev/null @@ -1,189 +0,0 @@ -import { CircleAlert, Edit, KeyRound, Loader2 } from "lucide-react"; -import type React from "react"; -import type { UseFormReturn } from "react-hook-form"; -import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; -import { Button } from "@/components/ui/button"; -import { Checkbox } from "@/components/ui/checkbox"; -import { - FormControl, - FormDescription, - FormField, - FormItem, - FormLabel, - FormMessage, -} from "@/components/ui/form"; -import { Input } from "@/components/ui/input"; -import { Skeleton } from "@/components/ui/skeleton"; - -// Types needed from parent -interface GithubRepo { - id: number; - name: string; - full_name: string; - private: boolean; - url: string; - description: string | null; - last_updated: string | null; -} -type GithubPatFormValues = { github_pat: string }; -type EditMode = "viewing" | "editing_repos"; - -interface EditGitHubConnectorConfigProps { - // State from parent - editMode: EditMode; - originalPat: string; - currentSelectedRepos: string[]; - fetchedRepos: GithubRepo[] | null; - newSelectedRepos: string[]; - isFetchingRepos: boolean; - // Forms from parent - patForm: UseFormReturn; - // Handlers from parent - setEditMode: (mode: EditMode) => void; - handleFetchRepositories: (values: GithubPatFormValues) => Promise; - handleRepoSelectionChange: (repoFullName: string, checked: boolean) => void; - setNewSelectedRepos: React.Dispatch>; - setFetchedRepos: React.Dispatch>; -} - -export function EditGitHubConnectorConfig({ - editMode, - originalPat, - currentSelectedRepos, - fetchedRepos, - newSelectedRepos, - isFetchingRepos, - patForm, - setEditMode, - handleFetchRepositories, - handleRepoSelectionChange, - setNewSelectedRepos, - setFetchedRepos, -}: EditGitHubConnectorConfigProps) { - return ( -
-

Repository Selection & Access

- - {/* Viewing Mode */} - {editMode === "viewing" && ( -
- Currently Indexed Repositories: - {currentSelectedRepos.length > 0 ? ( -
    - {currentSelectedRepos.map((repo) => ( -
  • {repo}
  • - ))} -
- ) : ( -

(No repositories currently selected)

- )} - - - To change repo selections or update the PAT, click above. - -
- )} - - {/* Editing Mode */} - {editMode === "editing_repos" && ( -
- {/* PAT Input */} -
- ( - - - GitHub PAT - - - - - - Enter PAT to fetch/update repos or if you need to update the stored token. - - - - )} - /> - -
- - {/* Repo List */} - {isFetchingRepos && } - {!isFetchingRepos && - fetchedRepos !== null && - (fetchedRepos.length === 0 ? ( - - - No Repositories Found - Check PAT & permissions. - - ) : ( -
- - Select Repositories to Index ({newSelectedRepos.length} selected): - -
- {fetchedRepos.map((repo) => ( -
- - handleRepoSelectionChange(repo.full_name, !!checked) - } - /> - -
- ))} -
-
- ))} - -
- )} -
- ); -} diff --git a/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx b/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx deleted file mode 100644 index 4ad654045..000000000 --- a/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx +++ /dev/null @@ -1,49 +0,0 @@ -"use client"; - -import { KeyRound } from "lucide-react"; -import type { Control } from "react-hook-form"; -import { - FormControl, - FormDescription, - FormField, - FormItem, - FormLabel, - FormMessage, -} from "@/components/ui/form"; -import { Input } from "@/components/ui/input"; - -// Assuming EditConnectorFormValues is defined elsewhere or passed as generic -interface EditSimpleTokenFormProps { - control: Control; - fieldName: string; // e.g., "SLACK_BOT_TOKEN" - fieldLabel: string; // e.g., "Slack Bot Token" - fieldDescription: string; - placeholder?: string; -} - -export function EditSimpleTokenForm({ - control, - fieldName, - fieldLabel, - fieldDescription, - placeholder, -}: EditSimpleTokenFormProps) { - return ( - ( - - - {fieldLabel} - - - - - {fieldDescription} - - - )} - /> - ); -} diff --git a/surfsense_web/components/editConnector/types.ts b/surfsense_web/components/editConnector/types.ts deleted file mode 100644 index e17a3513a..000000000 --- a/surfsense_web/components/editConnector/types.ts +++ /dev/null @@ -1,60 +0,0 @@ -import * as z from "zod"; - -// Types -export interface GithubRepo { - id: number; - name: string; - full_name: string; - private: boolean; - url: string; - description: string | null; - last_updated: string | null; -} - -export type EditMode = "viewing" | "editing_repos"; - -// Schemas -export const githubPatSchema = z.object({ - github_pat: z - .string() - .min(20, { message: "GitHub Personal Access Token seems too short." }) - .refine((pat) => pat.startsWith("ghp_") || pat.startsWith("github_pat_"), { - message: "GitHub PAT should start with 'ghp_' or 'github_pat_'", - }), -}); -export type GithubPatFormValues = z.infer; - -export const editConnectorSchema = z.object({ - name: z.string().min(3, { message: "Connector name must be at least 3 characters." }), - SLACK_BOT_TOKEN: z.string().optional(), - NOTION_INTEGRATION_TOKEN: z.string().optional(), - TAVILY_API_KEY: z.string().optional(), - SEARXNG_HOST: z.string().optional(), - SEARXNG_API_KEY: z.string().optional(), - SEARXNG_ENGINES: z.string().optional(), - SEARXNG_CATEGORIES: z.string().optional(), - SEARXNG_LANGUAGE: z.string().optional(), - SEARXNG_SAFESEARCH: z.string().optional(), - SEARXNG_VERIFY_SSL: z.string().optional(), - LINEAR_API_KEY: z.string().optional(), - LINKUP_API_KEY: z.string().optional(), - DISCORD_BOT_TOKEN: z.string().optional(), - CONFLUENCE_BASE_URL: z.string().optional(), - CONFLUENCE_EMAIL: z.string().optional(), - CONFLUENCE_API_TOKEN: z.string().optional(), - BOOKSTACK_BASE_URL: z.string().optional(), - BOOKSTACK_TOKEN_ID: z.string().optional(), - BOOKSTACK_TOKEN_SECRET: z.string().optional(), - JIRA_BASE_URL: z.string().optional(), - JIRA_EMAIL: z.string().optional(), - JIRA_API_TOKEN: z.string().optional(), - GOOGLE_CALENDAR_CLIENT_ID: z.string().optional(), - GOOGLE_CALENDAR_CLIENT_SECRET: z.string().optional(), - GOOGLE_CALENDAR_REFRESH_TOKEN: z.string().optional(), - GOOGLE_CALENDAR_CALENDAR_IDS: z.string().optional(), - LUMA_API_KEY: z.string().optional(), - ELASTICSEARCH_API_KEY: z.string().optional(), - FIRECRAWL_API_KEY: z.string().optional(), - INITIAL_URLS: z.string().optional(), -}); -export type EditConnectorFormValues = z.infer; diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index db7525881..a9cfdeba2 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -4,8 +4,31 @@ import Image from "next/image"; import Link from "next/link"; import React, { useEffect, useRef, useState } from "react"; import Balancer from "react-wrap-balancer"; +import { trackLoginAttempt } from "@/lib/posthog/events"; import { cn } from "@/lib/utils"; +// Official Google "G" logo with brand colors +const GoogleLogo = ({ className }: { className?: string }) => ( + + + + + + +); + export function HeroSection() { const containerRef = useRef(null); const parentRef = useRef(null); @@ -60,7 +83,7 @@ export function HeroSection() {

The AI Workspace{" "} -
+
Built for Teams
@@ -73,12 +96,7 @@ export function HeroSection() { your team.

- - Get Started - + {/* { + trackLoginAttempt("google"); + window.location.href = `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/auth/google/authorize-redirect`; + }; + + if (isGoogleAuth) { + return ( + + {/* Animated gradient background on hover */} + + {/* Google logo with subtle animation */} + + + + Continue with Google + + ); + } + + return ( + + + Get Started + + + ); +} + const BackgroundGrids = () => { return (
@@ -126,7 +203,7 @@ const BackgroundGrids = () => {
-
+
@@ -237,7 +314,7 @@ const CollisionMechanism = React.forwardRef< repeatDelay: beamOptions.repeatDelay || 0, }} className={cn( - "absolute left-96 top-20 m-auto h-14 w-px rounded-full bg-gradient-to-t from-orange-500 via-yellow-500 to-transparent", + "absolute left-96 top-20 m-auto h-14 w-px rounded-full bg-linear-to-t from-orange-500 via-yellow-500 to-transparent", beamOptions.className )} /> @@ -276,7 +353,7 @@ const Explosion = ({ ...props }: React.HTMLProps) => { animate={{ opacity: [0, 1, 0] }} exit={{ opacity: 0 }} transition={{ duration: 1, ease: "easeOut" }} - className="absolute -inset-x-10 top-0 m-auto h-[4px] w-10 rounded-full bg-gradient-to-r from-transparent via-orange-500 to-transparent blur-sm" + className="absolute -inset-x-10 top-0 m-auto h-[4px] w-10 rounded-full bg-linear-to-r from-transparent via-orange-500 to-transparent blur-sm" > {spans.map((span) => ( ) => { initial={{ x: span.initialX, y: span.initialY, opacity: 1 }} animate={{ x: span.directionX, y: span.directionY, opacity: 0 }} transition={{ duration: Math.random() * 1.5 + 0.5, ease: "easeOut" }} - className="absolute h-1 w-1 rounded-full bg-gradient-to-b from-orange-500 to-yellow-500" + className="absolute h-1 w-1 rounded-full bg-linear-to-b from-orange-500 to-yellow-500" /> ))}
@@ -307,11 +384,11 @@ const GridLineVertical = ({ className, offset }: { className?: string; offset?: } as React.CSSProperties } className={cn( - "absolute top-[calc(var(--offset)/2*-1)] h-[calc(100%+var(--offset))] w-[var(--width)]", + "absolute top-[calc(var(--offset)/2*-1)] h-[calc(100%+var(--offset))] w-(--width)", "bg-[linear-gradient(to_bottom,var(--color),var(--color)_50%,transparent_0,transparent)]", - "[background-size:var(--width)_var(--height)]", - "[mask:linear-gradient(to_top,var(--background)_var(--fade-stop),transparent),_linear-gradient(to_bottom,var(--background)_var(--fade-stop),transparent),_linear-gradient(black,black)]", - "[mask-composite:exclude]", + "bg-size-[var(--width)_var(--height)]", + "[mask:linear-gradient(to_top,var(--background)_var(--fade-stop),transparent),linear-gradient(to_bottom,var(--background)_var(--fade-stop),transparent),linear-gradient(black,black)]", + "mask-exclude", "z-30", "dark:bg-[linear-gradient(to_bottom,var(--color-dark),var(--color-dark)_50%,transparent_0,transparent)]", className diff --git a/surfsense_web/components/markdown-viewer.tsx b/surfsense_web/components/markdown-viewer.tsx index 5318ba5d1..93e3f26e1 100644 --- a/surfsense_web/components/markdown-viewer.tsx +++ b/surfsense_web/components/markdown-viewer.tsx @@ -1,6 +1,5 @@ import Image from "next/image"; -import type { Components } from "react-markdown"; -import { Streamdown } from "streamdown"; +import { Streamdown, type StreamdownProps } from "streamdown"; import { cn } from "@/lib/utils"; interface MarkdownViewerProps { @@ -9,7 +8,7 @@ interface MarkdownViewerProps { } export function MarkdownViewer({ content, className }: MarkdownViewerProps) { - const components: Components = { + const components: StreamdownProps["components"] = { // Define custom components for markdown elements p: ({ children, ...props }) => (

diff --git a/surfsense_web/components/new-chat/document-mention-picker.tsx b/surfsense_web/components/new-chat/document-mention-picker.tsx index 2d5d46267..7a9e7aaa5 100644 --- a/surfsense_web/components/new-chat/document-mention-picker.tsx +++ b/surfsense_web/components/new-chat/document-mention-picker.tsx @@ -12,7 +12,7 @@ import { useState, } from "react"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import type { Document } from "@/contracts/types/document.types"; +import type { Document, GetDocumentsResponse } from "@/contracts/types/document.types"; import { documentsApiService } from "@/lib/apis/documents-api.service"; import { cacheKeys } from "@/lib/query-client/cache-keys"; import { cn } from "@/lib/utils"; @@ -31,6 +31,8 @@ interface DocumentMentionPickerProps { externalSearch?: string; } +const PAGE_SIZE = 20; + function useDebounced(value: T, delay = 300) { const [debounced, setDebounced] = useState(value); useEffect(() => { @@ -52,12 +54,29 @@ export const DocumentMentionPicker = forwardRef< const debouncedSearch = useDebounced(search, 150); const [highlightedIndex, setHighlightedIndex] = useState(0); const itemRefs = useRef>(new Map()); + const scrollContainerRef = useRef(null); + // State for pagination + const [accumulatedDocuments, setAccumulatedDocuments] = useState([]); + const [currentPage, setCurrentPage] = useState(0); + const [hasMore, setHasMore] = useState(false); + const [isLoadingMore, setIsLoadingMore] = useState(false); + + // Reset pagination when search or search space changes + // biome-ignore lint/correctness/useExhaustiveDependencies: intentionally reset pagination when search/space changes + useEffect(() => { + setAccumulatedDocuments([]); + setCurrentPage(0); + setHasMore(false); + setHighlightedIndex(0); + }, [debouncedSearch, searchSpaceId]); + + // Query params for initial fetch (page 0) const fetchQueryParams = useMemo( () => ({ search_space_id: searchSpaceId, page: 0, - page_size: 20, + page_size: PAGE_SIZE, }), [searchSpaceId] ); @@ -66,31 +85,97 @@ export const DocumentMentionPicker = forwardRef< return { search_space_id: searchSpaceId, page: 0, - page_size: 20, + page_size: PAGE_SIZE, title: debouncedSearch, }; }, [debouncedSearch, searchSpaceId]); - // Use query for fetching documents + // Use query for fetching first page of documents const { data: documents, isLoading: isDocumentsLoading } = useQuery({ queryKey: cacheKeys.documents.withQueryParams(fetchQueryParams), queryFn: () => documentsApiService.getDocuments({ queryParams: fetchQueryParams }), staleTime: 3 * 60 * 1000, - enabled: !!searchSpaceId && !debouncedSearch.trim(), + enabled: !!searchSpaceId && !debouncedSearch.trim() && currentPage === 0, }); - // Searching + // Searching - first page const { data: searchedDocuments, isLoading: isSearchedDocumentsLoading } = useQuery({ queryKey: cacheKeys.documents.withQueryParams(searchQueryParams), queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }), staleTime: 3 * 60 * 1000, - enabled: !!searchSpaceId && !!debouncedSearch.trim(), + enabled: !!searchSpaceId && !!debouncedSearch.trim() && currentPage === 0, }); - const actualDocuments = debouncedSearch.trim() - ? searchedDocuments?.items || [] - : documents?.items || []; - const actualLoading = debouncedSearch.trim() ? isSearchedDocumentsLoading : isDocumentsLoading; + // Update accumulated documents when first page loads + useEffect(() => { + if (currentPage === 0) { + if (debouncedSearch.trim()) { + if (searchedDocuments) { + setAccumulatedDocuments(searchedDocuments.items); + setHasMore(searchedDocuments.has_more); + } + } else { + if (documents) { + setAccumulatedDocuments(documents.items); + setHasMore(documents.has_more); + } + } + } + }, [documents, searchedDocuments, debouncedSearch, currentPage]); + + // Function to load next page + const loadNextPage = useCallback(async () => { + if (isLoadingMore || !hasMore) return; + + const nextPage = currentPage + 1; + setIsLoadingMore(true); + + try { + let response: GetDocumentsResponse; + if (debouncedSearch.trim()) { + const queryParams = { + search_space_id: searchSpaceId, + page: nextPage, + page_size: PAGE_SIZE, + title: debouncedSearch, + }; + response = await documentsApiService.searchDocuments({ queryParams }); + } else { + const queryParams = { + search_space_id: searchSpaceId, + page: nextPage, + page_size: PAGE_SIZE, + }; + response = await documentsApiService.getDocuments({ queryParams }); + } + + setAccumulatedDocuments((prev) => [...prev, ...response.items]); + setHasMore(response.has_more); + setCurrentPage(nextPage); + } catch (error) { + console.error("Failed to load next page:", error); + } finally { + setIsLoadingMore(false); + } + }, [currentPage, hasMore, isLoadingMore, debouncedSearch, searchSpaceId]); + + // Infinite scroll handler + const handleScroll = useCallback( + (e: React.UIEvent) => { + const target = e.currentTarget; + const scrollBottom = target.scrollHeight - target.scrollTop - target.clientHeight; + + // Load more when within 50px of bottom + if (scrollBottom < 50 && hasMore && !isLoadingMore) { + loadNextPage(); + } + }, + [hasMore, isLoadingMore, loadNextPage] + ); + + const actualDocuments = accumulatedDocuments; + const actualLoading = + (debouncedSearch.trim() ? isSearchedDocumentsLoading : isDocumentsLoading) && currentPage === 0; // Track already selected document IDs const selectedIds = useMemo( @@ -184,8 +269,12 @@ export const DocumentMentionPicker = forwardRef< role="listbox" tabIndex={-1} > - {/* Document List - Shows max 3 items on mobile, 5 items on desktop */} -

+ {/* Document List - Shows max 5 items on mobile, 7-8 items on desktop */} +
{actualLoading ? (
@@ -235,6 +324,12 @@ export const DocumentMentionPicker = forwardRef< ); })} + {/* Loading indicator for additional pages */} + {isLoadingMore && ( +
+
+
+ )}
)}
diff --git a/surfsense_web/components/onboarding-tour.tsx b/surfsense_web/components/onboarding-tour.tsx new file mode 100644 index 000000000..958bb43b0 --- /dev/null +++ b/surfsense_web/components/onboarding-tour.tsx @@ -0,0 +1,742 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { useAtomValue } from "jotai"; +import { usePathname } from "next/navigation"; +import { useTheme } from "next-themes"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; +import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; +import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; +import { currentUserAtom } from "@/atoms/user/user-query.atoms"; +import { fetchThreads } from "@/lib/chat/thread-persistence"; + +interface TourStep { + target: string; + title: string; + content: string; + placement: "top" | "bottom" | "left" | "right"; +} + +const TOUR_STEPS: TourStep[] = [ + { + target: '[data-joyride="connector-icon"]', + title: "Connect your data sources", + content: "Connect and sync data from Gmail, Drive, Slack, Notion, Jira, Confluence, and more.", + placement: "bottom", + }, + { + target: '[data-joyride="documents-sidebar"]', + title: "Manage your documents", + content: "Access and manage all your uploaded documents.", + placement: "right", + }, +]; + +interface TooltipPosition { + top: number; + left: number; + pointerPosition: "top" | "bottom" | "left" | "right"; +} + +function calculatePosition(targetEl: Element, placement: TourStep["placement"]): TooltipPosition { + const rect = targetEl.getBoundingClientRect(); + const scrollTop = window.scrollY; + const scrollLeft = window.scrollX; + const tooltipWidth = 280; + const tooltipHeight = 120; + const offset = 16; + + let top = 0; + let left = 0; + let pointerPosition: TooltipPosition["pointerPosition"] = "left"; + + switch (placement) { + case "bottom": + top = rect.bottom + scrollTop + offset; + left = rect.left + scrollLeft + rect.width / 2 - tooltipWidth / 2; + pointerPosition = "top"; + break; + case "top": + top = rect.top + scrollTop - tooltipHeight - offset; + left = rect.left + scrollLeft + rect.width / 2 - tooltipWidth / 2; + pointerPosition = "bottom"; + break; + case "right": + top = rect.top + scrollTop + rect.height / 2 - tooltipHeight / 2; + left = rect.right + scrollLeft + offset; + pointerPosition = "left"; + break; + case "left": + top = rect.top + scrollTop + rect.height / 2 - tooltipHeight / 2; + left = rect.left + scrollLeft - tooltipWidth - offset; + pointerPosition = "right"; + break; + } + + // Ensure tooltip stays within viewport + left = Math.max(10, Math.min(left, window.innerWidth - tooltipWidth - 10)); + top = Math.max(10, top); + + return { top, left, pointerPosition }; +} + +function Spotlight({ + targetEl, + isDarkMode, + currentStepTarget, +}: { + targetEl: Element; + isDarkMode: boolean; + currentStepTarget: string; +}) { + const rect = targetEl.getBoundingClientRect(); + const padding = 6; + const shadowColor = isDarkMode ? "#172554" : "#3b82f6"; + + // Check if this is the connector icon step - verify both the selector matches AND the element matches + // This prevents the shape from changing before targetEl updates + const isConnectorSelector = currentStepTarget === '[data-joyride="connector-icon"]'; + const isConnectorElement = targetEl.matches('[data-joyride="connector-icon"]'); + const isConnectorStep = isConnectorSelector && isConnectorElement; + + // For circle, use the larger dimension to ensure it's a perfect circle + const circleSize = isConnectorStep ? Math.max(rect.width, rect.height) : 0; + const circleTop = isConnectorStep ? rect.top + (rect.height - circleSize) / 2 : rect.top; + const circleLeft = isConnectorStep ? rect.left + (rect.width - circleSize) / 2 : rect.left; + + return ( + <> + {/* Dark overlay with cutout using box-shadow technique */} +
+ {/* Blue shadow behind the button - starts from button border */} +
+ + ); +} + +function TourTooltip({ + step, + stepIndex, + totalSteps, + position, + onNext, + onPrev, + onSkip, + isDarkMode, +}: { + step: TourStep; + stepIndex: number; + totalSteps: number; + position: TooltipPosition; + targetRect: DOMRect; + onNext: () => void; + onPrev: () => void; + onSkip: () => void; + isDarkMode: boolean; +}) { + const [contentKey, setContentKey] = useState(stepIndex); + const [shouldAnimate, setShouldAnimate] = useState(false); + const prevStepIndexRef = useRef(stepIndex); + const isLastStep = stepIndex === totalSteps - 1; + const isFirstStep = stepIndex === 0; + + // Update content key when step changes to trigger animation + // Only animate if stepIndex actually changes (not on initial mount) + useEffect(() => { + if (prevStepIndexRef.current !== stepIndex) { + setShouldAnimate(true); + setContentKey(stepIndex); + prevStepIndexRef.current = stepIndex; + } + }, [stepIndex]); + + const bgColor = isDarkMode ? "#18181b" : "#ffffff"; + const textColor = isDarkMode ? "#ffffff" : "#18181b"; + const mutedTextColor = isDarkMode ? "#a1a1aa" : "#71717a"; + + // Calculate pointer line position + const getPointerStyles = (): React.CSSProperties => { + const lineLength = 16; + const dotSize = 6; + // Check if this is the documents step (stepIndex === 1) + const isDocumentsStep = stepIndex === 1; + + if (position.pointerPosition === "left") { + return { + position: "absolute", + left: -lineLength - dotSize, + top: isDocumentsStep ? "calc(50% - 8px)" : "50%", + transform: "translateY(-50%)", + display: "flex", + alignItems: "center", + }; + } + if (position.pointerPosition === "top") { + return { + position: "absolute", + top: -lineLength - dotSize, + left: "50%", + transform: "translateX(-50%)", + display: "flex", + flexDirection: "column", + alignItems: "center", + }; + } + return {}; + }; + + const renderPointer = () => { + const lineColor = isDarkMode ? "#18181B" : "#ffffff"; + + if (position.pointerPosition === "left") { + return ( +
+
+
+
+ ); + } + if (position.pointerPosition === "top") { + return ( +
+
+
+
+ ); + } + return null; + }; + + // Render step dots + const renderStepDots = () => { + return ( +
+ {Array.from({ length: totalSteps }).map((_, i) => ( +
+ ))} +
+ ); + }; + + return ( +
e.stopPropagation()} + onKeyDown={(e) => e.stopPropagation()} + > + {/* Pointer line */} + {renderPointer()} + +
+ {/* Content */} +
setShouldAnimate(false)} + > +

+ {step.title} +

+

+ {step.content} +

+
+ + {/* Footer */} +
+ {/* Step dots */} + {renderStepDots()} + + {/* Navigation buttons */} +
+ {!isFirstStep && ( + + )} + {isFirstStep && ( + + )} + +
+
+
+
+ ); +} + +export function OnboardingTour() { + const [isActive, setIsActive] = useState(false); + const [stepIndex, setStepIndex] = useState(0); + const [targetEl, setTargetEl] = useState(null); + const [spotlightTargetEl, setSpotlightTargetEl] = useState(null); + const [spotlightStepTarget, setSpotlightStepTarget] = useState(null); + const [position, setPosition] = useState(null); + const [targetRect, setTargetRect] = useState(null); + const [mounted, setMounted] = useState(false); + const { resolvedTheme } = useTheme(); + const pathname = usePathname(); + const retryCountRef = useRef(0); + const maxRetries = 10; + // Track previous user ID to detect user changes + const previousUserIdRef = useRef(null); + + // Get user data + const { data: user } = useAtomValue(currentUserAtom); + const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); + + // Fetch threads data + const { data: threadsData } = useQuery({ + queryKey: ["threads", searchSpaceId], + queryFn: () => fetchThreads(Number(searchSpaceId), 1), // Only need to check if any exist + enabled: !!searchSpaceId, + }); + + // Get document type counts + const { data: documentTypeCounts } = useAtomValue(documentTypeCountsAtom); + + // Get connectors + const { data: connectors = [] } = useAtomValue(connectorsAtom); + + const isDarkMode = resolvedTheme === "dark"; + const currentStep = TOUR_STEPS[stepIndex]; + + // Handle mounting for portal + useEffect(() => { + setMounted(true); + }, []); + + // Find and track target element with retry logic + const updateTarget = useCallback(() => { + if (!currentStep) return; + + const el = document.querySelector(currentStep.target); + if (el) { + setTargetEl(el); + setTargetRect(el.getBoundingClientRect()); + setPosition(calculatePosition(el, currentStep.placement)); + retryCountRef.current = 0; + } else if (retryCountRef.current < maxRetries) { + retryCountRef.current++; + setTimeout(() => { + const retryEl = document.querySelector(currentStep.target); + if (retryEl) { + setTargetEl(retryEl); + setTargetRect(retryEl.getBoundingClientRect()); + setPosition(calculatePosition(retryEl, currentStep.placement)); + retryCountRef.current = 0; + } + }, 200); + } + }, [currentStep]); + + // Check if tour should run: localStorage + data validation with user ID tracking + useEffect(() => { + // Don't check if not mounted or no user + if (!mounted || !user?.id || !searchSpaceId) return; + + // Check if on new-chat page + const isNewChatPage = pathname?.includes("/new-chat"); + if (!isNewChatPage) return; + + // Wait for all data to be loaded before making decision + // Data is considered loaded when: + // - threadsData is defined (query completed, even if empty) + // - documentTypeCounts is defined (query completed, even if empty object) + // - connectors is an array (always defined with default []) + // If searchSpaceId is not set, connectors query won't run, but that's okay + const dataLoaded = threadsData !== undefined && documentTypeCounts !== undefined; + if (!dataLoaded) return; + + const currentUserId = user.id; + const previousUserId = previousUserIdRef.current; + + // Detect user change - if user ID changed, reset tour state + if (previousUserId !== null && previousUserId !== currentUserId) { + // User changed - reset tour state and re-evaluate for new user + setIsActive(false); + setStepIndex(0); + setTargetEl(null); + setSpotlightTargetEl(null); + setSpotlightStepTarget(null); + setPosition(null); + setTargetRect(null); + retryCountRef.current = 0; + } + + // Update previous user ID ref + previousUserIdRef.current = currentUserId; + + // Check localStorage for CURRENT user ID (not stale cache) + // This ensures we check the correct user's tour status + const tourKey = `surfsense-tour-${currentUserId}`; + const hasSeenTour = localStorage.getItem(tourKey); + if (hasSeenTour === "true") { + return; // Current user has seen tour, don't show + } + + // Validate user is actually new (reliable check) + const threads = threadsData?.threads ?? []; + const hasThreads = threads.length > 0; + + // Check document counts - sum all document type counts + const totalDocuments = documentTypeCounts + ? Object.values(documentTypeCounts).reduce((sum, count) => sum + count, 0) + : 0; + const hasDocuments = totalDocuments > 0; + + const hasConnectors = connectors.length > 0; + + // User is new if they have no threads, documents, or connectors + const isNewUser = !hasThreads && !hasDocuments && !hasConnectors; + + // Only show tour if user is new and hasn't seen it + // Don't auto-mark as seen if user has data - let them explicitly dismiss it + if (!isNewUser) { + return; // User has data, don't show tour + } + + // User is new and hasn't seen tour - wait for DOM elements and start tour + const checkAndStartTour = () => { + // Check if both required elements exist + const connectorEl = document.querySelector(TOUR_STEPS[0].target); + const documentsEl = document.querySelector(TOUR_STEPS[1].target); + + if (connectorEl && documentsEl) { + // Both elements found, start tour + setIsActive(true); + setTargetEl(connectorEl); + setSpotlightTargetEl(connectorEl); + setSpotlightStepTarget(TOUR_STEPS[0].target); + setTargetRect(connectorEl.getBoundingClientRect()); + setPosition(calculatePosition(connectorEl, TOUR_STEPS[0].placement)); + } else { + // Retry after delay + setTimeout(checkAndStartTour, 200); + } + }; + + // Start checking after initial delay + const timer = setTimeout(checkAndStartTour, 500); + return () => clearTimeout(timer); + }, [mounted, user?.id, searchSpaceId, pathname, threadsData, documentTypeCounts, connectors]); + + // Update position on resize/scroll + useEffect(() => { + if (!isActive || !targetEl) return; + + const handleUpdate = () => { + const rect = targetEl.getBoundingClientRect(); + if (rect.width > 0 && rect.height > 0) { + setTargetRect(rect); + setPosition(calculatePosition(targetEl, currentStep?.placement || "bottom")); + } + }; + + window.addEventListener("resize", handleUpdate); + window.addEventListener("scroll", handleUpdate, true); + + return () => { + window.removeEventListener("resize", handleUpdate); + window.removeEventListener("scroll", handleUpdate, true); + }; + }, [isActive, targetEl, currentStep?.placement]); + + // Update target when step changes + useEffect(() => { + if (isActive && currentStep) { + // Try to find element synchronously first to prevent any delay + const el = document.querySelector(currentStep.target); + if (el) { + // Found immediately - update state synchronously to prevent flicker + const rect = el.getBoundingClientRect(); + const newPosition = calculatePosition(el, currentStep.placement); + // React 18+ automatically batches these updates + setTargetEl(el); + setTargetRect(rect); + setPosition(newPosition); + retryCountRef.current = 0; + } else { + // Not found immediately, use updateTarget with retry logic + // Use requestAnimationFrame to batch with next paint + const frameId = requestAnimationFrame(() => { + updateTarget(); + }); + return () => cancelAnimationFrame(frameId); + } + } + }, [isActive, updateTarget, currentStep]); + + // Delay spotlight update to sync with tooltip animation + useEffect(() => { + if (targetEl && currentStep) { + const timer = setTimeout(() => { + setSpotlightTargetEl(targetEl); + setSpotlightStepTarget(currentStep.target); + }, 100); + return () => clearTimeout(timer); + } + }, [targetEl, currentStep]); + + // Ensure target element is above overlay layers so content is fully visible + useEffect(() => { + if (!targetEl || !isActive) return; + + const originalZIndex = (targetEl as HTMLElement).style.zIndex; + const originalPosition = (targetEl as HTMLElement).style.position; + + // Ensure the element has a position that allows z-index + if (getComputedStyle(targetEl).position === "static") { + (targetEl as HTMLElement).style.position = "relative"; + } + (targetEl as HTMLElement).style.zIndex = "99999"; + + return () => { + (targetEl as HTMLElement).style.zIndex = originalZIndex; + if (originalPosition) { + (targetEl as HTMLElement).style.position = originalPosition; + } else if (getComputedStyle(targetEl).position === "relative" && originalPosition === "") { + (targetEl as HTMLElement).style.position = ""; + } + }; + }, [targetEl, isActive]); + + const handleNext = useCallback(() => { + if (stepIndex < TOUR_STEPS.length - 1) { + retryCountRef.current = 0; + setStepIndex(stepIndex + 1); + } else { + // Tour completed - save to localStorage + if (user?.id) { + const tourKey = `surfsense-tour-${user.id}`; + localStorage.setItem(tourKey, "true"); + } + setIsActive(false); + } + }, [stepIndex, user?.id]); + + const handlePrev = useCallback(() => { + if (stepIndex > 0) { + retryCountRef.current = 0; + setStepIndex(stepIndex - 1); + } + }, [stepIndex]); + + const handleSkip = useCallback(() => { + // Tour skipped - save to localStorage + if (user?.id) { + const tourKey = `surfsense-tour-${user.id}`; + localStorage.setItem(tourKey, "true"); + } + setIsActive(false); + }, [user?.id]); + + // Handle overlay click to close + const handleOverlayClick = useCallback(() => { + // Tour closed - save to localStorage + if (user?.id) { + const tourKey = `surfsense-tour-${user.id}`; + localStorage.setItem(tourKey, "true"); + } + setIsActive(false); + }, [user?.id]); + + // Handle escape key + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === "Escape" && isActive) { + // Tour closed via escape - save to localStorage + if (user?.id) { + const tourKey = `surfsense-tour-${user.id}`; + localStorage.setItem(tourKey, "true"); + } + setIsActive(false); + } + }; + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [isActive, user?.id]); + + // Don't render if not active or not mounted + if (!mounted || !isActive) { + return null; + } + + return createPortal( + <> + +
+ {/* Clickable backdrop to close */} +
+ , + document.body + ); +} diff --git a/surfsense_web/components/settings/llm-role-manager.tsx b/surfsense_web/components/settings/llm-role-manager.tsx index 1bf7a3629..ba4c4970c 100644 --- a/surfsense_web/components/settings/llm-role-manager.tsx +++ b/surfsense_web/components/settings/llm-role-manager.tsx @@ -45,7 +45,7 @@ const ROLE_DESCRIPTIONS = { document_summary: { icon: FileText, title: "Document Summary LLM", - description: "Handles document summarization, long context analysis, and query reformulation", + description: "Handles document summarization", color: "bg-purple-100 text-purple-800 border-purple-200", examples: "Document analysis, podcasts, research synthesis", characteristics: ["Large context window", "Deep reasoning", "Summarization"], @@ -74,7 +74,6 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { data: preferences = {}, isFetching: preferencesLoading, error: preferencesError, - refetch: refreshPreferences, } = useAtomValue(llmPreferencesAtom); const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom); @@ -187,19 +186,6 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { Refresh Configs Configs -
diff --git a/surfsense_web/components/sidebar/nav-main.tsx b/surfsense_web/components/sidebar/nav-main.tsx index 43c551875..a0dbe912f 100644 --- a/surfsense_web/components/sidebar/nav-main.tsx +++ b/surfsense_web/components/sidebar/nav-main.tsx @@ -131,7 +131,11 @@ export function NavMain({ items }: NavMainProps) { isActive={isActive} aria-label={`${translatedTitle} with submenu`} > - @@ -152,10 +156,18 @@ export function NavMain({ items }: NavMainProps) { {item.items?.map((subItem, subIndex) => { const translatedSubTitle = translateTitle(subItem.title); + const isDocumentsLink = + subItem.title === "Manage Documents" || + translatedSubTitle.toLowerCase().includes("documents"); return ( - + {translatedSubTitle} @@ -173,7 +185,13 @@ export function NavMain({ items }: NavMainProps) { isActive={isActive} aria-label={translatedTitle} > - + {translatedTitle} diff --git a/surfsense_web/components/sources/DocumentUploadTab.tsx b/surfsense_web/components/sources/DocumentUploadTab.tsx index 5280ea850..0b7f7b51f 100644 --- a/surfsense_web/components/sources/DocumentUploadTab.tsx +++ b/surfsense_web/components/sources/DocumentUploadTab.tsx @@ -31,6 +31,7 @@ import { GridPattern } from "./GridPattern"; interface DocumentUploadTabProps { searchSpaceId: string; onSuccess?: () => void; + onAccordionStateChange?: (isExpanded: boolean) => void; } const audioFileTypes = { @@ -109,11 +110,16 @@ const FILE_TYPE_CONFIG: Record> = { const cardClass = "border border-border bg-slate-400/5 dark:bg-white/5"; -export function DocumentUploadTab({ searchSpaceId, onSuccess }: DocumentUploadTabProps) { +export function DocumentUploadTab({ + searchSpaceId, + onSuccess, + onAccordionStateChange, +}: DocumentUploadTabProps) { const t = useTranslations("upload_documents"); const router = useRouter(); const [files, setFiles] = useState([]); const [uploadProgress, setUploadProgress] = useState(0); + const [accordionValue, setAccordionValue] = useState(""); const [uploadDocumentMutation] = useAtom(uploadDocumentMutationAtom); const { mutate: uploadDocuments, isPending: isUploading } = uploadDocumentMutation; const fileInputRef = useRef(null); @@ -154,6 +160,15 @@ export function DocumentUploadTab({ searchSpaceId, onSuccess }: DocumentUploadTa const totalFileSize = files.reduce((total, file) => total + file.size, 0); + // Track accordion state changes + const handleAccordionChange = useCallback( + (value: string) => { + setAccordionValue(value); + onAccordionStateChange?.(value === "supported-file-types"); + }, + [onAccordionStateChange] + ); + const handleUpload = async () => { setUploadProgress(0); trackDocumentUploadStarted(Number(searchSpaceId), files.length, totalFileSize); @@ -190,11 +205,13 @@ export function DocumentUploadTab({ searchSpaceId, onSuccess }: DocumentUploadTa initial={{ opacity: 0, y: 20 }} animate={{ opacity: 1, y: 0 }} transition={{ duration: 0.3 }} - className="space-y-3 sm:space-y-6 max-w-4xl mx-auto" + className="space-y-3 sm:space-y-6 max-w-4xl mx-auto pt-0" > - - - {t("file_size_limit")} + + + + {t("file_size_limit")} + @@ -366,11 +383,13 @@ export function DocumentUploadTab({ searchSpaceId, onSuccess }: DocumentUploadTa - -
+ +
diff --git a/surfsense_web/components/sources/types.ts b/surfsense_web/components/sources/types.ts deleted file mode 100644 index 230af7503..000000000 --- a/surfsense_web/components/sources/types.ts +++ /dev/null @@ -1,13 +0,0 @@ -export interface Connector { - id: string; - title: string; - description: string; - icon: React.ReactNode; - status: "available" | "coming-soon" | "connected"; -} - -export interface ConnectorCategory { - id: string; - title: string; - connectors: Connector[]; -} diff --git a/surfsense_web/components/tool-ui/shared/action-buttons.tsx b/surfsense_web/components/tool-ui/shared/action-buttons.tsx deleted file mode 100644 index 4ed280559..000000000 --- a/surfsense_web/components/tool-ui/shared/action-buttons.tsx +++ /dev/null @@ -1,41 +0,0 @@ -"use client"; - -import type { FC } from "react"; -import { Button } from "@/components/ui/button"; -import type { Action, ActionsConfig } from "./schema"; - -interface ActionButtonsProps { - actions?: Action[] | ActionsConfig; - onAction?: (actionId: string) => void; - disabled?: boolean; -} - -export const ActionButtons: FC = ({ actions, onAction, disabled }) => { - if (!actions) return null; - - // Normalize actions to array format - const actionArray: Action[] = Array.isArray(actions) - ? actions - : ([ - actions.confirm && { ...actions.confirm, id: "confirm" }, - actions.cancel && { ...actions.cancel, id: "cancel" }, - ].filter(Boolean) as Action[]); - - if (actionArray.length === 0) return null; - - return ( -
- {actionArray.map((action) => ( - - ))} -
- ); -}; diff --git a/surfsense_web/components/tool-ui/shared/index.ts b/surfsense_web/components/tool-ui/shared/index.ts deleted file mode 100644 index 23f5a27dd..000000000 --- a/surfsense_web/components/tool-ui/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./action-buttons"; -export * from "./schema"; diff --git a/surfsense_web/components/tool-ui/shared/schema.ts b/surfsense_web/components/tool-ui/shared/schema.ts deleted file mode 100644 index 8076a8e45..000000000 --- a/surfsense_web/components/tool-ui/shared/schema.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { z } from "zod"; - -/** - * Shared action schema for tool UI components - */ -export const ActionSchema = z.object({ - id: z.string(), - label: z.string(), - variant: z.enum(["default", "secondary", "destructive", "outline", "ghost", "link"]).optional(), - disabled: z.boolean().optional(), -}); - -export type Action = z.infer; - -/** - * Actions configuration schema - */ -export const ActionsConfigSchema = z.object({ - confirm: ActionSchema.optional(), - cancel: ActionSchema.optional(), -}); - -export type ActionsConfig = z.infer; diff --git a/surfsense_web/content/docs/connectors/airtable.mdx b/surfsense_web/content/docs/connectors/airtable.mdx new file mode 100644 index 000000000..366a6e8e5 --- /dev/null +++ b/surfsense_web/content/docs/connectors/airtable.mdx @@ -0,0 +1,101 @@ +--- +title: Airtable +description: Connect your Airtable bases to SurfSense +--- + +# Airtable OAuth Integration Setup Guide + +This guide walks you through setting up an Airtable OAuth integration for SurfSense. + +## Step 1: Access Airtable OAuth Integrations + +1. Navigate to [airtable.com/create/oauth](https://airtable.com/create/oauth) +2. In the **Builder Hub**, under **Developers**, click **"OAuth integrations"** +3. Click **"Register an OAuth integration"** + +![Airtable OAuth Integrations Page](/docs/connectors/airtable/airtable-oauth-integrations.png) + +## Step 2: Register an Integration + +Fill in the basic integration details: + +| Field | Value | +|-------|-------| +| **Name** | `SurfSense` | +| **OAuth redirect URL** | `http://localhost:8000/api/v1/auth/airtable/connector/callback` | + +Click **"Register integration"** + +![Register Integration Form](/docs/connectors/airtable/airtable-register-integration.png) + +## Step 3: Configure Scopes + +After registration, configure the required scopes (permissions) for your integration: + +### Record data and comments + +| Scope | Description | +|-------|-------------| +| ✅ `data.recordComments:read` | See comments in records | +| ✅ `data.records:read` | See the data in records | + +### Base schema + +| Scope | Description | +|-------|-------------| +| ✅ `schema.bases:read` | See the structure of a base, like table names or field types | + +### User metadata + +| Scope | Description | +|-------|-------------| +| ✅ `user.email:read` | See the user's email address | + +![Scopes Configuration](/docs/connectors/airtable/airtable-scopes.png) + +## Step 4: Configure Support Information + +Scroll down to configure the support information and authorization preview: + +| Field | Value | +|-------|-------| +| **Support email** | Your support email address | +| **Privacy policy URL** | Your privacy policy URL | +| **Terms of service URL** | Your terms of service URL | + +The preview shows what users will see when authorizing SurfSense: +- The data in your records +- Comments in your records +- The structure of your base, like table names or field types +- Your email address + +Click **"Save changes"** + +![Support Information & Preview](/docs/connectors/airtable/airtable-support-info.png) + +## Step 5: Get OAuth Credentials + +After saving, you'll find your OAuth credentials on the integration page: + +1. Copy your **Client ID** +2. Copy your **Client Secret** + +> ⚠️ Never share your client secret publicly. + +--- + +## Running SurfSense with Airtable Connector + +Add the Airtable environment variables to your Docker run command: + +```bash +docker run -d -p 3000:3000 -p 8000:8000 \ + -v surfsense-data:/data \ + # Airtable Connector + -e AIRTABLE_CLIENT_ID=your_airtable_client_id \ + -e AIRTABLE_CLIENT_SECRET=your_airtable_client_secret \ + -e AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` \ No newline at end of file diff --git a/surfsense_web/content/docs/connectors/bookstack.mdx b/surfsense_web/content/docs/connectors/bookstack.mdx new file mode 100644 index 000000000..8ee581948 --- /dev/null +++ b/surfsense_web/content/docs/connectors/bookstack.mdx @@ -0,0 +1,6 @@ +--- +title: Bookstack +description: Connect your Bookstack instance to SurfSense +--- + +# Documentation in progress \ No newline at end of file diff --git a/surfsense_web/content/docs/connectors/circleback.mdx b/surfsense_web/content/docs/connectors/circleback.mdx new file mode 100644 index 000000000..a5c90a28f --- /dev/null +++ b/surfsense_web/content/docs/connectors/circleback.mdx @@ -0,0 +1,8 @@ +--- +title: Circleback +description: Connect your circleback to SurfSense +--- + +# Documentation in progress + + diff --git a/surfsense_web/content/docs/connectors/clickup.mdx b/surfsense_web/content/docs/connectors/clickup.mdx new file mode 100644 index 000000000..1b732c968 --- /dev/null +++ b/surfsense_web/content/docs/connectors/clickup.mdx @@ -0,0 +1,57 @@ +--- +title: ClickUp +description: Connect your ClickUp workspace to SurfSense +--- + +# ClickUp OAuth Integration Setup Guide + +This guide walks you through setting up a ClickUp OAuth integration for SurfSense. + +## Step 1: Access ClickUp API Settings + +1. Open your ClickUp workspace +2. Navigate to **Settings** (gear icon) → **ClickUp API** +3. You'll see the **ClickUp API Settings** page + +![ClickUp API Settings Page](/docs/connectors/clickup/clickup-api-settings.png) + +## Step 2: Create an App + +1. Click **"+ Create an App"** in the top-right corner +2. Fill in the app details: + +| Field | Value | +|-------|-------| +| **App Name** | `SurfSense` | +| **Redirect URL(s)** | `localhost:8000` | + +3. Click **"Save"** to create the app + +![App Created with Credentials](/docs/connectors/clickup/clickup-app-credentials.png) + +## Step 3: Get OAuth Credentials + +After creating the app, you'll see your credentials: + +1. Copy your **Client ID** +2. Copy your **Client Secret** (click "Show" to reveal, or "Regenerate" if needed) + +> ⚠️ Never share your client secret publicly. + +--- + +## Running SurfSense with ClickUp Connector + +Add the ClickUp environment variables to your Docker run command: + +```bash +docker run -d -p 3000:3000 -p 8000:8000 \ + -v surfsense-data:/data \ + # ClickUp Connector + -e CLICKUP_CLIENT_ID=your_clickup_client_id \ + -e CLICKUP_CLIENT_SECRET=your_clickup_client_secret \ + -e CLICKUP_REDIRECT_URI=http://localhost:8000/api/v1/auth/clickup/connector/callback \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` \ No newline at end of file diff --git a/surfsense_web/content/docs/connectors/confluence.mdx b/surfsense_web/content/docs/connectors/confluence.mdx new file mode 100644 index 000000000..fad9f3e3d --- /dev/null +++ b/surfsense_web/content/docs/connectors/confluence.mdx @@ -0,0 +1,106 @@ +--- +title: Confluence +description: Connect your Confluence spaces to SurfSense +--- + +# Confluence OAuth Integration Setup Guide + +This guide walks you through setting up an Atlassian OAuth 2.0 (3LO) integration for SurfSense to connect your Confluence spaces. + +## Step 1: Access the Developer Console + +1. Navigate to [developer.atlassian.com](https://developer.atlassian.com) +2. Click your profile icon in the top-right corner +3. Select **"Developer console"** from the dropdown + +![Atlassian Developer Console Access](/docs/connectors/atlassian/atlassian-dev-console-access.png) + +## Step 2: Create a New OAuth 2.0 Integration + +1. In the Developer Console, under **My apps**, click the **"Create"** button +2. Select **"OAuth 2.0 integration"** from the dropdown + +![Create OAuth 2.0 Integration](/docs/connectors/atlassian/atlassian-create-app.png) + +## Step 3: Name Your Integration + +1. Enter **Name**: `SurfSense` +2. Check the box to agree to Atlassian's developer terms +3. Click **"Create"** + +> ℹ️ New OAuth 2.0 integrations use rotating refresh tokens, which improve security by limiting token validity and enabling automatic detection of token reuse. + +![Create New Integration Form](/docs/connectors/atlassian/atlassian-name-integration.png) + +## Step 4: Configure Callback URL + +1. In the left sidebar, click **"Authorization"** +2. Under **Callback URLs**, enter the redirect URI: + ``` + http://localhost:8000/api/v1/auth/confluence/connector/callback + ``` +3. Click **"Save changes"** + +> ℹ️ You can enter up to 10 redirect URIs, one per line. + +![Authorization Callback URLs](/docs/connectors/atlassian/atlassian-authorization.png) + +## Step 5: Configure API Permissions + +1. In the left sidebar, click **"Permissions"** +2. You'll see a list of available APIs including Confluence API + +![Permissions Overview](/docs/connectors/atlassian/atlassian-permissions.png) + +## Step 6: Configure Confluence API Scopes + +1. Click **"Configure"** next to **Confluence API** + +### Classic Scopes + +Select the **"Classic scopes"** tab and enable: + +| Scope Name | Code | Description | +|------------|------|-------------| +| ✅ Read user | `read:confluence-user` | View user information in Confluence that you have access to, including usernames, email addresses and profile pictures | + +![Confluence API Classic Scopes](/docs/connectors/atlassian/confluence/atlassian-confluence-classic-scopes.png) + +### Granular Scopes + +Select the **"Granular scopes"** tab and enable: + +| Scope Name | Code | Description | +|------------|------|-------------| +| ✅ View pages | `read:page:confluence` | View page content | +| ✅ View comments | `read:comment:confluence` | View comments on pages or blogposts | +| ✅ View spaces | `read:space:confluence` | View space details | + +4. Click **"Save"** + +![Confluence API Granular Scopes](/docs/connectors/atlassian/confluence/atlassian-confluence-granular-scopes.png) + +## Step 7: Get OAuth Credentials + +1. In the left sidebar, click **"Settings"** +2. Copy your **Client ID** and **Client Secret** + +> ⚠️ Never share your client secret publicly. + +--- + +## Running SurfSense with Confluence Connector + +Add the Atlassian environment variables to your Docker run command: + +```bash +docker run -d -p 3000:3000 -p 8000:8000 \ + -v surfsense-data:/data \ + # Confluence Connector + -e ATLASSIAN_CLIENT_ID=your_atlassian_client_id \ + -e ATLASSIAN_CLIENT_SECRET=your_atlassian_client_secret \ + -e CONFLUENCE_REDIRECT_URI=http://localhost:8000/api/v1/auth/confluence/connector/callback \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` diff --git a/surfsense_web/content/docs/connectors/discord.mdx b/surfsense_web/content/docs/connectors/discord.mdx new file mode 100644 index 000000000..6bb64e7e7 --- /dev/null +++ b/surfsense_web/content/docs/connectors/discord.mdx @@ -0,0 +1,76 @@ +--- +title: Discord +description: Connect your Discord servers to SurfSense +--- + +# Discord OAuth Integration Setup Guide + +This guide walks you through setting up a Discord OAuth integration for SurfSense. + +## Step 1: Create a New Discord Application + +1. Navigate to [discord.com/developers/applications](https://discord.com/developers/applications) +2. Click **"New Application"** +3. Enter the application name: `SurfSense` +4. Click **"Create"** + +## Step 2: Configure General Information + +On the **General Information** page, fill in the details: + +| Field | Value | +|-------|-------| +| **App Icon** | Upload an icon (1024x1024px, PNG/GIF/JPG/WEBP, max 10MB) | +| **Name** | `SurfSense` | +| **Description** | Connect any LLM to your internal knowledge sources and chat with it in real time alongside your team. | +| **Tags** | Add up to 5 tags (optional) | + +You'll also see your **Application ID** and **Public Key** on this page. + +![General Information](/docs/connectors/discord/discord-general-info.png) + +## Step 3: Configure OAuth2 Settings + +1. In the left sidebar, click **"OAuth2"** +2. Copy your **Client ID** and **Client Secret** (click to reveal) +3. Under **Redirects**, click **"Add Another"** and enter: + ``` + http://localhost:8000/api/v1/auth/discord/connector/callback + ``` + +> ⚠️ Keep **Public Client** disabled (off) since SurfSense uses a server to make requests. + +![OAuth2 Configuration](/docs/connectors/discord/discord-oauth2.png) + +## Step 4: Configure Bot Settings + +1. In the left sidebar, click **"Bot"** +2. Configure the **Authorization Flow**: + - ✅ **Public Bot** - Enable to allow anyone to add the bot to servers + +3. Enable **Privileged Gateway Intents**: + - ✅ **Server Members Intent** - Required to receive GUILD_MEMBERS events + - ✅ **Message Content Intent** - Required to receive message content + +> ⚠️ Once your bot reaches 100+ servers, these intents will require verification and approval. + +![Bot Settings](/docs/connectors/discord/discord-bot-settings.png) + +--- + +## Running SurfSense with Discord Connector + +Add the Discord environment variables to your Docker run command: + +```bash +docker run -d -p 3000:3000 -p 8000:8000 \ + -v surfsense-data:/data \ + # Discord Connector + -e DISCORD_CLIENT_ID=your_discord_client_id \ + -e DISCORD_CLIENT_SECRET=your_discord_client_secret \ + -e DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback \ + -e DISCORD_BOT_TOKEN=http://localhost:8000/api/v1/auth/discord/connector/callback \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` diff --git a/surfsense_web/content/docs/connectors/elasticsearch.mdx b/surfsense_web/content/docs/connectors/elasticsearch.mdx new file mode 100644 index 000000000..ac43cca4e --- /dev/null +++ b/surfsense_web/content/docs/connectors/elasticsearch.mdx @@ -0,0 +1,6 @@ +--- +title: Elasticsearch +description: Connect your Elasticsearch cluster to SurfSense +--- + +# Documentation in progress \ No newline at end of file diff --git a/surfsense_web/content/docs/connectors/github.mdx b/surfsense_web/content/docs/connectors/github.mdx new file mode 100644 index 000000000..bb2faca81 --- /dev/null +++ b/surfsense_web/content/docs/connectors/github.mdx @@ -0,0 +1,6 @@ +--- +title: GitHub +description: Connect your GitHub repositories to SurfSense +--- + +# Documentation in progress \ No newline at end of file diff --git a/surfsense_web/content/docs/connectors/gmail.mdx b/surfsense_web/content/docs/connectors/gmail.mdx new file mode 100644 index 000000000..434e6ae4d --- /dev/null +++ b/surfsense_web/content/docs/connectors/gmail.mdx @@ -0,0 +1,83 @@ +--- +title: Gmail +description: Connect your Gmail to SurfSense +--- + +# Gmail OAuth Integration Setup Guide + +This guide walks you through setting up a Google OAuth 2.0 integration for SurfSense to connect your Gmail account. + +## Step 1: Access the Google Cloud Console + +1. Navigate to [Google Cloud Console](https://console.cloud.google.com/) +2. Select an existing project or create a new one + +## Step 2: Enable Required APIs + +1. Go to **APIs & Services** > **Library** +2. Search for and enable the following APIs: + - **People API** (required for Google OAuth) + - **Gmail API** (required for Gmail connector) + +![Google Developer Console People API](/docs/connectors/google/google_oauth_people_api.png) + +## Step 3: Configure OAuth Consent Screen + +1. Go to **APIs & Services** > **OAuth consent screen** +2. Select **External** user type (or Internal if using Google Workspace) +3. Fill in the required information: + - **App name**: `SurfSense` + - **User support email**: Your email address + - **Developer contact information**: Your email address +4. Click **Save and Continue** + +![Google Developer Console OAuth consent screen](/docs/connectors/google/google_oauth_screen.png) + +### Add Scopes + +1. Click **Add or Remove Scopes** +2. Add the following scopes: + - `https://www.googleapis.com/auth/gmail.readonly` - Read Gmail messages + - `https://www.googleapis.com/auth/userinfo.email` - View user email address +3. Click **Update** and then **Save and Continue** + +## Step 4: Create OAuth Client ID + +1. Go to **APIs & Services** > **Credentials** +2. Click **Create Credentials** > **OAuth client ID** +3. Select **Web application** as the application type +4. Enter **Name**: `SurfSense` +5. Under **Authorized redirect URIs**, add: + ``` + http://localhost:8000/api/v1/auth/google/gmail/connector/callback + ``` +6. Click **Create** + +![Google Developer Console OAuth client ID](/docs/connectors/google/google_oauth_client.png) + +## Step 5: Get OAuth Credentials + +1. After creating the OAuth client, you'll see a dialog with your credentials +2. Copy your **Client ID** and **Client Secret** + +> ⚠️ Never share your client secret publicly. + +![Google Developer Console Config](/docs/connectors/google/google_oauth_config.png) + +--- + +## Running SurfSense with Gmail Connector + +Add the Google OAuth environment variables to your Docker run command: + +```bash +docker run -d -p 3000:3000 -p 8000:8000 \ + -v surfsense-data:/data \ + # Gmail Connector + -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \ + -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \ + -e GOOGLE_GMAIL_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/gmail/connector/callback \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` diff --git a/surfsense_web/content/docs/connectors/google-calendar.mdx b/surfsense_web/content/docs/connectors/google-calendar.mdx new file mode 100644 index 000000000..cc1eae545 --- /dev/null +++ b/surfsense_web/content/docs/connectors/google-calendar.mdx @@ -0,0 +1,82 @@ +--- +title: Google Calendar +description: Connect your Google Calendar to SurfSense +--- + +# Google Calendar OAuth Integration Setup Guide + +This guide walks you through setting up a Google OAuth 2.0 integration for SurfSense to connect your Google Calendar. + +## Step 1: Access the Google Cloud Console + +1. Navigate to [Google Cloud Console](https://console.cloud.google.com/) +2. Select an existing project or create a new one + +## Step 2: Enable Required APIs + +1. Go to **APIs & Services** > **Library** +2. Search for and enable the following APIs: + - **People API** (required for Google OAuth) + - **Google Calendar API** (required for Calendar connector) + +![Google Developer Console People API](/docs/connectors/google/google_oauth_people_api.png) + +## Step 3: Configure OAuth Consent Screen + +1. Go to **APIs & Services** > **OAuth consent screen** +2. Select **External** user type (or Internal if using Google Workspace) +3. Fill in the required information: + - **App name**: `SurfSense` + - **User support email**: Your email address + - **Developer contact information**: Your email address +4. Click **Save and Continue** + +![Google Developer Console OAuth consent screen](/docs/connectors/google/google_oauth_screen.png) + +### Add Scopes + +1. Click **Add or Remove Scopes** +2. Add the following scope: + - `https://www.googleapis.com/auth/calendar.readonly` - Read Google Calendar events +3. Click **Update** and then **Save and Continue** + +## Step 4: Create OAuth Client ID + +1. Go to **APIs & Services** > **Credentials** +2. Click **Create Credentials** > **OAuth client ID** +3. Select **Web application** as the application type +4. Enter **Name**: `SurfSense` +5. Under **Authorized redirect URIs**, add: + ``` + http://localhost:8000/api/v1/auth/google/calendar/connector/callback + ``` +6. Click **Create** + +![Google Developer Console OAuth client ID](/docs/connectors/google/google_oauth_client.png) + +## Step 5: Get OAuth Credentials + +1. After creating the OAuth client, you'll see a dialog with your credentials +2. Copy your **Client ID** and **Client Secret** + +> ⚠️ Never share your client secret publicly. + +![Google Developer Console Config](/docs/connectors/google/google_oauth_config.png) + +--- + +## Running SurfSense with Google Calendar Connector + +Add the Google OAuth environment variables to your Docker run command: + +```bash +docker run -d -p 3000:3000 -p 8000:8000 \ + -v surfsense-data:/data \ + # Google Calendar Connector + -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \ + -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \ + -e GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` diff --git a/surfsense_web/content/docs/connectors/google-drive.mdx b/surfsense_web/content/docs/connectors/google-drive.mdx new file mode 100644 index 000000000..00ea2f610 --- /dev/null +++ b/surfsense_web/content/docs/connectors/google-drive.mdx @@ -0,0 +1,83 @@ +--- +title: Google Drive +description: Connect your Google Drive to SurfSense +--- + +# Google Drive OAuth Integration Setup Guide + +This guide walks you through setting up a Google OAuth 2.0 integration for SurfSense to connect your Google Drive. + +## Step 1: Access the Google Cloud Console + +1. Navigate to [Google Cloud Console](https://console.cloud.google.com/) +2. Select an existing project or create a new one + +## Step 2: Enable Required APIs + +1. Go to **APIs & Services** > **Library** +2. Search for and enable the following APIs: + - **People API** (required for Google OAuth) + - **Google Drive API** (required for Drive connector) + +![Google Developer Console People API](/docs/connectors/google/google_oauth_people_api.png) + +## Step 3: Configure OAuth Consent Screen + +1. Go to **APIs & Services** > **OAuth consent screen** +2. Select **External** user type (or Internal if using Google Workspace) +3. Fill in the required information: + - **App name**: `SurfSense` + - **User support email**: Your email address + - **Developer contact information**: Your email address +4. Click **Save and Continue** + +![Google Developer Console OAuth consent screen](/docs/connectors/google/google_oauth_screen.png) + +### Add Scopes + +1. Click **Add or Remove Scopes** +2. Add the following scopes: + - `https://www.googleapis.com/auth/drive.readonly` - Read-only access to Google Drive + - `https://www.googleapis.com/auth/userinfo.email` - View user email address +3. Click **Update** and then **Save and Continue** + +## Step 4: Create OAuth Client ID + +1. Go to **APIs & Services** > **Credentials** +2. Click **Create Credentials** > **OAuth client ID** +3. Select **Web application** as the application type +4. Enter **Name**: `SurfSense` +5. Under **Authorized redirect URIs**, add: + ``` + http://localhost:8000/api/v1/auth/google/drive/connector/callback + ``` +6. Click **Create** + +![Google Developer Console OAuth client ID](/docs/connectors/google/google_oauth_client.png) + +## Step 5: Get OAuth Credentials + +1. After creating the OAuth client, you'll see a dialog with your credentials +2. Copy your **Client ID** and **Client Secret** + +> ⚠️ Never share your client secret publicly. + +![Google Developer Console Config](/docs/connectors/google/google_oauth_config.png) + +--- + +## Running SurfSense with Google Drive Connector + +Add the Google OAuth environment variables to your Docker run command: + +```bash +docker run -d -p 3000:3000 -p 8000:8000 \ + -v surfsense-data:/data \ + # Google Drive Connector + -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \ + -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \ + -e GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` diff --git a/surfsense_web/content/docs/connectors/jira.mdx b/surfsense_web/content/docs/connectors/jira.mdx new file mode 100644 index 000000000..ebe639d6d --- /dev/null +++ b/surfsense_web/content/docs/connectors/jira.mdx @@ -0,0 +1,93 @@ +--- +title: Jira +description: Connect your Jira projects to SurfSense +--- + +# Jira OAuth Integration Setup Guide + +This guide walks you through setting up an Atlassian OAuth 2.0 (3LO) integration for SurfSense to connect your Jira projects. + +## Step 1: Access the Developer Console + +1. Navigate to [developer.atlassian.com](https://developer.atlassian.com) +2. Click your profile icon in the top-right corner +3. Select **"Developer console"** from the dropdown + +![Atlassian Developer Console Access](/docs/connectors/atlassian/atlassian-dev-console-access.png) + +## Step 2: Create a New OAuth 2.0 Integration + +1. In the Developer Console, under **My apps**, click the **"Create"** button +2. Select **"OAuth 2.0 integration"** from the dropdown + +![Create OAuth 2.0 Integration](/docs/connectors/atlassian/atlassian-create-app.png) + +## Step 3: Name Your Integration + +1. Enter **Name**: `SurfSense` +2. Check the box to agree to Atlassian's developer terms +3. Click **"Create"** + +> ℹ️ New OAuth 2.0 integrations use rotating refresh tokens, which improve security by limiting token validity and enabling automatic detection of token reuse. + +![Create New Integration Form](/docs/connectors/atlassian/atlassian-name-integration.png) + +## Step 4: Configure Callback URL + +1. In the left sidebar, click **"Authorization"** +2. Under **Callback URLs**, enter the redirect URI: + ``` + http://localhost:8000/api/v1/auth/jira/connector/callback + ``` +3. Click **"Save changes"** + +> ℹ️ You can enter up to 10 redirect URIs, one per line. + +![Authorization Callback URLs](/docs/connectors/atlassian/atlassian-authorization.png) + +## Step 5: Configure API Permissions + +1. In the left sidebar, click **"Permissions"** +2. You'll see a list of available APIs including Jira API + +![Permissions Overview](/docs/connectors/atlassian/atlassian-permissions.png) + +## Step 6: Configure Jira API Scopes + +1. Click **"Configure"** next to **Jira API** +2. Select the **"Classic scopes"** tab +3. Under **Jira platform REST API**, select the following scopes: + +| Scope Name | Code | Description | +|------------|------|-------------| +| ✅ View Jira issue data | `read:jira-work` | Read Jira project and issue data, search for issues, and objects associated with issues like attachments and worklogs | +| ✅ View user profiles | `read:jira-user` | View user information in Jira that the user has access to, including usernames, email addresses, and avatars | + +4. Click **"Save"** + +![Jira API Scopes](/docs/connectors/atlassian/jira/atlassian-jira-scopes.png) + +## Step 7: Get OAuth Credentials + +1. In the left sidebar, click **"Settings"** +2. Copy your **Client ID** and **Client Secret** + +> ⚠️ Never share your client secret publicly. + +--- + +## Running SurfSense with Jira Connector + +Add the Atlassian environment variables to your Docker run command: + +```bash +docker run -d -p 3000:3000 -p 8000:8000 \ + -v surfsense-data:/data \ + # Jira Connector + -e ATLASSIAN_CLIENT_ID=your_atlassian_client_id \ + -e ATLASSIAN_CLIENT_SECRET=your_atlassian_client_secret \ + -e JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` diff --git a/surfsense_web/content/docs/connectors/linear.mdx b/surfsense_web/content/docs/connectors/linear.mdx new file mode 100644 index 000000000..f9dc9a62b --- /dev/null +++ b/surfsense_web/content/docs/connectors/linear.mdx @@ -0,0 +1,67 @@ +--- +title: Linear +description: Connect your Linear workspace to SurfSense +--- + +# Linear OAuth Integration Setup Guide + +This guide walks you through setting up a Linear OAuth integration for SurfSense. + +## Step 1: Access Linear API Settings + +1. Navigate to your workspace's API settings at `linear.app//settings/api` +2. Under **OAuth Applications**, click **"+ New OAuth application"** + +![Linear API Settings Page](/docs/connectors/linear/linear-api-settings.png) + +## Step 2: Create New Application + +Fill in the application details: + +| Field | Value | +|-------|-------| +| **Application icon** | Upload an icon (at least 256x256px) | +| **Application name** | `SurfSense` | +| **Developer name** | `SurfSense` | +| **Developer URL** | `https://www.surfsense.com/` | +| **Description** | Connect any LLM to your internal knowledge sources and chat with it in real time alongside your team. | +| **Callback URLs** | `http://localhost:8000/api/v1/auth/linear/connector/callback` | +| **GitHub username** | Your GitHub username (optional) | + +### Settings + +- ✅ **Public** - Enable this to allow the application to be installed by other workspaces + +Click **Create** to create the application. + +![Create New Application Form](/docs/connectors/linear/linear-new-application.png) + +## Step 3: Get OAuth Credentials + +After creating the application, you'll see your OAuth credentials: + +1. Copy your **Client ID** +2. Copy your **Client Secret** + +> ⚠️ Never share your client secret publicly. + +![OAuth Credentials](/docs/connectors/linear/linear-oauth-credentials.png) + +--- + +## Running SurfSense with Linear Connector + +Add the Linear environment variables to your Docker run command: + +```bash +docker run -d -p 3000:3000 -p 8000:8000 \ + -v surfsense-data:/data \ + # Linear Connector + -e LINEAR_CLIENT_ID=your_linear_client_id \ + -e LINEAR_CLIENT_SECRET=your_linear_client_secret \ + -e LINEAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/linear/connector/callback \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` + diff --git a/surfsense_web/content/docs/connectors/luma.mdx b/surfsense_web/content/docs/connectors/luma.mdx new file mode 100644 index 000000000..e16e5a949 --- /dev/null +++ b/surfsense_web/content/docs/connectors/luma.mdx @@ -0,0 +1,6 @@ +--- +title: Luma +description: Connect your Luma events to SurfSense +--- + +# Documentation in progress \ No newline at end of file diff --git a/surfsense_web/content/docs/connectors/meta.json b/surfsense_web/content/docs/connectors/meta.json new file mode 100644 index 000000000..7a075fbb5 --- /dev/null +++ b/surfsense_web/content/docs/connectors/meta.json @@ -0,0 +1,22 @@ +{ + "title": "Connectors", + "pages": [ + "google-drive", + "gmail", + "google-calendar", + "notion", + "slack", + "discord", + "jira", + "linear", + "confluence", + "airtable", + "clickup", + "github", + "luma", + "circleback", + "elasticsearch", + "bookstack" + ], + "defaultOpen": true +} diff --git a/surfsense_web/content/docs/connectors/notion.mdx b/surfsense_web/content/docs/connectors/notion.mdx new file mode 100644 index 000000000..936972f7e --- /dev/null +++ b/surfsense_web/content/docs/connectors/notion.mdx @@ -0,0 +1,84 @@ +--- +title: Notion +description: Connect your Notion workspaces to SurfSense +--- + +# Notion OAuth Integration Setup Guide + +This guide walks you through setting up a Notion OAuth integration for SurfSense. + +## Step 1: Access Notion Integrations + +1. Navigate to [notion.so/profile/integrations](https://notion.so/profile/integrations) +2. Click the **"New integration"** button + +![Notion Integrations Page](/docs/connectors/notion/notion-integrations-page.png) + +## Step 2: Configure New Integration + +Fill in the integration details: + +| Field | Value | +|-------|-------| +| **Integration Name** | `SurfSense` | +| **Associated workspace** | Select your workspace | +| **Type** | `Public` | +| **Company name** | Your company name | +| **Website** | Your website URL | +| **Tagline** | Brief description | +| **Privacy Policy URL** | Your privacy policy URL | +| **Terms of Use URL** | Your terms of use URL | +| **Email** | Your developer email | +| **Logo** | Upload a 512x512 logo | + +### OAuth Redirect URI + +Under **OAuth domains & URIs**, set the **Redirect URI** to: + +``` +http://localhost:8000/api/v1/auth/notion/connector/callback +``` + +Click **Save** to create the integration. + +![New Integration Form](/docs/connectors/notion/notion-new-integration-form.png) + +## Step 3: Get OAuth Credentials & Configure Capabilities + +After creating the integration, you'll see the configuration page with your credentials: + +1. Copy your **OAuth Client ID** +2. Copy your **OAuth Client Secret** (click Refresh if needed) + +### Set Required Capabilities + +Under **Content Capabilities**, enable: +- ✅ Read content + +Under **Comment Capabilities**, enable: +- ✅ Read comments + +Under **User Capabilities**, select: +- 🔘 Read user information including email addresses + +Click **Save** to apply the capabilities. + +![Integration Configuration](/docs/connectors/notion/notion-integration-config.png) + +--- + +## Running SurfSense with Notion Connector + +Add the Notion environment variables to your Docker run command: + +```bash +docker run -d -p 3000:3000 -p 8000:8000 \ + -v surfsense-data:/data \ + # Notion Connector + -e NOTION_OAUTH_CLIENT_ID=your_notion_client_id \ + -e NOTION_OAUTH_CLIENT_SECRET=your_notion_client_secret \ + -e NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` diff --git a/surfsense_web/content/docs/connectors/slack.mdx b/surfsense_web/content/docs/connectors/slack.mdx new file mode 100644 index 000000000..ccabe6f9e --- /dev/null +++ b/surfsense_web/content/docs/connectors/slack.mdx @@ -0,0 +1,91 @@ +--- +title: Slack +description: Connect your Slack workspace to SurfSense +--- + +# Slack OAuth Integration Setup Guide + +This guide walks you through setting up a Slack OAuth integration for SurfSense. + +## Step 1: Create a New Slack App + +1. Navigate to [api.slack.com/apps](https://api.slack.com/apps) +2. Click **"Create New App"** +3. Select **"From scratch"** to manually configure your app + +![Create an App Dialog](/docs/connectors/slack/slack-create-app.png) + +## Step 2: Name App & Choose Workspace + +1. Enter **App Name**: `SurfSense` +2. Select the workspace to develop your app in +3. Click **"Create App"** + +> ⚠️ You won't be able to change the workspace later. The workspace will control the app even if you leave it. + +![Name App & Choose Workspace](/docs/connectors/slack/slack-name-workspace.png) + +## Step 3: Get App Credentials + +After creating the app, you'll be taken to the **Basic Information** page. Here you'll find your credentials: + +1. Copy your **Client ID** +2. Copy your **Client Secret** (click Show to reveal) + +> ⚠️ Never share your app credentials publicly. + +![Basic Information - App Credentials](/docs/connectors/slack/slack-app-credentials.png) + +## Step 4: Configure Redirect URLs + +1. In the left sidebar, click **"OAuth & Permissions"** +2. Scroll down to **Redirect URLs** +3. Click **"Add New Redirect URL"** +4. Enter: `https://localhost:8000/api/v1/auth/slack/connector/callback` +5. Click **"Add"**, then **"Save URLs"** + +![Redirect URLs Configuration](/docs/connectors/slack/slack-redirect-urls.png) + +## Step 5: Configure Bot Token Scopes + +On the same **OAuth & Permissions** page, scroll to **Scopes** and add the following **Bot Token Scopes**: + +| OAuth Scope | Description | +|-------------|-------------| +| `channels:history` | View messages and other content in public channels | +| `channels:read` | View basic information about public channels | +| `groups:history` | View messages and other content in private channels | +| `groups:read` | View basic information about private channels | +| `im:history` | View messages and other content in direct messages | +| `mpim:history` | View messages and other content in group direct messages | +| `users:read` | View people in a workspace | + +Click **"Add an OAuth Scope"** to add each scope. + +![Bot Token Scopes](/docs/connectors/slack/slack-scopes.png) + +## Step 6: Enable Public Distribution + +1. In the left sidebar, click **"Manage Distribution"** +2. Under **Share Your App with Other Workspaces**, ensure distribution is enabled +3. You can use the **"Add to Slack"** button or **Sharable URL** to install the app + +![Manage Distribution](/docs/connectors/slack/slack-distribution.png) + +--- + +## Running SurfSense with Slack Connector + +Add the Slack environment variables to your Docker run command: + +```bash +docker run -d -p 3000:3000 -p 8000:8000 \ + -v surfsense-data:/data \ + # Slack Connector + -e SLACK_CLIENT_ID=your_slack_client_id \ + -e SLACK_CLIENT_SECRET=your_slack_client_secret \ + -e SLACK_REDIRECT_URI=https://localhost:8000/api/v1/auth/slack/connector/callback \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` diff --git a/surfsense_web/content/docs/connectors/web-crawler.mdx b/surfsense_web/content/docs/connectors/web-crawler.mdx new file mode 100644 index 000000000..6ea5b1c2b --- /dev/null +++ b/surfsense_web/content/docs/connectors/web-crawler.mdx @@ -0,0 +1,6 @@ +--- +title: Web Crawler +description: Crawl and index websites with SurfSense +--- + +# Documentation in progress \ No newline at end of file diff --git a/surfsense_web/content/docs/docker-installation.mdx b/surfsense_web/content/docs/docker-installation.mdx index 9957bbd37..6501c7783 100644 --- a/surfsense_web/content/docs/docker-installation.mdx +++ b/surfsense_web/content/docs/docker-installation.mdx @@ -1,7 +1,6 @@ --- title: Docker Installation description: Setting up SurfSense using Docker -full: true --- @@ -48,31 +47,29 @@ docker run -d -p 3000:3000 -p 8000:8000 ` ### With Custom Configuration -**Using OpenAI Embeddings:** +You can pass any [environment variable](/docs/manual-installation#backend-environment-variables) using `-e` flags: ```bash docker run -d -p 3000:3000 -p 8000:8000 \ -v surfsense-data:/data \ -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \ -e OPENAI_API_KEY=your_openai_api_key \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -**With Google OAuth:** - -```bash -docker run -d -p 3000:3000 -p 8000:8000 \ - -v surfsense-data:/data \ -e AUTH_TYPE=GOOGLE \ - -e GOOGLE_OAUTH_CLIENT_ID=your_client_id \ - -e GOOGLE_OAUTH_CLIENT_SECRET=your_client_secret \ + -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \ + -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \ + -e ETL_SERVICE=LLAMACLOUD \ + -e LLAMA_CLOUD_API_KEY=your_llama_cloud_key \ --name surfsense \ --restart unless-stopped \ ghcr.io/modsetter/surfsense:latest ``` + +- For Google OAuth, create credentials in the [Google Cloud Console](https://console.cloud.google.com/apis/credentials) +- For Airtable connector, create an OAuth integration in the [Airtable Developer Hub](https://airtable.com/create/oauth) +- If deploying behind a reverse proxy with HTTPS, add `-e BACKEND_URL=https://api.yourdomain.com` + + ### Quick Start with Docker Compose For easier management with environment files: diff --git a/surfsense_web/content/docs/index.mdx b/surfsense_web/content/docs/index.mdx index 2f0bd4f8f..bb07c5f68 100644 --- a/surfsense_web/content/docs/index.mdx +++ b/surfsense_web/content/docs/index.mdx @@ -1,7 +1,6 @@ --- title: Prerequisites description: Required setup's before setting up SurfSense -full: true --- @@ -16,15 +15,13 @@ To set up Google OAuth: 1. Login to your [Google Developer Console](https://console.cloud.google.com/) 2. Enable the required APIs: - **People API** (required for basic Google OAuth) - - **Gmail API** (required if you want to use the Gmail connector) - - **Google Calendar API** (required if you want to use the Google Calendar connector) -![Google Developer Console People API](/docs/google_oauth_people_api.png) +![Google Developer Console People API](/docs/connectors/google/google_oauth_people_api.png) 3. Set up OAuth consent screen. -![Google Developer Console OAuth consent screen](/docs/google_oauth_screen.png) +![Google Developer Console OAuth consent screen](/docs/connectors/google/google_oauth_screen.png) 4. Create OAuth client ID and secret. -![Google Developer Console OAuth client ID](/docs/google_oauth_client.png) +![Google Developer Console OAuth client ID](/docs/connectors/google/google_oauth_client.png) 5. It should look like this. -![Google Developer Console Config](/docs/google_oauth_config.png) +![Google Developer Console Config](/docs/connectors/google/google_oauth_config.png) --- diff --git a/surfsense_web/content/docs/installation.mdx b/surfsense_web/content/docs/installation.mdx index 1a3f4553b..f5e948b64 100644 --- a/surfsense_web/content/docs/installation.mdx +++ b/surfsense_web/content/docs/installation.mdx @@ -1,7 +1,6 @@ --- title: Installation description: Current ways to use SurfSense -full: true --- # Installing SurfSense diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index e7caf93a6..0dd703758 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -1,7 +1,6 @@ --- title: Manual Installation description: Setting up SurfSense manually for customized deployments (Preferred) -full: true --- # Manual Installation (Preferred) @@ -234,7 +233,7 @@ redis-cli ping In a new terminal window, start the Celery worker to handle background tasks: -**Linux/macOS/Windows:** +**If using uv:** ```bash # Make sure you're in the surfsense_backend directory @@ -244,13 +243,31 @@ cd surfsense_backend uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo ``` +**If using pip/venv:** + +```bash +# Make sure you're in the surfsense_backend directory +cd surfsense_backend + +# Activate virtual environment +source .venv/bin/activate # Linux/macOS +# OR +.venv\Scripts\activate # Windows + +# Start Celery worker +celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo +``` + **Optional: Start Flower for monitoring Celery tasks:** In another terminal window: ```bash -# Start Flower (Celery monitoring tool) +# If using uv uv run celery -A celery_worker.celery_app flower --port=5555 + +# If using pip/venv (activate venv first) +celery -A celery_worker.celery_app flower --port=5555 ``` Access Flower at [http://localhost:5555](http://localhost:5555) to monitor your Celery tasks. @@ -259,7 +276,7 @@ Access Flower at [http://localhost:5555](http://localhost:5555) to monitor your In another new terminal window, start Celery Beat to enable periodic tasks (like scheduled connector indexing): -**Linux/macOS/Windows:** +**If using uv:** ```bash # Make sure you're in the surfsense_backend directory @@ -269,13 +286,28 @@ cd surfsense_backend uv run celery -A celery_worker.celery_app beat --loglevel=info ``` +**If using pip/venv:** + +```bash +# Make sure you're in the surfsense_backend directory +cd surfsense_backend + +# Activate virtual environment +source .venv/bin/activate # Linux/macOS +# OR +.venv\Scripts\activate # Windows + +# Start Celery Beat +celery -A celery_worker.celery_app beat --loglevel=info +``` + **Important**: Celery Beat is required for the periodic indexing functionality to work. Without it, scheduled connector tasks won't run automatically. The schedule interval can be configured using the `SCHEDULE_CHECKER_INTERVAL` environment variable. ### 6. Run the Backend Start the backend server: -**Linux/macOS/Windows:** +**If using uv:** ```bash # Run without hot reloading @@ -285,6 +317,21 @@ uv run main.py uv run main.py --reload ``` +**If using pip/venv:** + +```bash +# Activate virtual environment if not already activated +source .venv/bin/activate # Linux/macOS +# OR +.venv\Scripts\activate # Windows + +# Run without hot reloading +python main.py + +# Or with hot reloading for development +python main.py --reload +``` + If everything is set up correctly, you should see output indicating the server is running on `http://localhost:8000`. ## Frontend Setup diff --git a/surfsense_web/content/docs/meta.json b/surfsense_web/content/docs/meta.json index c85ab5928..03790dd15 100644 --- a/surfsense_web/content/docs/meta.json +++ b/surfsense_web/content/docs/meta.json @@ -1,6 +1,13 @@ { - "title": "Setup", - "description": "The setup guide for Surfsense", + "title": "Documentation", + "description": "SurfSense Documentation", "root": true, - "pages": ["index", "installation", "docker-installation", "manual-installation"] + "pages": [ + "---Guides---", + "index", + "installation", + "docker-installation", + "manual-installation", + "connectors" + ] } diff --git a/surfsense_web/hooks/use-chat.ts b/surfsense_web/hooks/use-chat.ts deleted file mode 100644 index c31097e11..000000000 --- a/surfsense_web/hooks/use-chat.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { useEffect, useState } from "react"; -import type { ResearchMode } from "@/components/chat"; -import type { Document } from "@/contracts/types/document.types"; -import { getBearerToken } from "@/lib/auth-utils"; - -interface UseChatStateProps { - search_space_id: string; - chat_id?: string; -} - -export function useChatState({ chat_id }: UseChatStateProps) { - const [token, setToken] = useState(null); - const [isLoading, setIsLoading] = useState(false); - const [currentChatId, setCurrentChatId] = useState(chat_id || null); - - // Chat configuration state - const [researchMode, setResearchMode] = useState("QNA"); - const [selectedConnectors, setSelectedConnectors] = useState([]); - const [selectedDocuments, setSelectedDocuments] = useState([]); - const [topK, setTopK] = useState(5); - - useEffect(() => { - const bearerToken = getBearerToken(); - setToken(bearerToken); - }, []); - - return { - token, - setToken, - isLoading, - setIsLoading, - currentChatId, - setCurrentChatId, - researchMode, - setResearchMode, - selectedConnectors, - setSelectedConnectors, - selectedDocuments, - setSelectedDocuments, - topK, - setTopK, - }; -} diff --git a/surfsense_web/hooks/use-connector-edit-page.ts b/surfsense_web/hooks/use-connector-edit-page.ts deleted file mode 100644 index 8fa690d04..000000000 --- a/surfsense_web/hooks/use-connector-edit-page.ts +++ /dev/null @@ -1,683 +0,0 @@ -import { zodResolver } from "@hookform/resolvers/zod"; -import { useAtomValue } from "jotai"; -import { useRouter } from "next/navigation"; -import { useCallback, useEffect, useState } from "react"; -import { useForm } from "react-hook-form"; -import { toast } from "sonner"; -import { updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; -import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; -import { - type EditConnectorFormValues, - type EditMode, - editConnectorSchema, - type GithubPatFormValues, - type GithubRepo, - githubPatSchema, -} from "@/components/editConnector/types"; -import type { EnumConnectorName } from "@/contracts/enums/connector"; -import type { UpdateConnectorResponse } from "@/contracts/types/connector.types"; -import type { SearchSourceConnector } from "@/hooks/use-search-source-connectors"; -import { authenticatedFetch } from "@/lib/auth-utils"; - -const normalizeListInput = (value: unknown): string[] => { - if (Array.isArray(value)) { - return value.map((item) => String(item).trim()).filter((item) => item.length > 0); - } - if (typeof value === "string") { - return value - .split(",") - .map((item) => item.trim()) - .filter((item) => item.length > 0); - } - return []; -}; - -const arraysEqual = (a: string[], b: string[]): boolean => { - if (a.length !== b.length) return false; - return a.every((value, index) => value === b[index]); -}; - -const normalizeBoolean = (value: unknown): boolean | null => { - if (typeof value === "boolean") return value; - if (typeof value === "string") { - const lowered = value.trim().toLowerCase(); - if (["true", "1", "yes", "on"].includes(lowered)) return true; - if (["false", "0", "no", "off"].includes(lowered)) return false; - } - if (typeof value === "number") { - if (value === 1) return true; - if (value === 0) return false; - } - return null; -}; - -export function useConnectorEditPage(connectorId: number, searchSpaceId: string) { - const router = useRouter(); - const { data: connectors = [], isLoading: connectorsLoading } = useAtomValue(connectorsAtom); - const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom); - - // State managed by the hook - const [connector, setConnector] = useState(null); - const [originalConfig, setOriginalConfig] = useState | null>(null); - const [isSaving, setIsSaving] = useState(false); - const [currentSelectedRepos, setCurrentSelectedRepos] = useState([]); - const [originalPat, setOriginalPat] = useState(""); - const [editMode, setEditMode] = useState("viewing"); - const [fetchedRepos, setFetchedRepos] = useState(null); - const [newSelectedRepos, setNewSelectedRepos] = useState([]); - const [isFetchingRepos, setIsFetchingRepos] = useState(false); - - // Forms managed by the hook - const patForm = useForm({ - resolver: zodResolver(githubPatSchema), - defaultValues: { github_pat: "" }, - }); - const editForm = useForm({ - resolver: zodResolver(editConnectorSchema), - defaultValues: { - name: "", - SLACK_BOT_TOKEN: "", - NOTION_INTEGRATION_TOKEN: "", - TAVILY_API_KEY: "", - SEARXNG_HOST: "", - SEARXNG_API_KEY: "", - SEARXNG_ENGINES: "", - SEARXNG_CATEGORIES: "", - SEARXNG_LANGUAGE: "", - SEARXNG_SAFESEARCH: "", - SEARXNG_VERIFY_SSL: "", - LINEAR_API_KEY: "", - DISCORD_BOT_TOKEN: "", - CONFLUENCE_BASE_URL: "", - CONFLUENCE_EMAIL: "", - CONFLUENCE_API_TOKEN: "", - BOOKSTACK_BASE_URL: "", - BOOKSTACK_TOKEN_ID: "", - BOOKSTACK_TOKEN_SECRET: "", - JIRA_BASE_URL: "", - JIRA_EMAIL: "", - JIRA_API_TOKEN: "", - LUMA_API_KEY: "", - ELASTICSEARCH_API_KEY: "", - FIRECRAWL_API_KEY: "", - INITIAL_URLS: "", - }, - }); - - // Effect to load initial data - useEffect(() => { - if (!connectorsLoading && connectors.length > 0 && !connector) { - const currentConnector = connectors.find((c) => c.id === connectorId); - if (currentConnector) { - setConnector(currentConnector); - const config = currentConnector.config || {}; - setOriginalConfig(config); - editForm.reset({ - name: currentConnector.name, - SLACK_BOT_TOKEN: config.SLACK_BOT_TOKEN || "", - NOTION_INTEGRATION_TOKEN: config.NOTION_INTEGRATION_TOKEN || "", - TAVILY_API_KEY: config.TAVILY_API_KEY || "", - SEARXNG_HOST: config.SEARXNG_HOST || "", - SEARXNG_API_KEY: config.SEARXNG_API_KEY || "", - SEARXNG_ENGINES: Array.isArray(config.SEARXNG_ENGINES) - ? config.SEARXNG_ENGINES.join(", ") - : config.SEARXNG_ENGINES || "", - SEARXNG_CATEGORIES: Array.isArray(config.SEARXNG_CATEGORIES) - ? config.SEARXNG_CATEGORIES.join(", ") - : config.SEARXNG_CATEGORIES || "", - SEARXNG_LANGUAGE: config.SEARXNG_LANGUAGE || "", - SEARXNG_SAFESEARCH: - config.SEARXNG_SAFESEARCH !== undefined && config.SEARXNG_SAFESEARCH !== null - ? String(config.SEARXNG_SAFESEARCH) - : "", - SEARXNG_VERIFY_SSL: - config.SEARXNG_VERIFY_SSL !== undefined && config.SEARXNG_VERIFY_SSL !== null - ? String(config.SEARXNG_VERIFY_SSL) - : "", - LINEAR_API_KEY: config.LINEAR_API_KEY || "", - LINKUP_API_KEY: config.LINKUP_API_KEY || "", - DISCORD_BOT_TOKEN: config.DISCORD_BOT_TOKEN || "", - CONFLUENCE_BASE_URL: config.CONFLUENCE_BASE_URL || "", - CONFLUENCE_EMAIL: config.CONFLUENCE_EMAIL || "", - CONFLUENCE_API_TOKEN: config.CONFLUENCE_API_TOKEN || "", - BOOKSTACK_BASE_URL: config.BOOKSTACK_BASE_URL || "", - BOOKSTACK_TOKEN_ID: config.BOOKSTACK_TOKEN_ID || "", - BOOKSTACK_TOKEN_SECRET: config.BOOKSTACK_TOKEN_SECRET || "", - JIRA_BASE_URL: config.JIRA_BASE_URL || "", - JIRA_EMAIL: config.JIRA_EMAIL || "", - JIRA_API_TOKEN: config.JIRA_API_TOKEN || "", - LUMA_API_KEY: config.LUMA_API_KEY || "", - ELASTICSEARCH_API_KEY: config.ELASTICSEARCH_API_KEY || "", - FIRECRAWL_API_KEY: config.FIRECRAWL_API_KEY || "", - INITIAL_URLS: config.INITIAL_URLS || "", - }); - if (currentConnector.connector_type === "GITHUB_CONNECTOR") { - const savedRepos = config.repo_full_names || []; - const savedPat = config.GITHUB_PAT || ""; - setCurrentSelectedRepos(savedRepos); - setNewSelectedRepos(savedRepos); - setOriginalPat(savedPat); - patForm.reset({ github_pat: savedPat }); - setEditMode("viewing"); - } - } else { - toast.error("Connector not found."); - router.push(`/dashboard/${searchSpaceId}`); - } - } - }, [ - connectorId, - connectors, - connectorsLoading, - router, - searchSpaceId, - connector, - editForm.reset, - patForm.reset, - // Note: editForm and patForm are intentionally excluded from dependencies - // to prevent infinite loops. They are stable form objects from react-hook-form. - ]); - - // Handlers managed by the hook - const handleFetchRepositories = useCallback( - async (values: GithubPatFormValues) => { - setIsFetchingRepos(true); - setFetchedRepos(null); - try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/github/repositories`, - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ github_pat: values.github_pat }), - } - ); - if (!response.ok) { - const err = await response.json(); - throw new Error(err.detail || "Fetch failed"); - } - const data: GithubRepo[] = await response.json(); - setFetchedRepos(data); - setNewSelectedRepos(currentSelectedRepos); - toast.success(`Found ${data.length} repos.`); - } catch (error) { - console.error("Error fetching GitHub repositories:", error); - toast.error(error instanceof Error ? error.message : "Failed to fetch repositories."); - } finally { - setIsFetchingRepos(false); - } - }, - [currentSelectedRepos] - ); // Added dependency - - const handleRepoSelectionChange = useCallback((repoFullName: string, checked: boolean) => { - setNewSelectedRepos((prev) => - checked ? [...prev, repoFullName] : prev.filter((name) => name !== repoFullName) - ); - }, []); - - const handleSaveChanges = useCallback( - async (formData: EditConnectorFormValues) => { - if (!connector || !originalConfig) return; - setIsSaving(true); - const updatePayload: Partial = {}; - let configChanged = false; - let newConfig: Record | null = null; - - if (formData.name !== connector.name) { - updatePayload.name = formData.name; - } - - switch (connector.connector_type) { - case "GITHUB_CONNECTOR": { - const currentPatInForm = patForm.getValues("github_pat"); - const patChanged = currentPatInForm !== originalPat; - const initialRepoSet = new Set(currentSelectedRepos); - const newRepoSet = new Set(newSelectedRepos); - const reposChanged = - initialRepoSet.size !== newRepoSet.size || - ![...initialRepoSet].every((repo) => newRepoSet.has(repo)); - if ( - patChanged || - (editMode === "editing_repos" && reposChanged && fetchedRepos !== null) - ) { - if ( - !currentPatInForm || - !(currentPatInForm.startsWith("ghp_") || currentPatInForm.startsWith("github_pat_")) - ) { - toast.error("Invalid GitHub PAT format. Cannot save."); - setIsSaving(false); - return; - } - newConfig = { - GITHUB_PAT: currentPatInForm, - repo_full_names: newSelectedRepos, - }; - if (reposChanged && newSelectedRepos.length === 0) { - toast.warning("Warning: No repositories selected."); - } - } - break; - } - case "SLACK_CONNECTOR": - if (formData.SLACK_BOT_TOKEN !== originalConfig.SLACK_BOT_TOKEN) { - if (!formData.SLACK_BOT_TOKEN) { - toast.error("Slack Token empty."); - setIsSaving(false); - return; - } - newConfig = { SLACK_BOT_TOKEN: formData.SLACK_BOT_TOKEN }; - } - break; - case "NOTION_CONNECTOR": - if (formData.NOTION_INTEGRATION_TOKEN !== originalConfig.NOTION_INTEGRATION_TOKEN) { - if (!formData.NOTION_INTEGRATION_TOKEN) { - toast.error("Notion Token empty."); - setIsSaving(false); - return; - } - newConfig = { - NOTION_INTEGRATION_TOKEN: formData.NOTION_INTEGRATION_TOKEN, - }; - } - break; - case "TAVILY_API": - if (formData.TAVILY_API_KEY !== originalConfig.TAVILY_API_KEY) { - if (!formData.TAVILY_API_KEY) { - toast.error("Tavily Key empty."); - setIsSaving(false); - return; - } - newConfig = { TAVILY_API_KEY: formData.TAVILY_API_KEY }; - } - break; - case "SEARXNG_API": { - const host = (formData.SEARXNG_HOST || "").trim(); - if (!host) { - toast.error("SearxNG host is required."); - setIsSaving(false); - return; - } - - const candidateConfig: Record = { SEARXNG_HOST: host }; - const originalHost = - typeof originalConfig.SEARXNG_HOST === "string" ? originalConfig.SEARXNG_HOST : ""; - let hasChanges = host !== originalHost.trim(); - - const apiKey = (formData.SEARXNG_API_KEY || "").trim(); - const originalApiKey = - typeof originalConfig.SEARXNG_API_KEY === "string" - ? originalConfig.SEARXNG_API_KEY - : ""; - const originalApiKeyTrimmed = originalApiKey.trim(); - if (apiKey !== originalApiKeyTrimmed) { - candidateConfig.SEARXNG_API_KEY = apiKey || null; - hasChanges = true; - } - - const newEngines = normalizeListInput(formData.SEARXNG_ENGINES || ""); - const originalEngines = normalizeListInput(originalConfig.SEARXNG_ENGINES); - if (!arraysEqual(newEngines, originalEngines)) { - candidateConfig.SEARXNG_ENGINES = newEngines; - hasChanges = true; - } - - const newCategories = normalizeListInput(formData.SEARXNG_CATEGORIES || ""); - const originalCategories = normalizeListInput(originalConfig.SEARXNG_CATEGORIES); - if (!arraysEqual(newCategories, originalCategories)) { - candidateConfig.SEARXNG_CATEGORIES = newCategories; - hasChanges = true; - } - - const language = (formData.SEARXNG_LANGUAGE || "").trim(); - const originalLanguage = - typeof originalConfig.SEARXNG_LANGUAGE === "string" - ? originalConfig.SEARXNG_LANGUAGE - : ""; - const originalLanguageTrimmed = originalLanguage.trim(); - if (language !== originalLanguageTrimmed) { - candidateConfig.SEARXNG_LANGUAGE = language || null; - hasChanges = true; - } - - const safesearchRaw = (formData.SEARXNG_SAFESEARCH || "").trim(); - const originalSafesearch = originalConfig.SEARXNG_SAFESEARCH; - if (safesearchRaw) { - const parsed = Number(safesearchRaw); - if (Number.isNaN(parsed) || !Number.isInteger(parsed) || parsed < 0 || parsed > 2) { - toast.error("SearxNG SafeSearch must be 0, 1, or 2."); - setIsSaving(false); - return; - } - if (parsed !== Number(originalSafesearch)) { - candidateConfig.SEARXNG_SAFESEARCH = parsed; - hasChanges = true; - } - } else if (originalSafesearch !== undefined && originalSafesearch !== null) { - candidateConfig.SEARXNG_SAFESEARCH = null; - hasChanges = true; - } - - const verifyRaw = (formData.SEARXNG_VERIFY_SSL || "").trim().toLowerCase(); - const originalVerifyBool = normalizeBoolean(originalConfig.SEARXNG_VERIFY_SSL); - if (verifyRaw) { - let parsedBool: boolean | null = null; - if (["true", "1", "yes", "on"].includes(verifyRaw)) parsedBool = true; - else if (["false", "0", "no", "off"].includes(verifyRaw)) parsedBool = false; - if (parsedBool === null) { - toast.error("SearxNG SSL verification must be true or false."); - setIsSaving(false); - return; - } - if (parsedBool !== originalVerifyBool) { - candidateConfig.SEARXNG_VERIFY_SSL = parsedBool; - hasChanges = true; - } - } else if (originalVerifyBool !== null) { - candidateConfig.SEARXNG_VERIFY_SSL = null; - hasChanges = true; - } - - if (hasChanges) { - newConfig = candidateConfig; - } - break; - } - - case "LINEAR_CONNECTOR": - if (formData.LINEAR_API_KEY !== originalConfig.LINEAR_API_KEY) { - if (!formData.LINEAR_API_KEY) { - toast.error("Linear API Key cannot be empty."); - setIsSaving(false); - return; - } - newConfig = { LINEAR_API_KEY: formData.LINEAR_API_KEY }; - } - break; - case "LINKUP_API": - if (formData.LINKUP_API_KEY !== originalConfig.LINKUP_API_KEY) { - if (!formData.LINKUP_API_KEY) { - toast.error("Linkup API Key cannot be empty."); - setIsSaving(false); - return; - } - newConfig = { LINKUP_API_KEY: formData.LINKUP_API_KEY }; - } - break; - case "DISCORD_CONNECTOR": - if (formData.DISCORD_BOT_TOKEN !== originalConfig.DISCORD_BOT_TOKEN) { - if (!formData.DISCORD_BOT_TOKEN) { - toast.error("Discord Bot Token cannot be empty."); - setIsSaving(false); - return; - } - newConfig = { DISCORD_BOT_TOKEN: formData.DISCORD_BOT_TOKEN }; - } - break; - case "CONFLUENCE_CONNECTOR": - if ( - formData.CONFLUENCE_BASE_URL !== originalConfig.CONFLUENCE_BASE_URL || - formData.CONFLUENCE_EMAIL !== originalConfig.CONFLUENCE_EMAIL || - formData.CONFLUENCE_API_TOKEN !== originalConfig.CONFLUENCE_API_TOKEN - ) { - if ( - !formData.CONFLUENCE_BASE_URL || - !formData.CONFLUENCE_EMAIL || - !formData.CONFLUENCE_API_TOKEN - ) { - toast.error("All Confluence fields are required."); - setIsSaving(false); - return; - } - newConfig = { - CONFLUENCE_BASE_URL: formData.CONFLUENCE_BASE_URL, - CONFLUENCE_EMAIL: formData.CONFLUENCE_EMAIL, - CONFLUENCE_API_TOKEN: formData.CONFLUENCE_API_TOKEN, - }; - } - break; - case "BOOKSTACK_CONNECTOR": - if ( - formData.BOOKSTACK_BASE_URL !== originalConfig.BOOKSTACK_BASE_URL || - formData.BOOKSTACK_TOKEN_ID !== originalConfig.BOOKSTACK_TOKEN_ID || - formData.BOOKSTACK_TOKEN_SECRET !== originalConfig.BOOKSTACK_TOKEN_SECRET - ) { - if ( - !formData.BOOKSTACK_BASE_URL || - !formData.BOOKSTACK_TOKEN_ID || - !formData.BOOKSTACK_TOKEN_SECRET - ) { - toast.error("All BookStack fields are required."); - setIsSaving(false); - return; - } - newConfig = { - BOOKSTACK_BASE_URL: formData.BOOKSTACK_BASE_URL, - BOOKSTACK_TOKEN_ID: formData.BOOKSTACK_TOKEN_ID, - BOOKSTACK_TOKEN_SECRET: formData.BOOKSTACK_TOKEN_SECRET, - }; - } - break; - case "JIRA_CONNECTOR": - if ( - formData.JIRA_BASE_URL !== originalConfig.JIRA_BASE_URL || - formData.JIRA_EMAIL !== originalConfig.JIRA_EMAIL || - formData.JIRA_API_TOKEN !== originalConfig.JIRA_API_TOKEN - ) { - if (!formData.JIRA_BASE_URL || !formData.JIRA_EMAIL || !formData.JIRA_API_TOKEN) { - toast.error("All Jira fields are required."); - setIsSaving(false); - return; - } - newConfig = { - JIRA_BASE_URL: formData.JIRA_BASE_URL, - JIRA_EMAIL: formData.JIRA_EMAIL, - JIRA_API_TOKEN: formData.JIRA_API_TOKEN, - }; - } - break; - case "LUMA_CONNECTOR": - if (formData.LUMA_API_KEY !== originalConfig.LUMA_API_KEY) { - if (!formData.LUMA_API_KEY) { - toast.error("Luma API Key cannot be empty."); - setIsSaving(false); - return; - } - newConfig = { LUMA_API_KEY: formData.LUMA_API_KEY }; - } - break; - case "ELASTICSEARCH_CONNECTOR": - if (formData.ELASTICSEARCH_API_KEY !== originalConfig.ELASTICSEARCH_API_KEY) { - if (!formData.ELASTICSEARCH_API_KEY) { - toast.error("Elasticsearch API Key cannot be empty."); - setIsSaving(false); - return; - } - newConfig = { ELASTICSEARCH_API_KEY: formData.ELASTICSEARCH_API_KEY }; - } - break; - case "WEBCRAWLER_CONNECTOR": - if ( - formData.FIRECRAWL_API_KEY !== originalConfig.FIRECRAWL_API_KEY || - formData.INITIAL_URLS !== originalConfig.INITIAL_URLS - ) { - newConfig = {}; - - if (formData.FIRECRAWL_API_KEY?.trim()) { - if (!formData.FIRECRAWL_API_KEY.startsWith("fc-")) { - toast.warning( - "Firecrawl API keys typically start with 'fc-'. Please verify your key." - ); - } - newConfig.FIRECRAWL_API_KEY = formData.FIRECRAWL_API_KEY.trim(); - } else if (originalConfig.FIRECRAWL_API_KEY) { - toast.info( - "Firecrawl API key removed. Web crawler will use AsyncChromiumLoader as fallback." - ); - } - - if (formData.INITIAL_URLS !== undefined) { - if (formData.INITIAL_URLS?.trim()) { - newConfig.INITIAL_URLS = formData.INITIAL_URLS.trim(); - } else if (originalConfig.INITIAL_URLS) { - toast.info("URLs removed from crawler configuration."); - } - } - } - break; - } - - if (newConfig !== null) { - updatePayload.config = newConfig; - configChanged = true; - } - - if (Object.keys(updatePayload).length === 0) { - toast.info("No changes detected."); - setIsSaving(false); - if (connector.connector_type === "GITHUB_CONNECTOR") { - setEditMode("viewing"); - patForm.reset({ github_pat: originalPat }); - } - return; - } - - try { - const updatedConnector = (await updateConnector({ - id: connectorId, - data: { - ...updatePayload, - connector_type: connector.connector_type as EnumConnectorName, - }, - })) as UpdateConnectorResponse; - toast.success("Connector updated!"); - // Use the response from the API which has the full merged config - const newlySavedConfig = updatedConnector.config || originalConfig; - setOriginalConfig(newlySavedConfig); - // Update connector state with the full updated connector from the API - setConnector(updatedConnector); - if (configChanged) { - if (connector.connector_type === "GITHUB_CONNECTOR") { - const savedGitHubConfig = newlySavedConfig as { - GITHUB_PAT?: string; - repo_full_names?: string[]; - }; - setCurrentSelectedRepos(savedGitHubConfig.repo_full_names || []); - setOriginalPat(savedGitHubConfig.GITHUB_PAT || ""); - setNewSelectedRepos(savedGitHubConfig.repo_full_names || []); - patForm.reset({ github_pat: savedGitHubConfig.GITHUB_PAT || "" }); - } else if (connector.connector_type === "SLACK_CONNECTOR") { - editForm.setValue("SLACK_BOT_TOKEN", newlySavedConfig.SLACK_BOT_TOKEN || ""); - } else if (connector.connector_type === "NOTION_CONNECTOR") { - editForm.setValue( - "NOTION_INTEGRATION_TOKEN", - newlySavedConfig.NOTION_INTEGRATION_TOKEN || "" - ); - } else if (connector.connector_type === "TAVILY_API") { - editForm.setValue("TAVILY_API_KEY", newlySavedConfig.TAVILY_API_KEY || ""); - } else if (connector.connector_type === "SEARXNG_API") { - editForm.setValue("SEARXNG_HOST", newlySavedConfig.SEARXNG_HOST || ""); - editForm.setValue("SEARXNG_API_KEY", newlySavedConfig.SEARXNG_API_KEY || ""); - editForm.setValue( - "SEARXNG_ENGINES", - normalizeListInput(newlySavedConfig.SEARXNG_ENGINES).join(", ") - ); - editForm.setValue( - "SEARXNG_CATEGORIES", - normalizeListInput(newlySavedConfig.SEARXNG_CATEGORIES).join(", ") - ); - editForm.setValue("SEARXNG_LANGUAGE", newlySavedConfig.SEARXNG_LANGUAGE || ""); - editForm.setValue( - "SEARXNG_SAFESEARCH", - newlySavedConfig.SEARXNG_SAFESEARCH === null || - newlySavedConfig.SEARXNG_SAFESEARCH === undefined - ? "" - : String(newlySavedConfig.SEARXNG_SAFESEARCH) - ); - const verifyValue = normalizeBoolean(newlySavedConfig.SEARXNG_VERIFY_SSL); - editForm.setValue( - "SEARXNG_VERIFY_SSL", - verifyValue === null ? "" : String(verifyValue) - ); - } else if (connector.connector_type === "LINEAR_CONNECTOR") { - editForm.setValue("LINEAR_API_KEY", newlySavedConfig.LINEAR_API_KEY || ""); - } else if (connector.connector_type === "LINKUP_API") { - editForm.setValue("LINKUP_API_KEY", newlySavedConfig.LINKUP_API_KEY || ""); - } else if (connector.connector_type === "DISCORD_CONNECTOR") { - editForm.setValue("DISCORD_BOT_TOKEN", newlySavedConfig.DISCORD_BOT_TOKEN || ""); - } else if (connector.connector_type === "CONFLUENCE_CONNECTOR") { - editForm.setValue("CONFLUENCE_BASE_URL", newlySavedConfig.CONFLUENCE_BASE_URL || ""); - editForm.setValue("CONFLUENCE_EMAIL", newlySavedConfig.CONFLUENCE_EMAIL || ""); - editForm.setValue("CONFLUENCE_API_TOKEN", newlySavedConfig.CONFLUENCE_API_TOKEN || ""); - } else if (connector.connector_type === "BOOKSTACK_CONNECTOR") { - editForm.setValue("BOOKSTACK_BASE_URL", newlySavedConfig.BOOKSTACK_BASE_URL || ""); - editForm.setValue("BOOKSTACK_TOKEN_ID", newlySavedConfig.BOOKSTACK_TOKEN_ID || ""); - editForm.setValue( - "BOOKSTACK_TOKEN_SECRET", - newlySavedConfig.BOOKSTACK_TOKEN_SECRET || "" - ); - } else if (connector.connector_type === "JIRA_CONNECTOR") { - editForm.setValue("JIRA_BASE_URL", newlySavedConfig.JIRA_BASE_URL || ""); - editForm.setValue("JIRA_EMAIL", newlySavedConfig.JIRA_EMAIL || ""); - editForm.setValue("JIRA_API_TOKEN", newlySavedConfig.JIRA_API_TOKEN || ""); - } else if (connector.connector_type === "LUMA_CONNECTOR") { - editForm.setValue("LUMA_API_KEY", newlySavedConfig.LUMA_API_KEY || ""); - } else if (connector.connector_type === "ELASTICSEARCH_CONNECTOR") { - editForm.setValue( - "ELASTICSEARCH_API_KEY", - newlySavedConfig.ELASTICSEARCH_API_KEY || "" - ); - } else if (connector.connector_type === "WEBCRAWLER_CONNECTOR") { - editForm.setValue("FIRECRAWL_API_KEY", newlySavedConfig.FIRECRAWL_API_KEY || ""); - editForm.setValue("INITIAL_URLS", newlySavedConfig.INITIAL_URLS || ""); - } - } - if (connector.connector_type === "GITHUB_CONNECTOR") { - setEditMode("viewing"); - setFetchedRepos(null); - } - // Resetting simple form values is handled by useEffect if connector state updates - } catch (error) { - console.error("Error updating connector:", error); - toast.error(error instanceof Error ? error.message : "Failed to update connector."); - } finally { - setIsSaving(false); - } - }, - [ - connector, - originalConfig, - updateConnector, - connectorId, - patForm, - originalPat, - currentSelectedRepos, - newSelectedRepos, - editMode, - fetchedRepos, - editForm, - ] - ); // Added editForm to dependencies - - // Return values needed by the component - return { - connectorsLoading, - connector, - isSaving, - editForm, - patForm, - handleSaveChanges, - // GitHub specific props - editMode, - setEditMode, - originalPat, - currentSelectedRepos, - fetchedRepos, - setFetchedRepos, - newSelectedRepos, - setNewSelectedRepos, - isFetchingRepos, - handleFetchRepositories, - handleRepoSelectionChange, - }; -} diff --git a/surfsense_web/lib/auth-utils.ts b/surfsense_web/lib/auth-utils.ts index c1dc7194b..604843292 100644 --- a/surfsense_web/lib/auth-utils.ts +++ b/surfsense_web/lib/auth-utils.ts @@ -130,44 +130,3 @@ export async function authenticatedFetch( return response; } - -/** - * Type for the result of a fetch operation with built-in error handling - */ -export type FetchResult = - | { success: true; data: T; response: Response } - | { success: false; error: string; status?: number }; - -/** - * Authenticated fetch with JSON response handling - * Returns a result object instead of throwing on non-401 errors - */ -export async function authenticatedFetchJson( - url: string, - options?: RequestInit & { skipAuthRedirect?: boolean } -): Promise> { - try { - const response = await authenticatedFetch(url, options); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - return { - success: false, - error: errorData.detail || `Request failed: ${response.status}`, - status: response.status, - }; - } - - const data = await response.json(); - return { success: true, data, response }; - } catch (err: any) { - // Re-throw if it's the unauthorized redirect - if (err.message?.includes("Unauthorized")) { - throw err; - } - return { - success: false, - error: err.message || "Request failed", - }; - } -} diff --git a/surfsense_web/lib/connectors/utils.ts b/surfsense_web/lib/connectors/utils.ts index 75e81e2cc..a85b912ed 100644 --- a/surfsense_web/lib/connectors/utils.ts +++ b/surfsense_web/lib/connectors/utils.ts @@ -15,6 +15,7 @@ export const getConnectorTypeDisplay = (type: string): string => { CLICKUP_CONNECTOR: "ClickUp", GOOGLE_CALENDAR_CONNECTOR: "Google Calendar", GOOGLE_GMAIL_CONNECTOR: "Google Gmail", + GOOGLE_DRIVE_CONNECTOR: "Google Drive", AIRTABLE_CONNECTOR: "Airtable", LUMA_CONNECTOR: "Luma", ELASTICSEARCH_CONNECTOR: "Elasticsearch", diff --git a/surfsense_web/lib/utils.ts b/surfsense_web/lib/utils.ts index 1e29bb9a4..212ff1259 100644 --- a/surfsense_web/lib/utils.ts +++ b/surfsense_web/lib/utils.ts @@ -1,4 +1,3 @@ -import type { Message } from "@ai-sdk/react"; import { type ClassValue, clsx } from "clsx"; import { twMerge } from "tailwind-merge"; @@ -6,12 +5,6 @@ export function cn(...inputs: ClassValue[]) { return twMerge(clsx(inputs)); } -export function getChatTitleFromMessages(messages: Message[]) { - const userMessages = messages.filter((msg) => msg.role === "user"); - if (userMessages.length === 0) return "Untitled Chat"; - return userMessages[0].content; -} - export const formatDate = (date: Date): string => { return date.toLocaleDateString("en-US", { year: "numeric", diff --git a/surfsense_web/mdx-components.tsx b/surfsense_web/mdx-components.tsx index cea62b834..9dedbd20f 100644 --- a/surfsense_web/mdx-components.tsx +++ b/surfsense_web/mdx-components.tsx @@ -1,5 +1,6 @@ import defaultMdxComponents from "fumadocs-ui/mdx"; import type { MDXComponents } from "mdx/types"; +import Image, { type ImageProps } from "next/image"; import { Accordion, AccordionContent, @@ -11,8 +12,12 @@ import { cn } from "@/lib/utils"; export function getMDXComponents(components?: MDXComponents): MDXComponents { return { ...defaultMdxComponents, - img: ({ className, ...props }: React.ComponentProps<"img">) => ( - + img: ({ className, alt, ...props }: React.ComponentProps<"img">) => ( + {alt ), Video: ({ className, ...props }: React.ComponentProps<"video">) => (