diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 2cacedc21..d2c667178 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -44,6 +44,12 @@ AIRTABLE_CLIENT_ID=your_airtable_client_id AIRTABLE_CLIENT_SECRET=your_airtable_client_secret AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback +# Discord OAuth Configuration +DISCORD_CLIENT_ID=your_discord_client_id_here +DISCORD_CLIENT_SECRET=your_discord_client_secret_here +DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback +DISCORD_BOT_TOKEN=your_bot_token_from_developer_portal + # OAuth for Linear Connector LINEAR_CLIENT_ID=your_linear_client_id LINEAR_CLIENT_SECRET=your_linear_client_secret diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index f69d1c1a3..f65a94cc0 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -105,6 +105,12 @@ class Config: SLACK_CLIENT_SECRET = os.getenv("SLACK_CLIENT_SECRET") SLACK_REDIRECT_URI = os.getenv("SLACK_REDIRECT_URI") + # Discord OAuth + DISCORD_CLIENT_ID = os.getenv("DISCORD_CLIENT_ID") + DISCORD_CLIENT_SECRET = os.getenv("DISCORD_CLIENT_SECRET") + DISCORD_REDIRECT_URI = os.getenv("DISCORD_REDIRECT_URI") + DISCORD_BOT_TOKEN = os.getenv("DISCORD_BOT_TOKEN") + # LLM instances are now managed per-user through the LLMConfig system # Legacy environment variables removed in favor of user-specific configurations diff --git a/surfsense_backend/app/connectors/discord_connector.py b/surfsense_backend/app/connectors/discord_connector.py index 506b463a5..1e12cb9a4 100644 --- a/surfsense_backend/app/connectors/discord_connector.py +++ b/surfsense_backend/app/connectors/discord_connector.py @@ -3,7 +3,7 @@ Discord Connector A module for interacting with Discord's HTTP API to retrieve guilds, channels, and message history. -Requires a Discord bot token. +Supports both direct bot token and OAuth-based authentication with token refresh. """ import asyncio @@ -12,6 +12,14 @@ import logging import discord from discord.ext import commands +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.db import SearchSourceConnector +from app.routes.discord_add_connector_route import refresh_discord_token +from app.schemas.discord_auth_credentials import DiscordAuthCredentialsBase +from app.utils.oauth_security import TokenEncryption logger = logging.getLogger(__name__) @@ -19,12 +27,21 @@ logger = logging.getLogger(__name__) class DiscordConnector(commands.Bot): """Class for retrieving guild, channel, and message history from Discord.""" - def __init__(self, token: str | None = None): + def __init__( + self, + token: str | None = None, + session: AsyncSession | None = None, + connector_id: int | None = None, + credentials: DiscordAuthCredentialsBase | None = None, + ): """ - Initialize the DiscordConnector with a bot token. + Initialize the DiscordConnector with a bot token or OAuth credentials. Args: - token (str): The Discord bot token. + token: Discord bot token (optional, for backward compatibility) + session: Database session for token refresh (optional) + connector_id: Connector ID for token refresh (optional) + credentials: Discord OAuth credentials (optional, will be loaded from DB if not provided) """ intents = discord.Intents.default() intents.guilds = True # Required to fetch guilds and channels @@ -34,7 +51,14 @@ class DiscordConnector(commands.Bot): super().__init__( command_prefix="!", intents=intents ) # command_prefix is required but not strictly used here - self.token = token + self._session = session + self._connector_id = connector_id + self._credentials = credentials + # For backward compatibility, if token is provided directly, use it + if token: + self.token = token + else: + self.token = None self._bot_task = None # Holds the async bot task self._is_running = False # Flag to track if the bot is running @@ -57,12 +81,143 @@ class DiscordConnector(commands.Bot): async def on_resumed(): logger.debug("Bot resumed connection to Discord gateway.") + async def _get_valid_token(self) -> str: + """ + Get valid Discord bot token, refreshing if needed. + + Returns: + Valid bot token + + Raises: + ValueError: If credentials are missing or invalid + Exception: If token refresh fails + """ + # If we have a direct token (backward compatibility), use it + if ( + self.token + and self._session is None + and self._connector_id is None + and self._credentials is None + ): + # This means it was initialized with a direct token, use it + return self.token + + # Load credentials from DB if not provided + if self._credentials is None: + if not self._session or not self._connector_id: + raise ValueError( + "Cannot load credentials: session and connector_id required" + ) + + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise ValueError(f"Connector {self._connector_id} not found") + + config_data = connector.config.copy() + + # Decrypt credentials if they are encrypted + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt sensitive fields + if config_data.get("bot_token"): + config_data["bot_token"] = token_encryption.decrypt_token( + config_data["bot_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + logger.info( + f"Decrypted Discord credentials for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to decrypt Discord credentials for connector {self._connector_id}: {e!s}" + ) + raise ValueError( + f"Failed to decrypt Discord credentials: {e!s}" + ) from e + + try: + self._credentials = DiscordAuthCredentialsBase.from_dict(config_data) + except Exception as e: + raise ValueError(f"Invalid Discord credentials: {e!s}") from e + + # Check if token is expired and refreshable + if self._credentials.is_expired and self._credentials.is_refreshable: + try: + logger.info( + f"Discord token expired for connector {self._connector_id}, refreshing..." + ) + + # Get connector for refresh + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + connector = result.scalars().first() + + if not connector: + raise RuntimeError( + f"Connector {self._connector_id} not found; cannot refresh token." + ) + + # Refresh token + connector = await refresh_discord_token(self._session, connector) + + # Reload credentials after refresh + config_data = connector.config.copy() + token_encrypted = config_data.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY: + token_encryption = TokenEncryption(config.SECRET_KEY) + if config_data.get("bot_token"): + config_data["bot_token"] = token_encryption.decrypt_token( + config_data["bot_token"] + ) + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + + self._credentials = DiscordAuthCredentialsBase.from_dict(config_data) + + logger.info( + f"Successfully refreshed Discord token for connector {self._connector_id}" + ) + except Exception as e: + logger.error( + f"Failed to refresh Discord token for connector {self._connector_id}: {e!s}" + ) + raise Exception( + f"Failed to refresh Discord OAuth credentials: {e!s}" + ) from e + + return self._credentials.bot_token + async def start_bot(self): """Starts the bot to connect to Discord.""" logger.info("Starting Discord bot...") + # Get valid token (with auto-refresh if using OAuth) if not self.token: - raise ValueError("Discord bot token not set. Call set_token(token) first.") + # Try to get token from credentials + try: + self.token = await self._get_valid_token() + except ValueError as e: + raise ValueError( + f"Discord bot token not set. {e!s} Please authenticate via OAuth or provide a token." + ) from e try: if self._is_running: @@ -107,7 +262,7 @@ class DiscordConnector(commands.Bot): def set_token(self, token: str) -> None: """ - Set the discord bot token. + Set the discord bot token (for backward compatibility). Args: token (str): The Discord bot token. diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 05020deff..b35d743e0 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -27,6 +27,7 @@ from .rbac_routes import router as rbac_router from .search_source_connectors_routes import router as search_source_connectors_router from .search_spaces_routes import router as search_spaces_router from .slack_add_connector_route import router as slack_add_connector_router +from .discord_add_connector_route import router as discord_add_connector_router router = APIRouter() @@ -46,6 +47,7 @@ router.include_router(linear_add_connector_router) router.include_router(luma_add_connector_router) router.include_router(notion_add_connector_router) router.include_router(slack_add_connector_router) +router.include_router(discord_add_connector_router) router.include_router(new_llm_config_router) # LLM configs with prompt configuration router.include_router(logs_router) router.include_router(circleback_webhook_router) # Circleback meeting webhooks diff --git a/surfsense_backend/app/routes/discord_add_connector_route.py b/surfsense_backend/app/routes/discord_add_connector_route.py new file mode 100644 index 000000000..70a0046a3 --- /dev/null +++ b/surfsense_backend/app/routes/discord_add_connector_route.py @@ -0,0 +1,509 @@ +""" +Discord Connector OAuth Routes. + +Handles OAuth 2.0 authentication flow for Discord connector. +""" + +import logging +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.db import ( + SearchSourceConnector, + SearchSourceConnectorType, + User, + get_async_session, +) +from app.schemas.discord_auth_credentials import DiscordAuthCredentialsBase +from app.users import current_active_user +from app.utils.oauth_security import OAuthStateManager, TokenEncryption + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# Discord OAuth endpoints +AUTHORIZATION_URL = "https://discord.com/api/oauth2/authorize" +TOKEN_URL = "https://discord.com/api/oauth2/token" + +# OAuth scopes for Discord (Bot Token) +SCOPES = [ + "bot", # Basic bot scope + "guilds", # Access to guild information + "guilds.members.read", # Read member information +] + +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + +@router.get("/auth/discord/connector/add") +async def connect_discord(space_id: int, user: User = Depends(current_active_user)): + """ + Initiate Discord OAuth flow. + + Args: + space_id: The search space ID + user: Current authenticated user + + Returns: + Authorization URL for redirect + """ + try: + if not space_id: + raise HTTPException(status_code=400, detail="space_id is required") + + if not config.DISCORD_CLIENT_ID: + raise HTTPException(status_code=500, detail="Discord OAuth not configured.") + + if not config.DISCORD_BOT_TOKEN: + raise HTTPException( + status_code=500, + detail="Discord bot token not configured. Please set DISCORD_BOT_TOKEN in backend configuration.", + ) + + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) + + # Build authorization URL + from urllib.parse import urlencode + + auth_params = { + "client_id": config.DISCORD_CLIENT_ID, + "scope": " ".join(SCOPES), # Discord uses space-separated scopes + "redirect_uri": config.DISCORD_REDIRECT_URI, + "response_type": "code", + "state": state_encoded, + } + + auth_url = f"{AUTHORIZATION_URL}?{urlencode(auth_params)}" + + logger.info(f"Generated Discord OAuth URL for user {user.id}, space {space_id}") + return {"auth_url": auth_url} + + except Exception as e: + logger.error(f"Failed to initiate Discord OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to initiate Discord OAuth: {e!s}" + ) from e + + +@router.get("/auth/discord/connector/callback") +async def discord_callback( + request: Request, + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +): + """ + Handle Discord OAuth callback. + + Args: + request: FastAPI request object + code: Authorization code from Discord (if user granted access) + error: Error code from Discord (if user denied access or error occurred) + state: State parameter containing user/space info + session: Database session + + Returns: + Redirect response to frontend + """ + try: + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Discord OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=discord_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=discord_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e + + user_id = UUID(data["user_id"]) + space_id = data["space_id"] + + # Validate redirect URI (security: ensure it matches configured value) + if not config.DISCORD_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="DISCORD_REDIRECT_URI not configured" + ) + + # Exchange authorization code for access token + token_data = { + "client_id": config.DISCORD_CLIENT_ID, + "client_secret": config.DISCORD_CLIENT_SECRET, + "grant_type": "authorization_code", + "code": code, + "redirect_uri": config.DISCORD_REDIRECT_URI, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=token_data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error_description", error_json.get("error", error_detail)) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token exchange failed: {error_detail}" + ) + + token_json = token_response.json() + + # Log OAuth response for debugging (without sensitive data) + logger.info(f"Discord OAuth response received. Keys: {list(token_json.keys())}") + + # Discord OAuth with 'bot' scope returns access_token (user token), not bot token + # The bot token must come from backend config (DISCORD_BOT_TOKEN) + # OAuth is used to authorize bot installation to servers, not to get bot token + if not config.DISCORD_BOT_TOKEN: + raise HTTPException( + status_code=500, + detail="Discord bot token not configured. Please set DISCORD_BOT_TOKEN in backend configuration.", + ) + + # Use the bot token from backend config (not the OAuth access_token) + bot_token = config.DISCORD_BOT_TOKEN + + # Extract OAuth access_token and refresh_token (for reference, not used for bot operations) + oauth_access_token = token_json.get("access_token") + refresh_token = token_json.get("refresh_token") + + # Encrypt sensitive tokens before storing + token_encryption = get_token_encryption() + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + if token_json.get("expires_in"): + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(token_json["expires_in"])) + + # Extract guild info from OAuth response if available + guild_id = None + guild_name = None + if token_json.get("guild"): + guild_id = token_json["guild"].get("id") + guild_name = token_json["guild"].get("name") + + # Store the bot token from config and OAuth metadata + connector_config = { + "bot_token": token_encryption.encrypt_token(bot_token), # Use bot token from config + "oauth_access_token": token_encryption.encrypt_token(oauth_access_token) + if oauth_access_token + else None, # Store OAuth token for reference + "refresh_token": token_encryption.encrypt_token(refresh_token) + if refresh_token + else None, + "token_type": token_json.get("token_type", "Bearer"), + "expires_in": token_json.get("expires_in"), + "expires_at": expires_at.isoformat() if expires_at else None, + "scope": token_json.get("scope"), + "guild_id": guild_id, + "guild_name": guild_name, + # Mark that tokens are encrypted for backward compatibility + "_token_encrypted": True, + } + + # Check if connector already exists for this search space and user + existing_connector_result = await session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == space_id, + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.DISCORD_CONNECTOR, + ) + ) + existing_connector = existing_connector_result.scalars().first() + + if existing_connector: + # Update existing connector + existing_connector.config = connector_config + existing_connector.name = "Discord Connector" + existing_connector.is_indexable = True + logger.info( + f"Updated existing Discord connector for user {user_id} in space {space_id}" + ) + else: + # Create new connector + new_connector = SearchSourceConnector( + name="Discord Connector", + connector_type=SearchSourceConnectorType.DISCORD_CONNECTOR, + is_indexable=True, + config=connector_config, + search_space_id=space_id, + user_id=user_id, + ) + session.add(new_connector) + logger.info( + f"Created new Discord connector for user {user_id} in space {space_id}" + ) + + try: + await session.commit() + logger.info(f"Successfully saved Discord connector for user {user_id}") + + # Redirect to the frontend with success params + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=discord-connector" + ) + + except ValidationError as e: + await session.rollback() + raise HTTPException( + status_code=422, detail=f"Validation error: {e!s}" + ) from e + except IntegrityError as e: + await session.rollback() + raise HTTPException( + status_code=409, + detail=f"Integrity error: A connector with this type already exists. {e!s}", + ) from e + except Exception as e: + logger.error(f"Failed to create search source connector: {e!s}") + await session.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to create search source connector: {e!s}", + ) from e + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to complete Discord OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to complete Discord OAuth: {e!s}" + ) from e + + +async def refresh_discord_token( + session: AsyncSession, connector: SearchSourceConnector +) -> SearchSourceConnector: + """ + Refresh the Discord OAuth tokens for a connector. + + Note: Bot tokens from config don't expire, but OAuth access tokens might. + This function refreshes OAuth tokens if needed, but always uses bot token from config. + + Args: + session: Database session + connector: Discord connector to refresh + + Returns: + Updated connector object + """ + try: + logger.info(f"Refreshing Discord OAuth tokens for connector {connector.id}") + + # Bot token always comes from config, not from OAuth + if not config.DISCORD_BOT_TOKEN: + raise HTTPException( + status_code=500, + detail="Discord bot token not configured. Please set DISCORD_BOT_TOKEN in backend configuration.", + ) + + credentials = DiscordAuthCredentialsBase.from_dict(connector.config) + + # Decrypt tokens if they are encrypted + token_encryption = get_token_encryption() + is_encrypted = connector.config.get("_token_encrypted", False) + + refresh_token = credentials.refresh_token + if is_encrypted and refresh_token: + try: + refresh_token = token_encryption.decrypt_token(refresh_token) + except Exception as e: + logger.error(f"Failed to decrypt refresh token: {e!s}") + raise HTTPException( + status_code=500, detail="Failed to decrypt stored refresh token" + ) from e + + # If no refresh token, bot token from config is still valid (bot tokens don't expire) + # Just update the bot token from config in case it was changed + if not refresh_token: + logger.info( + f"No refresh token available for connector {connector.id}. Using bot token from config." + ) + # Update bot token from config (in case it was changed) + credentials.bot_token = token_encryption.encrypt_token(config.DISCORD_BOT_TOKEN) + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + return connector + + # Discord uses oauth2/token for token refresh with grant_type=refresh_token + refresh_data = { + "client_id": config.DISCORD_CLIENT_ID, + "client_secret": config.DISCORD_CLIENT_SECRET, + "grant_type": "refresh_token", + "refresh_token": refresh_token, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=refresh_data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error_description", error_json.get("error", error_detail)) + except Exception: + pass + # If refresh fails, bot token from config is still valid + logger.warning( + f"OAuth token refresh failed for connector {connector.id}: {error_detail}. " + "Using bot token from config." + ) + # Update bot token from config + credentials.bot_token = token_encryption.encrypt_token(config.DISCORD_BOT_TOKEN) + credentials.refresh_token = None # Clear invalid refresh token + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + return connector + + token_json = token_response.json() + + # Extract OAuth access token from refresh response (for reference) + oauth_access_token = token_json.get("access_token") + + # Get new refresh token if provided (Discord may rotate refresh tokens) + new_refresh_token = token_json.get("refresh_token") + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Always use bot token from config (bot tokens don't expire) + credentials.bot_token = token_encryption.encrypt_token(config.DISCORD_BOT_TOKEN) + + # Update OAuth tokens if available + if oauth_access_token: + # Store OAuth access token for reference + connector.config["oauth_access_token"] = token_encryption.encrypt_token( + oauth_access_token + ) + if new_refresh_token: + credentials.refresh_token = token_encryption.encrypt_token( + new_refresh_token + ) + credentials.expires_in = expires_in + credentials.expires_at = expires_at + credentials.scope = token_json.get("scope") + + # Preserve guild info if present + if not credentials.guild_id: + credentials.guild_id = connector.config.get("guild_id") + if not credentials.guild_name: + credentials.guild_name = connector.config.get("guild_name") + if not credentials.bot_user_id: + credentials.bot_user_id = connector.config.get("bot_user_id") + + # Update connector config with encrypted tokens + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + + logger.info(f"Successfully refreshed Discord OAuth tokens for connector {connector.id}") + + return connector + except HTTPException: + raise + except Exception as e: + logger.error( + f"Failed to refresh Discord tokens for connector {connector.id}: {e!s}", + exc_info=True, + ) + raise HTTPException( + status_code=500, detail=f"Failed to refresh Discord tokens: {e!s}" + ) from e + diff --git a/surfsense_backend/app/schemas/discord_auth_credentials.py b/surfsense_backend/app/schemas/discord_auth_credentials.py new file mode 100644 index 000000000..0c18a7554 --- /dev/null +++ b/surfsense_backend/app/schemas/discord_auth_credentials.py @@ -0,0 +1,76 @@ +from datetime import UTC, datetime + +from pydantic import BaseModel, field_validator + + +class DiscordAuthCredentialsBase(BaseModel): + bot_token: str + refresh_token: str | None = None + token_type: str = "Bearer" + expires_in: int | None = None + expires_at: datetime | None = None + scope: str | None = None + bot_user_id: str | None = None + guild_id: str | None = None + guild_name: str | None = None + + @property + def is_expired(self) -> bool: + """Check if the credentials have expired.""" + if self.expires_at is None: + return False # Long-lived token, treat as not expired + return self.expires_at <= datetime.now(UTC) + + @property + def is_refreshable(self) -> bool: + """Check if the credentials can be refreshed.""" + return self.refresh_token is not None + + def to_dict(self) -> dict: + """Convert credentials to dictionary for storage.""" + return { + "bot_token": self.bot_token, + "refresh_token": self.refresh_token, + "token_type": self.token_type, + "expires_in": self.expires_in, + "expires_at": self.expires_at.isoformat() if self.expires_at else None, + "scope": self.scope, + "bot_user_id": self.bot_user_id, + "guild_id": self.guild_id, + "guild_name": self.guild_name, + } + + @classmethod + def from_dict(cls, data: dict) -> "DiscordAuthCredentialsBase": + """Create credentials from dictionary.""" + expires_at = None + if data.get("expires_at"): + expires_at = datetime.fromisoformat(data["expires_at"]) + + return cls( + bot_token=data.get("bot_token", ""), + refresh_token=data.get("refresh_token"), + token_type=data.get("token_type", "Bearer"), + expires_in=data.get("expires_in"), + expires_at=expires_at, + scope=data.get("scope"), + bot_user_id=data.get("bot_user_id"), + guild_id=data.get("guild_id"), + guild_name=data.get("guild_name"), + ) + + @field_validator("expires_at", mode="before") + @classmethod + def ensure_aware_utc(cls, v): + # Strings like "2025-08-26T14:46:57.367184" + if isinstance(v, str): + # add +00:00 if missing tz info + if v.endswith("Z"): + return datetime.fromisoformat(v.replace("Z", "+00:00")) + dt = datetime.fromisoformat(v) + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + # datetime objects + if isinstance(v, datetime): + return v if v.tzinfo else v.replace(tzinfo=UTC) + return v + diff --git a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py index 9391be788..b3de1f4b5 100644 --- a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py @@ -8,6 +8,7 @@ from datetime import UTC, datetime, timedelta from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession +from app.config import config from app.connectors.discord_connector import DiscordConnector from app.db import Document, DocumentType, SearchSourceConnectorType from app.services.llm_service import get_user_long_context_llm @@ -69,6 +70,12 @@ async def index_discord_messages( ) try: + # Normalize date parameters - handle 'undefined' strings from frontend + if start_date and (start_date.lower() == "undefined" or start_date.strip() == ""): + start_date = None + if end_date and (end_date.lower() == "undefined" or end_date.strip() == ""): + end_date = None + # Get the connector await task_logger.log_task_progress( log_entry, @@ -92,27 +99,54 @@ async def index_discord_messages( f"Connector with ID {connector_id} not found or is not a Discord connector", ) - # Get the Discord token from the connector config - discord_token = connector.config.get("DISCORD_BOT_TOKEN") - if not discord_token: - await task_logger.log_task_failure( - log_entry, - f"Discord token not found in connector config for connector {connector_id}", - "Missing Discord token", - {"error_type": "MissingToken"}, - ) - return 0, "Discord token not found in connector config" - logger.info(f"Starting Discord indexing for connector {connector_id}") - # Initialize Discord client + # Initialize Discord client with OAuth credentials support await task_logger.log_task_progress( log_entry, f"Initializing Discord client for connector {connector_id}", {"stage": "client_initialization"}, ) - discord_client = DiscordConnector(token=discord_token) + # Check if using OAuth (has bot_token in config) or legacy (has DISCORD_BOT_TOKEN) + has_oauth = connector.config.get("bot_token") is not None + has_legacy = connector.config.get("DISCORD_BOT_TOKEN") is not None + + if has_oauth: + # Use OAuth credentials with auto-refresh + discord_client = DiscordConnector( + session=session, connector_id=connector_id + ) + elif has_legacy: + # Backward compatibility: use legacy token format + discord_token = connector.config.get("DISCORD_BOT_TOKEN") + + # Decrypt token if it's encrypted (legacy tokens might be encrypted) + token_encrypted = connector.config.get("_token_encrypted", False) + if token_encrypted and config.SECRET_KEY and discord_token: + try: + from app.utils.oauth_security import TokenEncryption + token_encryption = TokenEncryption(config.SECRET_KEY) + discord_token = token_encryption.decrypt_token(discord_token) + logger.info( + f"Decrypted legacy Discord token for connector {connector_id}" + ) + except Exception as e: + logger.warning( + f"Failed to decrypt legacy Discord token for connector {connector_id}: {e!s}. " + "Trying to use token as-is (might be unencrypted)." + ) + # Continue with token as-is - might be unencrypted legacy token + + discord_client = DiscordConnector(token=discord_token) + else: + await task_logger.log_task_failure( + log_entry, + f"Discord credentials not found in connector config for connector {connector_id}", + "Missing Discord credentials", + {"error_type": "MissingCredentials"}, + ) + return 0, "Discord credentials not found in connector config" # Calculate date range if start_date is None or end_date is None: @@ -135,32 +169,63 @@ async def index_discord_messages( if start_date is None: start_date_iso = calculated_start_date.isoformat() else: - # Convert YYYY-MM-DD to ISO format + # Validate and convert YYYY-MM-DD to ISO format + try: + start_date_iso = ( + datetime.strptime(start_date, "%Y-%m-%d") + .replace(tzinfo=UTC) + .isoformat() + ) + except ValueError as e: + logger.warning( + f"Invalid start_date format '{start_date}', using calculated start date: {e!s}" + ) + start_date_iso = calculated_start_date.isoformat() + + if end_date is None: + end_date_iso = calculated_end_date.isoformat() + else: + # Validate and convert YYYY-MM-DD to ISO format + try: + end_date_iso = ( + datetime.strptime(end_date, "%Y-%m-%d") + .replace(tzinfo=UTC) + .isoformat() + ) + except ValueError as e: + logger.warning( + f"Invalid end_date format '{end_date}', using calculated end date: {e!s}" + ) + end_date_iso = calculated_end_date.isoformat() + else: + # Convert provided dates to ISO format for Discord API + try: start_date_iso = ( datetime.strptime(start_date, "%Y-%m-%d") .replace(tzinfo=UTC) .isoformat() ) - - if end_date is None: - end_date_iso = calculated_end_date.isoformat() - else: - # Convert YYYY-MM-DD to ISO format - end_date_iso = ( - datetime.strptime(end_date, "%Y-%m-%d") - .replace(tzinfo=UTC) - .isoformat() + except ValueError as e: + await task_logger.log_task_failure( + log_entry, + f"Invalid start_date format: {start_date}", + f"Date parsing error: {e!s}", + {"error_type": "InvalidDateFormat", "start_date": start_date}, ) - else: - # Convert provided dates to ISO format for Discord API - start_date_iso = ( - datetime.strptime(start_date, "%Y-%m-%d") - .replace(tzinfo=UTC) - .isoformat() - ) - end_date_iso = ( - datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC).isoformat() - ) + return 0, f"Invalid start_date format: {start_date}. Expected YYYY-MM-DD format." + + try: + end_date_iso = ( + datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC).isoformat() + ) + except ValueError as e: + await task_logger.log_task_failure( + log_entry, + f"Invalid end_date format: {end_date}", + f"Date parsing error: {e!s}", + {"error_type": "InvalidDateFormat", "end_date": end_date}, + ) + return 0, f"Invalid end_date format: {end_date}. Expected YYYY-MM-DD format." logger.info( f"Indexing Discord messages from {start_date_iso} to {end_date_iso}" diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index 8db6ed4a3..f1620c0e5 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -537,7 +537,7 @@ def validate_connector_config( ) }, }, - "DISCORD_CONNECTOR": {"required": ["DISCORD_BOT_TOKEN"], "validators": {}}, + # "DISCORD_CONNECTOR": {"required": ["DISCORD_BOT_TOKEN"], "validators": {}}, "JIRA_CONNECTOR": { "required": ["JIRA_EMAIL", "JIRA_API_TOKEN", "JIRA_BASE_URL"], "validators": { diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/discord-connect-form.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/discord-connect-form.tsx deleted file mode 100644 index 8f4fa1a47..000000000 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/discord-connect-form.tsx +++ /dev/null @@ -1,409 +0,0 @@ -"use client"; - -import { zodResolver } from "@hookform/resolvers/zod"; -import { Info } from "lucide-react"; -import type { FC } from "react"; -import { useRef, useState } from "react"; -import { useForm } from "react-hook-form"; -import * as z from "zod"; -import { - Accordion, - AccordionContent, - AccordionItem, - AccordionTrigger, -} from "@/components/ui/accordion"; -import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; -import { - Form, - FormControl, - FormDescription, - FormField, - FormItem, - FormLabel, - FormMessage, -} from "@/components/ui/form"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { Switch } from "@/components/ui/switch"; -import { EnumConnectorName } from "@/contracts/enums/connector"; -import { DateRangeSelector } from "../../components/date-range-selector"; -import { getConnectorBenefits } from "../connector-benefits"; -import type { ConnectFormProps } from "../index"; - -const discordConnectorFormSchema = z.object({ - name: z.string().min(3, { - message: "Connector name must be at least 3 characters.", - }), - bot_token: z.string().min(10, { - message: "Discord Bot Token is required and must be valid.", - }), -}); - -type DiscordConnectorFormValues = z.infer; - -export const DiscordConnectForm: FC = ({ onSubmit, isSubmitting }) => { - const isSubmittingRef = useRef(false); - const [startDate, setStartDate] = useState(undefined); - const [endDate, setEndDate] = useState(undefined); - const [periodicEnabled, setPeriodicEnabled] = useState(false); - const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); - const form = useForm({ - resolver: zodResolver(discordConnectorFormSchema), - defaultValues: { - name: "Discord Connector", - bot_token: "", - }, - }); - - const handleSubmit = async (values: DiscordConnectorFormValues) => { - // Prevent multiple submissions - if (isSubmittingRef.current || isSubmitting) { - return; - } - - isSubmittingRef.current = true; - try { - await onSubmit({ - name: values.name, - connector_type: EnumConnectorName.DISCORD_CONNECTOR, - config: { - DISCORD_BOT_TOKEN: values.bot_token, - }, - is_indexable: true, - last_indexed_at: null, - periodic_indexing_enabled: periodicEnabled, - indexing_frequency_minutes: periodicEnabled ? parseInt(frequencyMinutes, 10) : null, - next_scheduled_at: null, - startDate, - endDate, - periodicEnabled, - frequencyMinutes, - }); - } finally { - isSubmittingRef.current = false; - } - }; - - return ( -
- - -
- Bot Token Required - - You'll need a Discord Bot Token to use this connector. You can create one from{" "} - - Discord Developer Portal - - -
- - -
-
- - ( - - Connector Name - - - - - A friendly name to identify this connector. - - - - )} - /> - - ( - - Discord Bot Token - - - - - Your Discord Bot Token will be encrypted and stored securely. - - - - )} - /> - - {/* Indexing Configuration */} -
-

Indexing Configuration

- - {/* Date Range Selector */} - - - {/* Periodic Sync Config */} -
-
-
-

Enable Periodic Sync

-

- Automatically re-index at regular intervals -

-
- -
- - {periodicEnabled && ( -
-
- - -
-
- )} -
-
- - -
- - {/* What you get section */} - {getConnectorBenefits(EnumConnectorName.DISCORD_CONNECTOR) && ( -
-

What you get with Discord integration:

-
    - {getConnectorBenefits(EnumConnectorName.DISCORD_CONNECTOR)?.map((benefit) => ( -
  • {benefit}
    • - ))} -
-
- )} - - {/* Documentation Section */} - - - - Documentation - - -
-

How it works

-

- The Discord connector uses the Discord API to fetch messages from all accessible - channels that the bot token has access to within a server. -

-
    -
  • - For follow up indexing runs, the connector retrieves messages that have been - updated since the last indexing attempt. -
    • -
  • - Indexing is configured to run periodically, so updates should appear in your - search results within minutes. -
    • -
-
- -
-
-

Authorization

- - - Bot Token Required - - You need to create a Discord application and bot to get a bot token. The bot - needs read access to channels and messages. - - - -
-
-

- Step 1: Create a Discord Application -

-
    -
  1. - Go to{" "} - - https://discord.com/developers/applications - -
    2. -
  2. - Click New Application -
    3. -
  3. - Enter an application name and click Create -
    4. -
-
- -
-

- Step 2: Create a Bot -

-
    -
  1. - Navigate to Bot in the sidebar -
    2. -
  2. - Click Add Bot and confirm -
    3. -
  3. - Under Privileged Gateway Intents, enable: -
      -
    • - - MESSAGE CONTENT INTENT - {" "} - - Required to read message content -
      • -
    -
    4. -
-
- -
-

- Step 3: Get Bot Token and Invite Bot -

-
    -
  1. - Under Token, click Reset Token and copy - the token -
    2. -
  2. - Navigate to OAuth2 → URL Generator -
    3. -
  3. - Select bot scope and Read Messages{" "} - permission -
    4. -
  4. Copy the generated URL and open it in your browser
    5. -
  5. Select your server and authorize the bot
    6. -
-
-
-
-
- -
-
-

Indexing

-
    -
  1. - Navigate to the Connector Dashboard and select the Discord{" "} - Connector. -
    2. -
  2. - Place the Bot Token in the form field. -
    3. -
  3. - Click Connect to establish the connection. -
    4. -
  4. Once connected, your Discord messages will be indexed automatically.
    5. -
- - - - What Gets Indexed - -

The Discord connector indexes the following data:

-
    -
  • Messages from all accessible channels
    • -
  • Direct messages (if bot has access)
    • -
  • Message timestamps and metadata
    • -
  • Thread replies and conversations
    • -
- - -
-
- - - -
- ); -}; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx index 807d4cb7a..81e5ee03f 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx @@ -4,7 +4,6 @@ import { BookStackConnectForm } from "./components/bookstack-connect-form"; import { CirclebackConnectForm } from "./components/circleback-connect-form"; import { ClickUpConnectForm } from "./components/clickup-connect-form"; import { ConfluenceConnectForm } from "./components/confluence-connect-form"; -import { DiscordConnectForm } from "./components/discord-connect-form"; import { ElasticsearchConnectForm } from "./components/elasticsearch-connect-form"; import { GithubConnectForm } from "./components/github-connect-form"; import { JiraConnectForm } from "./components/jira-connect-form"; @@ -50,8 +49,6 @@ export function getConnectFormComponent(connectorType: string): ConnectFormCompo return BaiduSearchApiConnectForm; case "ELASTICSEARCH_CONNECTOR": return ElasticsearchConnectForm; - case "DISCORD_CONNECTOR": - return DiscordConnectForm; case "CONFLUENCE_CONNECTOR": return ConfluenceConnectForm; case "BOOKSTACK_CONNECTOR": diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/discord-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/discord-config.tsx index 377987637..464bc438f 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/discord-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/discord-config.tsx @@ -1,88 +1,26 @@ "use client"; -import { KeyRound } from "lucide-react"; +import { Info } from "lucide-react"; import type { FC } from "react"; -import { useEffect, useState } from "react"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; import type { ConnectorConfigProps } from "../index"; export interface DiscordConfigProps extends ConnectorConfigProps { onNameChange?: (name: string) => void; } -export const DiscordConfig: FC = ({ - connector, - onConfigChange, - onNameChange, -}) => { - const [botToken, setBotToken] = useState( - (connector.config?.DISCORD_BOT_TOKEN as string) || "" - ); - const [name, setName] = useState(connector.name || ""); - - // Update bot token and name when connector changes - useEffect(() => { - const token = (connector.config?.DISCORD_BOT_TOKEN as string) || ""; - setBotToken(token); - setName(connector.name || ""); - }, [connector.config, connector.name]); - - const handleBotTokenChange = (value: string) => { - setBotToken(value); - if (onConfigChange) { - onConfigChange({ - ...connector.config, - DISCORD_BOT_TOKEN: value, - }); - } - }; - - const handleNameChange = (value: string) => { - setName(value); - if (onNameChange) { - onNameChange(value); - } - }; - +export const DiscordConfig: FC = () => { return (
- {/* Connector Name */} -
-
- - handleNameChange(e.target.value)} - placeholder="My Discord Connector" - className="border-slate-400/20 focus-visible:border-slate-400/40" - /> -

- A friendly name to identify this connector. -

+
+
+
-
- - {/* Configuration */} -
-
-

Configuration

-
- -
- - handleBotTokenChange(e.target.value)} - placeholder="Your Bot Token" - className="border-slate-400/20 focus-visible:border-slate-400/40" - /> -

- Update your Discord Bot Token if needed. +

+

Add Bot to Servers

+

+ Before indexing, make sure the Discord bot has been added to the servers (guilds) you want to + index. The bot can only access messages from servers it's been added to. Use the OAuth + authorization flow to add the bot to your servers.

diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx index 5437426c8..3ba03f956 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx @@ -52,7 +52,6 @@ export const ConnectorConnectView: FC = ({ LINKUP_API: "linkup-api-connect-form", BAIDU_SEARCH_API: "baidu-search-api-connect-form", ELASTICSEARCH_CONNECTOR: "elasticsearch-connect-form", - DISCORD_CONNECTOR: "discord-connect-form", CONFLUENCE_CONNECTOR: "confluence-connect-form", BOOKSTACK_CONNECTOR: "bookstack-connect-form", GITHUB_CONNECTOR: "github-connect-form", diff --git a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts index 111b7485d..9822ff6e6 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts @@ -51,6 +51,13 @@ export const OAUTH_CONNECTORS = [ connectorType: EnumConnectorName.SLACK_CONNECTOR, authEndpoint: "/api/v1/auth/slack/connector/add/", }, + { + id: "discord-connector", + title: "Discord", + description: "Search Discord messages", + connectorType: EnumConnectorName.DISCORD_CONNECTOR, + authEndpoint: "/api/v1/auth/discord/connector/add/", + }, ] as const; // Content Sources (tools that extract and import content from external sources) @@ -71,12 +78,6 @@ export const CRAWLERS = [ // Non-OAuth Connectors (redirect to old connector config pages) export const OTHER_CONNECTORS = [ - { - id: "discord-connector", - title: "Discord", - description: "Search Discord messages", - connectorType: EnumConnectorName.DISCORD_CONNECTOR, - }, { id: "confluence-connector", title: "Confluence",