From c5b31bb7b53d4c3737127c60c41fd83f3ec514ec Mon Sep 17 00:00:00 2001 From: melvnl Date: Fri, 2 Jan 2026 17:51:19 +0700 Subject: [PATCH 1/6] fix(ui): fix dashboard search spaces layout for 1024px and below --- surfsense_web/app/dashboard/searchspaces/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_web/app/dashboard/searchspaces/page.tsx b/surfsense_web/app/dashboard/searchspaces/page.tsx index d7d24ae03..b40eb5d82 100644 --- a/surfsense_web/app/dashboard/searchspaces/page.tsx +++ b/surfsense_web/app/dashboard/searchspaces/page.tsx @@ -28,7 +28,7 @@ export default function SearchSpacesPage() { return ( Date: Sat, 3 Jan 2026 01:09:33 +0700 Subject: [PATCH 2/6] ui: adjust dashboard search spaces heading text and padding responsiveness --- surfsense_web/components/search-space-form.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_web/components/search-space-form.tsx b/surfsense_web/components/search-space-form.tsx index a43370303..bcea48e3d 100644 --- a/surfsense_web/components/search-space-form.tsx +++ b/surfsense_web/components/search-space-form.tsx @@ -118,7 +118,7 @@ export function SearchSpaceForm({ >
-

+

{isEditing ? "Edit Search Space" : "Create Search Space"}

@@ -157,7 +157,7 @@ export function SearchSpaceForm({ mass: 0.2, }} /> -
+
From bfed9a31f822929cc2b59b6c78a6ebe58a6c97d8 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 5 Jan 2026 23:59:16 +0530 Subject: [PATCH 3/6] feat: implement Jira OAuth integration and connector routes - Added support for Jira OAuth with new environment variables for client ID, client secret, and redirect URI. - Implemented Jira connector routes for OAuth flow, including authorization and callback handling. - Enhanced Jira connector to support both OAuth 2.0 and legacy API token authentication. - Updated Jira indexing logic to utilize OAuth credentials with auto-refresh capabilities. - Removed outdated Jira UI components and adjusted frontend logic to reflect the new integration. --- surfsense_backend/.env.example | 5 + surfsense_backend/app/config/__init__.py | 5 + .../app/connectors/jira_connector.py | 103 +++- surfsense_backend/app/routes/__init__.py | 2 + .../app/routes/jira_add_connector_route.py | 495 ++++++++++++++++++ .../app/schemas/jira_auth_credentials.py | 73 +++ .../tasks/connector_indexers/jira_indexer.py | 142 ++++- surfsense_backend/app/utils/validators.py | 14 +- .../components/jira-connect-form.tsx | 450 ---------------- .../connector-popup/connect-forms/index.tsx | 3 - .../components/jira-config.tsx | 51 +- .../views/connector-connect-view.tsx | 1 - .../constants/connector-constants.ts | 13 +- .../hooks/use-connector-edit-page.ts | 10 + 14 files changed, 845 insertions(+), 522 deletions(-) create mode 100644 surfsense_backend/app/routes/jira_add_connector_route.py create mode 100644 surfsense_backend/app/schemas/jira_auth_credentials.py delete mode 100644 surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/jira-connect-form.tsx diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index d2c667178..a2f662c23 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -50,6 +50,11 @@ DISCORD_CLIENT_SECRET=your_discord_client_secret_here DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback DISCORD_BOT_TOKEN=your_bot_token_from_developer_portal +# Jira OAuth Configuration +JIRA_CLIENT_ID=our_jira_client_id +JIRA_CLIENT_SECRET=your_jira_client_secret +JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback + # OAuth for Linear Connector LINEAR_CLIENT_ID=your_linear_client_id LINEAR_CLIENT_SECRET=your_linear_client_secret diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index f65a94cc0..56641215d 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -111,6 +111,11 @@ class Config: DISCORD_REDIRECT_URI = os.getenv("DISCORD_REDIRECT_URI") DISCORD_BOT_TOKEN = os.getenv("DISCORD_BOT_TOKEN") + # Jira OAuth + JIRA_CLIENT_ID = os.getenv("JIRA_CLIENT_ID") + JIRA_CLIENT_SECRET = os.getenv("JIRA_CLIENT_SECRET") + JIRA_REDIRECT_URI = os.getenv("JIRA_REDIRECT_URI") + # LLM instances are now managed per-user through the LLMConfig system # Legacy environment variables removed in favor of user-specific configurations diff --git a/surfsense_backend/app/connectors/jira_connector.py b/surfsense_backend/app/connectors/jira_connector.py index e73198e79..8e9badf0b 100644 --- a/surfsense_backend/app/connectors/jira_connector.py +++ b/surfsense_backend/app/connectors/jira_connector.py @@ -3,6 +3,7 @@ Jira Connector Module A module for retrieving data from Jira. Allows fetching issue lists and their comments, projects and more. +Supports both OAuth 2.0 (preferred) and legacy API token authentication. """ import base64 @@ -18,6 +19,8 @@ class JiraConnector: def __init__( self, base_url: str | None = None, + access_token: str | None = None, + cloud_id: str | None = None, email: str | None = None, api_token: str | None = None, ): @@ -25,18 +28,39 @@ class JiraConnector: Initialize the JiraConnector class. Args: - base_url: Jira instance base URL (e.g., 'https://yourcompany.atlassian.net') (optional) - email: Jira account email address (optional) - api_token: Jira API token (optional) + base_url: Jira instance base URL (e.g., 'https://yourcompany.atlassian.net') + access_token: OAuth 2.0 access token (preferred method) + cloud_id: Atlassian cloud ID (used with OAuth for API URL construction) + email: Jira account email address (legacy method, used with api_token) + api_token: Jira API token (legacy method, used with email) """ self.base_url = base_url.rstrip("/") if base_url else None + self.access_token = access_token + self.cloud_id = cloud_id self.email = email self.api_token = api_token self.api_version = "3" # Jira Cloud API version + self._use_oauth = access_token is not None + + def set_oauth_credentials( + self, base_url: str, access_token: str, cloud_id: str | None = None + ) -> None: + """ + Set OAuth 2.0 credentials (preferred method). + + Args: + base_url: Jira instance base URL + access_token: OAuth 2.0 access token + cloud_id: Atlassian cloud ID (optional, used for API URL construction) + """ + self.base_url = base_url.rstrip("/") + self.access_token = access_token + self.cloud_id = cloud_id + self._use_oauth = True def set_credentials(self, base_url: str, email: str, api_token: str) -> None: """ - Set the Jira credentials. + Set the Jira credentials (legacy method using API token). Args: base_url: Jira instance base URL @@ -46,50 +70,69 @@ class JiraConnector: self.base_url = base_url.rstrip("/") self.email = email self.api_token = api_token + self._use_oauth = False def set_email(self, email: str) -> None: """ - Set the Jira account email. + Set the Jira account email (legacy method). Args: email: Jira account email address """ self.email = email + self._use_oauth = False def set_api_token(self, api_token: str) -> None: """ - Set the Jira API token. + Set the Jira API token (legacy method). Args: api_token: Jira API token """ self.api_token = api_token + self._use_oauth = False def get_headers(self) -> dict[str, str]: """ - Get headers for Jira API requests using Basic Authentication. + Get headers for Jira API requests. + + Uses OAuth Bearer token if available, otherwise falls back to Basic Auth. Returns: Dictionary of headers Raises: - ValueError: If email, api_token, or base_url have not been set + ValueError: If credentials have not been set """ - if not all([self.base_url, self.email, self.api_token]): - raise ValueError( - "Jira credentials not initialized. Call set_credentials() first." - ) + if self._use_oauth: + # OAuth 2.0 authentication + if not self.base_url or not self.access_token: + raise ValueError( + "Jira OAuth credentials not initialized. Call set_oauth_credentials() first." + ) - # Create Basic Auth header using email:api_token - auth_str = f"{self.email}:{self.api_token}" - auth_bytes = auth_str.encode("utf-8") - auth_header = "Basic " + base64.b64encode(auth_bytes).decode("ascii") + return { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.access_token}", + "Accept": "application/json", + } + else: + # Legacy Basic Auth + if not all([self.base_url, self.email, self.api_token]): + raise ValueError( + "Jira credentials not initialized. Call set_credentials() first." + ) - return { - "Content-Type": "application/json", - "Authorization": auth_header, - "Accept": "application/json", - } + # Create Basic Auth header using email:api_token + auth_str = f"{self.email}:{self.api_token}" + auth_bytes = auth_str.encode("utf-8") + auth_header = "Basic " + base64.b64encode(auth_bytes).decode("ascii") + + return { + "Content-Type": "application/json", + "Authorization": auth_header, + "Accept": "application/json", + } def make_api_request( self, @@ -104,22 +147,26 @@ class JiraConnector: Args: endpoint: API endpoint (without base URL) params: Query parameters for the request (optional) + method: HTTP method (GET or POST) + json_payload: JSON payload for POST requests (optional) Returns: Response data from the API Raises: - ValueError: If email, api_token, or base_url have not been set + ValueError: If credentials have not been set Exception: If the API request fails """ - if not all([self.base_url, self.email, self.api_token]): - raise ValueError( - "Jira credentials not initialized. Call set_credentials() first." - ) - - url = f"{self.base_url}/rest/api/{self.api_version}/{endpoint}" headers = self.get_headers() + # Construct API URL based on authentication method + if self._use_oauth and self.cloud_id: + # Use Atlassian API gateway with cloud_id for OAuth + url = f"https://api.atlassian.com/ex/jira/{self.cloud_id}/rest/api/{self.api_version}/{endpoint}" + else: + # Use direct base URL (works for both OAuth and legacy) + url = f"{self.base_url}/rest/api/{self.api_version}/{endpoint}" + if method.upper() == "POST": response = requests.post( url, headers=headers, json=json_payload, timeout=500 diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index b35d743e0..16cacfeb8 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -28,6 +28,7 @@ from .search_source_connectors_routes import router as search_source_connectors_ from .search_spaces_routes import router as search_spaces_router from .slack_add_connector_route import router as slack_add_connector_router from .discord_add_connector_route import router as discord_add_connector_router +from .jira_add_connector_route import router as jira_add_connector_router router = APIRouter() @@ -48,6 +49,7 @@ router.include_router(luma_add_connector_router) router.include_router(notion_add_connector_router) router.include_router(slack_add_connector_router) router.include_router(discord_add_connector_router) +router.include_router(jira_add_connector_router) router.include_router(new_llm_config_router) # LLM configs with prompt configuration router.include_router(logs_router) router.include_router(circleback_webhook_router) # Circleback meeting webhooks diff --git a/surfsense_backend/app/routes/jira_add_connector_route.py b/surfsense_backend/app/routes/jira_add_connector_route.py new file mode 100644 index 000000000..ac415e80e --- /dev/null +++ b/surfsense_backend/app/routes/jira_add_connector_route.py @@ -0,0 +1,495 @@ +""" +Jira Connector OAuth Routes. + +Handles OAuth 2.0 authentication flow for Jira connector. +Uses Atlassian OAuth 2.0 (3LO) with accessible-resources API to discover Jira instances. +""" + +import logging +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.db import ( + SearchSourceConnector, + SearchSourceConnectorType, + User, + get_async_session, +) +from app.schemas.jira_auth_credentials import JiraAuthCredentialsBase +from app.users import current_active_user +from app.utils.oauth_security import OAuthStateManager, TokenEncryption + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# Atlassian OAuth endpoints +AUTHORIZATION_URL = "https://auth.atlassian.com/authorize" +TOKEN_URL = "https://auth.atlassian.com/oauth/token" +ACCESSIBLE_RESOURCES_URL = "https://api.atlassian.com/oauth/token/accessible-resources" + +# OAuth scopes for Jira +SCOPES = [ + "read:jira-work", + "write:jira-work", + "read:jira-user", + "offline_access", # Required for refresh tokens +] + +# Initialize security utilities +_state_manager = None +_token_encryption = None + + +def get_state_manager() -> OAuthStateManager: + """Get or create OAuth state manager instance.""" + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for OAuth security") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def get_token_encryption() -> TokenEncryption: + """Get or create token encryption instance.""" + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise ValueError("SECRET_KEY must be set for token encryption") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + +@router.get("/auth/jira/connector/add") +async def connect_jira(space_id: int, user: User = Depends(current_active_user)): + """ + Initiate Jira OAuth flow. + + Args: + space_id: The search space ID + user: Current authenticated user + + Returns: + Authorization URL for redirect + """ + try: + if not space_id: + raise HTTPException(status_code=400, detail="space_id is required") + + if not config.JIRA_CLIENT_ID: + raise HTTPException(status_code=500, detail="Jira OAuth not configured.") + + if not config.SECRET_KEY: + raise HTTPException( + status_code=500, detail="SECRET_KEY not configured for OAuth security." + ) + + # Generate secure state parameter with HMAC signature + state_manager = get_state_manager() + state_encoded = state_manager.generate_secure_state(space_id, user.id) + + # Build authorization URL + from urllib.parse import urlencode + + auth_params = { + "audience": "api.atlassian.com", + "client_id": config.JIRA_CLIENT_ID, + "scope": " ".join(SCOPES), + "redirect_uri": config.JIRA_REDIRECT_URI, + "state": state_encoded, + "response_type": "code", + "prompt": "consent", # Force consent screen to get refresh token + } + + auth_url = f"{AUTHORIZATION_URL}?{urlencode(auth_params)}" + + logger.info(f"Generated Jira OAuth URL for user {user.id}, space {space_id}") + return {"auth_url": auth_url} + + except Exception as e: + logger.error(f"Failed to initiate Jira OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to initiate Jira OAuth: {e!s}" + ) from e + + +@router.get("/auth/jira/connector/callback") +async def jira_callback( + request: Request, + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +): + """ + Handle Jira OAuth callback. + + Args: + request: FastAPI request object + code: Authorization code from Atlassian (if user granted access) + error: Error code from Atlassian (if user denied access or error occurred) + state: State parameter containing user/space info + session: Database session + + Returns: + Redirect response to frontend + """ + try: + # Handle OAuth errors (e.g., user denied access) + if error: + logger.warning(f"Jira OAuth error: {error}") + # Try to decode state to get space_id for redirect, but don't fail if it's invalid + space_id = None + if state: + try: + state_manager = get_state_manager() + data = state_manager.validate_state(state) + space_id = data.get("space_id") + except Exception: + # If state is invalid, we'll redirect without space_id + logger.warning("Failed to validate state in error handler") + + # Redirect to frontend with error parameter + if space_id: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=jira_oauth_denied" + ) + else: + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=jira_oauth_denied" + ) + + # Validate required parameters for successful flow + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + if not state: + raise HTTPException(status_code=400, detail="Missing state parameter") + + # Validate and decode state with signature verification + state_manager = get_state_manager() + try: + data = state_manager.validate_state(state) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Invalid state parameter: {e!s}" + ) from e + + user_id = UUID(data["user_id"]) + space_id = data["space_id"] + + # Validate redirect URI (security: ensure it matches configured value) + if not config.JIRA_REDIRECT_URI: + raise HTTPException( + status_code=500, detail="JIRA_REDIRECT_URI not configured" + ) + + # Exchange authorization code for access token + token_data = { + "grant_type": "authorization_code", + "client_id": config.JIRA_CLIENT_ID, + "client_secret": config.JIRA_CLIENT_SECRET, + "code": code, + "redirect_uri": config.JIRA_REDIRECT_URI, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=token_data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error_description", error_json.get("error", error_detail)) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token exchange failed: {error_detail}" + ) + + token_json = token_response.json() + + # Encrypt sensitive tokens before storing + token_encryption = get_token_encryption() + access_token = token_json.get("access_token") + refresh_token = token_json.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from Atlassian" + ) + + # Fetch accessible resources to get Jira instance information + async with httpx.AsyncClient() as client: + resources_response = await client.get( + ACCESSIBLE_RESOURCES_URL, + headers={"Authorization": f"Bearer {access_token}"}, + timeout=30.0, + ) + + if resources_response.status_code != 200: + error_detail = resources_response.text + logger.error(f"Failed to fetch accessible resources: {error_detail}") + raise HTTPException( + status_code=400, + detail=f"Failed to fetch Jira instances: {error_detail}", + ) + + resources = resources_response.json() + + # Filter for Jira instances (resources with type "jira" or id field) + jira_instances = [ + r + for r in resources + if r.get("id") and (r.get("name") or r.get("url")) + ] + + if not jira_instances: + raise HTTPException( + status_code=400, + detail="No accessible Jira instances found. Please ensure you have access to at least one Jira instance.", + ) + + # For now, use the first Jira instance + # TODO: Support multiple instances by letting user choose during OAuth + jira_instance = jira_instances[0] + cloud_id = jira_instance["id"] + base_url = jira_instance.get("url") + + # If URL is not provided, construct it from cloud_id + if not base_url: + # Try to extract from name or construct default format + instance_name = jira_instance.get("name", "").lower().replace(" ", "") + if instance_name: + base_url = f"https://{instance_name}.atlassian.net" + else: + # Fallback: use cloud_id directly (though this may not work) + base_url = f"https://{cloud_id}.atlassian.net" + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Store the encrypted access token and refresh token in connector config + connector_config = { + "access_token": token_encryption.encrypt_token(access_token), + "refresh_token": token_encryption.encrypt_token(refresh_token) + if refresh_token + else None, + "token_type": token_json.get("token_type", "Bearer"), + "expires_in": expires_in, + "expires_at": expires_at.isoformat() if expires_at else None, + "scope": token_json.get("scope"), + "cloud_id": cloud_id, + "base_url": base_url.rstrip("/") if base_url else None, + # Mark that tokens are encrypted for backward compatibility + "_token_encrypted": True, + } + + # Check if connector already exists for this search space and user + existing_connector_result = await session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == space_id, + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.JIRA_CONNECTOR, + ) + ) + existing_connector = existing_connector_result.scalars().first() + + if existing_connector: + # Update existing connector + existing_connector.config = connector_config + existing_connector.name = "Jira Connector" + existing_connector.is_indexable = True + logger.info( + f"Updated existing Jira connector for user {user_id} in space {space_id}" + ) + else: + # Create new connector + new_connector = SearchSourceConnector( + name="Jira Connector", + connector_type=SearchSourceConnectorType.JIRA_CONNECTOR, + is_indexable=True, + config=connector_config, + search_space_id=space_id, + user_id=user_id, + ) + session.add(new_connector) + logger.info( + f"Created new Jira connector for user {user_id} in space {space_id}" + ) + + try: + await session.commit() + logger.info(f"Successfully saved Jira connector for user {user_id}") + + # Redirect to the frontend with success params + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=jira-connector" + ) + + except ValidationError as e: + await session.rollback() + raise HTTPException( + status_code=422, detail=f"Validation error: {e!s}" + ) from e + except IntegrityError as e: + await session.rollback() + raise HTTPException( + status_code=409, + detail=f"Integrity error: A connector with this type already exists. {e!s}", + ) from e + except Exception as e: + logger.error(f"Failed to create search source connector: {e!s}") + await session.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to create search source connector: {e!s}", + ) from e + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to complete Jira OAuth: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to complete Jira OAuth: {e!s}" + ) from e + + +async def refresh_jira_token( + session: AsyncSession, connector: SearchSourceConnector +) -> SearchSourceConnector: + """ + Refresh the Jira access token for a connector. + + Args: + session: Database session + connector: Jira connector to refresh + + Returns: + Updated connector object + """ + try: + logger.info(f"Refreshing Jira token for connector {connector.id}") + + credentials = JiraAuthCredentialsBase.from_dict(connector.config) + + # Decrypt tokens if they are encrypted + token_encryption = get_token_encryption() + is_encrypted = connector.config.get("_token_encrypted", False) + + refresh_token = credentials.refresh_token + if is_encrypted and refresh_token: + try: + refresh_token = token_encryption.decrypt_token(refresh_token) + except Exception as e: + logger.error(f"Failed to decrypt refresh token: {e!s}") + raise HTTPException( + status_code=500, detail="Failed to decrypt stored refresh token" + ) from e + + if not refresh_token: + raise HTTPException( + status_code=400, + detail="No refresh token available. Please re-authenticate.", + ) + + # Prepare token refresh data + refresh_data = { + "grant_type": "refresh_token", + "client_id": config.JIRA_CLIENT_ID, + "client_secret": config.JIRA_CLIENT_SECRET, + "refresh_token": refresh_token, + } + + async with httpx.AsyncClient() as client: + token_response = await client.post( + TOKEN_URL, + data=refresh_data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=30.0, + ) + + if token_response.status_code != 200: + error_detail = token_response.text + try: + error_json = token_response.json() + error_detail = error_json.get("error_description", error_json.get("error", error_detail)) + except Exception: + pass + raise HTTPException( + status_code=400, detail=f"Token refresh failed: {error_detail}" + ) + + token_json = token_response.json() + + # Calculate expiration time (UTC, tz-aware) + expires_at = None + expires_in = token_json.get("expires_in") + if expires_in: + now_utc = datetime.now(UTC) + expires_at = now_utc + timedelta(seconds=int(expires_in)) + + # Encrypt new tokens before storing + access_token = token_json.get("access_token") + new_refresh_token = token_json.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=400, detail="No access token received from Jira refresh" + ) + + # Update credentials object with encrypted tokens + credentials.access_token = token_encryption.encrypt_token(access_token) + if new_refresh_token: + credentials.refresh_token = token_encryption.encrypt_token( + new_refresh_token + ) + credentials.expires_in = expires_in + credentials.expires_at = expires_at + credentials.scope = token_json.get("scope") + + # Preserve cloud_id and base_url + if not credentials.cloud_id: + credentials.cloud_id = connector.config.get("cloud_id") + if not credentials.base_url: + credentials.base_url = connector.config.get("base_url") + + # Update connector config with encrypted tokens + credentials_dict = credentials.to_dict() + credentials_dict["_token_encrypted"] = True + connector.config = credentials_dict + await session.commit() + await session.refresh(connector) + + logger.info(f"Successfully refreshed Jira token for connector {connector.id}") + + return connector + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to refresh Jira token: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, detail=f"Failed to refresh Jira token: {e!s}" + ) from e + diff --git a/surfsense_backend/app/schemas/jira_auth_credentials.py b/surfsense_backend/app/schemas/jira_auth_credentials.py new file mode 100644 index 000000000..23d1ffcbf --- /dev/null +++ b/surfsense_backend/app/schemas/jira_auth_credentials.py @@ -0,0 +1,73 @@ +from datetime import UTC, datetime + +from pydantic import BaseModel, field_validator + + +class JiraAuthCredentialsBase(BaseModel): + access_token: str + refresh_token: str | None = None + token_type: str = "Bearer" + expires_in: int | None = None + expires_at: datetime | None = None + scope: str | None = None + cloud_id: str | None = None + base_url: str | None = None + + @property + def is_expired(self) -> bool: + """Check if the credentials have expired.""" + if self.expires_at is None: + return False + return self.expires_at <= datetime.now(UTC) + + @property + def is_refreshable(self) -> bool: + """Check if the credentials can be refreshed.""" + return self.refresh_token is not None + + def to_dict(self) -> dict: + """Convert credentials to dictionary for storage.""" + return { + "access_token": self.access_token, + "refresh_token": self.refresh_token, + "token_type": self.token_type, + "expires_in": self.expires_in, + "expires_at": self.expires_at.isoformat() if self.expires_at else None, + "scope": self.scope, + "cloud_id": self.cloud_id, + "base_url": self.base_url, + } + + @classmethod + def from_dict(cls, data: dict) -> "JiraAuthCredentialsBase": + """Create credentials from dictionary.""" + expires_at = None + if data.get("expires_at"): + expires_at = datetime.fromisoformat(data["expires_at"]) + + return cls( + access_token=data["access_token"], + refresh_token=data.get("refresh_token"), + token_type=data.get("token_type", "Bearer"), + expires_in=data.get("expires_in"), + expires_at=expires_at, + scope=data.get("scope"), + cloud_id=data.get("cloud_id"), + base_url=data.get("base_url"), + ) + + @field_validator("expires_at", mode="before") + @classmethod + def ensure_aware_utc(cls, v): + # Strings like "2025-08-26T14:46:57.367184" + if isinstance(v, str): + # add +00:00 if missing tz info + if v.endswith("Z"): + return datetime.fromisoformat(v.replace("Z", "+00:00")) + dt = datetime.fromisoformat(v) + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + # datetime objects + if isinstance(v, datetime): + return v if v.tzinfo else v.replace(tzinfo=UTC) + return v + diff --git a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py index 8c56b10ab..616927e6f 100644 --- a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py @@ -84,31 +84,137 @@ async def index_jira_issues( return 0, f"Connector with ID {connector_id} not found" # Get the Jira credentials from the connector config - jira_email = connector.config.get("JIRA_EMAIL") - jira_api_token = connector.config.get("JIRA_API_TOKEN") - jira_base_url = connector.config.get("JIRA_BASE_URL") + # Support both OAuth (preferred) and legacy API token authentication + config_data = connector.config.copy() + is_oauth = config_data.get("_token_encrypted", False) or config_data.get("access_token") - if not jira_email or not jira_api_token or not jira_base_url: - await task_logger.log_task_failure( + if is_oauth: + # OAuth 2.0 authentication + from app.utils.oauth_security import TokenEncryption + + if not config.SECRET_KEY: + await task_logger.log_task_failure( + log_entry, + f"SECRET_KEY not configured but tokens are marked as encrypted for connector {connector_id}", + "Missing SECRET_KEY for token decryption", + {"error_type": "MissingSecretKey"}, + ) + return 0, "SECRET_KEY not configured but tokens are marked as encrypted" + + try: + token_encryption = TokenEncryption(config.SECRET_KEY) + + # Decrypt access_token + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + logger.info( + f"Decrypted Jira access token for connector {connector_id}" + ) + + # Decrypt refresh_token if present + if config_data.get("refresh_token"): + config_data["refresh_token"] = token_encryption.decrypt_token( + config_data["refresh_token"] + ) + logger.info( + f"Decrypted Jira refresh token for connector {connector_id}" + ) + except Exception as e: + await task_logger.log_task_failure( + log_entry, + f"Failed to decrypt Jira tokens for connector {connector_id}: {e!s}", + "Token decryption failed", + {"error_type": "TokenDecryptionError"}, + ) + return 0, f"Failed to decrypt Jira tokens: {e!s}" + + try: + from app.schemas.jira_auth_credentials import JiraAuthCredentialsBase + credentials = JiraAuthCredentialsBase.from_dict(config_data) + except Exception as e: + await task_logger.log_task_failure( + log_entry, + f"Invalid Jira OAuth credentials in connector {connector_id}", + str(e), + {"error_type": "InvalidCredentials"}, + ) + return 0, f"Invalid Jira OAuth credentials: {e!s}" + + # Check if credentials are expired and refresh if needed + if credentials.is_expired: + await task_logger.log_task_progress( + log_entry, + f"Jira credentials expired for connector {connector_id}, refreshing token", + {"stage": "token_refresh"}, + ) + + from app.routes.jira_add_connector_route import refresh_jira_token + + try: + connector = await refresh_jira_token(session, connector) + # Re-fetch credentials after refresh + config_data = connector.config.copy() + if config_data.get("access_token"): + config_data["access_token"] = token_encryption.decrypt_token( + config_data["access_token"] + ) + credentials = JiraAuthCredentialsBase.from_dict(config_data) + except Exception as e: + await task_logger.log_task_failure( + log_entry, + f"Failed to refresh Jira token for connector {connector_id}: {e!s}", + "Token refresh failed", + {"error_type": "TokenRefreshError"}, + ) + return 0, f"Failed to refresh Jira token: {e!s}" + + # Initialize Jira client with OAuth credentials + await task_logger.log_task_progress( log_entry, - f"Jira credentials not found in connector config for connector {connector_id}", - "Missing Jira credentials", - {"error_type": "MissingCredentials"}, + f"Initializing Jira client with OAuth for connector {connector_id}", + {"stage": "client_initialization"}, ) - return 0, "Jira credentials not found in connector config" - # Initialize Jira client - await task_logger.log_task_progress( - log_entry, - f"Initializing Jira client for connector {connector_id}", - {"stage": "client_initialization"}, - ) + jira_client = JiraConnector( + base_url=credentials.base_url, + access_token=credentials.access_token, + cloud_id=credentials.cloud_id, + ) + else: + # Legacy API token authentication + jira_email = config_data.get("JIRA_EMAIL") + jira_api_token = config_data.get("JIRA_API_TOKEN") + jira_base_url = config_data.get("JIRA_BASE_URL") - jira_client = JiraConnector( - base_url=jira_base_url, email=jira_email, api_token=jira_api_token - ) + if not jira_email or not jira_api_token or not jira_base_url: + await task_logger.log_task_failure( + log_entry, + f"Jira credentials not found in connector config for connector {connector_id}", + "Missing Jira credentials", + {"error_type": "MissingCredentials"}, + ) + return 0, "Jira credentials not found in connector config" + + # Initialize Jira client with legacy credentials + await task_logger.log_task_progress( + log_entry, + f"Initializing Jira client with API token for connector {connector_id}", + {"stage": "client_initialization"}, + ) + + jira_client = JiraConnector( + base_url=jira_base_url, email=jira_email, api_token=jira_api_token + ) # Calculate date range + # Handle "undefined" strings from frontend + if start_date == "undefined" or start_date == "": + start_date = None + if end_date == "undefined" or end_date == "": + end_date = None + start_date_str, end_date_str = calculate_date_range( connector, start_date, end_date, default_days_back=365 ) diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index f1620c0e5..d1f416339 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -538,13 +538,13 @@ def validate_connector_config( }, }, # "DISCORD_CONNECTOR": {"required": ["DISCORD_BOT_TOKEN"], "validators": {}}, - "JIRA_CONNECTOR": { - "required": ["JIRA_EMAIL", "JIRA_API_TOKEN", "JIRA_BASE_URL"], - "validators": { - "JIRA_EMAIL": lambda: validate_email_field("JIRA_EMAIL", "JIRA"), - "JIRA_BASE_URL": lambda: validate_url_field("JIRA_BASE_URL", "JIRA"), - }, - }, + # "JIRA_CONNECTOR": { + # "required": ["JIRA_EMAIL", "JIRA_API_TOKEN", "JIRA_BASE_URL"], + # "validators": { + # "JIRA_EMAIL": lambda: validate_email_field("JIRA_EMAIL", "JIRA"), + # "JIRA_BASE_URL": lambda: validate_url_field("JIRA_BASE_URL", "JIRA"), + # }, + # }, "CONFLUENCE_CONNECTOR": { "required": [ "CONFLUENCE_BASE_URL", diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/jira-connect-form.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/jira-connect-form.tsx deleted file mode 100644 index 0499554b4..000000000 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/jira-connect-form.tsx +++ /dev/null @@ -1,450 +0,0 @@ -"use client"; - -import { zodResolver } from "@hookform/resolvers/zod"; -import { Info } from "lucide-react"; -import type { FC } from "react"; -import { useRef, useState } from "react"; -import { useForm } from "react-hook-form"; -import * as z from "zod"; -import { - Accordion, - AccordionContent, - AccordionItem, - AccordionTrigger, -} from "@/components/ui/accordion"; -import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; -import { - Form, - FormControl, - FormDescription, - FormField, - FormItem, - FormLabel, - FormMessage, -} from "@/components/ui/form"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { Switch } from "@/components/ui/switch"; -import { EnumConnectorName } from "@/contracts/enums/connector"; -import { DateRangeSelector } from "../../components/date-range-selector"; -import { getConnectorBenefits } from "../connector-benefits"; -import type { ConnectFormProps } from "../index"; - -const jiraConnectorFormSchema = z.object({ - name: z.string().min(3, { - message: "Connector name must be at least 3 characters.", - }), - base_url: z.string().url({ message: "Please enter a valid Jira base URL." }), - email: z.string().email({ message: "Please enter a valid email address." }), - api_token: z.string().min(10, { - message: "Jira API Token is required and must be valid.", - }), -}); - -type JiraConnectorFormValues = z.infer; - -export const JiraConnectForm: FC = ({ onSubmit, isSubmitting }) => { - const isSubmittingRef = useRef(false); - const [startDate, setStartDate] = useState(undefined); - const [endDate, setEndDate] = useState(undefined); - const [periodicEnabled, setPeriodicEnabled] = useState(false); - const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); - const form = useForm({ - resolver: zodResolver(jiraConnectorFormSchema), - defaultValues: { - name: "Jira Connector", - base_url: "", - email: "", - api_token: "", - }, - }); - - const handleSubmit = async (values: JiraConnectorFormValues) => { - // Prevent multiple submissions - if (isSubmittingRef.current || isSubmitting) { - return; - } - - isSubmittingRef.current = true; - try { - await onSubmit({ - name: values.name, - connector_type: EnumConnectorName.JIRA_CONNECTOR, - config: { - JIRA_BASE_URL: values.base_url, - JIRA_EMAIL: values.email, - JIRA_API_TOKEN: values.api_token, - }, - is_indexable: true, - last_indexed_at: null, - periodic_indexing_enabled: periodicEnabled, - indexing_frequency_minutes: periodicEnabled ? parseInt(frequencyMinutes, 10) : null, - next_scheduled_at: null, - startDate, - endDate, - periodicEnabled, - frequencyMinutes, - }); - } finally { - isSubmittingRef.current = false; - } - }; - - return ( -
- - -
- API Token Required - - You'll need a Jira API Token to use this connector. You can create one from{" "} - - Atlassian Account Settings - - -
- - -
-
- - ( - - Connector Name - - - - - A friendly name to identify this connector. - - - - )} - /> - - ( - - Jira Base URL - - - - - The base URL of your Jira instance (e.g., https://your-domain.atlassian.net). - - - - )} - /> - - ( - - Email Address - - - - - The email address associated with your Atlassian account. - - - - )} - /> - - ( - - API Token - - - - - Your Jira API Token will be encrypted and stored securely. - - - - )} - /> - - {/* Indexing Configuration */} -
-

Indexing Configuration

- - {/* Date Range Selector */} - - - {/* Periodic Sync Config */} -
-
-
-

Enable Periodic Sync

-

- Automatically re-index at regular intervals -

-
- -
- - {periodicEnabled && ( -
-
- - -
-
- )} -
-
- - -
- - {/* What you get section */} - {getConnectorBenefits(EnumConnectorName.JIRA_CONNECTOR) && ( -
-

What you get with Jira integration:

-
    - {getConnectorBenefits(EnumConnectorName.JIRA_CONNECTOR)?.map((benefit) => ( -
  • {benefit}
    • - ))} -
-
- )} - - {/* Documentation Section */} - - - - Documentation - - -
-

How it works

-

- The Jira connector uses the Jira REST API with Basic Authentication to fetch all - issues and comments that your account has access to within your Jira instance. -

-
    -
  • - For follow up indexing runs, the connector retrieves issues and comments that have - been updated since the last indexing attempt. -
    • -
  • - Indexing is configured to run periodically, so updates should appear in your - search results within minutes. -
    • -
-
- -
-
-

Authorization

- - - - Read-Only Access is Sufficient - - - You only need read access for this connector to work. The API Token will only be - used to read your Jira data. - - - -
-
-

- Step 1: Create an API Token -

-
    -
  1. Log in to your Atlassian account
    2. -
  2. - Navigate to{" "} - - https://id.atlassian.com/manage-profile/security/api-tokens - {" "} - in your browser. -
    3. -
  3. - Click Create API token -
    4. -
  4. Enter a label for your token (like "SurfSense Connector")
    5. -
  5. - Click Create -
    6. -
  6. Copy the generated token as it will only be shown once
    7. -
-
- -
-

- Step 2: Grant necessary access -

-

- The API Token will have access to all projects and issues that your user - account can see. Make sure your account has appropriate permissions for the - projects you want to index. -

- - - Data Privacy - - Only issues, comments, and basic metadata will be indexed. Jira attachments - and linked files are not indexed by this connector. - - -
-
-
-
- -
-
-

Indexing

-
    -
  1. - Navigate to the Connector Dashboard and select the Jira{" "} - Connector. -
    2. -
  2. - Enter your Jira Instance URL (e.g., - https://yourcompany.atlassian.net) -
    3. -
  3. - Enter your Email Address associated with your Atlassian account -
    4. -
  4. - Place your API Token in the form field. -
    5. -
  5. - Click Connect to establish the connection. -
    6. -
  6. Once connected, your Jira issues will be indexed automatically.
    7. -
- - - - What Gets Indexed - -

The Jira connector indexes the following data:

-
    -
  • Issue keys and summaries (e.g., PROJ-123)
    • -
  • Issue descriptions
    • -
  • Issue comments and discussion threads
    • -
  • Issue status, priority, and type information
    • -
  • Assignee and reporter information
    • -
  • Project information
    • -
- - -
-
- - - -
- ); -}; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx index 81e5ee03f..cda17ddfc 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx @@ -6,7 +6,6 @@ import { ClickUpConnectForm } from "./components/clickup-connect-form"; import { ConfluenceConnectForm } from "./components/confluence-connect-form"; import { ElasticsearchConnectForm } from "./components/elasticsearch-connect-form"; import { GithubConnectForm } from "./components/github-connect-form"; -import { JiraConnectForm } from "./components/jira-connect-form"; import { LinkupApiConnectForm } from "./components/linkup-api-connect-form"; import { LumaConnectForm } from "./components/luma-connect-form"; import { SearxngConnectForm } from "./components/searxng-connect-form"; @@ -55,8 +54,6 @@ export function getConnectFormComponent(connectorType: string): ConnectFormCompo return BookStackConnectForm; case "GITHUB_CONNECTOR": return GithubConnectForm; - case "JIRA_CONNECTOR": - return JiraConnectForm; case "CLICKUP_CONNECTOR": return ClickUpConnectForm; case "LUMA_CONNECTOR": diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx index 3ef16bdb4..158dfdf13 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx @@ -1,6 +1,6 @@ "use client"; -import { KeyRound } from "lucide-react"; +import { Info, KeyRound } from "lucide-react"; import type { FC } from "react"; import { useEffect, useState } from "react"; import { Input } from "@/components/ui/input"; @@ -12,6 +12,9 @@ export interface JiraConfigProps extends ConnectorConfigProps { } export const JiraConfig: FC = ({ connector, onConfigChange, onNameChange }) => { + // Check if this is an OAuth connector (has access_token or _token_encrypted flag) + const isOAuth = !!(connector.config?.access_token || connector.config?._token_encrypted); + const [baseUrl, setBaseUrl] = useState((connector.config?.JIRA_BASE_URL as string) || ""); const [email, setEmail] = useState((connector.config?.JIRA_EMAIL as string) || ""); const [apiToken, setApiToken] = useState( @@ -19,16 +22,18 @@ export const JiraConfig: FC = ({ connector, onConfigChange, onN ); const [name, setName] = useState(connector.name || ""); - // Update values when connector changes + // Update values when connector changes (only for legacy connectors) useEffect(() => { - const url = (connector.config?.JIRA_BASE_URL as string) || ""; - const emailVal = (connector.config?.JIRA_EMAIL as string) || ""; - const token = (connector.config?.JIRA_API_TOKEN as string) || ""; - setBaseUrl(url); - setEmail(emailVal); - setApiToken(token); + if (!isOAuth) { + const url = (connector.config?.JIRA_BASE_URL as string) || ""; + const emailVal = (connector.config?.JIRA_EMAIL as string) || ""; + const token = (connector.config?.JIRA_API_TOKEN as string) || ""; + setBaseUrl(url); + setEmail(emailVal); + setApiToken(token); + } setName(connector.name || ""); - }, [connector.config, connector.name]); + }, [connector.config, connector.name, isOAuth]); const handleBaseUrlChange = (value: string) => { setBaseUrl(value); @@ -67,6 +72,34 @@ export const JiraConfig: FC = ({ connector, onConfigChange, onN } }; + // For OAuth connectors, show simple info message + if (isOAuth) { + const baseUrl = (connector.config?.base_url as string) || "Unknown"; + return ( +
+ {/* OAuth Info */} +
+
+ +
+
+

Connected via OAuth

+

+ This connector is authenticated using OAuth 2.0. Your Jira instance is: +

+

+ {baseUrl} +

+

+ To update your connection, disconnect and reconnect through the OAuth flow. +

+
+
+
+ ); + } + + // For legacy API token connectors, show the form return (
{/* Connector Name */} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx index 3ba03f956..7b0c3e82f 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx @@ -55,7 +55,6 @@ export const ConnectorConnectView: FC = ({ CONFLUENCE_CONNECTOR: "confluence-connect-form", BOOKSTACK_CONNECTOR: "bookstack-connect-form", GITHUB_CONNECTOR: "github-connect-form", - JIRA_CONNECTOR: "jira-connect-form", CLICKUP_CONNECTOR: "clickup-connect-form", LUMA_CONNECTOR: "luma-connect-form", CIRCLEBACK_CONNECTOR: "circleback-connect-form", diff --git a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts index 9822ff6e6..0e942dd1e 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts @@ -58,6 +58,13 @@ export const OAUTH_CONNECTORS = [ connectorType: EnumConnectorName.DISCORD_CONNECTOR, authEndpoint: "/api/v1/auth/discord/connector/add/", }, + { + id: "jira-connector", + title: "Jira", + description: "Search Jira issues", + connectorType: EnumConnectorName.JIRA_CONNECTOR, + authEndpoint: "/api/v1/auth/jira/connector/add/", + }, ] as const; // Content Sources (tools that extract and import content from external sources) @@ -96,12 +103,6 @@ export const OTHER_CONNECTORS = [ description: "Search repositories", connectorType: EnumConnectorName.GITHUB_CONNECTOR, }, - { - id: "jira-connector", - title: "Jira", - description: "Search Jira issues", - connectorType: EnumConnectorName.JIRA_CONNECTOR, - }, { id: "clickup-connector", title: "ClickUp", diff --git a/surfsense_web/hooks/use-connector-edit-page.ts b/surfsense_web/hooks/use-connector-edit-page.ts index 3beb80247..ba4ba6b58 100644 --- a/surfsense_web/hooks/use-connector-edit-page.ts +++ b/surfsense_web/hooks/use-connector-edit-page.ts @@ -447,6 +447,16 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) } break; case "JIRA_CONNECTOR": + // Check if this is an OAuth connector (has access_token or _token_encrypted flag) + const isJiraOAuth = !!(originalConfig.access_token || originalConfig._token_encrypted); + + if (isJiraOAuth) { + // OAuth connectors don't allow editing credentials through the form + // Only allow name changes, which are handled separately + break; + } + + // Legacy API token connector - allow editing credentials if ( formData.JIRA_BASE_URL !== originalConfig.JIRA_BASE_URL || formData.JIRA_EMAIL !== originalConfig.JIRA_EMAIL || From f236110a08176986667c8dd1580a9301604c489e Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 6 Jan 2026 00:09:08 +0530 Subject: [PATCH 4/6] Revert "feat: implement Jira OAuth integration and connector routes" This reverts commit bfed9a31f822929cc2b59b6c78a6ebe58a6c97d8. --- surfsense_backend/.env.example | 5 - surfsense_backend/app/config/__init__.py | 5 - .../app/connectors/jira_connector.py | 103 +--- surfsense_backend/app/routes/__init__.py | 2 - .../app/routes/jira_add_connector_route.py | 495 ------------------ .../app/schemas/jira_auth_credentials.py | 73 --- .../tasks/connector_indexers/jira_indexer.py | 142 +---- surfsense_backend/app/utils/validators.py | 14 +- .../components/jira-connect-form.tsx | 450 ++++++++++++++++ .../connector-popup/connect-forms/index.tsx | 3 + .../components/jira-config.tsx | 51 +- .../views/connector-connect-view.tsx | 1 + .../constants/connector-constants.ts | 13 +- .../hooks/use-connector-edit-page.ts | 10 - 14 files changed, 522 insertions(+), 845 deletions(-) delete mode 100644 surfsense_backend/app/routes/jira_add_connector_route.py delete mode 100644 surfsense_backend/app/schemas/jira_auth_credentials.py create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/jira-connect-form.tsx diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index a2f662c23..d2c667178 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -50,11 +50,6 @@ DISCORD_CLIENT_SECRET=your_discord_client_secret_here DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback DISCORD_BOT_TOKEN=your_bot_token_from_developer_portal -# Jira OAuth Configuration -JIRA_CLIENT_ID=our_jira_client_id -JIRA_CLIENT_SECRET=your_jira_client_secret -JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback - # OAuth for Linear Connector LINEAR_CLIENT_ID=your_linear_client_id LINEAR_CLIENT_SECRET=your_linear_client_secret diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 56641215d..f65a94cc0 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -111,11 +111,6 @@ class Config: DISCORD_REDIRECT_URI = os.getenv("DISCORD_REDIRECT_URI") DISCORD_BOT_TOKEN = os.getenv("DISCORD_BOT_TOKEN") - # Jira OAuth - JIRA_CLIENT_ID = os.getenv("JIRA_CLIENT_ID") - JIRA_CLIENT_SECRET = os.getenv("JIRA_CLIENT_SECRET") - JIRA_REDIRECT_URI = os.getenv("JIRA_REDIRECT_URI") - # LLM instances are now managed per-user through the LLMConfig system # Legacy environment variables removed in favor of user-specific configurations diff --git a/surfsense_backend/app/connectors/jira_connector.py b/surfsense_backend/app/connectors/jira_connector.py index 8e9badf0b..e73198e79 100644 --- a/surfsense_backend/app/connectors/jira_connector.py +++ b/surfsense_backend/app/connectors/jira_connector.py @@ -3,7 +3,6 @@ Jira Connector Module A module for retrieving data from Jira. Allows fetching issue lists and their comments, projects and more. -Supports both OAuth 2.0 (preferred) and legacy API token authentication. """ import base64 @@ -19,8 +18,6 @@ class JiraConnector: def __init__( self, base_url: str | None = None, - access_token: str | None = None, - cloud_id: str | None = None, email: str | None = None, api_token: str | None = None, ): @@ -28,39 +25,18 @@ class JiraConnector: Initialize the JiraConnector class. Args: - base_url: Jira instance base URL (e.g., 'https://yourcompany.atlassian.net') - access_token: OAuth 2.0 access token (preferred method) - cloud_id: Atlassian cloud ID (used with OAuth for API URL construction) - email: Jira account email address (legacy method, used with api_token) - api_token: Jira API token (legacy method, used with email) + base_url: Jira instance base URL (e.g., 'https://yourcompany.atlassian.net') (optional) + email: Jira account email address (optional) + api_token: Jira API token (optional) """ self.base_url = base_url.rstrip("/") if base_url else None - self.access_token = access_token - self.cloud_id = cloud_id self.email = email self.api_token = api_token self.api_version = "3" # Jira Cloud API version - self._use_oauth = access_token is not None - - def set_oauth_credentials( - self, base_url: str, access_token: str, cloud_id: str | None = None - ) -> None: - """ - Set OAuth 2.0 credentials (preferred method). - - Args: - base_url: Jira instance base URL - access_token: OAuth 2.0 access token - cloud_id: Atlassian cloud ID (optional, used for API URL construction) - """ - self.base_url = base_url.rstrip("/") - self.access_token = access_token - self.cloud_id = cloud_id - self._use_oauth = True def set_credentials(self, base_url: str, email: str, api_token: str) -> None: """ - Set the Jira credentials (legacy method using API token). + Set the Jira credentials. Args: base_url: Jira instance base URL @@ -70,69 +46,50 @@ class JiraConnector: self.base_url = base_url.rstrip("/") self.email = email self.api_token = api_token - self._use_oauth = False def set_email(self, email: str) -> None: """ - Set the Jira account email (legacy method). + Set the Jira account email. Args: email: Jira account email address """ self.email = email - self._use_oauth = False def set_api_token(self, api_token: str) -> None: """ - Set the Jira API token (legacy method). + Set the Jira API token. Args: api_token: Jira API token """ self.api_token = api_token - self._use_oauth = False def get_headers(self) -> dict[str, str]: """ - Get headers for Jira API requests. - - Uses OAuth Bearer token if available, otherwise falls back to Basic Auth. + Get headers for Jira API requests using Basic Authentication. Returns: Dictionary of headers Raises: - ValueError: If credentials have not been set + ValueError: If email, api_token, or base_url have not been set """ - if self._use_oauth: - # OAuth 2.0 authentication - if not self.base_url or not self.access_token: - raise ValueError( - "Jira OAuth credentials not initialized. Call set_oauth_credentials() first." - ) + if not all([self.base_url, self.email, self.api_token]): + raise ValueError( + "Jira credentials not initialized. Call set_credentials() first." + ) - return { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.access_token}", - "Accept": "application/json", - } - else: - # Legacy Basic Auth - if not all([self.base_url, self.email, self.api_token]): - raise ValueError( - "Jira credentials not initialized. Call set_credentials() first." - ) + # Create Basic Auth header using email:api_token + auth_str = f"{self.email}:{self.api_token}" + auth_bytes = auth_str.encode("utf-8") + auth_header = "Basic " + base64.b64encode(auth_bytes).decode("ascii") - # Create Basic Auth header using email:api_token - auth_str = f"{self.email}:{self.api_token}" - auth_bytes = auth_str.encode("utf-8") - auth_header = "Basic " + base64.b64encode(auth_bytes).decode("ascii") - - return { - "Content-Type": "application/json", - "Authorization": auth_header, - "Accept": "application/json", - } + return { + "Content-Type": "application/json", + "Authorization": auth_header, + "Accept": "application/json", + } def make_api_request( self, @@ -147,25 +104,21 @@ class JiraConnector: Args: endpoint: API endpoint (without base URL) params: Query parameters for the request (optional) - method: HTTP method (GET or POST) - json_payload: JSON payload for POST requests (optional) Returns: Response data from the API Raises: - ValueError: If credentials have not been set + ValueError: If email, api_token, or base_url have not been set Exception: If the API request fails """ - headers = self.get_headers() + if not all([self.base_url, self.email, self.api_token]): + raise ValueError( + "Jira credentials not initialized. Call set_credentials() first." + ) - # Construct API URL based on authentication method - if self._use_oauth and self.cloud_id: - # Use Atlassian API gateway with cloud_id for OAuth - url = f"https://api.atlassian.com/ex/jira/{self.cloud_id}/rest/api/{self.api_version}/{endpoint}" - else: - # Use direct base URL (works for both OAuth and legacy) - url = f"{self.base_url}/rest/api/{self.api_version}/{endpoint}" + url = f"{self.base_url}/rest/api/{self.api_version}/{endpoint}" + headers = self.get_headers() if method.upper() == "POST": response = requests.post( diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 16cacfeb8..b35d743e0 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -28,7 +28,6 @@ from .search_source_connectors_routes import router as search_source_connectors_ from .search_spaces_routes import router as search_spaces_router from .slack_add_connector_route import router as slack_add_connector_router from .discord_add_connector_route import router as discord_add_connector_router -from .jira_add_connector_route import router as jira_add_connector_router router = APIRouter() @@ -49,7 +48,6 @@ router.include_router(luma_add_connector_router) router.include_router(notion_add_connector_router) router.include_router(slack_add_connector_router) router.include_router(discord_add_connector_router) -router.include_router(jira_add_connector_router) router.include_router(new_llm_config_router) # LLM configs with prompt configuration router.include_router(logs_router) router.include_router(circleback_webhook_router) # Circleback meeting webhooks diff --git a/surfsense_backend/app/routes/jira_add_connector_route.py b/surfsense_backend/app/routes/jira_add_connector_route.py deleted file mode 100644 index ac415e80e..000000000 --- a/surfsense_backend/app/routes/jira_add_connector_route.py +++ /dev/null @@ -1,495 +0,0 @@ -""" -Jira Connector OAuth Routes. - -Handles OAuth 2.0 authentication flow for Jira connector. -Uses Atlassian OAuth 2.0 (3LO) with accessible-resources API to discover Jira instances. -""" - -import logging -from datetime import UTC, datetime, timedelta -from uuid import UUID - -import httpx -from fastapi import APIRouter, Depends, HTTPException, Request -from fastapi.responses import RedirectResponse -from pydantic import ValidationError -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.config import config -from app.db import ( - SearchSourceConnector, - SearchSourceConnectorType, - User, - get_async_session, -) -from app.schemas.jira_auth_credentials import JiraAuthCredentialsBase -from app.users import current_active_user -from app.utils.oauth_security import OAuthStateManager, TokenEncryption - -logger = logging.getLogger(__name__) - -router = APIRouter() - -# Atlassian OAuth endpoints -AUTHORIZATION_URL = "https://auth.atlassian.com/authorize" -TOKEN_URL = "https://auth.atlassian.com/oauth/token" -ACCESSIBLE_RESOURCES_URL = "https://api.atlassian.com/oauth/token/accessible-resources" - -# OAuth scopes for Jira -SCOPES = [ - "read:jira-work", - "write:jira-work", - "read:jira-user", - "offline_access", # Required for refresh tokens -] - -# Initialize security utilities -_state_manager = None -_token_encryption = None - - -def get_state_manager() -> OAuthStateManager: - """Get or create OAuth state manager instance.""" - global _state_manager - if _state_manager is None: - if not config.SECRET_KEY: - raise ValueError("SECRET_KEY must be set for OAuth security") - _state_manager = OAuthStateManager(config.SECRET_KEY) - return _state_manager - - -def get_token_encryption() -> TokenEncryption: - """Get or create token encryption instance.""" - global _token_encryption - if _token_encryption is None: - if not config.SECRET_KEY: - raise ValueError("SECRET_KEY must be set for token encryption") - _token_encryption = TokenEncryption(config.SECRET_KEY) - return _token_encryption - - -@router.get("/auth/jira/connector/add") -async def connect_jira(space_id: int, user: User = Depends(current_active_user)): - """ - Initiate Jira OAuth flow. - - Args: - space_id: The search space ID - user: Current authenticated user - - Returns: - Authorization URL for redirect - """ - try: - if not space_id: - raise HTTPException(status_code=400, detail="space_id is required") - - if not config.JIRA_CLIENT_ID: - raise HTTPException(status_code=500, detail="Jira OAuth not configured.") - - if not config.SECRET_KEY: - raise HTTPException( - status_code=500, detail="SECRET_KEY not configured for OAuth security." - ) - - # Generate secure state parameter with HMAC signature - state_manager = get_state_manager() - state_encoded = state_manager.generate_secure_state(space_id, user.id) - - # Build authorization URL - from urllib.parse import urlencode - - auth_params = { - "audience": "api.atlassian.com", - "client_id": config.JIRA_CLIENT_ID, - "scope": " ".join(SCOPES), - "redirect_uri": config.JIRA_REDIRECT_URI, - "state": state_encoded, - "response_type": "code", - "prompt": "consent", # Force consent screen to get refresh token - } - - auth_url = f"{AUTHORIZATION_URL}?{urlencode(auth_params)}" - - logger.info(f"Generated Jira OAuth URL for user {user.id}, space {space_id}") - return {"auth_url": auth_url} - - except Exception as e: - logger.error(f"Failed to initiate Jira OAuth: {e!s}", exc_info=True) - raise HTTPException( - status_code=500, detail=f"Failed to initiate Jira OAuth: {e!s}" - ) from e - - -@router.get("/auth/jira/connector/callback") -async def jira_callback( - request: Request, - code: str | None = None, - error: str | None = None, - state: str | None = None, - session: AsyncSession = Depends(get_async_session), -): - """ - Handle Jira OAuth callback. - - Args: - request: FastAPI request object - code: Authorization code from Atlassian (if user granted access) - error: Error code from Atlassian (if user denied access or error occurred) - state: State parameter containing user/space info - session: Database session - - Returns: - Redirect response to frontend - """ - try: - # Handle OAuth errors (e.g., user denied access) - if error: - logger.warning(f"Jira OAuth error: {error}") - # Try to decode state to get space_id for redirect, but don't fail if it's invalid - space_id = None - if state: - try: - state_manager = get_state_manager() - data = state_manager.validate_state(state) - space_id = data.get("space_id") - except Exception: - # If state is invalid, we'll redirect without space_id - logger.warning("Failed to validate state in error handler") - - # Redirect to frontend with error parameter - if space_id: - return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=jira_oauth_denied" - ) - else: - return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=jira_oauth_denied" - ) - - # Validate required parameters for successful flow - if not code: - raise HTTPException(status_code=400, detail="Missing authorization code") - if not state: - raise HTTPException(status_code=400, detail="Missing state parameter") - - # Validate and decode state with signature verification - state_manager = get_state_manager() - try: - data = state_manager.validate_state(state) - except HTTPException: - raise - except Exception as e: - raise HTTPException( - status_code=400, detail=f"Invalid state parameter: {e!s}" - ) from e - - user_id = UUID(data["user_id"]) - space_id = data["space_id"] - - # Validate redirect URI (security: ensure it matches configured value) - if not config.JIRA_REDIRECT_URI: - raise HTTPException( - status_code=500, detail="JIRA_REDIRECT_URI not configured" - ) - - # Exchange authorization code for access token - token_data = { - "grant_type": "authorization_code", - "client_id": config.JIRA_CLIENT_ID, - "client_secret": config.JIRA_CLIENT_SECRET, - "code": code, - "redirect_uri": config.JIRA_REDIRECT_URI, - } - - async with httpx.AsyncClient() as client: - token_response = await client.post( - TOKEN_URL, - data=token_data, - headers={"Content-Type": "application/x-www-form-urlencoded"}, - timeout=30.0, - ) - - if token_response.status_code != 200: - error_detail = token_response.text - try: - error_json = token_response.json() - error_detail = error_json.get("error_description", error_json.get("error", error_detail)) - except Exception: - pass - raise HTTPException( - status_code=400, detail=f"Token exchange failed: {error_detail}" - ) - - token_json = token_response.json() - - # Encrypt sensitive tokens before storing - token_encryption = get_token_encryption() - access_token = token_json.get("access_token") - refresh_token = token_json.get("refresh_token") - - if not access_token: - raise HTTPException( - status_code=400, detail="No access token received from Atlassian" - ) - - # Fetch accessible resources to get Jira instance information - async with httpx.AsyncClient() as client: - resources_response = await client.get( - ACCESSIBLE_RESOURCES_URL, - headers={"Authorization": f"Bearer {access_token}"}, - timeout=30.0, - ) - - if resources_response.status_code != 200: - error_detail = resources_response.text - logger.error(f"Failed to fetch accessible resources: {error_detail}") - raise HTTPException( - status_code=400, - detail=f"Failed to fetch Jira instances: {error_detail}", - ) - - resources = resources_response.json() - - # Filter for Jira instances (resources with type "jira" or id field) - jira_instances = [ - r - for r in resources - if r.get("id") and (r.get("name") or r.get("url")) - ] - - if not jira_instances: - raise HTTPException( - status_code=400, - detail="No accessible Jira instances found. Please ensure you have access to at least one Jira instance.", - ) - - # For now, use the first Jira instance - # TODO: Support multiple instances by letting user choose during OAuth - jira_instance = jira_instances[0] - cloud_id = jira_instance["id"] - base_url = jira_instance.get("url") - - # If URL is not provided, construct it from cloud_id - if not base_url: - # Try to extract from name or construct default format - instance_name = jira_instance.get("name", "").lower().replace(" ", "") - if instance_name: - base_url = f"https://{instance_name}.atlassian.net" - else: - # Fallback: use cloud_id directly (though this may not work) - base_url = f"https://{cloud_id}.atlassian.net" - - # Calculate expiration time (UTC, tz-aware) - expires_at = None - expires_in = token_json.get("expires_in") - if expires_in: - now_utc = datetime.now(UTC) - expires_at = now_utc + timedelta(seconds=int(expires_in)) - - # Store the encrypted access token and refresh token in connector config - connector_config = { - "access_token": token_encryption.encrypt_token(access_token), - "refresh_token": token_encryption.encrypt_token(refresh_token) - if refresh_token - else None, - "token_type": token_json.get("token_type", "Bearer"), - "expires_in": expires_in, - "expires_at": expires_at.isoformat() if expires_at else None, - "scope": token_json.get("scope"), - "cloud_id": cloud_id, - "base_url": base_url.rstrip("/") if base_url else None, - # Mark that tokens are encrypted for backward compatibility - "_token_encrypted": True, - } - - # Check if connector already exists for this search space and user - existing_connector_result = await session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.JIRA_CONNECTOR, - ) - ) - existing_connector = existing_connector_result.scalars().first() - - if existing_connector: - # Update existing connector - existing_connector.config = connector_config - existing_connector.name = "Jira Connector" - existing_connector.is_indexable = True - logger.info( - f"Updated existing Jira connector for user {user_id} in space {space_id}" - ) - else: - # Create new connector - new_connector = SearchSourceConnector( - name="Jira Connector", - connector_type=SearchSourceConnectorType.JIRA_CONNECTOR, - is_indexable=True, - config=connector_config, - search_space_id=space_id, - user_id=user_id, - ) - session.add(new_connector) - logger.info( - f"Created new Jira connector for user {user_id} in space {space_id}" - ) - - try: - await session.commit() - logger.info(f"Successfully saved Jira connector for user {user_id}") - - # Redirect to the frontend with success params - return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=jira-connector" - ) - - except ValidationError as e: - await session.rollback() - raise HTTPException( - status_code=422, detail=f"Validation error: {e!s}" - ) from e - except IntegrityError as e: - await session.rollback() - raise HTTPException( - status_code=409, - detail=f"Integrity error: A connector with this type already exists. {e!s}", - ) from e - except Exception as e: - logger.error(f"Failed to create search source connector: {e!s}") - await session.rollback() - raise HTTPException( - status_code=500, - detail=f"Failed to create search source connector: {e!s}", - ) from e - - except HTTPException: - raise - except Exception as e: - logger.error(f"Failed to complete Jira OAuth: {e!s}", exc_info=True) - raise HTTPException( - status_code=500, detail=f"Failed to complete Jira OAuth: {e!s}" - ) from e - - -async def refresh_jira_token( - session: AsyncSession, connector: SearchSourceConnector -) -> SearchSourceConnector: - """ - Refresh the Jira access token for a connector. - - Args: - session: Database session - connector: Jira connector to refresh - - Returns: - Updated connector object - """ - try: - logger.info(f"Refreshing Jira token for connector {connector.id}") - - credentials = JiraAuthCredentialsBase.from_dict(connector.config) - - # Decrypt tokens if they are encrypted - token_encryption = get_token_encryption() - is_encrypted = connector.config.get("_token_encrypted", False) - - refresh_token = credentials.refresh_token - if is_encrypted and refresh_token: - try: - refresh_token = token_encryption.decrypt_token(refresh_token) - except Exception as e: - logger.error(f"Failed to decrypt refresh token: {e!s}") - raise HTTPException( - status_code=500, detail="Failed to decrypt stored refresh token" - ) from e - - if not refresh_token: - raise HTTPException( - status_code=400, - detail="No refresh token available. Please re-authenticate.", - ) - - # Prepare token refresh data - refresh_data = { - "grant_type": "refresh_token", - "client_id": config.JIRA_CLIENT_ID, - "client_secret": config.JIRA_CLIENT_SECRET, - "refresh_token": refresh_token, - } - - async with httpx.AsyncClient() as client: - token_response = await client.post( - TOKEN_URL, - data=refresh_data, - headers={"Content-Type": "application/x-www-form-urlencoded"}, - timeout=30.0, - ) - - if token_response.status_code != 200: - error_detail = token_response.text - try: - error_json = token_response.json() - error_detail = error_json.get("error_description", error_json.get("error", error_detail)) - except Exception: - pass - raise HTTPException( - status_code=400, detail=f"Token refresh failed: {error_detail}" - ) - - token_json = token_response.json() - - # Calculate expiration time (UTC, tz-aware) - expires_at = None - expires_in = token_json.get("expires_in") - if expires_in: - now_utc = datetime.now(UTC) - expires_at = now_utc + timedelta(seconds=int(expires_in)) - - # Encrypt new tokens before storing - access_token = token_json.get("access_token") - new_refresh_token = token_json.get("refresh_token") - - if not access_token: - raise HTTPException( - status_code=400, detail="No access token received from Jira refresh" - ) - - # Update credentials object with encrypted tokens - credentials.access_token = token_encryption.encrypt_token(access_token) - if new_refresh_token: - credentials.refresh_token = token_encryption.encrypt_token( - new_refresh_token - ) - credentials.expires_in = expires_in - credentials.expires_at = expires_at - credentials.scope = token_json.get("scope") - - # Preserve cloud_id and base_url - if not credentials.cloud_id: - credentials.cloud_id = connector.config.get("cloud_id") - if not credentials.base_url: - credentials.base_url = connector.config.get("base_url") - - # Update connector config with encrypted tokens - credentials_dict = credentials.to_dict() - credentials_dict["_token_encrypted"] = True - connector.config = credentials_dict - await session.commit() - await session.refresh(connector) - - logger.info(f"Successfully refreshed Jira token for connector {connector.id}") - - return connector - except HTTPException: - raise - except Exception as e: - logger.error(f"Failed to refresh Jira token: {e!s}", exc_info=True) - raise HTTPException( - status_code=500, detail=f"Failed to refresh Jira token: {e!s}" - ) from e - diff --git a/surfsense_backend/app/schemas/jira_auth_credentials.py b/surfsense_backend/app/schemas/jira_auth_credentials.py deleted file mode 100644 index 23d1ffcbf..000000000 --- a/surfsense_backend/app/schemas/jira_auth_credentials.py +++ /dev/null @@ -1,73 +0,0 @@ -from datetime import UTC, datetime - -from pydantic import BaseModel, field_validator - - -class JiraAuthCredentialsBase(BaseModel): - access_token: str - refresh_token: str | None = None - token_type: str = "Bearer" - expires_in: int | None = None - expires_at: datetime | None = None - scope: str | None = None - cloud_id: str | None = None - base_url: str | None = None - - @property - def is_expired(self) -> bool: - """Check if the credentials have expired.""" - if self.expires_at is None: - return False - return self.expires_at <= datetime.now(UTC) - - @property - def is_refreshable(self) -> bool: - """Check if the credentials can be refreshed.""" - return self.refresh_token is not None - - def to_dict(self) -> dict: - """Convert credentials to dictionary for storage.""" - return { - "access_token": self.access_token, - "refresh_token": self.refresh_token, - "token_type": self.token_type, - "expires_in": self.expires_in, - "expires_at": self.expires_at.isoformat() if self.expires_at else None, - "scope": self.scope, - "cloud_id": self.cloud_id, - "base_url": self.base_url, - } - - @classmethod - def from_dict(cls, data: dict) -> "JiraAuthCredentialsBase": - """Create credentials from dictionary.""" - expires_at = None - if data.get("expires_at"): - expires_at = datetime.fromisoformat(data["expires_at"]) - - return cls( - access_token=data["access_token"], - refresh_token=data.get("refresh_token"), - token_type=data.get("token_type", "Bearer"), - expires_in=data.get("expires_in"), - expires_at=expires_at, - scope=data.get("scope"), - cloud_id=data.get("cloud_id"), - base_url=data.get("base_url"), - ) - - @field_validator("expires_at", mode="before") - @classmethod - def ensure_aware_utc(cls, v): - # Strings like "2025-08-26T14:46:57.367184" - if isinstance(v, str): - # add +00:00 if missing tz info - if v.endswith("Z"): - return datetime.fromisoformat(v.replace("Z", "+00:00")) - dt = datetime.fromisoformat(v) - return dt if dt.tzinfo else dt.replace(tzinfo=UTC) - # datetime objects - if isinstance(v, datetime): - return v if v.tzinfo else v.replace(tzinfo=UTC) - return v - diff --git a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py index 616927e6f..8c56b10ab 100644 --- a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py @@ -84,137 +84,31 @@ async def index_jira_issues( return 0, f"Connector with ID {connector_id} not found" # Get the Jira credentials from the connector config - # Support both OAuth (preferred) and legacy API token authentication - config_data = connector.config.copy() - is_oauth = config_data.get("_token_encrypted", False) or config_data.get("access_token") + jira_email = connector.config.get("JIRA_EMAIL") + jira_api_token = connector.config.get("JIRA_API_TOKEN") + jira_base_url = connector.config.get("JIRA_BASE_URL") - if is_oauth: - # OAuth 2.0 authentication - from app.utils.oauth_security import TokenEncryption - - if not config.SECRET_KEY: - await task_logger.log_task_failure( - log_entry, - f"SECRET_KEY not configured but tokens are marked as encrypted for connector {connector_id}", - "Missing SECRET_KEY for token decryption", - {"error_type": "MissingSecretKey"}, - ) - return 0, "SECRET_KEY not configured but tokens are marked as encrypted" - - try: - token_encryption = TokenEncryption(config.SECRET_KEY) - - # Decrypt access_token - if config_data.get("access_token"): - config_data["access_token"] = token_encryption.decrypt_token( - config_data["access_token"] - ) - logger.info( - f"Decrypted Jira access token for connector {connector_id}" - ) - - # Decrypt refresh_token if present - if config_data.get("refresh_token"): - config_data["refresh_token"] = token_encryption.decrypt_token( - config_data["refresh_token"] - ) - logger.info( - f"Decrypted Jira refresh token for connector {connector_id}" - ) - except Exception as e: - await task_logger.log_task_failure( - log_entry, - f"Failed to decrypt Jira tokens for connector {connector_id}: {e!s}", - "Token decryption failed", - {"error_type": "TokenDecryptionError"}, - ) - return 0, f"Failed to decrypt Jira tokens: {e!s}" - - try: - from app.schemas.jira_auth_credentials import JiraAuthCredentialsBase - credentials = JiraAuthCredentialsBase.from_dict(config_data) - except Exception as e: - await task_logger.log_task_failure( - log_entry, - f"Invalid Jira OAuth credentials in connector {connector_id}", - str(e), - {"error_type": "InvalidCredentials"}, - ) - return 0, f"Invalid Jira OAuth credentials: {e!s}" - - # Check if credentials are expired and refresh if needed - if credentials.is_expired: - await task_logger.log_task_progress( - log_entry, - f"Jira credentials expired for connector {connector_id}, refreshing token", - {"stage": "token_refresh"}, - ) - - from app.routes.jira_add_connector_route import refresh_jira_token - - try: - connector = await refresh_jira_token(session, connector) - # Re-fetch credentials after refresh - config_data = connector.config.copy() - if config_data.get("access_token"): - config_data["access_token"] = token_encryption.decrypt_token( - config_data["access_token"] - ) - credentials = JiraAuthCredentialsBase.from_dict(config_data) - except Exception as e: - await task_logger.log_task_failure( - log_entry, - f"Failed to refresh Jira token for connector {connector_id}: {e!s}", - "Token refresh failed", - {"error_type": "TokenRefreshError"}, - ) - return 0, f"Failed to refresh Jira token: {e!s}" - - # Initialize Jira client with OAuth credentials - await task_logger.log_task_progress( + if not jira_email or not jira_api_token or not jira_base_url: + await task_logger.log_task_failure( log_entry, - f"Initializing Jira client with OAuth for connector {connector_id}", - {"stage": "client_initialization"}, + f"Jira credentials not found in connector config for connector {connector_id}", + "Missing Jira credentials", + {"error_type": "MissingCredentials"}, ) + return 0, "Jira credentials not found in connector config" - jira_client = JiraConnector( - base_url=credentials.base_url, - access_token=credentials.access_token, - cloud_id=credentials.cloud_id, - ) - else: - # Legacy API token authentication - jira_email = config_data.get("JIRA_EMAIL") - jira_api_token = config_data.get("JIRA_API_TOKEN") - jira_base_url = config_data.get("JIRA_BASE_URL") + # Initialize Jira client + await task_logger.log_task_progress( + log_entry, + f"Initializing Jira client for connector {connector_id}", + {"stage": "client_initialization"}, + ) - if not jira_email or not jira_api_token or not jira_base_url: - await task_logger.log_task_failure( - log_entry, - f"Jira credentials not found in connector config for connector {connector_id}", - "Missing Jira credentials", - {"error_type": "MissingCredentials"}, - ) - return 0, "Jira credentials not found in connector config" - - # Initialize Jira client with legacy credentials - await task_logger.log_task_progress( - log_entry, - f"Initializing Jira client with API token for connector {connector_id}", - {"stage": "client_initialization"}, - ) - - jira_client = JiraConnector( - base_url=jira_base_url, email=jira_email, api_token=jira_api_token - ) + jira_client = JiraConnector( + base_url=jira_base_url, email=jira_email, api_token=jira_api_token + ) # Calculate date range - # Handle "undefined" strings from frontend - if start_date == "undefined" or start_date == "": - start_date = None - if end_date == "undefined" or end_date == "": - end_date = None - start_date_str, end_date_str = calculate_date_range( connector, start_date, end_date, default_days_back=365 ) diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index d1f416339..f1620c0e5 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -538,13 +538,13 @@ def validate_connector_config( }, }, # "DISCORD_CONNECTOR": {"required": ["DISCORD_BOT_TOKEN"], "validators": {}}, - # "JIRA_CONNECTOR": { - # "required": ["JIRA_EMAIL", "JIRA_API_TOKEN", "JIRA_BASE_URL"], - # "validators": { - # "JIRA_EMAIL": lambda: validate_email_field("JIRA_EMAIL", "JIRA"), - # "JIRA_BASE_URL": lambda: validate_url_field("JIRA_BASE_URL", "JIRA"), - # }, - # }, + "JIRA_CONNECTOR": { + "required": ["JIRA_EMAIL", "JIRA_API_TOKEN", "JIRA_BASE_URL"], + "validators": { + "JIRA_EMAIL": lambda: validate_email_field("JIRA_EMAIL", "JIRA"), + "JIRA_BASE_URL": lambda: validate_url_field("JIRA_BASE_URL", "JIRA"), + }, + }, "CONFLUENCE_CONNECTOR": { "required": [ "CONFLUENCE_BASE_URL", diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/jira-connect-form.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/jira-connect-form.tsx new file mode 100644 index 000000000..0499554b4 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/jira-connect-form.tsx @@ -0,0 +1,450 @@ +"use client"; + +import { zodResolver } from "@hookform/resolvers/zod"; +import { Info } from "lucide-react"; +import type { FC } from "react"; +import { useRef, useState } from "react"; +import { useForm } from "react-hook-form"; +import * as z from "zod"; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@/components/ui/accordion"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { EnumConnectorName } from "@/contracts/enums/connector"; +import { DateRangeSelector } from "../../components/date-range-selector"; +import { getConnectorBenefits } from "../connector-benefits"; +import type { ConnectFormProps } from "../index"; + +const jiraConnectorFormSchema = z.object({ + name: z.string().min(3, { + message: "Connector name must be at least 3 characters.", + }), + base_url: z.string().url({ message: "Please enter a valid Jira base URL." }), + email: z.string().email({ message: "Please enter a valid email address." }), + api_token: z.string().min(10, { + message: "Jira API Token is required and must be valid.", + }), +}); + +type JiraConnectorFormValues = z.infer; + +export const JiraConnectForm: FC = ({ onSubmit, isSubmitting }) => { + const isSubmittingRef = useRef(false); + const [startDate, setStartDate] = useState(undefined); + const [endDate, setEndDate] = useState(undefined); + const [periodicEnabled, setPeriodicEnabled] = useState(false); + const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); + const form = useForm({ + resolver: zodResolver(jiraConnectorFormSchema), + defaultValues: { + name: "Jira Connector", + base_url: "", + email: "", + api_token: "", + }, + }); + + const handleSubmit = async (values: JiraConnectorFormValues) => { + // Prevent multiple submissions + if (isSubmittingRef.current || isSubmitting) { + return; + } + + isSubmittingRef.current = true; + try { + await onSubmit({ + name: values.name, + connector_type: EnumConnectorName.JIRA_CONNECTOR, + config: { + JIRA_BASE_URL: values.base_url, + JIRA_EMAIL: values.email, + JIRA_API_TOKEN: values.api_token, + }, + is_indexable: true, + last_indexed_at: null, + periodic_indexing_enabled: periodicEnabled, + indexing_frequency_minutes: periodicEnabled ? parseInt(frequencyMinutes, 10) : null, + next_scheduled_at: null, + startDate, + endDate, + periodicEnabled, + frequencyMinutes, + }); + } finally { + isSubmittingRef.current = false; + } + }; + + return ( +
+ + +
+ API Token Required + + You'll need a Jira API Token to use this connector. You can create one from{" "} + + Atlassian Account Settings + + +
+ + +
+
+ + ( + + Connector Name + + + + + A friendly name to identify this connector. + + + + )} + /> + + ( + + Jira Base URL + + + + + The base URL of your Jira instance (e.g., https://your-domain.atlassian.net). + + + + )} + /> + + ( + + Email Address + + + + + The email address associated with your Atlassian account. + + + + )} + /> + + ( + + API Token + + + + + Your Jira API Token will be encrypted and stored securely. + + + + )} + /> + + {/* Indexing Configuration */} +
+

Indexing Configuration

+ + {/* Date Range Selector */} + + + {/* Periodic Sync Config */} +
+
+
+

Enable Periodic Sync

+

+ Automatically re-index at regular intervals +

+
+ +
+ + {periodicEnabled && ( +
+
+ + +
+
+ )} +
+
+ + +
+ + {/* What you get section */} + {getConnectorBenefits(EnumConnectorName.JIRA_CONNECTOR) && ( +
+

What you get with Jira integration:

+
    + {getConnectorBenefits(EnumConnectorName.JIRA_CONNECTOR)?.map((benefit) => ( +
  • {benefit}
    • + ))} +
+
+ )} + + {/* Documentation Section */} + + + + Documentation + + +
+

How it works

+

+ The Jira connector uses the Jira REST API with Basic Authentication to fetch all + issues and comments that your account has access to within your Jira instance. +

+
    +
  • + For follow up indexing runs, the connector retrieves issues and comments that have + been updated since the last indexing attempt. +
    • +
  • + Indexing is configured to run periodically, so updates should appear in your + search results within minutes. +
    • +
+
+ +
+
+

Authorization

+ + + + Read-Only Access is Sufficient + + + You only need read access for this connector to work. The API Token will only be + used to read your Jira data. + + + +
+
+

+ Step 1: Create an API Token +

+
    +
  1. Log in to your Atlassian account
    2. +
  2. + Navigate to{" "} + + https://id.atlassian.com/manage-profile/security/api-tokens + {" "} + in your browser. +
    3. +
  3. + Click Create API token +
    4. +
  4. Enter a label for your token (like "SurfSense Connector")
    5. +
  5. + Click Create +
    6. +
  6. Copy the generated token as it will only be shown once
    7. +
+
+ +
+

+ Step 2: Grant necessary access +

+

+ The API Token will have access to all projects and issues that your user + account can see. Make sure your account has appropriate permissions for the + projects you want to index. +

+ + + Data Privacy + + Only issues, comments, and basic metadata will be indexed. Jira attachments + and linked files are not indexed by this connector. + + +
+
+
+
+ +
+
+

Indexing

+
    +
  1. + Navigate to the Connector Dashboard and select the Jira{" "} + Connector. +
    2. +
  2. + Enter your Jira Instance URL (e.g., + https://yourcompany.atlassian.net) +
    3. +
  3. + Enter your Email Address associated with your Atlassian account +
    4. +
  4. + Place your API Token in the form field. +
    5. +
  5. + Click Connect to establish the connection. +
    6. +
  6. Once connected, your Jira issues will be indexed automatically.
    7. +
+ + + + What Gets Indexed + +

The Jira connector indexes the following data:

+
    +
  • Issue keys and summaries (e.g., PROJ-123)
    • +
  • Issue descriptions
    • +
  • Issue comments and discussion threads
    • +
  • Issue status, priority, and type information
    • +
  • Assignee and reporter information
    • +
  • Project information
    • +
+ + +
+
+ + + +
+ ); +}; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx index cda17ddfc..81e5ee03f 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx @@ -6,6 +6,7 @@ import { ClickUpConnectForm } from "./components/clickup-connect-form"; import { ConfluenceConnectForm } from "./components/confluence-connect-form"; import { ElasticsearchConnectForm } from "./components/elasticsearch-connect-form"; import { GithubConnectForm } from "./components/github-connect-form"; +import { JiraConnectForm } from "./components/jira-connect-form"; import { LinkupApiConnectForm } from "./components/linkup-api-connect-form"; import { LumaConnectForm } from "./components/luma-connect-form"; import { SearxngConnectForm } from "./components/searxng-connect-form"; @@ -54,6 +55,8 @@ export function getConnectFormComponent(connectorType: string): ConnectFormCompo return BookStackConnectForm; case "GITHUB_CONNECTOR": return GithubConnectForm; + case "JIRA_CONNECTOR": + return JiraConnectForm; case "CLICKUP_CONNECTOR": return ClickUpConnectForm; case "LUMA_CONNECTOR": diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx index 158dfdf13..3ef16bdb4 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/jira-config.tsx @@ -1,6 +1,6 @@ "use client"; -import { Info, KeyRound } from "lucide-react"; +import { KeyRound } from "lucide-react"; import type { FC } from "react"; import { useEffect, useState } from "react"; import { Input } from "@/components/ui/input"; @@ -12,9 +12,6 @@ export interface JiraConfigProps extends ConnectorConfigProps { } export const JiraConfig: FC = ({ connector, onConfigChange, onNameChange }) => { - // Check if this is an OAuth connector (has access_token or _token_encrypted flag) - const isOAuth = !!(connector.config?.access_token || connector.config?._token_encrypted); - const [baseUrl, setBaseUrl] = useState((connector.config?.JIRA_BASE_URL as string) || ""); const [email, setEmail] = useState((connector.config?.JIRA_EMAIL as string) || ""); const [apiToken, setApiToken] = useState( @@ -22,18 +19,16 @@ export const JiraConfig: FC = ({ connector, onConfigChange, onN ); const [name, setName] = useState(connector.name || ""); - // Update values when connector changes (only for legacy connectors) + // Update values when connector changes useEffect(() => { - if (!isOAuth) { - const url = (connector.config?.JIRA_BASE_URL as string) || ""; - const emailVal = (connector.config?.JIRA_EMAIL as string) || ""; - const token = (connector.config?.JIRA_API_TOKEN as string) || ""; - setBaseUrl(url); - setEmail(emailVal); - setApiToken(token); - } + const url = (connector.config?.JIRA_BASE_URL as string) || ""; + const emailVal = (connector.config?.JIRA_EMAIL as string) || ""; + const token = (connector.config?.JIRA_API_TOKEN as string) || ""; + setBaseUrl(url); + setEmail(emailVal); + setApiToken(token); setName(connector.name || ""); - }, [connector.config, connector.name, isOAuth]); + }, [connector.config, connector.name]); const handleBaseUrlChange = (value: string) => { setBaseUrl(value); @@ -72,34 +67,6 @@ export const JiraConfig: FC = ({ connector, onConfigChange, onN } }; - // For OAuth connectors, show simple info message - if (isOAuth) { - const baseUrl = (connector.config?.base_url as string) || "Unknown"; - return ( -
- {/* OAuth Info */} -
-
- -
-
-

Connected via OAuth

-

- This connector is authenticated using OAuth 2.0. Your Jira instance is: -

-

- {baseUrl} -

-

- To update your connection, disconnect and reconnect through the OAuth flow. -

-
-
-
- ); - } - - // For legacy API token connectors, show the form return (
{/* Connector Name */} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx index 7b0c3e82f..3ba03f956 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx @@ -55,6 +55,7 @@ export const ConnectorConnectView: FC = ({ CONFLUENCE_CONNECTOR: "confluence-connect-form", BOOKSTACK_CONNECTOR: "bookstack-connect-form", GITHUB_CONNECTOR: "github-connect-form", + JIRA_CONNECTOR: "jira-connect-form", CLICKUP_CONNECTOR: "clickup-connect-form", LUMA_CONNECTOR: "luma-connect-form", CIRCLEBACK_CONNECTOR: "circleback-connect-form", diff --git a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts index 0e942dd1e..9822ff6e6 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts @@ -58,13 +58,6 @@ export const OAUTH_CONNECTORS = [ connectorType: EnumConnectorName.DISCORD_CONNECTOR, authEndpoint: "/api/v1/auth/discord/connector/add/", }, - { - id: "jira-connector", - title: "Jira", - description: "Search Jira issues", - connectorType: EnumConnectorName.JIRA_CONNECTOR, - authEndpoint: "/api/v1/auth/jira/connector/add/", - }, ] as const; // Content Sources (tools that extract and import content from external sources) @@ -103,6 +96,12 @@ export const OTHER_CONNECTORS = [ description: "Search repositories", connectorType: EnumConnectorName.GITHUB_CONNECTOR, }, + { + id: "jira-connector", + title: "Jira", + description: "Search Jira issues", + connectorType: EnumConnectorName.JIRA_CONNECTOR, + }, { id: "clickup-connector", title: "ClickUp", diff --git a/surfsense_web/hooks/use-connector-edit-page.ts b/surfsense_web/hooks/use-connector-edit-page.ts index ba4ba6b58..3beb80247 100644 --- a/surfsense_web/hooks/use-connector-edit-page.ts +++ b/surfsense_web/hooks/use-connector-edit-page.ts @@ -447,16 +447,6 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) } break; case "JIRA_CONNECTOR": - // Check if this is an OAuth connector (has access_token or _token_encrypted flag) - const isJiraOAuth = !!(originalConfig.access_token || originalConfig._token_encrypted); - - if (isJiraOAuth) { - // OAuth connectors don't allow editing credentials through the form - // Only allow name changes, which are handled separately - break; - } - - // Legacy API token connector - allow editing credentials if ( formData.JIRA_BASE_URL !== originalConfig.JIRA_BASE_URL || formData.JIRA_EMAIL !== originalConfig.JIRA_EMAIL || From ebbb3ca02325fcae75826c507309299f86bcb093 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 5 Jan 2026 13:27:32 -0800 Subject: [PATCH 5/6] update: readmes with clearer descriptions of SurfSense. --- README.md | 6 ++++-- README.zh-CN.md | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7a10a71a7..acd900588 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,9 @@
# SurfSense -While tools like NotebookLM and Perplexity are impressive and highly effective for conducting research on any topic/query, SurfSense elevates this capability by integrating with your personal knowledge base. It is a highly customizable AI research agent, connected to external sources such as Search Engines (SearxNG, Tavily, LinkUp), Google Drive, Slack, Linear, Jira, ClickUp, Confluence, BookStack, Gmail, Notion, YouTube, GitHub, Discord, Airtable, Google Calendar, Luma, Circleback, Elasticsearch and more to come. +Connect any LLM to your internal knowledge sources and chat with it in real time alongside your team. OSS alternative to NotebookLM, Perplexity, and Glean. + +SurfSense is a highly customizable AI research agent, connected to external sources such as Search Engines (SearxNG, Tavily, LinkUp), Google Drive, Slack, Linear, Jira, ClickUp, Confluence, BookStack, Gmail, Notion, YouTube, GitHub, Discord, Airtable, Google Calendar, Luma, Circleback, Elasticsearch and more to come.
MODSetter%2FSurfSense | Trendshift @@ -38,7 +40,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 ## Key Features ### 💡 **Idea**: -- Have your own highly customizable private NotebookLM and Perplexity integrated with external sources. +- Open source alternative to NotebookLM, Perplexity, and Glean. Connect any LLM to your internal knowledge sources and collaborate with your team in real time. ### 📁 **Multiple File Format Uploading Support** - Save content from your own personal files *(Documents, images, videos and supports **50+ file extensions**)* to your own personal knowledge base . ### 🔍 **Powerful Search** diff --git a/README.zh-CN.md b/README.zh-CN.md index e4322ca27..4e4b0174b 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -16,7 +16,9 @@ # SurfSense -虽然像 NotebookLM 和 Perplexity 这样的工具在对任何主题/查询进行研究时令人印象深刻且非常有效，但 SurfSense 通过与您的个人知识库集成，将这一能力提升到了新的高度。它是一个高度可定制的 AI 研究助手，可以连接外部数据源，如搜索引擎（SearxNG、Tavily、LinkUp、Google Drive、Slack、Linear、Jira、ClickUp、Confluence、BookStack、Gmail、Notion、YouTube、GitHub、Discord、Airtable、Google Calendar、Luma、Circleback、Elasticsearch 等，未来还会支持更多。 +将任何 LLM 连接到您的内部知识源，并与团队成员实时聊天。NotebookLM、Perplexity 和 Glean 的开源替代方案。 + +SurfSense 是一个高度可定制的 AI 研究助手，可以连接外部数据源，如搜索引擎（SearxNG、Tavily、LinkUp）、Google Drive、Slack、Linear、Jira、ClickUp、Confluence、BookStack、Gmail、Notion、YouTube、GitHub、Discord、Airtable、Google Calendar、Luma、Circleback、Elasticsearch 等，未来还会支持更多。
MODSetter%2FSurfSense | Trendshift @@ -38,7 +40,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 ## 核心功能 ### 💡 **理念**: -- 拥有您自己的高度可定制的私有 NotebookLM 和 Perplexity，并与外部数据源集成。 +- NotebookLM、Perplexity 和 Glean 的开源替代方案。将任何 LLM 连接到您的内部知识源，并与团队实时协作。 ### 📁 **支持多种文件格式上传** - 将您个人文件中的内容（文档、图像、视频，支持 **50+ 种文件扩展名**）保存到您自己的个人知识库。 From aac04320236db523a98f6e70235eb90ff08afdca Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 5 Jan 2026 22:18:25 -0800 Subject: [PATCH 6/6] refactor: update Discord message indexing logic - Enhanced the indexing process for Discord messages to treat each message as an individual document, improving metadata handling and content management. - Replaced the announcement banner component and related state management with a more streamlined approach, removing unnecessary files and simplifying the dashboard layout. - Updated logging messages for clarity and accuracy regarding processed messages. --- .../connector_indexers/discord_indexer.py | 312 ++++++++---------- surfsense_web/app/dashboard/layout.tsx | 2 - surfsense_web/atoms/announcement.atom.ts | 5 - .../components/announcement-banner.tsx | 47 --- 4 files changed, 129 insertions(+), 237 deletions(-) delete mode 100644 surfsense_web/atoms/announcement.atom.ts delete mode 100644 surfsense_web/components/announcement-banner.tsx diff --git a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py index b3de1f4b5..5c92d2601 100644 --- a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py @@ -11,17 +11,15 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.config import config from app.connectors.discord_connector import DiscordConnector from app.db import Document, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) from .base import ( - build_document_metadata_string, + build_document_metadata_markdown, check_document_by_unique_identifier, get_connector_by_id, get_current_timestamp, @@ -336,207 +334,155 @@ async def index_discord_messages( documents_skipped += 1 continue - # Convert messages to markdown format - channel_content = ( - f"# Discord Channel: {guild_name} / {channel_name}\n\n" - ) + # Process each message as an individual document (like Slack) for msg in formatted_messages: - user_name = msg.get("author_name", "Unknown User") - timestamp = msg.get("created_at", "Unknown Time") - text = msg.get("content", "") - channel_content += ( - f"## {user_name} ({timestamp})\n\n{text}\n\n---\n\n" + msg_id = msg.get("id", "") + msg_user_name = msg.get("author_name", "Unknown User") + msg_timestamp = msg.get("created_at", "Unknown Time") + msg_text = msg.get("content", "") + + # Format document metadata (similar to Slack) + metadata_sections = [ + ( + "METADATA", + [ + f"GUILD_NAME: {guild_name}", + f"GUILD_ID: {guild_id}", + f"CHANNEL_NAME: {channel_name}", + f"CHANNEL_ID: {channel_id}", + f"MESSAGE_TIMESTAMP: {msg_timestamp}", + f"MESSAGE_USER_NAME: {msg_user_name}", + ], + ), + ( + "CONTENT", + [ + "FORMAT: markdown", + "TEXT_START", + msg_text, + "TEXT_END", + ], + ), + ] + + # Build the document string + combined_document_string = build_document_metadata_markdown( + metadata_sections ) - # Metadata sections - metadata_sections = [ - ( - "METADATA", - [ - f"GUILD_NAME: {guild_name}", - f"GUILD_ID: {guild_id}", - f"CHANNEL_NAME: {channel_name}", - f"CHANNEL_ID: {channel_id}", - f"MESSAGE_COUNT: {len(formatted_messages)}", - ], - ), - ( - "CONTENT", - [ - "FORMAT: markdown", - "TEXT_START", - channel_content, - "TEXT_END", - ], - ), - ] + # Generate unique identifier hash for this Discord message + unique_identifier = f"{channel_id}_{msg_id}" + unique_identifier_hash = generate_unique_identifier_hash( + DocumentType.DISCORD_CONNECTOR, + unique_identifier, + search_space_id, + ) - combined_document_string = build_document_metadata_string( - metadata_sections - ) + # Generate content hash + content_hash = generate_content_hash( + combined_document_string, search_space_id + ) - # Generate unique identifier hash for this Discord channel - unique_identifier_hash = generate_unique_identifier_hash( - DocumentType.DISCORD_CONNECTOR, channel_id, search_space_id - ) + # Check if document with this unique identifier already exists + existing_document = await check_document_by_unique_identifier( + session, unique_identifier_hash + ) - # Generate content hash - content_hash = generate_content_hash( - combined_document_string, search_space_id - ) - - # Check if document with this unique identifier already exists - existing_document = await check_document_by_unique_identifier( - session, unique_identifier_hash - ) - - if existing_document: - # Document exists - check if content has changed - if existing_document.content_hash == content_hash: - logger.info( - f"Document for Discord channel {guild_name}#{channel_name} unchanged. Skipping." - ) - documents_skipped += 1 - continue - else: - # Content has changed - update the existing document - logger.info( - f"Content changed for Discord channel {guild_name}#{channel_name}. Updating document." - ) - - # Get user's long context LLM - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) - if not user_llm: - logger.error( - f"No long context LLM configured for user {user_id}" - ) - skipped_channels.append( - f"{guild_name}#{channel_name} (no LLM configured)" + if existing_document: + # Document exists - check if content has changed + if existing_document.content_hash == content_hash: + logger.info( + f"Document for Discord message {msg_id} in {guild_name}#{channel_name} unchanged. Skipping." ) documents_skipped += 1 continue + else: + # Content has changed - update the existing document + logger.info( + f"Content changed for Discord message {msg_id} in {guild_name}#{channel_name}. Updating document." + ) - # Generate summary with metadata - document_metadata = { - "guild_name": guild_name, - "channel_name": channel_name, - "message_count": len(formatted_messages), - "document_type": "Discord Channel Messages", - "connector_type": "Discord", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - combined_document_string, - user_llm, - document_metadata, - ) + # Update chunks and embedding + chunks = await create_document_chunks( + combined_document_string + ) + doc_embedding = config.embedding_model_instance.embed( + combined_document_string + ) - # Chunks from channel content - chunks = await create_document_chunks(channel_content) + # Update existing document + existing_document.content = combined_document_string + existing_document.content_hash = content_hash + existing_document.embedding = doc_embedding + existing_document.document_metadata = { + "guild_name": guild_name, + "guild_id": guild_id, + "channel_name": channel_name, + "channel_id": channel_id, + "message_id": msg_id, + "message_timestamp": msg_timestamp, + "message_user_name": msg_user_name, + "indexed_at": datetime.now(UTC).strftime( + "%Y-%m-%d %H:%M:%S" + ), + } - # Update existing document - existing_document.title = ( - f"Discord - {guild_name}#{channel_name}" - ) - existing_document.content = summary_content - existing_document.content_hash = content_hash - existing_document.embedding = summary_embedding - existing_document.document_metadata = { + # Delete old chunks and add new ones + existing_document.chunks = chunks + existing_document.updated_at = get_current_timestamp() + + documents_indexed += 1 + logger.info( + f"Successfully updated Discord message {msg_id}" + ) + continue + + # Document doesn't exist - create new one + # Process chunks + chunks = await create_document_chunks(combined_document_string) + doc_embedding = config.embedding_model_instance.embed( + combined_document_string + ) + + # Create and store new document + document = Document( + search_space_id=search_space_id, + title=f"Discord - {guild_name}#{channel_name}", + document_type=DocumentType.DISCORD_CONNECTOR, + document_metadata={ "guild_name": guild_name, "guild_id": guild_id, "channel_name": channel_name, "channel_id": channel_id, - "message_count": len(formatted_messages), - "start_date": start_date_iso, - "end_date": end_date_iso, + "message_id": msg_id, + "message_timestamp": msg_timestamp, + "message_user_name": msg_user_name, "indexed_at": datetime.now(UTC).strftime( "%Y-%m-%d %H:%M:%S" ), - } - existing_document.chunks = chunks - existing_document.updated_at = get_current_timestamp() + }, + content=combined_document_string, + embedding=doc_embedding, + chunks=chunks, + content_hash=content_hash, + unique_identifier_hash=unique_identifier_hash, + updated_at=get_current_timestamp(), + ) - documents_indexed += 1 + session.add(document) + documents_indexed += 1 + + # Batch commit every 10 documents + if documents_indexed % 10 == 0: logger.info( - f"Successfully updated Discord channel {guild_name}#{channel_name}" + f"Committing batch: {documents_indexed} Discord messages processed so far" ) - continue + await session.commit() - # Document doesn't exist - create new one - # Get user's long context LLM - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) - if not user_llm: - logger.error( - f"No long context LLM configured for user {user_id}" - ) - skipped_channels.append( - f"{guild_name}#{channel_name} (no LLM configured)" - ) - documents_skipped += 1 - continue - - # Generate summary with metadata - document_metadata = { - "guild_name": guild_name, - "channel_name": channel_name, - "message_count": len(formatted_messages), - "document_type": "Discord Channel Messages", - "connector_type": "Discord", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - combined_document_string, user_llm, document_metadata - ) - - # Chunks from channel content - chunks = await create_document_chunks(channel_content) - - # Create and store new document - document = Document( - search_space_id=search_space_id, - title=f"Discord - {guild_name}#{channel_name}", - document_type=DocumentType.DISCORD_CONNECTOR, - document_metadata={ - "guild_name": guild_name, - "guild_id": guild_id, - "channel_name": channel_name, - "channel_id": channel_id, - "message_count": len(formatted_messages), - "start_date": start_date_iso, - "end_date": end_date_iso, - "indexed_at": datetime.now(UTC).strftime( - "%Y-%m-%d %H:%M:%S" - ), - }, - content=summary_content, - content_hash=content_hash, - unique_identifier_hash=unique_identifier_hash, - embedding=summary_embedding, - chunks=chunks, - updated_at=get_current_timestamp(), - ) - - session.add(document) - documents_indexed += 1 logger.info( - f"Successfully indexed new channel {guild_name}#{channel_name} with {len(formatted_messages)} messages" + f"Successfully indexed channel {guild_name}#{channel_name} with {len(formatted_messages)} messages" ) - # Batch commit every 10 documents - if documents_indexed % 10 == 0: - logger.info( - f"Committing batch: {documents_indexed} Discord channels processed so far" - ) - await session.commit() - except Exception as e: logger.error( f"Error processing guild {guild_name}: {e!s}", exc_info=True @@ -553,7 +499,7 @@ async def index_discord_messages( # Final commit for any remaining documents not yet committed in batches logger.info( - f"Final commit: Total {documents_indexed} Discord channels processed" + f"Final commit: Total {documents_indexed} Discord messages processed" ) await session.commit() @@ -561,18 +507,18 @@ async def index_discord_messages( result_message = None if skipped_channels: result_message = ( - f"Processed {documents_indexed} channels. Skipped {len(skipped_channels)} channels: " + f"Processed {documents_indexed} messages. Skipped {len(skipped_channels)} channels: " + ", ".join(skipped_channels) ) else: - result_message = f"Processed {documents_indexed} channels." + result_message = f"Processed {documents_indexed} messages." # Log success await task_logger.log_task_success( log_entry, f"Successfully completed Discord indexing for connector {connector_id}", { - "channels_processed": documents_indexed, + "messages_processed": documents_indexed, "documents_indexed": documents_indexed, "documents_skipped": documents_skipped, "skipped_channels_count": len(skipped_channels), @@ -582,7 +528,7 @@ async def index_discord_messages( ) logger.info( - f"Discord indexing completed: {documents_indexed} new channels, {documents_skipped} skipped" + f"Discord indexing completed: {documents_indexed} new messages, {documents_skipped} skipped" ) return documents_indexed, result_message diff --git a/surfsense_web/app/dashboard/layout.tsx b/surfsense_web/app/dashboard/layout.tsx index 8763a622f..71cd6275f 100644 --- a/surfsense_web/app/dashboard/layout.tsx +++ b/surfsense_web/app/dashboard/layout.tsx @@ -2,7 +2,6 @@ import { Loader2 } from "lucide-react"; import { useEffect, useState } from "react"; -import { AnnouncementBanner } from "@/components/announcement-banner"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { getBearerToken, redirectToLogin } from "@/lib/auth-utils"; @@ -43,7 +42,6 @@ export default function DashboardLayout({ children }: DashboardLayoutProps) { return (
-
{children}
); diff --git a/surfsense_web/atoms/announcement.atom.ts b/surfsense_web/atoms/announcement.atom.ts deleted file mode 100644 index 31e032978..000000000 --- a/surfsense_web/atoms/announcement.atom.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { atomWithStorage } from "jotai/utils"; - -// Atom to track whether the announcement banner has been dismissed -// Persists to localStorage automatically -export const announcementDismissedAtom = atomWithStorage("surfsense_announcement_dismissed", false); diff --git a/surfsense_web/components/announcement-banner.tsx b/surfsense_web/components/announcement-banner.tsx deleted file mode 100644 index 537aa6da7..000000000 --- a/surfsense_web/components/announcement-banner.tsx +++ /dev/null @@ -1,47 +0,0 @@ -"use client"; - -import { useAtom } from "jotai"; -import { ExternalLink, Info, X } from "lucide-react"; -import { announcementDismissedAtom } from "@/atoms/announcement.atom"; -import { Button } from "@/components/ui/button"; - -export function AnnouncementBanner() { - const [isDismissed, setIsDismissed] = useAtom(announcementDismissedAtom); - - const handleDismiss = () => { - setIsDismissed(true); - }; - - if (isDismissed) return null; - - return ( -
-
-
- -

- SurfSense is a work in progress.{" "} - - Report issues on GitHub - - -

- -
-
-
- ); -}