apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-04-25 00:36:31 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #670 from AnishSarkar22/fix/connector

Browse source 
feat: Clickup OAuth Connector, fixed Airtable OAuth Connector
This commit is contained in:
Rohan Verma 2026-01-07 12:39:41 -08:00 • committed by GitHub
commit fabbae2b48
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 1277 additions and 561 deletions
Download patch file Download diff file Expand all files Collapse all files

35
surfsense_backend/.env.example
View file

@ -34,16 +34,21 @@ REGISTRATION_ENABLED=TRUE or FALSE
GOOGLE_OAUTH_CLIENT_ID=924507538m
GOOGLE_OAUTH_CLIENT_SECRET=GOCSV
# Connector Specific Configs
# Google Connector Specific Configurations
GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback
GOOGLE_GMAIL_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/gmail/connector/callback
GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback
# OAuth for Aitable Connector
AIRTABLE_CLIENT_ID=your_airtable_client_id
AIRTABLE_CLIENT_SECRET=your_airtable_client_secret
# Aitable OAuth Configuration
AIRTABLE_CLIENT_ID=your_airtable_client_id_here
AIRTABLE_CLIENT_SECRET=your_airtable_client_secret_here
AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback
# ClickUp OAuth Configuration
CLICKUP_CLIENT_ID=your_clickup_client_id_here
CLICKUP_CLIENT_SECRET=your_clickup_client_secret_here
CLICKUP_REDIRECT_URI=http://localhost:8000/api/v1/auth/clickup/connector/callback
# Discord OAuth Configuration
DISCORD_CLIENT_ID=your_discord_client_id_here
DISCORD_CLIENT_SECRET=your_discord_client_secret_here
@ -51,23 +56,23 @@ DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callbac
DISCORD_BOT_TOKEN=your_bot_token_from_developer_portal
# Jira OAuth Configuration
JIRA_CLIENT_ID=our_jira_client_id
JIRA_CLIENT_SECRET=your_jira_client_secret
JIRA_CLIENT_ID=your_jira_client_id_here
JIRA_CLIENT_SECRET=your_jira_client_secret_here
JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback
# OAuth for Linear Connector
LINEAR_CLIENT_ID=your_linear_client_id
LINEAR_CLIENT_SECRET=your_linear_client_secret
# Linear OAuth Configuration
LINEAR_CLIENT_ID=your_linear_client_id_here
LINEAR_CLIENT_SECRET=your_linear_client_secret_here
LINEAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/linear/connector/callback
# OAuth for Notion Connector
NOTION_CLIENT_ID=your_notion_client_id
NOTION_CLIENT_SECRET=your_notion_client_secret
# Notion OAuth Configuration
NOTION_CLIENT_ID=your_notion_client_id_here
NOTION_CLIENT_SECRET=your_notion_client_secret_here
NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback
# OAuth for Slack connector
SLACK_CLIENT_ID=1234567890.1234567890123
SLACK_CLIENT_SECRET=abcdefghijklmnopqrstuvwxyz1234567890
# Slack OAuth Configuration
SLACK_CLIENT_ID=your_slack_client_id_here
SLACK_CLIENT_SECRET=your_slack_client_secret_here
SLACK_REDIRECT_URI=http://localhost:8000/api/v1/auth/slack/connector/callback
# Embedding Model

5
surfsense_backend/app/config/__init__.py
View file

@ -117,6 +117,11 @@ class Config:
DISCORD_REDIRECT_URI = os.getenv("DISCORD_REDIRECT_URI")
DISCORD_BOT_TOKEN = os.getenv("DISCORD_BOT_TOKEN")
# ClickUp OAuth
CLICKUP_CLIENT_ID = os.getenv("CLICKUP_CLIENT_ID")
CLICKUP_CLIENT_SECRET = os.getenv("CLICKUP_CLIENT_SECRET")
CLICKUP_REDIRECT_URI = os.getenv("CLICKUP_REDIRECT_URI")
# LLM instances are now managed per-user through the LLMConfig system
# Legacy environment variables removed in favor of user-specific configurations

6
surfsense_backend/app/connectors/airtable_connector.py
View file

@ -294,6 +294,12 @@ class AirtableConnector:
Tuple of (records, error_message)
"""
try:
# Validate date strings before parsing
if not start_date or start_date.lower() in ("undefined", "null", "none"):
return [], "Invalid start_date: date string is required"
if not end_date or end_date.lower() in ("undefined", "null", "none"):
return [], "Invalid end_date: date string is required"
# Parse and validate dates
start_dt = isoparse(start_date)
end_dt = isoparse(end_date)

175
surfsense_backend/app/connectors/airtable_history.py Normal file
View file

@ -0,0 +1,175 @@
"""
Airtable OAuth Connector.
Handles OAuth-based authentication and token refresh for Airtable API access.
"""
import logging
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.connectors.airtable_connector import AirtableConnector
from app.db import SearchSourceConnector
from app.routes.airtable_add_connector_route import refresh_airtable_token
from app.schemas.airtable_auth_credentials import AirtableAuthCredentialsBase
from app.utils.oauth_security import TokenEncryption
logger = logging.getLogger(__name__)
class AirtableHistoryConnector:
"""
Airtable connector with OAuth support and automatic token refresh.
This connector uses OAuth 2.0 access tokens to authenticate with the
Airtable API. It automatically refreshes expired tokens when needed.
"""
def __init__(
self,
session: AsyncSession,
connector_id: int,
credentials: AirtableAuthCredentialsBase | None = None,
):
"""
Initialize the AirtableHistoryConnector with auto-refresh capability.
Args:
session: Database session for updating connector
connector_id: Connector ID for direct updates
credentials: Airtable OAuth credentials (optional, will be loaded from DB if not provided)
"""
self._session = session
self._connector_id = connector_id
self._credentials = credentials
self._airtable_connector: AirtableConnector | None = None
async def _get_valid_token(self) -> str:
"""
Get valid Airtable access token, refreshing if needed.
Returns:
Valid access token
Raises:
ValueError: If credentials are missing or invalid
Exception: If token refresh fails
"""
# Load credentials from DB if not provided
if self._credentials is None:
result = await self._session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.id == self._connector_id
)
)
connector = result.scalars().first()
if not connector:
raise ValueError(f"Connector {self._connector_id} not found")
config_data = connector.config.copy()
# Decrypt credentials if they are encrypted
token_encrypted = config_data.get("_token_encrypted", False)
if token_encrypted and config.SECRET_KEY:
try:
token_encryption = TokenEncryption(config.SECRET_KEY)
# Decrypt sensitive fields
if config_data.get("access_token"):
config_data["access_token"] = token_encryption.decrypt_token(
config_data["access_token"]
)
if config_data.get("refresh_token"):
config_data["refresh_token"] = token_encryption.decrypt_token(
config_data["refresh_token"]
)
logger.info(
f"Decrypted Airtable credentials for connector {self._connector_id}"
)
except Exception as e:
logger.error(
f"Failed to decrypt Airtable credentials for connector {self._connector_id}: {e!s}"
)
raise ValueError(
f"Failed to decrypt Airtable credentials: {e!s}"
) from e
try:
self._credentials = AirtableAuthCredentialsBase.from_dict(config_data)
except Exception as e:
raise ValueError(f"Invalid Airtable credentials: {e!s}") from e
# Check if token is expired and refreshable
if self._credentials.is_expired and self._credentials.is_refreshable:
try:
logger.info(
f"Airtable token expired for connector {self._connector_id}, refreshing..."
)
# Get connector for refresh
result = await self._session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.id == self._connector_id
)
)
connector = result.scalars().first()
if not connector:
raise RuntimeError(
f"Connector {self._connector_id} not found; cannot refresh token."
)
# Refresh token
connector = await refresh_airtable_token(self._session, connector)
# Reload credentials after refresh
config_data = connector.config.copy()
token_encrypted = config_data.get("_token_encrypted", False)
if token_encrypted and config.SECRET_KEY:
token_encryption = TokenEncryption(config.SECRET_KEY)
if config_data.get("access_token"):
config_data["access_token"] = token_encryption.decrypt_token(
config_data["access_token"]
)
if config_data.get("refresh_token"):
config_data["refresh_token"] = token_encryption.decrypt_token(
config_data["refresh_token"]
)
self._credentials = AirtableAuthCredentialsBase.from_dict(config_data)
# Invalidate cached connector so it's recreated with new token
self._airtable_connector = None
logger.info(
f"Successfully refreshed Airtable token for connector {self._connector_id}"
)
except Exception as e:
logger.error(
f"Failed to refresh Airtable token for connector {self._connector_id}: {e!s}"
)
raise Exception(
f"Failed to refresh Airtable OAuth credentials: {e!s}"
) from e
return self._credentials.access_token
async def _get_connector(self) -> AirtableConnector:
"""
Get or create AirtableConnector with valid token.
Returns:
AirtableConnector instance
"""
if self._airtable_connector is None:
# Ensure we have valid credentials (this will refresh if needed)
await self._get_valid_token()
# Use the credentials object which is now guaranteed to be valid
if not self._credentials:
raise ValueError("Credentials not loaded")
self._airtable_connector = AirtableConnector(self._credentials)
return self._airtable_connector

349
surfsense_backend/app/connectors/clickup_history.py Normal file
View file

@ -0,0 +1,349 @@
"""
ClickUp History Module
A module for retrieving data from ClickUp with OAuth support and backward compatibility.
Allows fetching tasks from workspaces and lists with automatic token refresh.
"""
import logging
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.connectors.clickup_connector import ClickUpConnector
from app.db import SearchSourceConnector
from app.routes.clickup_add_connector_route import refresh_clickup_token
from app.schemas.clickup_auth_credentials import ClickUpAuthCredentialsBase
from app.utils.oauth_security import TokenEncryption
logger = logging.getLogger(__name__)
class ClickUpHistoryConnector:
"""
Class for retrieving data from ClickUp with OAuth support and backward compatibility.
"""
def __init__(
self,
session: AsyncSession,
connector_id: int,
credentials: ClickUpAuthCredentialsBase | None = None,
api_token: str | None = None, # For backward compatibility
):
"""
Initialize the ClickUpHistoryConnector.
Args:
session: Database session for token refresh
connector_id: Connector ID for direct updates
credentials: ClickUp OAuth credentials (optional, will be loaded from DB if not provided)
api_token: Legacy API token for backward compatibility (optional)
"""
self._session = session
self._connector_id = connector_id
self._credentials = credentials
self._api_token = api_token # Legacy API token
self._use_oauth = False
self._use_legacy = api_token is not None
self._clickup_client: ClickUpConnector | None = None
async def _get_valid_token(self) -> str:
"""
Get valid ClickUp access token, refreshing if needed.
For legacy API tokens, returns the token directly.
Returns:
Valid access token or API token
Raises:
ValueError: If credentials are missing or invalid
Exception: If token refresh fails
"""
# If using legacy API token, return it directly
if self._use_legacy and self._api_token:
return self._api_token
# Load credentials from DB if not provided
if self._credentials is None:
result = await self._session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.id == self._connector_id
)
)
connector = result.scalars().first()
if not connector:
raise ValueError(f"Connector {self._connector_id} not found")
config_data = connector.config.copy()
# Check if using OAuth or legacy API token
is_oauth = config_data.get("_token_encrypted", False) or config_data.get(
"access_token"
)
has_legacy_token = config_data.get("CLICKUP_API_TOKEN") is not None
if is_oauth:
# OAuth 2.0 authentication
self._use_oauth = True
# Decrypt credentials if they are encrypted
token_encrypted = config_data.get("_token_encrypted", False)
if token_encrypted and config.SECRET_KEY:
try:
token_encryption = TokenEncryption(config.SECRET_KEY)
# Decrypt sensitive fields
if config_data.get("access_token"):
config_data["access_token"] = (
token_encryption.decrypt_token(
config_data["access_token"]
)
)
if config_data.get("refresh_token"):
config_data["refresh_token"] = (
token_encryption.decrypt_token(
config_data["refresh_token"]
)
)
logger.info(
f"Decrypted ClickUp OAuth credentials for connector {self._connector_id}"
)
except Exception as e:
logger.error(
f"Failed to decrypt ClickUp OAuth credentials for connector {self._connector_id}: {e!s}"
)
raise ValueError(
f"Failed to decrypt ClickUp OAuth credentials: {e!s}"
) from e
try:
self._credentials = ClickUpAuthCredentialsBase.from_dict(
config_data
)
except Exception as e:
raise ValueError(f"Invalid ClickUp OAuth credentials: {e!s}") from e
elif has_legacy_token:
# Legacy API token authentication (backward compatibility)
self._use_legacy = True
self._api_token = config_data.get("CLICKUP_API_TOKEN")
# Decrypt token if it's encrypted (legacy tokens might be encrypted)
token_encrypted = config_data.get("_token_encrypted", False)
if token_encrypted and config.SECRET_KEY and self._api_token:
try:
token_encryption = TokenEncryption(config.SECRET_KEY)
self._api_token = token_encryption.decrypt_token(
self._api_token
)
logger.info(
f"Decrypted legacy ClickUp API token for connector {self._connector_id}"
)
except Exception as e:
logger.warning(
f"Failed to decrypt legacy ClickUp API token for connector {self._connector_id}: {e!s}. "
"Trying to use token as-is (might be unencrypted)."
)
# Continue with token as-is - might be unencrypted legacy token
if not self._api_token:
raise ValueError("ClickUp API token not found in connector config")
# Return legacy token directly (no refresh needed)
return self._api_token
else:
raise ValueError(
"ClickUp credentials not found in connector config (neither OAuth nor API token)"
)
# Check if token is expired and refreshable (only for OAuth)
if (
self._use_oauth
and self._credentials.is_expired
and self._credentials.is_refreshable
):
try:
logger.info(
f"ClickUp token expired for connector {self._connector_id}, refreshing..."
)
# Get connector for refresh
result = await self._session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.id == self._connector_id
)
)
connector = result.scalars().first()
if not connector:
raise RuntimeError(
f"Connector {self._connector_id} not found; cannot refresh token."
)
# Refresh token
connector = await refresh_clickup_token(self._session, connector)
# Reload credentials after refresh
config_data = connector.config.copy()
token_encrypted = config_data.get("_token_encrypted", False)
if token_encrypted and config.SECRET_KEY:
token_encryption = TokenEncryption(config.SECRET_KEY)
if config_data.get("access_token"):
config_data["access_token"] = token_encryption.decrypt_token(
config_data["access_token"]
)
if config_data.get("refresh_token"):
config_data["refresh_token"] = token_encryption.decrypt_token(
config_data["refresh_token"]
)
self._credentials = ClickUpAuthCredentialsBase.from_dict(config_data)
# Invalidate cached client so it's recreated with new token
self._clickup_client = None
logger.info(
f"Successfully refreshed ClickUp token for connector {self._connector_id}"
)
except Exception as e:
logger.error(
f"Failed to refresh ClickUp token for connector {self._connector_id}: {e!s}"
)
raise Exception(
f"Failed to refresh ClickUp OAuth credentials: {e!s}"
) from e
if self._use_oauth:
return self._credentials.access_token
else:
return self._api_token
async def _get_client(self) -> ClickUpConnector:
"""
Get or create ClickUpConnector with valid token.
Returns:
ClickUpConnector instance
"""
if self._clickup_client is None:
token = await self._get_valid_token()
# ClickUp API uses Bearer token for OAuth, or direct token for legacy
if self._use_oauth:
# For OAuth, use Bearer token format (ClickUp OAuth expects "Bearer {token}")
self._clickup_client = ClickUpConnector(api_token=f"Bearer {token}")
else:
# For legacy API token, use token directly (format: "pk_...")
self._clickup_client = ClickUpConnector(api_token=token)
return self._clickup_client
async def close(self):
"""Close any open connections."""
self._clickup_client = None
async def __aenter__(self):
"""Async context manager entry."""
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
"""Async context manager exit."""
await self.close()
async def get_authorized_workspaces(self) -> dict[str, Any]:
"""
Fetch authorized workspaces (teams) from ClickUp.
Returns:
Dictionary containing teams data
Raises:
ValueError: If credentials have not been set
Exception: If the API request fails
"""
client = await self._get_client()
return client.get_authorized_workspaces()
async def get_workspace_tasks(
self, workspace_id: str, include_closed: bool = False
) -> list[dict[str, Any]]:
"""
Fetch all tasks from a ClickUp workspace.
Args:
workspace_id: ClickUp workspace (team) ID
include_closed: Whether to include closed tasks (default: False)
Returns:
List of task objects
Raises:
ValueError: If credentials have not been set
Exception: If the API request fails
"""
client = await self._get_client()
return client.get_workspace_tasks(
workspace_id=workspace_id, include_closed=include_closed
)
async def get_tasks_in_date_range(
self,
workspace_id: str,
start_date: str,
end_date: str,
include_closed: bool = False,
) -> tuple[list[dict[str, Any]], str | None]:
"""
Fetch tasks from ClickUp within a specific date range.
Args:
workspace_id: ClickUp workspace (team) ID
start_date: Start date in YYYY-MM-DD format
end_date: End date in YYYY-MM-DD format
include_closed: Whether to include closed tasks (default: False)
Returns:
Tuple containing (tasks list, error message or None)
"""
client = await self._get_client()
return client.get_tasks_in_date_range(
workspace_id=workspace_id,
start_date=start_date,
end_date=end_date,
include_closed=include_closed,
)
async def get_task_details(self, task_id: str) -> dict[str, Any]:
"""
Fetch detailed information about a specific task.
Args:
task_id: ClickUp task ID
Returns:
Task details
Raises:
ValueError: If credentials have not been set
Exception: If the API request fails
"""
client = await self._get_client()
return client.get_task_details(task_id)
async def get_task_comments(self, task_id: str) -> dict[str, Any]:
"""
Fetch comments for a specific task.
Args:
task_id: ClickUp task ID
Returns:
Task comments
Raises:
ValueError: If credentials have not been set
Exception: If the API request fails
"""
client = await self._get_client()
return client.get_task_comments(task_id)

2
surfsense_backend/app/connectors/slack_history.py
View file

@ -377,7 +377,7 @@ class SlackHistory:
else:
raise # Re-raise to outer handler for not_in_channel or other SlackApiErrors
if not current_api_call_successful:
if not current_api_call_successful or result is None:
continue # Retry the current page fetch due to handled rate limit
# Process result if successful

2
surfsense_backend/app/routes/__init__.py
View file

@ -4,6 +4,7 @@ from .airtable_add_connector_route import (
router as airtable_add_connector_router,
)
from .circleback_webhook_route import router as circleback_webhook_router
from .clickup_add_connector_route import router as clickup_add_connector_router
from .confluence_add_connector_route import router as confluence_add_connector_router
from .discord_add_connector_route import router as discord_add_connector_router
from .documents_routes import router as documents_router
@ -52,6 +53,7 @@ router.include_router(slack_add_connector_router)
router.include_router(discord_add_connector_router)
router.include_router(jira_add_connector_router)
router.include_router(confluence_add_connector_router)
router.include_router(clickup_add_connector_router)
router.include_router(new_llm_config_router) # LLM configs with prompt configuration
router.include_router(logs_router)
router.include_router(circleback_webhook_router) # Circleback meeting webhooks

18
surfsense_backend/app/routes/airtable_add_connector_route.py
View file

@ -381,7 +381,7 @@ async def airtable_callback(
async def refresh_airtable_token(
session: AsyncSession, connector: SearchSourceConnector
):
) -> SearchSourceConnector:
"""
Refresh the Airtable access token for a connector.
@ -411,6 +411,12 @@ async def refresh_airtable_token(
status_code=500, detail="Failed to decrypt stored refresh token"
) from e
if not refresh_token:
raise HTTPException(
status_code=400,
detail="No refresh token available. Please re-authenticate.",
)
auth_header = make_basic_auth_header(
config.AIRTABLE_CLIENT_ID, config.AIRTABLE_CLIENT_SECRET
)
@ -435,8 +441,14 @@ async def refresh_airtable_token(
)
if token_response.status_code != 200:
error_detail = token_response.text
try:
error_json = token_response.json()
error_detail = error_json.get("error_description", error_detail)
except Exception:
pass
raise HTTPException(
status_code=400, detail="Token refresh failed: {token_response.text}"
status_code=400, detail=f"Token refresh failed: {error_detail}"
)
token_json = token_response.json()
@ -478,6 +490,8 @@ async def refresh_airtable_token(
)
return connector
except HTTPException:
raise
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Failed to refresh Airtable token: {e!s}"

481
surfsense_backend/app/routes/clickup_add_connector_route.py Normal file
View file

@ -0,0 +1,481 @@
"""
ClickUp Connector OAuth Routes.
Handles OAuth 2.0 authentication flow for ClickUp connector.
"""
import logging
from datetime import UTC, datetime, timedelta
from uuid import UUID
import httpx
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.responses import RedirectResponse
from pydantic import ValidationError
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.db import (
SearchSourceConnector,
SearchSourceConnectorType,
User,
get_async_session,
)
from app.schemas.clickup_auth_credentials import ClickUpAuthCredentialsBase
from app.users import current_active_user
from app.utils.oauth_security import OAuthStateManager, TokenEncryption
logger = logging.getLogger(__name__)
router = APIRouter()
# ClickUp OAuth endpoints
AUTHORIZATION_URL = "https://app.clickup.com/api"
TOKEN_URL = "https://api.clickup.com/api/v2/oauth/token"
# Initialize security utilities
_state_manager = None
_token_encryption = None
def get_state_manager() -> OAuthStateManager:
"""Get or create OAuth state manager instance."""
global _state_manager
if _state_manager is None:
if not config.SECRET_KEY:
raise ValueError("SECRET_KEY must be set for OAuth security")
_state_manager = OAuthStateManager(config.SECRET_KEY)
return _state_manager
def get_token_encryption() -> TokenEncryption:
"""Get or create token encryption instance."""
global _token_encryption
if _token_encryption is None:
if not config.SECRET_KEY:
raise ValueError("SECRET_KEY must be set for token encryption")
_token_encryption = TokenEncryption(config.SECRET_KEY)
return _token_encryption
@router.get("/auth/clickup/connector/add")
async def connect_clickup(space_id: int, user: User = Depends(current_active_user)):
"""
Initiate ClickUp OAuth flow.
Args:
space_id: The search space ID
user: Current authenticated user
Returns:
Authorization URL for redirect
"""
try:
if not space_id:
raise HTTPException(status_code=400, detail="space_id is required")
if not config.CLICKUP_CLIENT_ID:
raise HTTPException(status_code=500, detail="ClickUp OAuth not configured.")
if not config.SECRET_KEY:
raise HTTPException(
status_code=500, detail="SECRET_KEY not configured for OAuth security."
)
# Generate secure state parameter with HMAC signature
state_manager = get_state_manager()
state_encoded = state_manager.generate_secure_state(space_id, user.id)
# Build authorization URL
from urllib.parse import urlencode
auth_params = {
"client_id": config.CLICKUP_CLIENT_ID,
"redirect_uri": config.CLICKUP_REDIRECT_URI,
"state": state_encoded,
}
auth_url = f"{AUTHORIZATION_URL}?{urlencode(auth_params)}"
logger.info(f"Generated ClickUp OAuth URL for user {user.id}, space {space_id}")
return {"auth_url": auth_url}
except Exception as e:
logger.error(f"Failed to initiate ClickUp OAuth: {e!s}", exc_info=True)
raise HTTPException(
status_code=500, detail=f"Failed to initiate ClickUp OAuth: {e!s}"
) from e
@router.get("/auth/clickup/connector/callback")
async def clickup_callback(
request: Request,
code: str | None = None,
error: str | None = None,
state: str | None = None,
session: AsyncSession = Depends(get_async_session),
):
"""
Handle ClickUp OAuth callback.
Args:
request: FastAPI request object
code: Authorization code from ClickUp (if user granted access)
error: Error code from ClickUp (if user denied access or error occurred)
state: State parameter containing user/space info
session: Database session
Returns:
Redirect response to frontend
"""
try:
# Handle OAuth errors (e.g., user denied access)
if error:
logger.warning(f"ClickUp OAuth error: {error}")
# Try to decode state to get space_id for redirect, but don't fail if it's invalid
space_id = None
if state:
try:
state_manager = get_state_manager()
data = state_manager.validate_state(state)
space_id = data.get("space_id")
except Exception:
# If state is invalid, we'll redirect without space_id
logger.warning("Failed to validate state in error handler")
# Redirect to frontend with error parameter
if space_id:
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=clickup_oauth_denied"
)
else:
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard?error=clickup_oauth_denied"
)
# Validate required parameters for successful flow
if not code:
raise HTTPException(status_code=400, detail="Missing authorization code")
if not state:
raise HTTPException(status_code=400, detail="Missing state parameter")
# Validate and decode state with signature verification
state_manager = get_state_manager()
try:
data = state_manager.validate_state(state)
except HTTPException:
raise
except Exception as e:
raise HTTPException(
status_code=400, detail=f"Invalid state parameter: {e!s}"
) from e
user_id = UUID(data["user_id"])
space_id = data["space_id"]
# Validate redirect URI (security: ensure it matches configured value)
if not config.CLICKUP_REDIRECT_URI:
raise HTTPException(
status_code=500, detail="CLICKUP_REDIRECT_URI not configured"
)
# Exchange authorization code for access token
token_data = {
"client_id": config.CLICKUP_CLIENT_ID,
"client_secret": config.CLICKUP_CLIENT_SECRET,
"code": code,
}
async with httpx.AsyncClient() as client:
token_response = await client.post(
TOKEN_URL,
json=token_data,
headers={"Content-Type": "application/json"},
timeout=30.0,
)
if token_response.status_code != 200:
error_detail = token_response.text
try:
error_json = token_response.json()
error_detail = error_json.get("error", error_detail)
except Exception:
pass
raise HTTPException(
status_code=400, detail=f"Token exchange failed: {error_detail}"
)
token_json = token_response.json()
# Extract access token
access_token = token_json.get("access_token")
if not access_token:
raise HTTPException(
status_code=400, detail="No access token received from ClickUp"
)
# Extract refresh token if available
refresh_token = token_json.get("refresh_token")
# Encrypt sensitive tokens before storing
token_encryption = get_token_encryption()
# Calculate expiration time (UTC, tz-aware)
expires_at = None
expires_in = token_json.get("expires_in")
if expires_in:
now_utc = datetime.now(UTC)
expires_at = now_utc + timedelta(seconds=int(expires_in))
# Get user information and workspace information from ClickUp API
user_info = {}
workspace_info = {}
try:
async with httpx.AsyncClient() as client:
# Get user info
user_response = await client.get(
"https://api.clickup.com/api/v2/user",
headers={"Authorization": f"Bearer {access_token}"},
timeout=30.0,
)
if user_response.status_code == 200:
user_data = user_response.json().get("user", {})
user_info = {
"user_id": str(user_data.get("id"))
if user_data.get("id") is not None
else None,
"user_email": user_data.get("email"),
"user_name": user_data.get("username"),
}
# Get workspace (team) info - get the first workspace
team_response = await client.get(
"https://api.clickup.com/api/v2/team",
headers={"Authorization": f"Bearer {access_token}"},
timeout=30.0,
)
if team_response.status_code == 200:
teams_data = team_response.json().get("teams", [])
if teams_data and len(teams_data) > 0:
first_team = teams_data[0]
workspace_info = {
"workspace_id": str(first_team.get("id"))
if first_team.get("id") is not None
else None,
"workspace_name": first_team.get("name"),
}
except Exception as e:
logger.warning(f"Failed to fetch user/workspace info from ClickUp: {e!s}")
# Store the encrypted tokens and user/workspace info in connector config
connector_config = {
"access_token": token_encryption.encrypt_token(access_token),
"refresh_token": token_encryption.encrypt_token(refresh_token)
if refresh_token
else None,
"expires_in": expires_in,
"expires_at": expires_at.isoformat() if expires_at else None,
"user_id": user_info.get("user_id"),
"user_email": user_info.get("user_email"),
"user_name": user_info.get("user_name"),
"workspace_id": workspace_info.get("workspace_id"),
"workspace_name": workspace_info.get("workspace_name"),
# Mark that token is encrypted for backward compatibility
"_token_encrypted": True,
}
# Check if connector already exists for this search space and user
existing_connector_result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.CLICKUP_CONNECTOR,
)
)
existing_connector = existing_connector_result.scalars().first()
if existing_connector:
# Update existing connector
existing_connector.config = connector_config
existing_connector.name = "ClickUp Connector"
existing_connector.is_indexable = True
logger.info(
f"Updated existing ClickUp connector for user {user_id} in space {space_id}"
)
else:
# Create new connector
new_connector = SearchSourceConnector(
name="ClickUp Connector",
connector_type=SearchSourceConnectorType.CLICKUP_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new ClickUp connector for user {user_id} in space {space_id}"
)
try:
await session.commit()
logger.info(f"Successfully saved ClickUp connector for user {user_id}")
# Redirect to the frontend with success params
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=clickup-connector"
)
except ValidationError as e:
await session.rollback()
raise HTTPException(
status_code=422, detail=f"Validation error: {e!s}"
) from e
except IntegrityError as e:
await session.rollback()
raise HTTPException(
status_code=409,
detail=f"Integrity error: A connector with this type already exists. {e!s}",
) from e
except Exception as e:
logger.error(f"Failed to create search source connector: {e!s}")
await session.rollback()
raise HTTPException(
status_code=500,
detail=f"Failed to create search source connector: {e!s}",
) from e
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to complete ClickUp OAuth: {e!s}", exc_info=True)
raise HTTPException(
status_code=500, detail=f"Failed to complete ClickUp OAuth: {e!s}"
) from e
async def refresh_clickup_token(
session: AsyncSession, connector: SearchSourceConnector
) -> SearchSourceConnector:
"""
Refresh the ClickUp access token for a connector.
Args:
session: Database session
connector: ClickUp connector to refresh
Returns:
Updated connector object
"""
try:
logger.info(f"Refreshing ClickUp token for connector {connector.id}")
credentials = ClickUpAuthCredentialsBase.from_dict(connector.config)
# Decrypt tokens if they are encrypted
token_encryption = get_token_encryption()
is_encrypted = connector.config.get("_token_encrypted", False)
refresh_token = credentials.refresh_token
if is_encrypted and refresh_token:
try:
refresh_token = token_encryption.decrypt_token(refresh_token)
except Exception as e:
logger.error(f"Failed to decrypt refresh token: {e!s}")
raise HTTPException(
status_code=500, detail="Failed to decrypt stored refresh token"
) from e
if not refresh_token:
raise HTTPException(
status_code=400,
detail="No refresh token available. Please re-authenticate.",
)
# Prepare token refresh data
refresh_data = {
"client_id": config.CLICKUP_CLIENT_ID,
"client_secret": config.CLICKUP_CLIENT_SECRET,
"refresh_token": refresh_token,
}
async with httpx.AsyncClient() as client:
token_response = await client.post(
TOKEN_URL,
json=refresh_data,
headers={"Content-Type": "application/json"},
timeout=30.0,
)
if token_response.status_code != 200:
error_detail = token_response.text
try:
error_json = token_response.json()
error_detail = error_json.get("error", error_detail)
except Exception:
pass
raise HTTPException(
status_code=400, detail=f"Token refresh failed: {error_detail}"
)
token_json = token_response.json()
# Calculate expiration time (UTC, tz-aware)
expires_at = None
expires_in = token_json.get("expires_in")
if expires_in:
now_utc = datetime.now(UTC)
expires_at = now_utc + timedelta(seconds=int(expires_in))
# Encrypt new tokens before storing
access_token = token_json.get("access_token")
new_refresh_token = token_json.get("refresh_token")
if not access_token:
raise HTTPException(
status_code=400, detail="No access token received from ClickUp refresh"
)
# Update credentials object with encrypted tokens
credentials.access_token = token_encryption.encrypt_token(access_token)
if new_refresh_token:
credentials.refresh_token = token_encryption.encrypt_token(
new_refresh_token
)
credentials.expires_in = expires_in
credentials.expires_at = expires_at
# Preserve user and workspace info
if not credentials.user_id:
credentials.user_id = connector.config.get("user_id")
if not credentials.user_email:
credentials.user_email = connector.config.get("user_email")
if not credentials.user_name:
credentials.user_name = connector.config.get("user_name")
if not credentials.workspace_id:
credentials.workspace_id = connector.config.get("workspace_id")
if not credentials.workspace_name:
credentials.workspace_name = connector.config.get("workspace_name")
# Update connector config with encrypted tokens
credentials_dict = credentials.to_dict()
credentials_dict["_token_encrypted"] = True
connector.config = credentials_dict
await session.commit()
await session.refresh(connector)
logger.info(
f"Successfully refreshed ClickUp token for connector {connector.id}"
)
return connector
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to refresh ClickUp token: {e!s}", exc_info=True)
raise HTTPException(
status_code=500, detail=f"Failed to refresh ClickUp token: {e!s}"
) from e

85
surfsense_backend/app/schemas/clickup_auth_credentials.py Normal file
View file

@ -0,0 +1,85 @@
from datetime import UTC, datetime
from pydantic import BaseModel, field_validator
class ClickUpAuthCredentialsBase(BaseModel):
access_token: str
refresh_token: str | None = None
expires_in: int | None = None
expires_at: datetime | None = None
user_id: str | None = None
user_email: str | None = None
user_name: str | None = None
workspace_id: str | None = None
workspace_name: str | None = None
@property
def is_expired(self) -> bool:
"""Check if the credentials have expired."""
if self.expires_at is None:
return False # Long-lived token, treat as not expired
return self.expires_at <= datetime.now(UTC)
@property
def is_refreshable(self) -> bool:
"""Check if the credentials can be refreshed."""
return self.refresh_token is not None
def to_dict(self) -> dict:
"""Convert credentials to dictionary for storage."""
return {
"access_token": self.access_token,
"refresh_token": self.refresh_token,
"expires_in": self.expires_in,
"expires_at": self.expires_at.isoformat() if self.expires_at else None,
"user_id": self.user_id,
"user_email": self.user_email,
"user_name": self.user_name,
"workspace_id": self.workspace_id,
"workspace_name": self.workspace_name,
}
@classmethod
def from_dict(cls, data: dict) -> "ClickUpAuthCredentialsBase":
"""Create credentials from dictionary."""
expires_at = None
if data.get("expires_at"):
expires_at = datetime.fromisoformat(data["expires_at"])
# Convert user_id to string if it's an integer (for backward compatibility)
user_id = data.get("user_id")
if user_id is not None and not isinstance(user_id, str):
user_id = str(user_id)
# Convert workspace_id to string if it's an integer (for backward compatibility)
workspace_id = data.get("workspace_id")
if workspace_id is not None and not isinstance(workspace_id, str):
workspace_id = str(workspace_id)
return cls(
access_token=data.get("access_token", ""),
refresh_token=data.get("refresh_token"),
expires_in=data.get("expires_in"),
expires_at=expires_at,
user_id=user_id,
user_email=data.get("user_email"),
user_name=data.get("user_name"),
workspace_id=workspace_id,
workspace_name=data.get("workspace_name"),
)
@field_validator("expires_at", mode="before")
@classmethod
def ensure_aware_utc(cls, v):
# Strings like "2025-08-26T14:46:57.367184"
if isinstance(v, str):
# add +00:00 if missing tz info
if v.endswith("Z"):
return datetime.fromisoformat(v.replace("Z", "+00:00"))
dt = datetime.fromisoformat(v)
return dt if dt.tzinfo else dt.replace(tzinfo=UTC)
# datetime objects
if isinstance(v, datetime):
return v if v.tzinfo else v.replace(tzinfo=UTC)
return v

156
surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py
View file

@ -6,10 +6,8 @@ from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import config
from app.connectors.airtable_connector import AirtableConnector
from app.connectors.airtable_history import AirtableHistoryConnector
from app.db import Document, DocumentType, SearchSourceConnectorType
from app.routes.airtable_add_connector_route import refresh_airtable_token
from app.schemas.airtable_auth_credentials import AirtableAuthCredentialsBase
from app.services.llm_service import get_user_long_context_llm
from app.services.task_logging_service import TaskLoggingService
from app.utils.document_converters import (
@ -18,7 +16,6 @@ from app.utils.document_converters import (
generate_document_summary,
generate_unique_identifier_hash,
)
from app.utils.oauth_security import TokenEncryption
from .base import (
calculate_date_range,
@ -85,76 +82,11 @@ async def index_airtable_records(
)
return 0, f"Connector with ID {connector_id} not found"
# Create credentials from connector config
config_data = (
connector.config.copy()
) # Work with a copy to avoid modifying original
# Decrypt tokens if they are encrypted (only when explicitly marked)
token_encrypted = config_data.get("_token_encrypted", False)
if token_encrypted:
# Tokens are explicitly marked as encrypted, attempt decryption
if not config.SECRET_KEY:
await task_logger.log_task_failure(
log_entry,
f"SECRET_KEY not configured but tokens are marked as encrypted for connector {connector_id}",
"Missing SECRET_KEY for token decryption",
{"error_type": "MissingSecretKey"},
)
return 0, "SECRET_KEY not configured but tokens are marked as encrypted"
try:
token_encryption = TokenEncryption(config.SECRET_KEY)
# Decrypt access_token
if config_data.get("access_token"):
config_data["access_token"] = token_encryption.decrypt_token(
config_data["access_token"]
)
logger.info(
f"Decrypted Airtable access token for connector {connector_id}"
)
# Decrypt refresh_token if present
if config_data.get("refresh_token"):
config_data["refresh_token"] = token_encryption.decrypt_token(
config_data["refresh_token"]
)
logger.info(
f"Decrypted Airtable refresh token for connector {connector_id}"
)
except Exception as e:
await task_logger.log_task_failure(
log_entry,
f"Failed to decrypt Airtable tokens for connector {connector_id}: {e!s}",
"Token decryption failed",
{"error_type": "TokenDecryptionError"},
)
return 0, f"Failed to decrypt Airtable tokens: {e!s}"
# If _token_encrypted is False or not set, treat tokens as plaintext
try:
credentials = AirtableAuthCredentialsBase.from_dict(config_data)
except Exception as e:
await task_logger.log_task_failure(
log_entry,
f"Invalid Airtable credentials in connector {connector_id}",
str(e),
{"error_type": "InvalidCredentials"},
)
return 0, f"Invalid Airtable credentials: {e!s}"
# Check if credentials are expired
if credentials.is_expired:
await task_logger.log_task_failure(
log_entry,
f"Airtable credentials expired for connector {connector_id}",
"Credentials expired",
{"error_type": "ExpiredCredentials"},
)
connector = await refresh_airtable_token(session, connector)
# return 0, "Airtable credentials have expired. Please re-authenticate."
# Normalize "undefined" strings to None (from frontend)
if start_date == "undefined" or start_date == "":
start_date = None
if end_date == "undefined" or end_date == "":
end_date = None
# Calculate date range for indexing
start_date_str, end_date_str = calculate_date_range(
@ -166,8 +98,9 @@ async def index_airtable_records(
f"from {start_date_str} to {end_date_str}"
)
# Initialize Airtable connector
airtable_connector = AirtableConnector(credentials)
# Initialize Airtable history connector with auto-refresh capability
airtable_history = AirtableHistoryConnector(session, connector_id)
airtable_connector = await airtable_history._get_connector()
total_processed = 0
try:
@ -459,47 +392,56 @@ async def index_airtable_records(
documents_skipped += 1
continue # Skip this message and continue with others
# Update the last_indexed_at timestamp for the connector only if requested
total_processed = documents_indexed
if total_processed > 0:
await update_connector_last_indexed(
session, connector, update_last_indexed
)
# Accumulate total processed across all tables
total_processed += documents_indexed
# Final commit for any remaining documents not yet committed in batches
logger.info(
f"Final commit: Total {documents_indexed} Airtable records processed"
)
await session.commit()
logger.info(
"Successfully committed all Airtable document changes to database"
)
if documents_indexed > 0:
logger.info(
f"Final commit for table {table_name}: {documents_indexed} Airtable records processed"
)
await session.commit()
logger.info(
f"Successfully committed all Airtable document changes for table {table_name}"
)
# Log success
await task_logger.log_task_success(
log_entry,
f"Successfully completed Airtable indexing for connector {connector_id}",
{
"events_processed": total_processed,
"documents_indexed": documents_indexed,
"documents_skipped": documents_skipped,
"skipped_messages_count": len(skipped_messages),
},
)
# Update the last_indexed_at timestamp for the connector only if requested
# (after all tables in all bases are processed)
if total_processed > 0:
await update_connector_last_indexed(
session, connector, update_last_indexed
)
logger.info(
f"Airtable indexing completed: {documents_indexed} new records, {documents_skipped} skipped"
)
return (
total_processed,
None,
) # Return None as the error message to indicate success
# Log success after processing all bases and tables
await task_logger.log_task_success(
log_entry,
f"Successfully completed Airtable indexing for connector {connector_id}",
{
"events_processed": total_processed,
"documents_indexed": total_processed,
},
)
logger.info(
f"Airtable indexing completed: {total_processed} total records processed"
)
return (
total_processed,
None,
) # Return None as the error message to indicate success
except Exception as e:
logger.error(
f"Fetching Airtable bases for connector {connector_id} failed: {e!s}",
exc_info=True,
)
await task_logger.log_task_failure(
log_entry,
f"Failed to fetch Airtable bases for connector {connector_id}",
str(e),
{"error_type": type(e).__name__},
)
return 0, f"Failed to fetch Airtable bases: {e!s}"
except SQLAlchemyError as db_error:
await session.rollback()

46
surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py
View file

@ -2,13 +2,14 @@
ClickUp connector indexer.
"""
import contextlib
from datetime import datetime
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import config
from app.connectors.clickup_connector import ClickUpConnector
from app.connectors.clickup_history import ClickUpHistoryConnector
from app.db import Document, DocumentType, SearchSourceConnectorType
from app.services.llm_service import get_user_long_context_llm
from app.services.task_logging_service import TaskLoggingService
@ -82,26 +83,30 @@ async def index_clickup_tasks(
)
return 0, error_msg
# Extract ClickUp configuration
clickup_api_token = connector.config.get("CLICKUP_API_TOKEN")
# Check if using OAuth (has access_token in config) or legacy (has CLICKUP_API_TOKEN)
has_oauth = connector.config.get("access_token") is not None
has_legacy = connector.config.get("CLICKUP_API_TOKEN") is not None
if not clickup_api_token:
error_msg = "ClickUp API token not found in connector configuration"
if not has_oauth and not has_legacy:
error_msg = "ClickUp credentials not found in connector configuration (neither OAuth nor API token)"
await task_logger.log_task_failure(
log_entry,
f"ClickUp API token not found in connector config for connector {connector_id}",
"Missing ClickUp token",
{"error_type": "MissingToken"},
f"ClickUp credentials not found in connector config for connector {connector_id}",
"Missing ClickUp credentials",
{"error_type": "MissingCredentials"},
)
return 0, error_msg
await task_logger.log_task_progress(
log_entry,
f"Initializing ClickUp client for connector {connector_id}",
f"Initializing ClickUp client for connector {connector_id} ({'OAuth' if has_oauth else 'API Token'})",
{"stage": "client_initialization"},
)
clickup_client = ClickUpConnector(api_token=clickup_api_token)
# Use history connector which supports both OAuth and legacy API tokens
clickup_client = ClickUpHistoryConnector(
session=session, connector_id=connector_id
)
# Get authorized workspaces
await task_logger.log_task_progress(
@ -110,7 +115,7 @@ async def index_clickup_tasks(
{"stage": "workspace_fetching"},
)
workspaces_response = clickup_client.get_authorized_workspaces()
workspaces_response = await clickup_client.get_authorized_workspaces()
workspaces = workspaces_response.get("teams", [])
if not workspaces:
@ -141,7 +146,7 @@ async def index_clickup_tasks(
# Fetch tasks for date range if provided
if start_date and end_date:
tasks, error = clickup_client.get_tasks_in_date_range(
tasks, error = await clickup_client.get_tasks_in_date_range(
workspace_id=workspace_id,
start_date=start_date,
end_date=end_date,
@ -153,7 +158,7 @@ async def index_clickup_tasks(
)
continue
else:
tasks = clickup_client.get_workspace_tasks(
tasks = await clickup_client.get_workspace_tasks(
workspace_id=workspace_id, include_closed=True
)
@ -393,10 +398,21 @@ async def index_clickup_tasks(
logger.info(
f"clickup indexing completed: {documents_indexed} new tasks, {documents_skipped} skipped"
)
# Close client connection
try:
await clickup_client.close()
except Exception as e:
logger.warning(f"Error closing ClickUp client: {e!s}")
return total_processed, None
except SQLAlchemyError as db_error:
await session.rollback()
# Clean up the connector in case of error
if "clickup_client" in locals():
with contextlib.suppress(Exception):
await clickup_client.close()
await task_logger.log_task_failure(
log_entry,
f"Database error during ClickUp indexing for connector {connector_id}",
@ -407,6 +423,10 @@ async def index_clickup_tasks(
return 0, f"Database error: {db_error!s}"
except Exception as e:
await session.rollback()
# Clean up the connector in case of error
if "clickup_client" in locals():
with contextlib.suppress(Exception):
await clickup_client.close()
await task_logger.log_task_failure(
log_entry,
f"Failed to index ClickUp tasks for connector {connector_id}",

21
surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py
View file

@ -2,6 +2,7 @@
Confluence connector indexer.
"""
import contextlib
from datetime import datetime
from sqlalchemy.exc import SQLAlchemyError
@ -142,10 +143,8 @@ async def index_confluence_pages(
)
# Close client before returning
if confluence_client:
try:
with contextlib.suppress(Exception):
await confluence_client.close()
except Exception:
pass
return 0, None
else:
await task_logger.log_task_failure(
@ -156,10 +155,8 @@ async def index_confluence_pages(
)
# Close client on error
if confluence_client:
try:
with contextlib.suppress(Exception):
await confluence_client.close()
except Exception:
pass
return 0, f"Failed to get Confluence pages: {error}"
logger.info(f"Retrieved {len(pages)} pages from Confluence API")
@ -168,10 +165,8 @@ async def index_confluence_pages(
logger.error(f"Error fetching Confluence pages: {e!s}", exc_info=True)
# Close client on error
if confluence_client:
try:
with contextlib.suppress(Exception):
await confluence_client.close()
except Exception:
pass
return 0, f"Error fetching Confluence pages: {e!s}"
# Process and index each page
@ -437,10 +432,8 @@ async def index_confluence_pages(
await session.rollback()
# Close client if it exists
if confluence_client:
try:
with contextlib.suppress(Exception):
await confluence_client.close()
except Exception:
pass
await task_logger.log_task_failure(
log_entry,
f"Database error during Confluence indexing for connector {connector_id}",
@ -453,10 +446,8 @@ async def index_confluence_pages(
await session.rollback()
# Close client if it exists
if confluence_client:
try:
with contextlib.suppress(Exception):
await confluence_client.close()
except Exception:
pass
await task_logger.log_task_failure(
log_entry,
f"Failed to index Confluence pages for connector {connector_id}",

9
surfsense_backend/app/tasks/connector_indexers/jira_indexer.py
View file

@ -2,6 +2,7 @@
Jira connector indexer.
"""
import contextlib
from datetime import datetime
from sqlalchemy.exc import SQLAlchemyError
@ -413,10 +414,8 @@ async def index_jira_issues(
logger.error(f"Database error: {db_error!s}", exc_info=True)
# Clean up the connector in case of error
if "jira_client" in locals():
try:
with contextlib.suppress(Exception):
await jira_client.close()
except Exception:
pass
return 0, f"Database error: {db_error!s}"
except Exception as e:
await session.rollback()
@ -429,8 +428,6 @@ async def index_jira_issues(
logger.error(f"Failed to index JIRA issues: {e!s}", exc_info=True)
# Clean up the connector in case of error
if "jira_client" in locals():
try:
with contextlib.suppress(Exception):
await jira_client.close()
except Exception:
pass
return 0, f"Failed to index JIRA issues: {e!s}"

2
surfsense_backend/app/utils/validators.py
View file

@ -551,7 +551,7 @@ def validate_connector_config(
# ],
# "validators": {},
# },
"CLICKUP_CONNECTOR": {"required": ["CLICKUP_API_TOKEN"], "validators": {}},
# "CLICKUP_CONNECTOR": {"required": ["CLICKUP_API_TOKEN"], "validators": {}},
# "GOOGLE_CALENDAR_CONNECTOR": {
# "required": ["token", "refresh_token", "token_uri", "client_id", "expiry", "scopes", "client_secret"],
# "validators": {},

385
surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/clickup-connect-form.tsx
View file

@ -1,385 +0,0 @@
"use client";
import { zodResolver } from "@hookform/resolvers/zod";
import { Info } from "lucide-react";
import type { FC } from "react";
import { useRef, useState } from "react";
import { useForm } from "react-hook-form";
import * as z from "zod";
import {
Accordion,
AccordionContent,
AccordionItem,
AccordionTrigger,
} from "@/components/ui/accordion";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import {
Form,
FormControl,
FormDescription,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
import { EnumConnectorName } from "@/contracts/enums/connector";
import { DateRangeSelector } from "../../components/date-range-selector";
import { getConnectorBenefits } from "../connector-benefits";
import type { ConnectFormProps } from "../index";
const clickupConnectorFormSchema = z.object({
name: z.string().min(3, {
message: "Connector name must be at least 3 characters.",
}),
api_token: z.string().min(10, {
message: "ClickUp API Token is required and must be valid.",
}),
});
type ClickUpConnectorFormValues = z.infer<typeof clickupConnectorFormSchema>;
export const ClickUpConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting }) => {
const isSubmittingRef = useRef(false);
const [startDate, setStartDate] = useState<Date | undefined>(undefined);
const [endDate, setEndDate] = useState<Date | undefined>(undefined);
const [periodicEnabled, setPeriodicEnabled] = useState(false);
const [frequencyMinutes, setFrequencyMinutes] = useState("1440");
const form = useForm<ClickUpConnectorFormValues>({
resolver: zodResolver(clickupConnectorFormSchema),
defaultValues: {
name: "ClickUp Connector",
api_token: "",
},
});
const handleSubmit = async (values: ClickUpConnectorFormValues) => {
// Prevent multiple submissions
if (isSubmittingRef.current || isSubmitting) {
return;
}
isSubmittingRef.current = true;
try {
await onSubmit({
name: values.name,
connector_type: EnumConnectorName.CLICKUP_CONNECTOR,
config: {
CLICKUP_API_TOKEN: values.api_token,
},
is_indexable: true,
last_indexed_at: null,
periodic_indexing_enabled: periodicEnabled,
indexing_frequency_minutes: periodicEnabled ? parseInt(frequencyMinutes, 10) : null,
next_scheduled_at: null,
startDate,
endDate,
periodicEnabled,
frequencyMinutes,
});
} finally {
isSubmittingRef.current = false;
}
};
return (
<div className="space-y-6 pb-6">
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 p-2 sm:p-3 flex items-center [&>svg]:relative [&>svg]:left-0 [&>svg]:top-0 [&>svg+div]:translate-y-0">
<Info className="h-3 w-3 sm:h-4 sm:w-4 shrink-0 ml-1" />
<div className="-ml-1">
<AlertTitle className="text-xs sm:text-sm">API Token Required</AlertTitle>
<AlertDescription className="text-[10px] sm:text-xs !pl-0">
You'll need a ClickUp API Token to use this connector. You can create one from{" "}
<a
href="https://app.clickup.com/settings/apps"
target="_blank"
rel="noopener noreferrer"
className="font-medium underline underline-offset-4"
>
ClickUp Settings
</a>
</AlertDescription>
</div>
</Alert>
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-3 sm:space-y-4">
<Form {...form}>
<form
id="clickup-connect-form"
onSubmit={form.handleSubmit(handleSubmit)}
className="space-y-4 sm:space-y-6"
>
<FormField
control={form.control}
name="name"
render={({ field }) => (
<FormItem>
<FormLabel className="text-xs sm:text-sm">Connector Name</FormLabel>
<FormControl>
<Input
placeholder="My ClickUp Connector"
className="h-8 sm:h-10 px-2 sm:px-3 text-xs sm:text-sm border-slate-400/20 focus-visible:border-slate-400/40"
disabled={isSubmitting}
{...field}
/>
</FormControl>
<FormDescription className="text-[10px] sm:text-xs">
A friendly name to identify this connector.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="api_token"
render={({ field }) => (
<FormItem>
<FormLabel className="text-xs sm:text-sm">ClickUp API Token</FormLabel>
<FormControl>
<Input
type="password"
placeholder="pk_..."
className="h-8 sm:h-10 px-2 sm:px-3 text-xs sm:text-sm border-slate-400/20 focus-visible:border-slate-400/40"
disabled={isSubmitting}
{...field}
/>
</FormControl>
<FormDescription className="text-[10px] sm:text-xs">
Your ClickUp API Token will be encrypted and stored securely.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
{/* Indexing Configuration */}
<div className="space-y-4 pt-4 border-t border-slate-400/20">
<h3 className="text-sm sm:text-base font-medium">Indexing Configuration</h3>
{/* Date Range Selector */}
<DateRangeSelector
startDate={startDate}
endDate={endDate}
onStartDateChange={setStartDate}
onEndDateChange={setEndDate}
/>
{/* Periodic Sync Config */}
<div className="rounded-xl bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6">
<div className="flex items-center justify-between">
<div className="space-y-1">
<h3 className="font-medium text-sm sm:text-base">Enable Periodic Sync</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Automatically re-index at regular intervals
</p>
</div>
<Switch
checked={periodicEnabled}
onCheckedChange={setPeriodicEnabled}
disabled={isSubmitting}
/>
</div>
{periodicEnabled && (
<div className="mt-4 pt-4 border-t border-slate-400/20 space-y-3">
<div className="space-y-2">
<Label htmlFor="frequency" className="text-xs sm:text-sm">
Sync Frequency
</Label>
<Select
value={frequencyMinutes}
onValueChange={setFrequencyMinutes}
disabled={isSubmitting}
>
<SelectTrigger
id="frequency"
className="w-full bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
>
<SelectValue placeholder="Select frequency" />
</SelectTrigger>
<SelectContent className="z-[100]">
<SelectItem value="5" className="text-xs sm:text-sm">
Every 5 minutes
</SelectItem>
<SelectItem value="15" className="text-xs sm:text-sm">
Every 15 minutes
</SelectItem>
<SelectItem value="60" className="text-xs sm:text-sm">
Every hour
</SelectItem>
<SelectItem value="360" className="text-xs sm:text-sm">
Every 6 hours
</SelectItem>
<SelectItem value="720" className="text-xs sm:text-sm">
Every 12 hours
</SelectItem>
<SelectItem value="1440" className="text-xs sm:text-sm">
Daily
</SelectItem>
<SelectItem value="10080" className="text-xs sm:text-sm">
Weekly
</SelectItem>
</SelectContent>
</Select>
</div>
</div>
)}
</div>
</div>
</form>
</Form>
</div>
{/* What you get section */}
{getConnectorBenefits(EnumConnectorName.CLICKUP_CONNECTOR) && (
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 px-3 sm:px-6 py-4 space-y-2">
<h4 className="text-xs sm:text-sm font-medium">What you get with ClickUp integration:</h4>
<ul className="list-disc pl-5 text-[10px] sm:text-xs text-muted-foreground space-y-1">
{getConnectorBenefits(EnumConnectorName.CLICKUP_CONNECTOR)?.map((benefit) => (
<li key={benefit}>{benefit}</li>
))}
</ul>
</div>
)}
{/* Documentation Section */}
<Accordion
type="single"
collapsible
className="w-full border border-border rounded-xl bg-slate-400/5 dark:bg-white/5"
>
<AccordionItem value="documentation" className="border-0">
<AccordionTrigger className="text-sm sm:text-base font-medium px-3 sm:px-6 no-underline hover:no-underline">
Documentation
</AccordionTrigger>
<AccordionContent className="px-3 sm:px-6 pb-3 sm:pb-6 space-y-6">
<div>
<h3 className="text-sm sm:text-base font-semibold mb-2">How it works</h3>
<p className="text-[10px] sm:text-xs text-muted-foreground">
The ClickUp connector uses the ClickUp API to fetch all tasks and projects that your
API token has access to within your workspace.
</p>
<ul className="mt-2 list-disc pl-5 text-[10px] sm:text-xs text-muted-foreground space-y-1">
<li>
For follow up indexing runs, the connector retrieves tasks that have been updated
since the last indexing attempt.
</li>
<li>
Indexing is configured to run periodically, so updates should appear in your
search results within minutes.
</li>
</ul>
</div>
<div className="space-y-4">
<div>
<h3 className="text-sm sm:text-base font-semibold mb-2">Authorization</h3>
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 mb-4">
<Info className="h-3 w-3 sm:h-4 sm:w-4" />
<AlertTitle className="text-[10px] sm:text-xs">API Token Required</AlertTitle>
<AlertDescription className="text-[9px] sm:text-[10px]">
You need a ClickUp personal API token to use this connector. The token will be
used to read your ClickUp data.
</AlertDescription>
</Alert>
<div className="space-y-4 sm:space-y-6">
<div>
<h4 className="text-[10px] sm:text-xs font-medium mb-2">
Step 1: Get Your API Token
</h4>
<ol className="list-decimal pl-5 space-y-2 text-[10px] sm:text-xs text-muted-foreground">
<li>Log in to your ClickUp account</li>
<li>Click your avatar in the upper-right corner and select "Settings"</li>
<li>In the sidebar, click "Apps"</li>
<li>
Under "API Token", click <strong>Generate</strong> or{" "}
<strong>Regenerate</strong>
</li>
<li>Copy the generated token (it typically starts with "pk_")</li>
<li>
Paste it in the form above. You can also visit{" "}
<a
href="https://app.clickup.com/settings/apps"
target="_blank"
rel="noopener noreferrer"
className="font-medium underline underline-offset-4"
>
ClickUp API Settings
</a>{" "}
directly.
</li>
</ol>
</div>
<div>
<h4 className="text-[10px] sm:text-xs font-medium mb-2">
Step 2: Grant necessary access
</h4>
<p className="text-[10px] sm:text-xs text-muted-foreground mb-3">
The API Token will have access to all tasks and projects that your user
account can see. Make sure your account has appropriate permissions for the
workspaces you want to index.
</p>
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20">
<Info className="h-3 w-3 sm:h-4 sm:w-4" />
<AlertTitle className="text-[10px] sm:text-xs">Data Privacy</AlertTitle>
<AlertDescription className="text-[9px] sm:text-[10px]">
Only tasks, comments, and basic metadata will be indexed. ClickUp
attachments and linked files are not indexed by this connector.
</AlertDescription>
</Alert>
</div>
</div>
</div>
</div>
<div className="space-y-4">
<div>
<h3 className="text-sm sm:text-base font-semibold mb-2">Indexing</h3>
<ol className="list-decimal pl-5 space-y-2 text-[10px] sm:text-xs text-muted-foreground mb-4">
<li>
Navigate to the Connector Dashboard and select the <strong>ClickUp</strong>{" "}
Connector.
</li>
<li>
Place your <strong>API Token</strong> in the form field.
</li>
<li>
Click <strong>Connect</strong> to establish the connection.
</li>
<li>Once connected, your ClickUp tasks will be indexed automatically.</li>
</ol>
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20">
<Info className="h-3 w-3 sm:h-4 sm:w-4" />
<AlertTitle className="text-[10px] sm:text-xs">What Gets Indexed</AlertTitle>
<AlertDescription className="text-[9px] sm:text-[10px]">
<p className="mb-2">The ClickUp connector indexes the following data:</p>
<ul className="list-disc pl-5 space-y-1">
<li>Task names and descriptions</li>
<li>Task comments and discussion threads</li>
<li>Task status, priority, and assignee information</li>
<li>Project and workspace information</li>
</ul>
</AlertDescription>
</Alert>
</div>
</div>
</AccordionContent>
</AccordionItem>
</Accordion>
</div>
);
};

3
surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx
View file

@ -2,7 +2,6 @@ import type { FC } from "react";
import { BaiduSearchApiConnectForm } from "./components/baidu-search-api-connect-form";
import { BookStackConnectForm } from "./components/bookstack-connect-form";
import { CirclebackConnectForm } from "./components/circleback-connect-form";
import { ClickUpConnectForm } from "./components/clickup-connect-form";
import { ElasticsearchConnectForm } from "./components/elasticsearch-connect-form";
import { GithubConnectForm } from "./components/github-connect-form";
import { LinkupApiConnectForm } from "./components/linkup-api-connect-form";
@ -51,8 +50,6 @@ export function getConnectFormComponent(connectorType: string): ConnectFormCompo
return BookStackConnectForm;
case "GITHUB_CONNECTOR":
return GithubConnectForm;
case "CLICKUP_CONNECTOR":
return ClickUpConnectForm;
case "LUMA_CONNECTOR":
return LumaConnectForm;
case "CIRCLEBACK_CONNECTOR":

44
surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/clickup-config.tsx
View file

@ -1,6 +1,6 @@
"use client";
import { KeyRound } from "lucide-react";
import { Info, KeyRound } from "lucide-react";
import type { FC } from "react";
import { useEffect, useState } from "react";
import { Input } from "@/components/ui/input";
@ -16,17 +16,22 @@ export const ClickUpConfig: FC<ClickUpConfigProps> = ({
onConfigChange,
onNameChange,
}) => {
// Check if this is an OAuth connector (has access_token or _token_encrypted flag)
const isOAuth = !!(connector.config?.access_token || connector.config?._token_encrypted);
const [apiToken, setApiToken] = useState<string>(
(connector.config?.CLICKUP_API_TOKEN as string) || ""
);
const [name, setName] = useState<string>(connector.name || "");
// Update API token and name when connector changes
// Update values when connector changes (only for legacy connectors)
useEffect(() => {
const token = (connector.config?.CLICKUP_API_TOKEN as string) || "";
setApiToken(token);
if (!isOAuth) {
const token = (connector.config?.CLICKUP_API_TOKEN as string) || "";
setApiToken(token);
}
setName(connector.name || "");
}, [connector.config, connector.name]);
}, [connector.config, connector.name, isOAuth]);
const handleApiTokenChange = (value: string) => {
setApiToken(value);
@ -45,6 +50,32 @@ export const ClickUpConfig: FC<ClickUpConfigProps> = ({
}
};
// For OAuth connectors, show simple info message
if (isOAuth) {
const workspaceName = (connector.config?.workspace_name as string) || "Unknown Workspace";
return (
<div className="space-y-6">
{/* OAuth Info */}
<div className="rounded-xl border border-border bg-primary/5 p-4 flex items-start gap-3">
<div className="flex h-8 w-8 items-center justify-center rounded-lg bg-primary/10 shrink-0 mt-0.5">
<Info className="size-4" />
</div>
<div className="text-xs sm:text-sm">
<p className="font-medium text-xs sm:text-sm">Connected via OAuth</p>
<p className="text-muted-foreground mt-1 text-[10px] sm:text-sm">
Workspace:{" "}
<code className="bg-muted px-1 py-0.5 rounded text-inherit">{workspaceName}</code>
</p>
<p className="text-muted-foreground mt-1 text-[10px] sm:text-sm">
To update your connection, reconnect this connector.
</p>
</div>
</div>
</div>
);
}
// For legacy API token connectors, show the form
return (
<div className="space-y-6">
{/* Connector Name */}
@ -82,7 +113,8 @@ export const ClickUpConfig: FC<ClickUpConfigProps> = ({
className="border-slate-400/20 focus-visible:border-slate-400/40"
/>
<p className="text-[10px] sm:text-xs text-muted-foreground">
Update your ClickUp API Token if needed.
Update your ClickUp API Token if needed. For better security and automatic token
refresh, consider disconnecting and reconnecting using OAuth 2.0.
</p>
</div>
</div>

1
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx
View file

@ -54,7 +54,6 @@ export const ConnectorConnectView: FC<ConnectorConnectViewProps> = ({
ELASTICSEARCH_CONNECTOR: "elasticsearch-connect-form",
BOOKSTACK_CONNECTOR: "bookstack-connect-form",
GITHUB_CONNECTOR: "github-connect-form",
CLICKUP_CONNECTOR: "clickup-connect-form",
LUMA_CONNECTOR: "luma-connect-form",
CIRCLEBACK_CONNECTOR: "circleback-connect-form",
};

13
surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts
View file

@ -72,6 +72,13 @@ export const OAUTH_CONNECTORS = [
connectorType: EnumConnectorName.CONFLUENCE_CONNECTOR,
authEndpoint: "/api/v1/auth/confluence/connector/add/",
},
{
id: "clickup-connector",
title: "ClickUp",
description: "Search ClickUp tasks",
connectorType: EnumConnectorName.CLICKUP_CONNECTOR,
authEndpoint: "/api/v1/auth/clickup/connector/add/",
},
] as const;
// Content Sources (tools that extract and import content from external sources)
@ -104,12 +111,6 @@ export const OTHER_CONNECTORS = [
description: "Search repositories",
connectorType: EnumConnectorName.GITHUB_CONNECTOR,
},
{
id: "clickup-connector",
title: "ClickUp",
description: "Search ClickUp tasks",
connectorType: EnumConnectorName.CLICKUP_CONNECTOR,
},
{
id: "luma-connector",
title: "Luma",