chore: merge upstream with local feature additions

- Merged dexscreener connector, composio connectors, crypto realtime tools from upstream
- Kept local additions: dropbox/onedrive connectors, memory routes, model_list routes, RefreshToken model
- Resolved frontend conflicts: kept tool UIs from both sides
- Accepted upstream lock files (uv.lock, pnpm-lock.yaml)

surfsense_backend/app/routes/__init__.py

@@ -30,6 +30,7 @@ from .jira_add_connector_route import router as jira_add_connector_router
 from .linear_add_connector_route import router as linear_add_connector_router
 from .logs_routes import router as logs_router
 from .luma_add_connector_route import router as luma_add_connector_router
+from .dexscreener_add_connector_route import router as dexscreener_add_connector_router
 from .memory_routes import router as memory_router
 from .model_list_routes import router as model_list_router
 from .new_chat_routes import router as new_chat_router
@@ -80,6 +81,7 @@ router.include_router(google_drive_add_connector_router)
 router.include_router(airtable_add_connector_router)
 router.include_router(linear_add_connector_router)
 router.include_router(luma_add_connector_router)
+router.include_router(dexscreener_add_connector_router)
 router.include_router(notion_add_connector_router)
 router.include_router(slack_add_connector_router)
 router.include_router(teams_add_connector_router)

surfsense_backend/app/routes/airtable_add_connector_route.py

@@ -1,5 +1,7 @@
 import base64
+import hashlib
 import logging
+import secrets
 from datetime import UTC, datetime, timedelta
 from uuid import UUID
 
@@ -20,15 +22,12 @@ from app.db import (
 )
 from app.schemas.airtable_auth_credentials import AirtableAuthCredentialsBase
 from app.users import current_active_user
+from app.utils.airtable_token_utils import refresh_airtable_token
 from app.utils.connector_naming import (
     check_duplicate_connector,
     generate_unique_connector_name,
 )
-from app.utils.oauth_security import (
-    OAuthStateManager,
-    TokenEncryption,
-    generate_pkce_pair,
-)
+from app.utils.oauth_security import OAuthStateManager, TokenEncryption
 
 logger = logging.getLogger(__name__)
 
@@ -77,6 +76,28 @@ def make_basic_auth_header(client_id: str, client_secret: str) -> str:
     return f"Basic {b64}"
 
 
+def generate_pkce_pair() -> tuple[str, str]:
+    """
+    Generate PKCE code verifier and code challenge.
+
+    Returns:
+        Tuple of (code_verifier, code_challenge)
+    """
+    # Generate code verifier (43-128 characters)
+    code_verifier = (
+        base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("utf-8").rstrip("=")
+    )
+
+    # Generate code challenge (SHA256 hash of verifier, base64url encoded)
+    code_challenge = (
+        base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode("utf-8")).digest())
+        .decode("utf-8")
+        .rstrip("=")
+    )
+
+    return code_verifier, code_challenge
+
+
 @router.get("/auth/airtable/connector/add")
 async def connect_airtable(space_id: int, user: User = Depends(current_active_user)):
     """
@@ -179,7 +200,7 @@ async def airtable_callback(
             # Redirect to frontend with error parameter
             if space_id:
                 return RedirectResponse(
-                    url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/callback?error=airtable_oauth_denied"
+                    url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=airtable_oauth_denied"
                 )
             else:
                 return RedirectResponse(
@@ -296,7 +317,7 @@ async def airtable_callback(
                 f"Duplicate Airtable connector detected for user {user_id} with email {user_email}"
             )
             return RedirectResponse(
-                url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/callback?error=duplicate_account&connector=airtable-connector"
+                url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=airtable-connector"
             )
 
         # Generate a unique, user-friendly connector name
@@ -328,7 +349,7 @@ async def airtable_callback(
             # Redirect to the frontend with success params for indexing config
             # Using query params to auto-open the popup with config view on new-chat page
             return RedirectResponse(
-                url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/callback?success=true&connector=airtable-connector&connectorId={new_connector.id}"
+                url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=airtable-connector&connectorId={new_connector.id}"
             )
 
         except ValidationError as e:
@@ -358,134 +379,3 @@ async def airtable_callback(
             status_code=500, detail=f"Failed to complete Airtable OAuth: {e!s}"
         ) from e
 
-
-async def refresh_airtable_token(
-    session: AsyncSession, connector: SearchSourceConnector
-) -> SearchSourceConnector:
-    """
-    Refresh the Airtable access token for a connector.
-
-    Args:
-        session: Database session
-        connector: Airtable connector to refresh
-
-    Returns:
-        Updated connector object
-    """
-    try:
-        logger.info(f"Refreshing Airtable token for connector {connector.id}")
-
-        credentials = AirtableAuthCredentialsBase.from_dict(connector.config)
-
-        # Decrypt tokens if they are encrypted
-        token_encryption = get_token_encryption()
-        is_encrypted = connector.config.get("_token_encrypted", False)
-
-        refresh_token = credentials.refresh_token
-        if is_encrypted and refresh_token:
-            try:
-                refresh_token = token_encryption.decrypt_token(refresh_token)
-            except Exception as e:
-                logger.error(f"Failed to decrypt refresh token: {e!s}")
-                raise HTTPException(
-                    status_code=500, detail="Failed to decrypt stored refresh token"
-                ) from e
-
-        if not refresh_token:
-            raise HTTPException(
-                status_code=400,
-                detail="No refresh token available. Please re-authenticate.",
-            )
-
-        auth_header = make_basic_auth_header(
-            config.AIRTABLE_CLIENT_ID, config.AIRTABLE_CLIENT_SECRET
-        )
-
-        # Prepare token refresh data
-        refresh_data = {
-            "grant_type": "refresh_token",
-            "refresh_token": refresh_token,
-            "client_id": config.AIRTABLE_CLIENT_ID,
-            "client_secret": config.AIRTABLE_CLIENT_SECRET,
-        }
-
-        async with httpx.AsyncClient() as client:
-            token_response = await client.post(
-                TOKEN_URL,
-                data=refresh_data,
-                headers={
-                    "Content-Type": "application/x-www-form-urlencoded",
-                    "Authorization": auth_header,
-                },
-                timeout=30.0,
-            )
-
-        if token_response.status_code != 200:
-            error_detail = token_response.text
-            error_code = ""
-            try:
-                error_json = token_response.json()
-                error_detail = error_json.get("error_description", error_detail)
-                error_code = error_json.get("error", "")
-            except Exception:
-                pass
-            # Check if this is a token expiration/revocation error
-            error_lower = (error_detail + error_code).lower()
-            if (
-                "invalid_grant" in error_lower
-                or "expired" in error_lower
-                or "revoked" in error_lower
-            ):
-                raise HTTPException(
-                    status_code=401,
-                    detail="Airtable authentication failed. Please re-authenticate.",
-                )
-            raise HTTPException(
-                status_code=400, detail=f"Token refresh failed: {error_detail}"
-            )
-
-        token_json = token_response.json()
-
-        # Calculate expiration time (UTC, tz-aware)
-        expires_at = None
-        if token_json.get("expires_in"):
-            now_utc = datetime.now(UTC)
-            expires_at = now_utc + timedelta(seconds=int(token_json["expires_in"]))
-
-        # Encrypt new tokens before storing
-        access_token = token_json.get("access_token")
-        new_refresh_token = token_json.get("refresh_token")
-
-        if not access_token:
-            raise HTTPException(
-                status_code=400, detail="No access token received from Airtable refresh"
-            )
-
-        # Update credentials object with encrypted tokens
-        credentials.access_token = token_encryption.encrypt_token(access_token)
-        if new_refresh_token:
-            credentials.refresh_token = token_encryption.encrypt_token(
-                new_refresh_token
-            )
-        credentials.expires_in = token_json.get("expires_in")
-        credentials.expires_at = expires_at
-        credentials.scope = token_json.get("scope")
-
-        # Update connector config with encrypted tokens
-        credentials_dict = credentials.to_dict()
-        credentials_dict["_token_encrypted"] = True
-        connector.config = credentials_dict
-        await session.commit()
-        await session.refresh(connector)
-
-        logger.info(
-            f"Successfully refreshed Airtable token for connector {connector.id}"
-        )
-
-        return connector
-    except HTTPException:
-        raise
-    except Exception as e:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to refresh Airtable token: {e!s}"
-        ) from e

surfsense_backend/app/routes/dexscreener_add_connector_route.py

@@ -0,0 +1,312 @@
+import logging
+import re
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel, Field, field_validator
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.future import select
+
+from app.db import (
+    SearchSourceConnector,
+    SearchSourceConnectorType,
+    User,
+    get_async_session,
+)
+from app.users import current_active_user
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class TokenConfig(BaseModel):
+    """Configuration for a single token to track."""
+
+    chain: str = Field(..., description="Blockchain network (e.g., ethereum, bsc, solana)", pattern=r"^[a-z0-9-]+$")
+    address: str = Field(..., description="Token contract address")
+    name: str | None = Field(None, description="Optional token name for display")
+    
+    @field_validator("address")
+    @classmethod
+    def validate_address(cls, v: str) -> str:
+        """Validate token address format (EVM or Solana)."""
+        # EVM address: 0x + 40 hex characters
+        if v.startswith("0x"):
+            if not re.match(r"^0x[a-fA-F0-9]{40}$", v):
+                raise ValueError("Invalid EVM address format. Must be 0x followed by 40 hex characters.")
+            return v
+        # Solana address: 32-44 base58 characters
+        if len(v) < 32 or len(v) > 44:
+            raise ValueError("Invalid Solana address format. Must be 32-44 characters.")
+        # Allow base58 chars only for Solana
+        if not re.match(r"^[1-9A-HJ-NP-Za-km-z]{32,44}$", v):
+            raise ValueError("Invalid Solana address format. Contains invalid characters.")
+        return v
+
+
+class AddDexScreenerConnectorRequest(BaseModel):
+    """Request model for adding a DexScreener connector."""
+
+    tokens: list[TokenConfig] = Field(
+        ..., description="List of tokens to track (max 50)", min_length=1, max_length=50
+    )
+    space_id: int = Field(..., description="Search space ID")
+
+
+@router.post("/connectors/dexscreener/add")
+async def add_dexscreener_connector(
+    request: AddDexScreenerConnectorRequest,
+    user: User = Depends(current_active_user),
+    session: AsyncSession = Depends(get_async_session),
+):
+    """
+    Add a new DexScreener connector for the authenticated user.
+
+    Args:
+        request: The request containing tokens configuration and space_id
+        user: Current authenticated user
+        session: Database session
+
+    Returns:
+        Success message and connector details
+
+    Raises:
+        HTTPException: If connector already exists or validation fails
+    """
+    try:
+        # Check if a DexScreener connector already exists for this search space and user
+        result = await session.execute(
+            select(SearchSourceConnector).filter(
+                SearchSourceConnector.search_space_id == request.space_id,
+                SearchSourceConnector.user_id == user.id,
+                SearchSourceConnector.connector_type
+                == SearchSourceConnectorType.DEXSCREENER_CONNECTOR,
+            )
+        )
+        existing_connector = result.scalars().first()
+
+        # Convert tokens to dict format for storage
+        tokens_config = [token.model_dump() for token in request.tokens]
+
+        if existing_connector:
+            # Update existing connector with new tokens
+            existing_connector.config = {"tokens": tokens_config}
+            existing_connector.is_indexable = True
+            await session.commit()
+            await session.refresh(existing_connector)
+
+            logger.info(
+                f"Updated existing DexScreener connector for user {user.id} in space {request.space_id}"
+            )
+
+            return {
+                "message": "DexScreener connector updated successfully",
+                "connector_id": existing_connector.id,
+                "connector_type": "DEXSCREENER_CONNECTOR",
+                "tokens_count": len(tokens_config),
+            }
+
+        # Create new DexScreener connector
+        db_connector = SearchSourceConnector(
+            name="DexScreener Connector",
+            connector_type=SearchSourceConnectorType.DEXSCREENER_CONNECTOR,
+            config={"tokens": tokens_config},
+            search_space_id=request.space_id,
+            user_id=user.id,
+            is_indexable=True,
+        )
+
+        session.add(db_connector)
+        await session.commit()
+        await session.refresh(db_connector)
+
+        logger.info(
+            f"Successfully created DexScreener connector for user {user.id} with ID {db_connector.id}"
+        )
+
+        return {
+            "message": "DexScreener connector added successfully",
+            "connector_id": db_connector.id,
+            "connector_type": "DEXSCREENER_CONNECTOR",
+            "tokens_count": len(tokens_config),
+        }
+
+    except IntegrityError as e:
+        await session.rollback()
+        logger.error(f"Database integrity error: {e!s}")
+        raise HTTPException(
+            status_code=409,
+            detail="A DexScreener connector already exists for this user.",
+        ) from e
+    except Exception as e:
+        await session.rollback()
+        logger.error(f"Unexpected error adding DexScreener connector: {e!s}", exc_info=True)
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to add DexScreener connector: {e!s}",
+        ) from e
+
+
+@router.delete("/connectors/dexscreener")
+async def delete_dexscreener_connector(
+    space_id: int,
+    user: User = Depends(current_active_user),
+    session: AsyncSession = Depends(get_async_session),
+):
+    """
+    Delete the DexScreener connector for the authenticated user in a specific search space.
+
+    Args:
+        space_id: Search space ID
+        user: Current authenticated user
+        session: Database session
+
+    Returns:
+        Success message
+
+    Raises:
+        HTTPException: If connector doesn't exist
+    """
+    try:
+        result = await session.execute(
+            select(SearchSourceConnector).filter(
+                SearchSourceConnector.search_space_id == space_id,
+                SearchSourceConnector.user_id == user.id,
+                SearchSourceConnector.connector_type
+                == SearchSourceConnectorType.DEXSCREENER_CONNECTOR,
+            )
+        )
+        connector = result.scalars().first()
+
+        if not connector:
+            raise HTTPException(
+                status_code=404,
+                detail="DexScreener connector not found for this user.",
+            )
+
+        await session.delete(connector)
+        await session.commit()
+
+        logger.info(f"Successfully deleted DexScreener connector for user {user.id}")
+
+        return {"message": "DexScreener connector deleted successfully"}
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        await session.rollback()
+        logger.error(f"Unexpected error deleting DexScreener connector: {e!s}", exc_info=True)
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to delete DexScreener connector: {e!s}",
+        ) from e
+
+
+@router.get("/connectors/dexscreener/test")
+async def test_dexscreener_connector(
+    space_id: int,
+    user: User = Depends(current_active_user),
+    session: AsyncSession = Depends(get_async_session),
+):
+    """
+    Test the DexScreener connector for the authenticated user in a specific search space.
+
+    Args:
+        space_id: Search space ID
+        user: Current authenticated user
+        session: Database session
+
+    Returns:
+        Test results including token count and sample pair data
+
+    Raises:
+        HTTPException: If connector doesn't exist or test fails
+    """
+    try:
+        # Get the DexScreener connector for this search space and user
+        result = await session.execute(
+            select(SearchSourceConnector).filter(
+                SearchSourceConnector.search_space_id == space_id,
+                SearchSourceConnector.user_id == user.id,
+                SearchSourceConnector.connector_type
+                == SearchSourceConnectorType.DEXSCREENER_CONNECTOR,
+            )
+        )
+        connector = result.scalars().first()
+
+        if not connector:
+            raise HTTPException(
+                status_code=404,
+                detail="DexScreener connector not found. Please add a connector first.",
+            )
+
+        # Import DexScreenerConnector
+        from app.connectors.dexscreener_connector import DexScreenerConnector
+
+        # Initialize the connector
+        tokens = connector.config.get("tokens", [])
+        if not tokens:
+            raise HTTPException(
+                status_code=400,
+                detail="Invalid connector configuration: No tokens configured.",
+            )
+
+        dexscreener = DexScreenerConnector()
+
+        # Test the connection by fetching pairs for the first token
+        first_token = tokens[0]
+        chain = first_token.get("chain")
+        address = first_token.get("address")
+        token_name = first_token.get("name", "Unknown")
+
+        if not chain or not address:
+            raise HTTPException(
+                status_code=400,
+                detail="Invalid token configuration: Missing chain or address.",
+            )
+
+        # Try to fetch pairs for the first token
+        pairs, error = await dexscreener.get_token_pairs(chain, address)
+
+        if error:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Failed to connect to DexScreener: {error}",
+            )
+
+        # Get sample pair info if available
+        sample_pair = None
+        if pairs and len(pairs) > 0:
+            pair = pairs[0]
+            base_token = pair.get("baseToken", {})
+            quote_token = pair.get("quoteToken", {})
+            sample_pair = {
+                "pair_address": pair.get("pairAddress"),
+                "base_symbol": base_token.get("symbol", "Unknown"),
+                "quote_symbol": quote_token.get("symbol", "Unknown"),
+                "dex": pair.get("dexId", "Unknown"),
+                "price_usd": pair.get("priceUsd", "N/A"),
+                "liquidity_usd": pair.get("liquidity", {}).get("usd", 0),
+            }
+
+        return {
+            "message": "DexScreener connector is working correctly",
+            "tokens_configured": len(tokens),
+            "test_token": {
+                "name": token_name,
+                "chain": chain,
+                "address": address,
+            },
+            "pairs_found": len(pairs) if pairs else 0,
+            "sample_pair": sample_pair,
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Unexpected error testing DexScreener connector: {e!s}", exc_info=True)
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to test DexScreener connector: {e!s}",
+        ) from e

surfsense_backend/app/routes/search_source_connectors_routes.py

@@ -1106,6 +1106,17 @@ async def index_connector_content(
             )
             response_message = "Luma indexing started in the background."
 
+        elif connector.connector_type == SearchSourceConnectorType.DEXSCREENER_CONNECTOR:
+            from app.tasks.celery_tasks.connector_tasks import index_dexscreener_pairs_task
+
+            logger.info(
+                f"Triggering DexScreener indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}"
+            )
+            index_dexscreener_pairs_task.delay(
+                connector_id, search_space_id, str(user.id), indexing_from, indexing_to
+            )
+            response_message = "DexScreener indexing started in the background."
+
         elif (
             connector.connector_type
             == SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR