apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-15 18:25:18 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #669 from CREDO23/sur-66-feature-support-multiple-connectors-of-the-same-type-per

Browse source 
[Feature] Support multiple connectors of the same type per search space
This commit is contained in:
Rohan Verma 2026-01-07 12:38:37 -08:00 • committed by GitHub
commit b53ea565db
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
30 changed files with 1420 additions and 362 deletions
Download patch file Download diff file Expand all files Collapse all files

55
surfsense_backend/alembic/versions/57_allow_multiple_connectors_per_type.py Normal file
View file

@ -0,0 +1,55 @@
"""Allow multiple connectors of same type per search space
Revision ID: 57
Revises: 56
Create Date: 2026-01-06 12:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "57"
down_revision: str | None = "56"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
from sqlalchemy import text
def upgrade() -> None:
connection = op.get_bind()
constraint_exists = connection.execute(
text("""
SELECT 1 FROM information_schema.table_constraints
WHERE table_name='search_source_connectors'
AND constraint_type='UNIQUE'
AND constraint_name='uq_searchspace_user_connector_type'
""")
).scalar()
if constraint_exists:
op.drop_constraint(
"uq_searchspace_user_connector_type",
"search_source_connectors",
type_="unique",
)
def downgrade() -> None:
connection = op.get_bind()
constraint_exists = connection.execute(
text("""
SELECT 1 FROM information_schema.table_constraints
WHERE table_name='search_source_connectors'
AND constraint_type='UNIQUE'
AND constraint_name='uq_searchspace_user_connector_type'
""")
).scalar()
if not constraint_exists:
op.create_unique_constraint(
"uq_searchspace_user_connector_type",
"search_source_connectors",
["search_space_id", "user_id", "connector_type"],
)

55
surfsense_backend/alembic/versions/58_unique_connector_name_per_space_user.py Normal file
View file

@ -0,0 +1,55 @@
"""
Add unique constraint for (search_space_id, user_id, name) on search_source_connectors.
Revision ID: 58
Revises: 57
Create Date: 2026-01-06 14:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
revision: str = "58"
down_revision: str | None = "57"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
from sqlalchemy import text
def upgrade() -> None:
connection = op.get_bind()
constraint_exists = connection.execute(
text("""
SELECT 1 FROM information_schema.table_constraints
WHERE table_name='search_source_connectors'
AND constraint_type='UNIQUE'
AND constraint_name='uq_searchspace_user_connector_name'
""")
).scalar()
if not constraint_exists:
op.create_unique_constraint(
"uq_searchspace_user_connector_name",
"search_source_connectors",
["search_space_id", "user_id", "name"],
)
def downgrade() -> None:
connection = op.get_bind()
constraint_exists = connection.execute(
text("""
SELECT 1 FROM information_schema.table_constraints
WHERE table_name='search_source_connectors'
AND constraint_type='UNIQUE'
AND constraint_name='uq_searchspace_user_connector_name'
""")
).scalar()
if constraint_exists:
op.drop_constraint(
"uq_searchspace_user_connector_name",
"search_source_connectors",
type_="unique",
)

40
surfsense_backend/app/connectors/airtable_connector.py
View file

@ -382,3 +382,43 @@ class AirtableConnector:
markdown_parts.append("")
return "\n".join(markdown_parts)
# --- OAuth User Info ---
AIRTABLE_WHOAMI_URL = "https://api.airtable.com/v0/meta/whoami"
async def fetch_airtable_user_email(access_token: str) -> str | None:
"""
Fetch user email from Airtable whoami API.
Args:
access_token: The Airtable OAuth access token
Returns:
User's email address or None if fetch fails
"""
try:
async with httpx.AsyncClient() as client:
response = await client.get(
AIRTABLE_WHOAMI_URL,
headers={"Authorization": f"Bearer {access_token}"},
timeout=10.0,
)
if response.status_code == 200:
data = response.json()
email = data.get("email")
if email:
logger.debug(f"Fetched Airtable user email: {email}")
return email
logger.warning(
f"Failed to fetch Airtable user info: {response.status_code}"
)
return None
except Exception as e:
logger.warning(f"Error fetching Airtable user email: {e!s}")
return None

29
surfsense_backend/app/connectors/google_gmail_connector.py
View file

@ -6,6 +6,7 @@ Allows fetching emails from Gmail mailbox using Google OAuth credentials.
import base64
import json
import logging
import re
from typing import Any
@ -21,6 +22,34 @@ from app.db import (
SearchSourceConnectorType,
)
logger = logging.getLogger(__name__)
def fetch_google_user_email(credentials: Credentials) -> str | None:
"""
Fetch user email from Gmail API using Google credentials.
Uses the Gmail users.getProfile endpoint which returns the authenticated
user's email address.
Args:
credentials: Google OAuth Credentials object (not encrypted)
Returns:
User's email address or None if fetch fails
"""
try:
service = build("gmail", "v1", credentials=credentials)
profile = service.users().getProfile(userId="me").execute()
email = profile.get("emailAddress")
if email:
logger.debug(f"Fetched Google user email: {email}")
return email
return None
except Exception as e:
logger.warning(f"Error fetching Google user email: {e!s}")
return None
class GoogleGmailConnector:
"""Class for retrieving emails from Gmail using Google OAuth credentials."""

52
surfsense_backend/app/connectors/linear_connector.py
View file

@ -9,18 +9,65 @@ import logging
from datetime import datetime
from typing import Any
import httpx
import requests
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.db import SearchSourceConnector
from app.routes.linear_add_connector_route import refresh_linear_token
from app.schemas.linear_auth_credentials import LinearAuthCredentialsBase
from app.utils.oauth_security import TokenEncryption
logger = logging.getLogger(__name__)
LINEAR_GRAPHQL_URL = "https://api.linear.app/graphql"
ORGANIZATION_QUERY = """
query {
organization {
name
}
}
"""
async def fetch_linear_organization_name(access_token: str) -> str | None:
"""
Fetch organization/workspace name from Linear GraphQL API.
Args:
access_token: The Linear OAuth access token
Returns:
Organization name or None if fetch fails
"""
try:
async with httpx.AsyncClient() as client:
response = await client.post(
LINEAR_GRAPHQL_URL,
headers={
"Authorization": access_token,
"Content-Type": "application/json",
},
json={"query": ORGANIZATION_QUERY},
timeout=10.0,
)
if response.status_code == 200:
data = response.json()
org_name = data.get("data", {}).get("organization", {}).get("name")
if org_name:
logger.debug(f"Fetched Linear organization name: {org_name}")
return org_name
logger.warning(f"Failed to fetch Linear org info: {response.status_code}")
return None
except Exception as e:
logger.warning(f"Error fetching Linear organization name: {e!s}")
return None
class LinearConnector:
"""Class for retrieving issues and comments from Linear."""
@ -121,6 +168,9 @@ class LinearConnector:
f"Connector {self._connector_id} not found; cannot refresh token."
)
# Lazy import to avoid circular dependency
from app.routes.linear_add_connector_route import refresh_linear_token
# Refresh token
connector = await refresh_linear_token(self._session, connector)

78
surfsense_backend/app/routes/airtable_add_connector_route.py
View file

@ -11,9 +11,9 @@ from fastapi.responses import RedirectResponse
from pydantic import ValidationError
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.connectors.airtable_connector import fetch_airtable_user_email
from app.db import (
SearchSourceConnector,
SearchSourceConnectorType,
@ -22,6 +22,10 @@ from app.db import (
)
from app.schemas.airtable_auth_credentials import AirtableAuthCredentialsBase
from app.users import current_active_user
from app.utils.connector_naming import (
check_duplicate_connector,
generate_unique_connector_name,
)
from app.utils.oauth_security import OAuthStateManager, TokenEncryption
logger = logging.getLogger(__name__)
@ -275,6 +279,8 @@ async def airtable_callback(
status_code=400, detail="No access token received from Airtable"
)
user_email = await fetch_airtable_user_email(access_token)
# Calculate expiration time (UTC, tz-aware)
expires_at = None
if token_json.get("expires_in"):
@ -297,39 +303,43 @@ async def airtable_callback(
credentials_dict = credentials.to_dict()
credentials_dict["_token_encrypted"] = True
# Check if connector already exists for this search space and user
existing_connector_result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.AIRTABLE_CONNECTOR,
)
# Check for duplicate connector (same account already connected)
is_duplicate = await check_duplicate_connector(
session,
SearchSourceConnectorType.AIRTABLE_CONNECTOR,
space_id,
user_id,
user_email,
)
existing_connector = existing_connector_result.scalars().first()
if is_duplicate:
logger.warning(
f"Duplicate Airtable connector detected for user {user_id} with email {user_email}"
)
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=airtable-connector"
)
if existing_connector:
# Update existing connector
existing_connector.config = credentials_dict
existing_connector.name = "Airtable Connector"
existing_connector.is_indexable = True
logger.info(
f"Updated existing Airtable connector for user {user_id} in space {space_id}"
)
else:
# Create new connector
new_connector = SearchSourceConnector(
name="Airtable Connector",
connector_type=SearchSourceConnectorType.AIRTABLE_CONNECTOR,
is_indexable=True,
config=credentials_dict,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Airtable connector for user {user_id} in space {space_id}"
)
# Generate a unique, user-friendly connector name
connector_name = await generate_unique_connector_name(
session,
SearchSourceConnectorType.AIRTABLE_CONNECTOR,
space_id,
user_id,
user_email,
)
# Create new connector
new_connector = SearchSourceConnector(
name=connector_name,
connector_type=SearchSourceConnectorType.AIRTABLE_CONNECTOR,
is_indexable=True,
config=credentials_dict,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Airtable connector for user {user_id} in space {space_id}"
)
try:
await session.commit()
@ -338,7 +348,7 @@ async def airtable_callback(
# Redirect to the frontend with success params for indexing config
# Using query params to auto-open the popup with config view on new-chat page
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=airtable-connector"
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=airtable-connector&connectorId={new_connector.id}"
)
except ValidationError as e:
@ -350,7 +360,7 @@ async def airtable_callback(
await session.rollback()
raise HTTPException(
status_code=409,
detail=f"Integrity error: A connector with this type already exists. {e!s}",
detail=f"Database integrity error: {e!s}",
) from e
except Exception as e:
logger.error(f"Failed to create search source connector: {e!s}")

77
surfsense_backend/app/routes/confluence_add_connector_route.py
View file

@ -14,7 +14,6 @@ from fastapi.responses import RedirectResponse
from pydantic import ValidationError
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.db import (
@ -25,6 +24,11 @@ from app.db import (
)
from app.schemas.atlassian_auth_credentials import AtlassianAuthCredentialsBase
from app.users import current_active_user
from app.utils.connector_naming import (
check_duplicate_connector,
extract_identifier_from_credentials,
generate_unique_connector_name,
)
from app.utils.oauth_security import OAuthStateManager, TokenEncryption
logger = logging.getLogger(__name__)
@ -288,47 +292,56 @@ async def confluence_callback(
"_token_encrypted": True,
}
# Check if connector already exists for this search space and user
existing_connector_result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.CONFLUENCE_CONNECTOR,
)
# Extract unique identifier from connector credentials
connector_identifier = extract_identifier_from_credentials(
SearchSourceConnectorType.CONFLUENCE_CONNECTOR, connector_config
)
existing_connector = existing_connector_result.scalars().first()
if existing_connector:
# Update existing connector
existing_connector.config = connector_config
existing_connector.name = "Confluence Connector"
existing_connector.is_indexable = True
logger.info(
f"Updated existing Confluence connector for user {user_id} in space {space_id}"
# Check for duplicate connector (same Confluence instance already connected)
is_duplicate = await check_duplicate_connector(
session,
SearchSourceConnectorType.CONFLUENCE_CONNECTOR,
space_id,
user_id,
connector_identifier,
)
if is_duplicate:
logger.warning(
f"Duplicate Confluence connector detected for user {user_id} with instance {connector_identifier}"
)
else:
# Create new connector
new_connector = SearchSourceConnector(
name="Confluence Connector",
connector_type=SearchSourceConnectorType.CONFLUENCE_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Confluence connector for user {user_id} in space {space_id}"
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=confluence-connector"
)
# Generate a unique, user-friendly connector name
connector_name = await generate_unique_connector_name(
session,
SearchSourceConnectorType.CONFLUENCE_CONNECTOR,
space_id,
user_id,
connector_identifier,
)
# Create new connector
new_connector = SearchSourceConnector(
name=connector_name,
connector_type=SearchSourceConnectorType.CONFLUENCE_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Confluence connector for user {user_id} in space {space_id}"
)
try:
await session.commit()
logger.info(f"Successfully saved Confluence connector for user {user_id}")
# Redirect to the frontend with success params
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=confluence-connector"
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=confluence-connector&connectorId={new_connector.id}"
)
except ValidationError as e:
@ -340,7 +353,7 @@ async def confluence_callback(
await session.rollback()
raise HTTPException(
status_code=409,
detail=f"Integrity error: A connector with this type already exists. {e!s}",
detail=f"Database integrity error: {e!s}",
) from e
except Exception as e:
logger.error(f"Failed to create search source connector: {e!s}")

77
surfsense_backend/app/routes/discord_add_connector_route.py
View file

@ -14,7 +14,6 @@ from fastapi.responses import RedirectResponse
from pydantic import ValidationError
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.db import (
@ -25,6 +24,11 @@ from app.db import (
)
from app.schemas.discord_auth_credentials import DiscordAuthCredentialsBase
from app.users import current_active_user
from app.utils.connector_naming import (
check_duplicate_connector,
extract_identifier_from_credentials,
generate_unique_connector_name,
)
from app.utils.oauth_security import OAuthStateManager, TokenEncryption
logger = logging.getLogger(__name__)
@ -284,47 +288,56 @@ async def discord_callback(
"_token_encrypted": True,
}
# Check if connector already exists for this search space and user
existing_connector_result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.DISCORD_CONNECTOR,
)
# Extract unique identifier from connector credentials
connector_identifier = extract_identifier_from_credentials(
SearchSourceConnectorType.DISCORD_CONNECTOR, connector_config
)
existing_connector = existing_connector_result.scalars().first()
if existing_connector:
# Update existing connector
existing_connector.config = connector_config
existing_connector.name = "Discord Connector"
existing_connector.is_indexable = True
logger.info(
f"Updated existing Discord connector for user {user_id} in space {space_id}"
# Check for duplicate connector (same server already connected)
is_duplicate = await check_duplicate_connector(
session,
SearchSourceConnectorType.DISCORD_CONNECTOR,
space_id,
user_id,
connector_identifier,
)
if is_duplicate:
logger.warning(
f"Duplicate Discord connector detected for user {user_id} with server {connector_identifier}"
)
else:
# Create new connector
new_connector = SearchSourceConnector(
name="Discord Connector",
connector_type=SearchSourceConnectorType.DISCORD_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Discord connector for user {user_id} in space {space_id}"
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=discord-connector"
)
# Generate a unique, user-friendly connector name
connector_name = await generate_unique_connector_name(
session,
SearchSourceConnectorType.DISCORD_CONNECTOR,
space_id,
user_id,
connector_identifier,
)
# Create new connector
new_connector = SearchSourceConnector(
name=connector_name,
connector_type=SearchSourceConnectorType.DISCORD_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Discord connector for user {user_id} in space {space_id}"
)
try:
await session.commit()
logger.info(f"Successfully saved Discord connector for user {user_id}")
# Redirect to the frontend with success params
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=discord-connector"
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=discord-connector&connectorId={new_connector.id}"
)
except ValidationError as e:
@ -336,7 +349,7 @@ async def discord_callback(
await session.rollback()
raise HTTPException(
status_code=409,
detail=f"Integrity error: A connector with this type already exists. {e!s}",
detail=f"Database integrity error: {e!s}",
) from e
except Exception as e:
logger.error(f"Failed to create search source connector: {e!s}")

54
surfsense_backend/app/routes/google_calendar_add_connector_route.py
View file

@ -12,9 +12,9 @@ from google_auth_oauthlib.flow import Flow
from pydantic import ValidationError
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.connectors.google_gmail_connector import fetch_google_user_email
from app.db import (
SearchSourceConnector,
SearchSourceConnectorType,
@ -22,6 +22,10 @@ from app.db import (
get_async_session,
)
from app.users import current_active_user
from app.utils.connector_naming import (
check_duplicate_connector,
generate_unique_connector_name,
)
from app.utils.oauth_security import OAuthStateManager, TokenEncryption
logger = logging.getLogger(__name__)
@ -172,6 +176,9 @@ async def calendar_callback(
creds = flow.credentials
creds_dict = json.loads(creds.to_json())
# Fetch user email
user_email = fetch_google_user_email(creds)
# Encrypt sensitive credentials before storing
token_encryption = get_token_encryption()
@ -190,24 +197,33 @@ async def calendar_callback(
# Mark that credentials are encrypted for backward compatibility
creds_dict["_token_encrypted"] = True
try:
# Check if a connector with the same type already exists for this search space and user
result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR,
)
# Check for duplicate connector (same account already connected)
is_duplicate = await check_duplicate_connector(
session,
SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR,
space_id,
user_id,
user_email,
)
if is_duplicate:
logger.warning(
f"Duplicate Google Calendar connector detected for user {user_id} with email {user_email}"
)
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=google-calendar-connector"
)
try:
# Generate a unique, user-friendly connector name
connector_name = await generate_unique_connector_name(
session,
SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR,
space_id,
user_id,
user_email,
)
existing_connector = result.scalars().first()
if existing_connector:
raise HTTPException(
status_code=409,
detail="A GOOGLE_CALENDAR_CONNECTOR connector already exists in this search space. Each search space can have only one connector of each type per user.",
)
db_connector = SearchSourceConnector(
name="Google Calendar Connector",
name=connector_name,
connector_type=SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR,
config=creds_dict,
search_space_id=space_id,
@ -220,7 +236,7 @@ async def calendar_callback(
# Redirect to the frontend with success params for indexing config
# Using query params to auto-open the popup with config view on new-chat page
return RedirectResponse(
f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-calendar-connector"
f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-calendar-connector&connectorId={db_connector.id}"
)
except ValidationError as e:
await session.rollback()
@ -231,7 +247,7 @@ async def calendar_callback(
await session.rollback()
raise HTTPException(
status_code=409,
detail=f"Integrity error: A connector with this type already exists. {e!s}",
detail=f"Database integrity error: {e!s}",
) from e
except HTTPException:
await session.rollback()

51
surfsense_backend/app/routes/google_drive_add_connector_route.py
View file

@ -29,6 +29,7 @@ from app.connectors.google_drive import (
get_start_page_token,
list_folder_contents,
)
from app.connectors.google_gmail_connector import fetch_google_user_email
from app.db import (
SearchSourceConnector,
SearchSourceConnectorType,
@ -36,6 +37,10 @@ from app.db import (
get_async_session,
)
from app.users import current_active_user
from app.utils.connector_naming import (
check_duplicate_connector,
generate_unique_connector_name,
)
from app.utils.oauth_security import OAuthStateManager, TokenEncryption
# Relax token scope validation for Google OAuth
@ -227,6 +232,9 @@ async def drive_callback(
creds = flow.credentials
creds_dict = json.loads(creds.to_json())
# Fetch user email
user_email = fetch_google_user_email(creds)
# Encrypt sensitive credentials before storing
token_encryption = get_token_encryption()
@ -245,26 +253,33 @@ async def drive_callback(
# Mark that credentials are encrypted for backward compatibility
creds_dict["_token_encrypted"] = True
# Check if connector already exists for this space/user
result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR,
)
# Check for duplicate connector (same account already connected)
is_duplicate = await check_duplicate_connector(
session,
SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR,
space_id,
user_id,
user_email,
)
existing_connector = result.scalars().first()
if existing_connector:
raise HTTPException(
status_code=409,
detail="A GOOGLE_DRIVE_CONNECTOR already exists in this search space. Each search space can have only one connector of each type per user.",
if is_duplicate:
logger.warning(
f"Duplicate Google Drive connector detected for user {user_id} with email {user_email}"
)
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=google-drive-connector"
)
# Create new connector (NO folder selection here - happens at index time)
# Generate a unique, user-friendly connector name
connector_name = await generate_unique_connector_name(
session,
SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR,
space_id,
user_id,
user_email,
)
db_connector = SearchSourceConnector(
name="Google Drive Connector",
name=connector_name,
connector_type=SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR,
config={
**creds_dict,
@ -301,7 +316,7 @@ async def drive_callback(
)
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-drive-connector"
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-drive-connector&connectorId={db_connector.id}"
)
except HTTPException:
@ -318,7 +333,7 @@ async def drive_callback(
logger.error(f"Database integrity error: {e!s}", exc_info=True)
raise HTTPException(
status_code=409,
detail="A connector with this configuration already exists.",
detail=f"Database integrity error: {e!s}",
) from e
except Exception as e:
await session.rollback()

54
surfsense_backend/app/routes/google_gmail_add_connector_route.py
View file

@ -12,9 +12,9 @@ from google_auth_oauthlib.flow import Flow
from pydantic import ValidationError
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.connectors.google_gmail_connector import fetch_google_user_email
from app.db import (
SearchSourceConnector,
SearchSourceConnectorType,
@ -22,6 +22,10 @@ from app.db import (
get_async_session,
)
from app.users import current_active_user
from app.utils.connector_naming import (
check_duplicate_connector,
generate_unique_connector_name,
)
from app.utils.oauth_security import OAuthStateManager, TokenEncryption
logger = logging.getLogger(__name__)
@ -203,6 +207,9 @@ async def gmail_callback(
creds = flow.credentials
creds_dict = json.loads(creds.to_json())
# Fetch user email
user_email = fetch_google_user_email(creds)
# Encrypt sensitive credentials before storing
token_encryption = get_token_encryption()
@ -221,24 +228,33 @@ async def gmail_callback(
# Mark that credentials are encrypted for backward compatibility
creds_dict["_token_encrypted"] = True
try:
# Check if a connector with the same type already exists for this search space and user
result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR,
)
# Check for duplicate connector (same account already connected)
is_duplicate = await check_duplicate_connector(
session,
SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR,
space_id,
user_id,
user_email,
)
if is_duplicate:
logger.warning(
f"Duplicate Gmail connector detected for user {user_id} with email {user_email}"
)
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=google-gmail-connector"
)
try:
# Generate a unique, user-friendly connector name
connector_name = await generate_unique_connector_name(
session,
SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR,
space_id,
user_id,
user_email,
)
existing_connector = result.scalars().first()
if existing_connector:
raise HTTPException(
status_code=409,
detail="A GOOGLE_GMAIL_CONNECTOR connector already exists in this search space. Each search space can have only one connector of each type per user.",
)
db_connector = SearchSourceConnector(
name="Google Gmail Connector",
name=connector_name,
connector_type=SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR,
config=creds_dict,
search_space_id=space_id,
@ -256,7 +272,7 @@ async def gmail_callback(
# Redirect to the frontend with success params for indexing config
# Using query params to auto-open the popup with config view on new-chat page
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-gmail-connector"
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-gmail-connector&connectorId={db_connector.id}"
)
except IntegrityError as e:
@ -264,7 +280,7 @@ async def gmail_callback(
logger.error(f"Database integrity error: {e!s}")
raise HTTPException(
status_code=409,
detail="A connector with this configuration already exists.",
detail=f"Database integrity error: {e!s}",
) from e
except ValidationError as e:
await session.rollback()

77
surfsense_backend/app/routes/jira_add_connector_route.py
View file

@ -15,7 +15,6 @@ from fastapi.responses import RedirectResponse
from pydantic import ValidationError
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.db import (
@ -26,6 +25,11 @@ from app.db import (
)
from app.schemas.atlassian_auth_credentials import AtlassianAuthCredentialsBase
from app.users import current_active_user
from app.utils.connector_naming import (
check_duplicate_connector,
extract_identifier_from_credentials,
generate_unique_connector_name,
)
from app.utils.oauth_security import OAuthStateManager, TokenEncryption
logger = logging.getLogger(__name__)
@ -306,47 +310,56 @@ async def jira_callback(
"_token_encrypted": True,
}
# Check if connector already exists for this search space and user
existing_connector_result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.JIRA_CONNECTOR,
)
# Extract unique identifier from connector credentials
connector_identifier = extract_identifier_from_credentials(
SearchSourceConnectorType.JIRA_CONNECTOR, connector_config
)
existing_connector = existing_connector_result.scalars().first()
if existing_connector:
# Update existing connector
existing_connector.config = connector_config
existing_connector.name = "Jira Connector"
existing_connector.is_indexable = True
logger.info(
f"Updated existing Jira connector for user {user_id} in space {space_id}"
# Check for duplicate connector (same Jira instance already connected)
is_duplicate = await check_duplicate_connector(
session,
SearchSourceConnectorType.JIRA_CONNECTOR,
space_id,
user_id,
connector_identifier,
)
if is_duplicate:
logger.warning(
f"Duplicate Jira connector detected for user {user_id} with instance {connector_identifier}"
)
else:
# Create new connector
new_connector = SearchSourceConnector(
name="Jira Connector",
connector_type=SearchSourceConnectorType.JIRA_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Jira connector for user {user_id} in space {space_id}"
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=jira-connector"
)
# Generate a unique, user-friendly connector name
connector_name = await generate_unique_connector_name(
session,
SearchSourceConnectorType.JIRA_CONNECTOR,
space_id,
user_id,
connector_identifier,
)
# Create new connector
new_connector = SearchSourceConnector(
name=connector_name,
connector_type=SearchSourceConnectorType.JIRA_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Jira connector for user {user_id} in space {space_id}"
)
try:
await session.commit()
logger.info(f"Successfully saved Jira connector for user {user_id}")
# Redirect to the frontend with success params
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=jira-connector"
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=jira-connector&connectorId={new_connector.id}"
)
except ValidationError as e:
@ -358,7 +371,7 @@ async def jira_callback(
await session.rollback()
raise HTTPException(
status_code=409,
detail=f"Integrity error: A connector with this type already exists. {e!s}",
detail=f"Database integrity error: {e!s}",
) from e
except Exception as e:
logger.error(f"Failed to create search source connector: {e!s}")

79
surfsense_backend/app/routes/linear_add_connector_route.py
View file

@ -14,9 +14,9 @@ from fastapi.responses import RedirectResponse
from pydantic import ValidationError
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.connectors.linear_connector import fetch_linear_organization_name
from app.db import (
SearchSourceConnector,
SearchSourceConnectorType,
@ -25,6 +25,10 @@ from app.db import (
)
from app.schemas.linear_auth_credentials import LinearAuthCredentialsBase
from app.users import current_active_user
from app.utils.connector_naming import (
check_duplicate_connector,
generate_unique_connector_name,
)
from app.utils.oauth_security import OAuthStateManager, TokenEncryption
logger = logging.getLogger(__name__)
@ -240,6 +244,9 @@ async def linear_callback(
status_code=400, detail="No access token received from Linear"
)
# Fetch organization name
org_name = await fetch_linear_organization_name(access_token)
# Calculate expiration time (UTC, tz-aware)
expires_at = None
if token_json.get("expires_in"):
@ -260,39 +267,43 @@ async def linear_callback(
"_token_encrypted": True,
}
# Check if connector already exists for this search space and user
existing_connector_result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.LINEAR_CONNECTOR,
)
# Check for duplicate connector (same organization already connected)
is_duplicate = await check_duplicate_connector(
session,
SearchSourceConnectorType.LINEAR_CONNECTOR,
space_id,
user_id,
org_name,
)
existing_connector = existing_connector_result.scalars().first()
if is_duplicate:
logger.warning(
f"Duplicate Linear connector detected for user {user_id} with org {org_name}"
)
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=linear-connector"
)
if existing_connector:
# Update existing connector
existing_connector.config = connector_config
existing_connector.name = "Linear Connector"
existing_connector.is_indexable = True
logger.info(
f"Updated existing Linear connector for user {user_id} in space {space_id}"
)
else:
# Create new connector
new_connector = SearchSourceConnector(
name="Linear Connector",
connector_type=SearchSourceConnectorType.LINEAR_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Linear connector for user {user_id} in space {space_id}"
)
# Generate a unique, user-friendly connector name
connector_name = await generate_unique_connector_name(
session,
SearchSourceConnectorType.LINEAR_CONNECTOR,
space_id,
user_id,
org_name,
)
# Create new connector
new_connector = SearchSourceConnector(
name=connector_name,
connector_type=SearchSourceConnectorType.LINEAR_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Linear connector for user {user_id} in space {space_id}"
)
try:
await session.commit()
@ -300,7 +311,7 @@ async def linear_callback(
# Redirect to the frontend with success params
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=linear-connector"
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=linear-connector&connectorId={new_connector.id}"
)
except ValidationError as e:
@ -312,7 +323,7 @@ async def linear_callback(
await session.rollback()
raise HTTPException(
status_code=409,
detail=f"Integrity error: A connector with this type already exists. {e!s}",
detail=f"Database integrity error: {e!s}",
) from e
except Exception as e:
logger.error(f"Failed to create search source connector: {e!s}")

77
surfsense_backend/app/routes/notion_add_connector_route.py
View file

@ -14,7 +14,6 @@ from fastapi.responses import RedirectResponse
from pydantic import ValidationError
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.db import (
@ -25,6 +24,11 @@ from app.db import (
)
from app.schemas.notion_auth_credentials import NotionAuthCredentialsBase
from app.users import current_active_user
from app.utils.connector_naming import (
check_duplicate_connector,
extract_identifier_from_credentials,
generate_unique_connector_name,
)
from app.utils.oauth_security import OAuthStateManager, TokenEncryption
logger = logging.getLogger(__name__)
@ -262,47 +266,56 @@ async def notion_callback(
"_token_encrypted": True,
}
# Check if connector already exists for this search space and user
existing_connector_result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.NOTION_CONNECTOR,
)
# Extract unique identifier from connector credentials
connector_identifier = extract_identifier_from_credentials(
SearchSourceConnectorType.NOTION_CONNECTOR, connector_config
)
existing_connector = existing_connector_result.scalars().first()
if existing_connector:
# Update existing connector
existing_connector.config = connector_config
existing_connector.name = "Notion Connector"
existing_connector.is_indexable = True
logger.info(
f"Updated existing Notion connector for user {user_id} in space {space_id}"
# Check for duplicate connector (same workspace already connected)
is_duplicate = await check_duplicate_connector(
session,
SearchSourceConnectorType.NOTION_CONNECTOR,
space_id,
user_id,
connector_identifier,
)
if is_duplicate:
logger.warning(
f"Duplicate Notion connector detected for user {user_id} with workspace {connector_identifier}"
)
else:
# Create new connector
new_connector = SearchSourceConnector(
name="Notion Connector",
connector_type=SearchSourceConnectorType.NOTION_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Notion connector for user {user_id} in space {space_id}"
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=notion-connector"
)
# Generate a unique, user-friendly connector name
connector_name = await generate_unique_connector_name(
session,
SearchSourceConnectorType.NOTION_CONNECTOR,
space_id,
user_id,
connector_identifier,
)
# Create new connector
new_connector = SearchSourceConnector(
name=connector_name,
connector_type=SearchSourceConnectorType.NOTION_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Notion connector for user {user_id} in space {space_id}"
)
try:
await session.commit()
logger.info(f"Successfully saved Notion connector for user {user_id}")
# Redirect to the frontend with success params
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=notion-connector"
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=notion-connector&connectorId={new_connector.id}"
)
except ValidationError as e:
@ -314,7 +327,7 @@ async def notion_callback(
await session.rollback()
raise HTTPException(
status_code=409,
detail=f"Integrity error: A connector with this type already exists. {e!s}",
detail=f"Database integrity error: {e!s}",
) from e
except Exception as e:
logger.error(f"Failed to create search source connector: {e!s}")

4
surfsense_backend/app/routes/search_source_connectors_routes.py
View file

@ -7,7 +7,8 @@ PUT /search-source-connectors/{connector_id} - Update a specific connector
DELETE /search-source-connectors/{connector_id} - Delete a specific connector
POST /search-source-connectors/{connector_id}/index - Index content from a connector to a search space
Note: Each search space can have only one connector of each type per user (based on search_space_id, user_id, and connector_type).
Note: OAuth connectors (Gmail, Drive, Slack, etc.) support multiple accounts per search space.
Non-OAuth connectors (BookStack, GitHub, etc.) are limited to one per search space.
"""
import logging
@ -125,6 +126,7 @@ async def create_search_source_connector(
)
# Check if a connector with the same type already exists for this search space
# (for non-OAuth connectors that don't support multiple accounts)
result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == search_space_id,

78
surfsense_backend/app/routes/slack_add_connector_route.py
View file

@ -14,7 +14,6 @@ from fastapi.responses import RedirectResponse
from pydantic import ValidationError
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.config import config
from app.db import (
@ -25,6 +24,11 @@ from app.db import (
)
from app.schemas.slack_auth_credentials import SlackAuthCredentialsBase
from app.users import current_active_user
from app.utils.connector_naming import (
check_duplicate_connector,
extract_identifier_from_credentials,
generate_unique_connector_name,
)
from app.utils.oauth_security import OAuthStateManager, TokenEncryption
logger = logging.getLogger(__name__)
@ -272,47 +276,57 @@ async def slack_callback(
"_token_encrypted": True,
}
# Check if connector already exists for this search space and user
existing_connector_result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.search_space_id == space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.SLACK_CONNECTOR,
)
# Extract unique identifier from connector credentials
connector_identifier = extract_identifier_from_credentials(
SearchSourceConnectorType.SLACK_CONNECTOR, connector_config
)
existing_connector = existing_connector_result.scalars().first()
if existing_connector:
# Update existing connector
existing_connector.config = connector_config
existing_connector.name = "Slack Connector"
existing_connector.is_indexable = True
logger.info(
f"Updated existing Slack connector for user {user_id} in space {space_id}"
# Check for duplicate connector (same workspace already connected)
is_duplicate = await check_duplicate_connector(
session,
SearchSourceConnectorType.SLACK_CONNECTOR,
space_id,
user_id,
connector_identifier,
)
if is_duplicate:
logger.warning(
f"Duplicate Slack connector detected for user {user_id} with workspace {connector_identifier}"
)
else:
# Create new connector
new_connector = SearchSourceConnector(
name="Slack Connector",
connector_type=SearchSourceConnectorType.SLACK_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Slack connector for user {user_id} in space {space_id}"
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&error=duplicate_account&connector=slack-connector"
)
# Generate a unique, user-friendly connector name
connector_name = await generate_unique_connector_name(
session,
SearchSourceConnectorType.SLACK_CONNECTOR,
space_id,
user_id,
connector_identifier,
)
# Create new connector
new_connector = SearchSourceConnector(
name=connector_name,
connector_type=SearchSourceConnectorType.SLACK_CONNECTOR,
is_indexable=True,
config=connector_config,
search_space_id=space_id,
user_id=user_id,
)
session.add(new_connector)
logger.info(
f"Created new Slack connector for user {user_id} in space {space_id}"
)
try:
await session.commit()
logger.info(f"Successfully saved Slack connector for user {user_id}")
# Redirect to the frontend with success params
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=slack-connector"
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=slack-connector&connectorId={new_connector.id}"
)
except ValidationError as e:
@ -324,7 +338,7 @@ async def slack_callback(
await session.rollback()
raise HTTPException(
status_code=409,
detail=f"Integrity error: A connector with this type already exists. {e!s}",
detail=f"Database integrity error: {e!s}",
) from e
except Exception as e:
logger.error(f"Failed to create search source connector: {e!s}")

189
surfsense_backend/app/utils/connector_naming.py Normal file
View file

@ -0,0 +1,189 @@
"""
Connector Naming Utilities.
Provides functions for generating unique, user-friendly connector names.
"""
from typing import Any
from urllib.parse import urlparse
from uuid import UUID
from sqlalchemy import func
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.db import SearchSourceConnector, SearchSourceConnectorType
# Friendly display names for connector types
BASE_NAME_FOR_TYPE = {
SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR: "Gmail",
SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR: "Google Drive",
SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR: "Google Calendar",
SearchSourceConnectorType.SLACK_CONNECTOR: "Slack",
SearchSourceConnectorType.NOTION_CONNECTOR: "Notion",
SearchSourceConnectorType.LINEAR_CONNECTOR: "Linear",
SearchSourceConnectorType.JIRA_CONNECTOR: "Jira",
SearchSourceConnectorType.DISCORD_CONNECTOR: "Discord",
SearchSourceConnectorType.CONFLUENCE_CONNECTOR: "Confluence",
SearchSourceConnectorType.AIRTABLE_CONNECTOR: "Airtable",
}
def get_base_name_for_type(connector_type: SearchSourceConnectorType) -> str:
"""Get a friendly display name for a connector type."""
return BASE_NAME_FOR_TYPE.get(
connector_type, connector_type.replace("_", " ").title()
)
def extract_identifier_from_credentials(
connector_type: SearchSourceConnectorType,
credentials: dict[str, Any],
) -> str | None:
"""
Extract a unique identifier from connector credentials.
Args:
connector_type: The type of connector
credentials: The connector credentials dict
Returns:
Identifier string (workspace name, email, etc.) or None
"""
if connector_type == SearchSourceConnectorType.SLACK_CONNECTOR:
return credentials.get("team_name")
if connector_type == SearchSourceConnectorType.NOTION_CONNECTOR:
return credentials.get("workspace_name")
if connector_type == SearchSourceConnectorType.DISCORD_CONNECTOR:
return credentials.get("guild_name")
if connector_type in (
SearchSourceConnectorType.JIRA_CONNECTOR,
SearchSourceConnectorType.CONFLUENCE_CONNECTOR,
):
base_url = credentials.get("base_url", "")
if base_url:
try:
parsed = urlparse(base_url)
hostname = parsed.netloc or parsed.path
if ".atlassian.net" in hostname:
return hostname.replace(".atlassian.net", "")
return hostname
except Exception:
pass
return None
# Google, Linear, Airtable require API calls - return None
return None
def generate_connector_name_with_identifier(
connector_type: SearchSourceConnectorType,
identifier: str | None,
) -> str:
"""
Generate a connector name with an identifier.
Args:
connector_type: The type of connector
identifier: User identifier (email, workspace name, etc.)
Returns:
Name like "Gmail - john@example.com" or just "Gmail" if no identifier
"""
base = get_base_name_for_type(connector_type)
if identifier:
return f"{base} - {identifier}"
return base
async def count_connectors_of_type(
session: AsyncSession,
connector_type: SearchSourceConnectorType,
search_space_id: int,
user_id: UUID,
) -> int:
"""Count existing connectors of a type for a user in a search space."""
result = await session.execute(
select(func.count(SearchSourceConnector.id)).where(
SearchSourceConnector.connector_type == connector_type,
SearchSourceConnector.search_space_id == search_space_id,
SearchSourceConnector.user_id == user_id,
)
)
return result.scalar() or 0
async def check_duplicate_connector(
session: AsyncSession,
connector_type: SearchSourceConnectorType,
search_space_id: int,
user_id: UUID,
identifier: str | None,
) -> bool:
"""
Check if a connector with the same identifier already exists.
Args:
session: Database session
connector_type: The type of connector
search_space_id: The search space ID
user_id: The user ID
identifier: User identifier (email, workspace name, etc.)
Returns:
True if a duplicate exists, False otherwise
"""
if not identifier:
return False
expected_name = f"{get_base_name_for_type(connector_type)} - {identifier}"
result = await session.execute(
select(func.count(SearchSourceConnector.id)).where(
SearchSourceConnector.connector_type == connector_type,
SearchSourceConnector.search_space_id == search_space_id,
SearchSourceConnector.user_id == user_id,
SearchSourceConnector.name == expected_name,
)
)
return (result.scalar() or 0) > 0
async def generate_unique_connector_name(
session: AsyncSession,
connector_type: SearchSourceConnectorType,
search_space_id: int,
user_id: UUID,
identifier: str | None = None,
) -> str:
"""
Generate a unique connector name.
If an identifier is provided (email, workspace name, etc.), uses it with base name.
Otherwise, falls back to counting existing connectors for uniqueness.
Args:
session: Database session
connector_type: The type of connector
search_space_id: The search space ID
user_id: The user ID
identifier: Optional user identifier (email, workspace name, etc.)
Returns:
Unique name like "Gmail - john@example.com" or "Gmail (2)"
"""
base = get_base_name_for_type(connector_type)
if identifier:
return f"{base} - {identifier}"
# Fallback: use counter for uniqueness
count = await count_connectors_of_type(
session, connector_type, search_space_id, user_id
)
if count == 0:
return base
return f"{base} ({count + 1})"

26
surfsense_web/components/assistant-ui/connector-popup.tsx
View file

@ -19,9 +19,11 @@ import { ConnectorDialogHeader } from "./connector-popup/components/connector-di
import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view";
import { ConnectorEditView } from "./connector-popup/connector-configs/views/connector-edit-view";
import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view";
import { OAUTH_CONNECTORS } from "./connector-popup/constants/connector-constants";
import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog";
import { ActiveConnectorsTab } from "./connector-popup/tabs/active-connectors-tab";
import { AllConnectorsTab } from "./connector-popup/tabs/all-connectors-tab";
import { ConnectorAccountsListView } from "./connector-popup/views/connector-accounts-list-view";
import { YouTubeCrawlerView } from "./connector-popup/views/youtube-crawler-view";
export const ConnectorIndicator: FC = () => {
@ -60,6 +62,7 @@ export const ConnectorIndicator: FC = () => {
periodicEnabled,
frequencyMinutes,
allConnectors,
viewingAccountsType,
setSearchQuery,
setStartDate,
setEndDate,
@ -81,6 +84,8 @@ export const ConnectorIndicator: FC = () => {
handleBackFromEdit,
handleBackFromConnect,
handleBackFromYouTube,
handleViewAccountsList,
handleBackFromAccountsList,
handleQuickIndexConnector,
connectorConfig,
setConnectorConfig,
@ -194,6 +199,25 @@ export const ConnectorIndicator: FC = () => {
{/* YouTube Crawler View - shown when adding YouTube videos */}
{isYouTubeView && searchSpaceId ? (
<YouTubeCrawlerView searchSpaceId={searchSpaceId} onBack={handleBackFromYouTube} />
) : viewingAccountsType ? (
<ConnectorAccountsListView
connectorType={viewingAccountsType.connectorType}
connectorTitle={viewingAccountsType.connectorTitle}
connectors={(allConnectors || []) as SearchSourceConnector[]}
indexingConnectorIds={indexingConnectorIds}
logsSummary={logsSummary}
onBack={handleBackFromAccountsList}
onManage={handleStartEdit}
onAddAccount={() => {
const oauthConnector = OAUTH_CONNECTORS.find(
(c) => c.connectorType === viewingAccountsType.connectorType
);
if (oauthConnector) {
handleConnectOAuth(oauthConnector);
}
}}
isConnecting={connectingId !== null}
/>
) : connectingConnectorType ? (
<ConnectorConnectView
connectorType={connectingConnectorType}
@ -289,6 +313,7 @@ export const ConnectorIndicator: FC = () => {
onCreateWebcrawler={handleCreateWebcrawler}
onCreateYouTubeCrawler={handleCreateYouTubeCrawler}
onManage={handleStartEdit}
onViewAccountsList={handleViewAccountsList}
/>
</TabsContent>
@ -303,6 +328,7 @@ export const ConnectorIndicator: FC = () => {
searchSpaceId={searchSpaceId}
onTabChange={handleTabChange}
onManage={handleStartEdit}
onViewAccountsList={handleViewAccountsList}
/>
</div>
</div>

20
surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx
View file

@ -17,6 +17,7 @@ interface ConnectorCardProps {
isConnected?: boolean;
isConnecting?: boolean;
documentCount?: number;
accountCount?: number;
lastIndexedAt?: string | null;
isIndexing?: boolean;
activeTask?: LogActiveTask;
@ -96,6 +97,7 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
isConnected = false,
isConnecting = false,
documentCount,
accountCount,
lastIndexedAt,
isIndexing = false,
activeTask,
@ -139,7 +141,7 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
return (
<div className="group relative flex items-center gap-4 p-4 rounded-xl text-left transition-all duration-200 w-full border border-border bg-slate-400/5 dark:bg-white/5 hover:bg-slate-400/10 dark:hover:bg-white/10">
<div className="flex h-12 w-12 items-center justify-center rounded-lg transition-colors flex-shrink-0 bg-slate-400/5 dark:bg-white/5 border border-slate-400/5 dark:border-white/5">
<div className="flex h-12 w-12 items-center justify-center rounded-lg transition-colors shrink-0 bg-slate-400/5 dark:bg-white/5 border border-slate-400/5 dark:border-white/5">
{connectorType ? (
getConnectorIcon(connectorType, "size-6")
) : id === "youtube-crawler" ? (
@ -150,12 +152,20 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
</div>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2">
<span className="text-[14px] font-semibold leading-tight">{title}</span>
<span className="text-[14px] font-semibold leading-tight truncate">{title}</span>
</div>
<div className="text-[10px] text-muted-foreground mt-1">{getStatusContent()}</div>
{isConnected && documentCount !== undefined && (
<p className="text-[10px] text-muted-foreground mt-0.5">
{formatDocumentCount(documentCount)}
<p className="text-[10px] text-muted-foreground mt-0.5 flex items-center gap-1.5">
<span>{formatDocumentCount(documentCount)}</span>
{accountCount !== undefined && accountCount > 0 && (
<>
<span className="text-muted-foreground/50">•</span>
<span>
{accountCount} {accountCount === 1 ? "Account" : "Accounts"}
</span>
</>
)}
</p>
)}
</div>
@ -163,7 +173,7 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
size="sm"
variant={isConnected ? "secondary" : "default"}
className={cn(
"h-8 text-[11px] px-3 rounded-lg flex-shrink-0 font-medium",
"h-8 text-[11px] px-3 rounded-lg shrink-0 font-medium",
isConnected &&
"bg-white text-slate-700 hover:bg-slate-50 border-0 shadow-xs dark:bg-secondary dark:text-secondary-foreground dark:hover:bg-secondary/80",
!isConnected && "shadow-xs"

8
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx
View file

@ -143,12 +143,14 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
{/* Connector header */}
<div className="flex flex-col sm:flex-row items-start sm:items-center gap-4 mb-6">
<div className="flex items-center gap-4 flex-1 w-full sm:w-auto">
<div className="flex h-14 w-14 items-center justify-center rounded-xl bg-primary/10 border border-primary/20 flex-shrink-0">
<div className="flex gap-4 flex-1 w-full sm:w-auto">
<div className="flex h-14 w-14 items-center justify-center rounded-xl bg-primary/10 border border-primary/20 shrink-0">
{getConnectorIcon(connector.connector_type, "size-7")}
</div>
<div className="flex-1 min-w-0">
<h2 className="text-xl sm:text-2xl font-semibold tracking-tight">{connector.name}</h2>
<h2 className="text-xl sm:text-2xl font-semibold tracking-tight text-wrap whitespace-normal wrap-break-word">
{connector.name}
</h2>
<p className="text-xs sm:text-base text-muted-foreground mt-1">
Manage your connector settings and sync configuration
</p>

23
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx
View file

@ -1,14 +1,16 @@
"use client";
import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
import { type FC, useCallback, useEffect, useMemo, useRef, useState } from "react";
import { useSearchParams } from "next/navigation";
import { type FC, useCallback, useEffect, useMemo, useRef, useState } from "react";
import { Button } from "@/components/ui/button";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { getConnectorTypeDisplay } from "@/lib/connectors/utils";
import { cn } from "@/lib/utils";
import { DateRangeSelector } from "../../components/date-range-selector";
import { PeriodicSyncConfig } from "../../components/periodic-sync-config";
import type { IndexingConfigState } from "../../constants/connector-constants";
import { type IndexingConfigState, OAUTH_CONNECTORS } from "../../constants/connector-constants";
import { getConnectorDisplayName } from "../../tabs/all-connectors-tab";
import { getConnectorConfigComponent } from "../index";
interface IndexingConfigurationViewProps {
@ -89,12 +91,14 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
};
}, [checkScrollState]);
const authConnector = OAUTH_CONNECTORS.find((c) => c.connectorType === connector?.connector_type);
return (
<div className="flex-1 flex flex-col min-h-0 overflow-hidden">
{/* Fixed Header */}
<div
className={cn(
"flex-shrink-0 px-6 sm:px-12 pt-8 sm:pt-10 transition-shadow duration-200 relative z-10",
"shrink-0 px-6 sm:px-12 pt-8 sm:pt-10 transition-shadow duration-200 relative z-10",
isScrolled && "shadow-sm"
)}
>
@ -111,14 +115,19 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
)}
{/* Success header */}
<div className="flex items-center gap-4 mb-6">
<div className="flex gap-4 mb-6">
<div className="flex h-14 w-14 items-center justify-center rounded-xl bg-green-500/10 border border-green-500/20">
<Check className="size-7 text-green-500" />
</div>
<div>
<h2 className="text-xl sm:text-2xl font-semibold tracking-tight">
{config.connectorTitle} Connected!
</h2>
<div className="flex flex-col">
<span className="text-xl sm:text-2xl font-semibold tracking-tight text-wrap whitespace-normal wrap-break-word">
{getConnectorTypeDisplay(connector?.connector_type || "")} Connected !
</span>{" "}
<span className="text-xl sm:text-xl font-semibold text-muted-foreground tracking-tight text-wrap whitespace-normal wrap-break-word">
{getConnectorDisplayName(connector?.name || "")}
</span>
</div>
<p className="text-xs sm:text-base text-muted-foreground mt-1">
Configure when to start syncing your data
</p>

3
surfsense_web/components/assistant-ui/connector-popup/constants/connector-popup.schemas.ts
View file

@ -7,11 +7,12 @@ import { searchSourceConnectorTypeEnum } from "@/contracts/types/connector.types
export const connectorPopupQueryParamsSchema = z.object({
modal: z.enum(["connectors"]).optional(),
tab: z.enum(["all", "active"]).optional(),
view: z.enum(["configure", "edit", "connect", "youtube"]).optional(),
view: z.enum(["configure", "edit", "connect", "youtube", "accounts"]).optional(),
connector: z.string().optional(),
connectorId: z.string().optional(),
connectorType: z.string().optional(),
success: z.enum(["true", "false"]).optional(),
error: z.string().optional(),
});
export type ConnectorPopupQueryParams = z.infer<typeof connectorPopupQueryParamsSchema>;

138
surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts
View file

@ -66,6 +66,12 @@ export const useConnectorDialog = () => {
const [isCreatingConnector, setIsCreatingConnector] = useState(false);
const isCreatingConnectorRef = useRef(false);
// Accounts list view state (for OAuth connectors with multiple accounts)
const [viewingAccountsType, setViewingAccountsType] = useState<{
connectorType: string;
connectorTitle: string;
} | null>(null);
// Helper function to get frequency label
const getFrequencyLabel = useCallback((minutes: string): string => {
switch (minutes) {
@ -114,24 +120,50 @@ export const useConnectorDialog = () => {
setConnectingConnectorType(null);
}
// Clear viewing accounts type if view is not "accounts" anymore
if (params.view !== "accounts" && viewingAccountsType) {
setViewingAccountsType(null);
}
// Handle connect view
if (params.view === "connect" && params.connectorType && !connectingConnectorType) {
setConnectingConnectorType(params.connectorType);
}
// Handle accounts view
if (params.view === "accounts" && params.connectorType && !viewingAccountsType) {
const oauthConnector = OAUTH_CONNECTORS.find(
(c) => c.connectorType === params.connectorType
);
if (oauthConnector) {
setViewingAccountsType({
connectorType: oauthConnector.connectorType,
connectorTitle: oauthConnector.title,
});
}
}
// Handle YouTube view
if (params.view === "youtube") {
// YouTube view is active - no additional state needed
}
if (params.view === "configure" && params.connector && !indexingConfig) {
// Handle configure view (for page refresh support)
if (params.view === "configure" && params.connector && !indexingConfig && allConnectors) {
const oauthConnector = OAUTH_CONNECTORS.find((c) => c.id === params.connector);
if (oauthConnector && allConnectors) {
const existingConnector = allConnectors.find(
(c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType
);
if (oauthConnector) {
let existingConnector: SearchSourceConnector | undefined;
if (params.connectorId) {
const connectorId = parseInt(params.connectorId, 10);
existingConnector = allConnectors.find(
(c: SearchSourceConnector) => c.id === connectorId
);
} else {
existingConnector = allConnectors.find(
(c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType
);
}
if (existingConnector) {
// Validate connector data before setting state
const connectorValidation = searchSourceConnector.safeParse(existingConnector);
if (connectorValidation.success) {
const config = validateIndexingConfigState({
@ -200,6 +232,10 @@ export const useConnectorDialog = () => {
if (connectingConnectorType) {
setConnectingConnectorType(null);
}
// Clear viewing accounts type when modal is closed
if (viewingAccountsType) {
setViewingAccountsType(null);
}
// Clear YouTube view when modal is closed (handled by view param check)
}
} catch (error) {
@ -207,13 +243,48 @@ export const useConnectorDialog = () => {
console.warn("Invalid connector popup query params:", error);
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [searchParams, allConnectors, editingConnector, indexingConfig, connectingConnectorType]);
}, [
searchParams,
allConnectors,
editingConnector,
indexingConfig,
connectingConnectorType,
viewingAccountsType,
]);
// Detect OAuth success and transition to config view
// Detect OAuth success / Failure and transition to config view
useEffect(() => {
try {
const params = parseConnectorPopupQueryParams(searchParams);
// Handle OAuth errors (e.g., duplicate account)
if (params.error && params.modal === "connectors") {
const oauthConnector = params.connector
? OAUTH_CONNECTORS.find((c) => c.id === params.connector)
: null;
const connectorName = oauthConnector?.title || "connector";
if (params.error === "duplicate_account") {
toast.error(`This ${connectorName} account is already connected`, {
description: "Please use a different account or manage the existing connection.",
});
} else {
toast.error(`Failed to connect ${connectorName}`, {
description: params.error.replace(/_/g, " "),
});
}
// Clean up error params from URL
const url = new URL(window.location.href);
url.searchParams.delete("error");
url.searchParams.delete("connector");
window.history.replaceState({}, "", url.toString());
// Open the popup to show the connectors
setIsOpen(true);
return;
}
if (
params.success === "true" &&
params.connector &&
@ -225,11 +296,17 @@ export const useConnectorDialog = () => {
refetchAllConnectors().then((result) => {
if (!result.data) return;
const newConnector = result.data.find(
(c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType
);
let newConnector: SearchSourceConnector | undefined;
if (params.connectorId) {
const connectorId = parseInt(params.connectorId, 10);
newConnector = result.data.find((c: SearchSourceConnector) => c.id === connectorId);
} else {
newConnector = result.data.find(
(c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType
);
}
if (newConnector) {
// Validate connector data before setting state
const connectorValidation = searchSourceConnector.safeParse(newConnector);
if (connectorValidation.success) {
const config = validateIndexingConfigState({
@ -243,6 +320,7 @@ export const useConnectorDialog = () => {
setIsOpen(true);
const url = new URL(window.location.href);
url.searchParams.delete("success");
url.searchParams.set("connectorId", newConnector.id.toString());
url.searchParams.set("view", "configure");
window.history.replaceState({}, "", url.toString());
} else {
@ -632,6 +710,38 @@ export const useConnectorDialog = () => {
router.replace(url.pathname + url.search, { scroll: false });
}, [router]);
// Handle viewing accounts list for OAuth connector type
const handleViewAccountsList = useCallback(
(connectorType: string, connectorTitle: string) => {
if (!searchSpaceId) return;
setViewingAccountsType({
connectorType,
connectorTitle,
});
// Update URL to show accounts view, preserving current tab
const url = new URL(window.location.href);
url.searchParams.set("modal", "connectors");
url.searchParams.set("view", "accounts");
url.searchParams.set("connectorType", connectorType);
// Keep the current tab in URL so we can go back to it
window.history.pushState({ modal: true }, "", url.toString());
},
[searchSpaceId]
);
// Handle going back from accounts list view
const handleBackFromAccountsList = useCallback(() => {
setViewingAccountsType(null);
const url = new URL(window.location.href);
url.searchParams.set("modal", "connectors");
// Keep the current tab (don't change it) - just remove view-specific params
url.searchParams.delete("view");
url.searchParams.delete("connectorType");
router.replace(url.pathname + url.search, { scroll: false });
}, [router]);
// Handle starting indexing
const handleStartIndexing = useCallback(
async (refreshConnectors: () => void) => {
@ -1081,6 +1191,7 @@ export const useConnectorDialog = () => {
setConnectorName(null);
setConnectorConfig(null);
setConnectingConnectorType(null);
setViewingAccountsType(null);
setStartDate(undefined);
setEndDate(undefined);
setPeriodicEnabled(false);
@ -1126,6 +1237,7 @@ export const useConnectorDialog = () => {
frequencyMinutes,
searchSpaceId,
allConnectors,
viewingAccountsType,
// Setters
setSearchQuery,
@ -1152,6 +1264,8 @@ export const useConnectorDialog = () => {
handleBackFromEdit,
handleBackFromConnect,
handleBackFromYouTube,
handleViewAccountsList,
handleBackFromAccountsList,
handleQuickIndexConnector,
connectorConfig,
setConnectorConfig,

186
surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx
View file

@ -11,6 +11,7 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
import { cn } from "@/lib/utils";
import { OAUTH_CONNECTORS } from "../constants/connector-constants";
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
interface ActiveConnectorsTabProps {
@ -24,6 +25,7 @@ interface ActiveConnectorsTabProps {
searchSpaceId: string;
onTabChange: (value: string) => void;
onManage?: (connector: SearchSourceConnector) => void;
onViewAccountsList?: (connectorType: string, connectorTitle: string) => void;
}
export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
@ -36,6 +38,7 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
searchSpaceId,
onTabChange,
onManage,
onViewAccountsList,
}) => {
const router = useRouter();
@ -71,38 +74,26 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
const minutesAgo = differenceInMinutes(now, date);
const daysAgo = differenceInDays(now, date);
// Just now (within last minute)
if (minutesAgo < 1) {
return "Just now";
}
// X minutes ago (less than 1 hour)
if (minutesAgo < 60) {
return `${minutesAgo} ${minutesAgo === 1 ? "minute" : "minutes"} ago`;
}
// Today at [time]
if (isToday(date)) {
return `Today at ${format(date, "h:mm a")}`;
}
// Yesterday at [time]
if (isYesterday(date)) {
return `Yesterday at ${format(date, "h:mm a")}`;
}
// X days ago (less than 7 days)
if (daysAgo < 7) {
return `${daysAgo} ${daysAgo === 1 ? "day" : "days"} ago`;
}
// Full date for older entries
if (minutesAgo < 1) return "Just now";
if (minutesAgo < 60) return `${minutesAgo} ${minutesAgo === 1 ? "minute" : "minutes"} ago`;
if (isToday(date)) return `Today at ${format(date, "h:mm a")}`;
if (isYesterday(date)) return `Yesterday at ${format(date, "h:mm a")}`;
if (daysAgo < 7) return `${daysAgo} ${daysAgo === 1 ? "day" : "days"} ago`;
return format(date, "MMM d, yyyy");
};
// Document types that should be shown as cards (not from connectors)
// These are: EXTENSION (browser extension), FILE (uploaded files), NOTE (editor notes),
// YOUTUBE_VIDEO (YouTube videos), and CRAWLED_URL (web pages - shown separately even though it can come from WEBCRAWLER_CONNECTOR)
// Get most recent last indexed date from a list of connectors
const getMostRecentLastIndexed = (
connectorsList: SearchSourceConnector[]
): string | undefined => {
return connectorsList.reduce<string | undefined>((latest, c) => {
if (!c.last_indexed_at) return latest;
if (!latest) return c.last_indexed_at;
return new Date(c.last_indexed_at) > new Date(latest) ? c.last_indexed_at : latest;
}, undefined);
};
// Document types that should be shown as standalone cards (not from connectors)
const standaloneDocumentTypes = ["EXTENSION", "FILE", "NOTE", "YOUTUBE_VIDEO", "CRAWLED_URL"];
// Filter to only show standalone document types that have documents (count > 0)
@ -118,8 +109,54 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
return doc.label.toLowerCase().includes(searchQuery.toLowerCase());
});
// Filter connectors based on search query
const filteredConnectors = connectors.filter((connector) => {
// Get OAuth connector types set for quick lookup
const oauthConnectorTypes = new Set<string>(OAUTH_CONNECTORS.map((c) => c.connectorType));
// Separate OAuth and non-OAuth connectors
const oauthConnectors = connectors.filter((c) => oauthConnectorTypes.has(c.connector_type));
const nonOauthConnectors = connectors.filter((c) => !oauthConnectorTypes.has(c.connector_type));
// Group OAuth connectors by type
const oauthConnectorsByType = oauthConnectors.reduce(
(acc, connector) => {
const type = connector.connector_type;
if (!acc[type]) {
acc[type] = [];
}
acc[type].push(connector);
return acc;
},
{} as Record<string, SearchSourceConnector[]>
);
// Get display info for OAuth connector type
const getOAuthConnectorTypeInfo = (connectorType: string) => {
const oauthConnector = OAUTH_CONNECTORS.find((c) => c.connectorType === connectorType);
return {
title:
oauthConnector?.title ||
connectorType
.replace(/_/g, " ")
.replace(/connector/gi, "")
.trim(),
};
};
// Filter OAuth connector types based on search query
const filteredOAuthConnectorTypes = Object.entries(oauthConnectorsByType).filter(
([connectorType]) => {
if (!searchQuery) return true;
const searchLower = searchQuery.toLowerCase();
const { title } = getOAuthConnectorTypeInfo(connectorType);
return (
title.toLowerCase().includes(searchLower) ||
connectorType.toLowerCase().includes(searchLower)
);
}
);
// Filter non-OAuth connectors based on search query
const filteredNonOAuthConnectors = nonOauthConnectors.filter((connector) => {
if (!searchQuery) return true;
const searchLower = searchQuery.toLowerCase();
return (
@ -128,18 +165,97 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
);
});
const hasActiveConnectors =
filteredOAuthConnectorTypes.length > 0 || filteredNonOAuthConnectors.length > 0;
return (
<TabsContent value="active" className="m-0">
{hasSources ? (
<div className="space-y-6">
{/* Active Connectors Section */}
{filteredConnectors.length > 0 && (
{hasActiveConnectors && (
<div className="space-y-4">
<div className="flex items-center gap-2">
<h3 className="text-sm font-semibold text-muted-foreground">Active Connectors</h3>
</div>
<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">
{filteredConnectors.map((connector) => {
{/* OAuth Connectors - Grouped by Type */}
{filteredOAuthConnectorTypes.map(([connectorType, typeConnectors]) => {
const { title } = getOAuthConnectorTypeInfo(connectorType);
const isAnyIndexing = typeConnectors.some((c: SearchSourceConnector) =>
indexingConnectorIds.has(c.id)
);
const documentCount = getDocumentCountForConnector(
connectorType,
documentTypeCounts
);
const accountCount = typeConnectors.length;
const mostRecentLastIndexed = getMostRecentLastIndexed(typeConnectors);
const handleManageClick = () => {
if (onViewAccountsList) {
onViewAccountsList(connectorType, title);
} else if (onManage && typeConnectors[0]) {
onManage(typeConnectors[0]);
}
};
return (
<div
key={`oauth-type-${connectorType}`}
className={cn(
"relative flex items-center gap-4 p-4 rounded-xl border border-border transition-all",
isAnyIndexing
? "bg-primary/5 border-primary/20"
: "bg-slate-400/5 dark:bg-white/5 hover:bg-slate-400/10 dark:hover:bg-white/10"
)}
>
<div
className={cn(
"flex h-12 w-12 items-center justify-center rounded-lg border shrink-0",
isAnyIndexing
? "bg-primary/10 border-primary/20"
: "bg-slate-400/5 dark:bg-white/5 border-slate-400/5 dark:border-white/5"
)}
>
{getConnectorIcon(connectorType, "size-6")}
</div>
<div className="flex-1 min-w-0">
<p className="text-[14px] font-semibold leading-tight truncate">{title}</p>
{isAnyIndexing ? (
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
<Loader2 className="size-3 animate-spin" />
Indexing...
</p>
) : (
<p className="text-[10px] text-muted-foreground mt-1 whitespace-nowrap">
{mostRecentLastIndexed
? `Last indexed: ${formatLastIndexedDate(mostRecentLastIndexed)}`
: "Never indexed"}
</p>
)}
<p className="text-[10px] text-muted-foreground mt-0.5 flex items-center gap-1.5">
<span>{formatDocumentCount(documentCount)}</span>
<span className="text-muted-foreground/50">•</span>
<span>
{accountCount} {accountCount === 1 ? "Account" : "Accounts"}
</span>
</p>
</div>
<Button
variant="secondary"
size="sm"
className="h-8 text-[11px] px-3 rounded-lg font-medium bg-white text-slate-700 hover:bg-slate-50 border-0 shadow-xs dark:bg-secondary dark:text-secondary-foreground dark:hover:bg-secondary/80 shrink-0"
onClick={handleManageClick}
>
Manage
</Button>
</div>
);
})}
{/* Non-OAuth Connectors - Individual Cards */}
{filteredNonOAuthConnectors.map((connector) => {
const isIndexing = indexingConnectorIds.has(connector.id);
const activeTask = logsSummary?.active_tasks?.find(
(task: LogActiveTask) => task.connector_id === connector.id
@ -161,7 +277,7 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
>
<div
className={cn(
"flex h-12 w-12 items-center justify-center rounded-lg border",
"flex h-12 w-12 items-center justify-center rounded-lg border shrink-0",
isIndexing
? "bg-primary/10 border-primary/20"
: "bg-slate-400/5 dark:bg-white/5 border-slate-400/5 dark:border-white/5"
@ -197,7 +313,7 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
<Button
variant="secondary"
size="sm"
className="h-8 text-[11px] px-3 rounded-lg font-medium bg-white text-slate-700 hover:bg-slate-50 border-0 shadow-xs dark:bg-secondary dark:text-secondary-foreground dark:hover:bg-secondary/80"
className="h-8 text-[11px] px-3 rounded-lg font-medium bg-white text-slate-700 hover:bg-slate-50 border-0 shadow-xs dark:bg-secondary dark:text-secondary-foreground dark:hover:bg-secondary/80 shrink-0"
onClick={onManage ? () => onManage(connector) : undefined}
>
Manage

57
surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx
View file

@ -1,12 +1,27 @@
"use client";
import { Plus } from "lucide-react";
import type { FC } from "react";
import { Button } from "@/components/ui/button";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
import { ConnectorCard } from "../components/connector-card";
import { CRAWLERS, OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants";
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
/**
* Extract the display name from a full connector name.
* Full names are in format "Base Name - identifier" (e.g., "Gmail - john@example.com").
* Returns just the identifier (e.g : john@example.com).
*/
export function getConnectorDisplayName(fullName: string): string {
const separatorIndex = fullName.indexOf(" - ");
if (separatorIndex !== -1) {
return fullName.substring(separatorIndex + 3);
}
return fullName;
}
interface AllConnectorsTabProps {
searchQuery: string;
searchSpaceId: string;
@ -21,6 +36,7 @@ interface AllConnectorsTabProps {
onCreateWebcrawler?: () => void;
onCreateYouTubeCrawler?: () => void;
onManage?: (connector: SearchSourceConnector) => void;
onViewAccountsList?: (connectorType: string, connectorTitle: string) => void;
}
export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
@ -37,6 +53,7 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
onCreateWebcrawler,
onCreateYouTubeCrawler,
onManage,
onViewAccountsList,
}) => {
// Helper to find active task for a connector
const getActiveTaskForConnector = (connectorId: number): LogActiveTask | undefined => {
@ -77,22 +94,39 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
{filteredOAuth.map((connector) => {
const isConnected = connectedTypes.has(connector.connectorType);
const isConnecting = connectingId === connector.id;
// Find the actual connector object if connected
const actualConnector =
// Find all connectors of this type
const typeConnectors =
isConnected && allConnectors
? allConnectors.find(
? allConnectors.filter(
(c: SearchSourceConnector) => c.connector_type === connector.connectorType
)
: undefined;
: [];
// Get the most recent last_indexed_at across all accounts
const mostRecentLastIndexed = typeConnectors.reduce<string | undefined>(
(latest, c) => {
if (!c.last_indexed_at) return latest;
if (!latest) return c.last_indexed_at;
return new Date(c.last_indexed_at) > new Date(latest)
? c.last_indexed_at
: latest;
},
undefined
);
const documentCount = getDocumentCountForConnector(
connector.connectorType,
documentTypeCounts
);
const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id);
const activeTask = actualConnector
? getActiveTaskForConnector(actualConnector.id)
: undefined;
// Check if any account is currently indexing
const isIndexing = typeConnectors.some((c) => indexingConnectorIds?.has(c.id));
// Get active task from any indexing account
const activeTask = typeConnectors
.map((c) => getActiveTaskForConnector(c.id))
.find((task) => task !== undefined);
return (
<ConnectorCard
@ -104,12 +138,15 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
isConnected={isConnected}
isConnecting={isConnecting}
documentCount={documentCount}
lastIndexedAt={actualConnector?.last_indexed_at}
accountCount={typeConnectors.length}
lastIndexedAt={mostRecentLastIndexed}
isIndexing={isIndexing}
activeTask={activeTask}
onConnect={() => onConnectOAuth(connector)}
onManage={
actualConnector && onManage ? () => onManage(actualConnector) : undefined
isConnected && onViewAccountsList
? () => onViewAccountsList(connector.connectorType, connector.title)
: undefined
}
/>
);

189
surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx Normal file
View file

@ -0,0 +1,189 @@
"use client";
import { differenceInDays, differenceInMinutes, format, isToday, isYesterday } from "date-fns";
import { ArrowLeft, Loader2, Plus } from "lucide-react";
import type { FC } from "react";
import { Button } from "@/components/ui/button";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
import { cn } from "@/lib/utils";
import { getConnectorDisplayName } from "../tabs/all-connectors-tab";
interface ConnectorAccountsListViewProps {
connectorType: string;
connectorTitle: string;
connectors: SearchSourceConnector[];
indexingConnectorIds: Set<number>;
logsSummary: LogSummary | undefined;
onBack: () => void;
onManage: (connector: SearchSourceConnector) => void;
onAddAccount: () => void;
isConnecting?: boolean;
}
/**
* Format last indexed date with contextual messages
*/
function formatLastIndexedDate(dateString: string): string {
const date = new Date(dateString);
const now = new Date();
const minutesAgo = differenceInMinutes(now, date);
const daysAgo = differenceInDays(now, date);
if (minutesAgo < 1) {
return "Just now";
}
if (minutesAgo < 60) {
return `${minutesAgo} ${minutesAgo === 1 ? "minute" : "minutes"} ago`;
}
if (isToday(date)) {
return `Today at ${format(date, "h:mm a")}`;
}
if (isYesterday(date)) {
return `Yesterday at ${format(date, "h:mm a")}`;
}
if (daysAgo < 7) {
return `${daysAgo} ${daysAgo === 1 ? "day" : "days"} ago`;
}
return format(date, "MMM d, yyyy");
}
export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
connectorType,
connectorTitle,
connectors,
indexingConnectorIds,
logsSummary,
onBack,
onManage,
onAddAccount,
isConnecting = false,
}) => {
// Filter connectors to only show those of this type
const typeConnectors = connectors.filter((c) => c.connector_type === connectorType);
return (
<div className="flex flex-col h-full">
{/* Header */}
<div className="px-4 sm:px-12 pt-6 sm:pt-10 pb-4 border-b border-border/50 bg-muted">
<div className="flex items-center justify-between gap-4 sm:pr-4">
<div className="flex items-center gap-4">
<Button
variant="ghost"
size="icon"
className="size-8 rounded-full shrink-0"
onClick={onBack}
>
<ArrowLeft className="size-4" />
</Button>
<div className="flex items-center gap-3">
<div className="flex h-10 w-10 items-center justify-center rounded-lg bg-slate-400/5 dark:bg-white/5 border border-slate-400/5 dark:border-white/5">
{getConnectorIcon(connectorType, "size-5")}
</div>
<div>
<h2 className="text-lg font-semibold">{connectorTitle} Accounts</h2>
<p className="text-xs text-muted-foreground">
{typeConnectors.length} connected account{typeConnectors.length !== 1 ? "s" : ""}
</p>
</div>
</div>
</div>
{/* Add Account Button with dashed border */}
<button
type="button"
onClick={onAddAccount}
disabled={isConnecting}
className={cn(
"flex items-center gap-2 px-3 py-2 rounded-lg mr-4 border-2 border-dashed border-border/70 text-left transition-all duration-200",
"border-primary/50 hover:bg-primary/5",
isConnecting && "opacity-50 cursor-not-allowed"
)}
>
<div className="flex h-6 w-6 items-center justify-center rounded-md bg-primary/10 shrink-0">
{isConnecting ? (
<Loader2 className="size-3.5 animate-spin text-primary" />
) : (
<Plus className="size-3.5 text-primary" />
)}
</div>
<span className="text-[12px] font-medium">
{isConnecting ? "Connecting..." : "Add Account"}
</span>
</button>
</div>
</div>
{/* Content */}
<div className="flex-1 overflow-y-auto px-4 sm:px-12 py-6 sm:py-8">
{/* Connected Accounts Grid */}
<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">
{typeConnectors.map((connector) => {
const isIndexing = indexingConnectorIds.has(connector.id);
const activeTask = logsSummary?.active_tasks?.find(
(task: LogActiveTask) => task.connector_id === connector.id
);
return (
<div
key={connector.id}
className={cn(
"flex items-center gap-4 p-4 rounded-xl border border-border transition-all",
isIndexing
? "bg-primary/5 border-primary/20"
: "bg-slate-400/5 dark:bg-white/5 hover:bg-slate-400/10 dark:hover:bg-white/10"
)}
>
<div
className={cn(
"flex h-12 w-12 items-center justify-center rounded-lg border shrink-0",
isIndexing
? "bg-primary/10 border-primary/20"
: "bg-slate-400/5 dark:bg-white/5 border-slate-400/5 dark:border-white/5"
)}
>
{getConnectorIcon(connector.connector_type, "size-6")}
</div>
<div className="flex-1 min-w-0">
<p className="text-[14px] font-semibold leading-tight truncate">
{getConnectorDisplayName(connector.name)}
</p>
{isIndexing ? (
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
<Loader2 className="size-3 animate-spin" />
Indexing...
{activeTask?.message && (
<span className="text-muted-foreground truncate max-w-[100px]">
• {activeTask.message}
</span>
)}
</p>
) : (
<p className="text-[10px] text-muted-foreground mt-1 whitespace-nowrap truncate">
{connector.last_indexed_at
? `Last indexed: ${formatLastIndexedDate(connector.last_indexed_at)}`
: "Never indexed"}
</p>
)}
</div>
<Button
variant="secondary"
size="sm"
className="h-8 text-[11px] px-3 rounded-lg font-medium bg-white text-slate-700 hover:bg-slate-50 border-0 shadow-xs dark:bg-secondary dark:text-secondary-foreground dark:hover:bg-secondary/80 shrink-0"
onClick={() => onManage(connector)}
>
Manage
</Button>
</div>
);
})}
</div>
</div>
</div>
);
};

2
surfsense_web/components/assistant-ui/document-upload-popup.tsx
View file

@ -1,7 +1,7 @@
"use client";
import { Upload } from "lucide-react";
import { useAtomValue } from "jotai";
import { Upload } from "lucide-react";
import { useRouter } from "next/navigation";
import {
createContext,

2
surfsense_web/components/markdown-viewer.tsx
View file

@ -1,5 +1,5 @@
import Image from "next/image";
import { type StreamdownProps, Streamdown } from "streamdown";
import { Streamdown, type StreamdownProps } from "streamdown";
import { cn } from "@/lib/utils";
interface MarkdownViewerProps {

1
surfsense_web/content/docs/connectors/meta.json
View file

@ -20,4 +20,3 @@
],
"defaultOpen": true
}

1
surfsense_web/lib/connectors/utils.ts
View file

@ -15,6 +15,7 @@ export const getConnectorTypeDisplay = (type: string): string => {
CLICKUP_CONNECTOR: "ClickUp",
GOOGLE_CALENDAR_CONNECTOR: "Google Calendar",
GOOGLE_GMAIL_CONNECTOR: "Google Gmail",
GOOGLE_DRIVE_CONNECTOR: "Google Drive",
AIRTABLE_CONNECTOR: "Airtable",
LUMA_CONNECTOR: "Luma",
ELASTICSEARCH_CONNECTOR: "Elasticsearch",