apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-06-10 20:35:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor: remove unused COMPOSIO_CONNECTOR migration and linting

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-01-22 16:43:08 -08:00
parent eb509810f1
commit 8b81507739
24 changed files with 401 additions and 251 deletions
Download patch file Download diff file Expand all files Collapse all files

81
surfsense_backend/alembic/versions/74_add_composio_connector_enums.py
View file

@ -1,81 +0,0 @@
"""Add COMPOSIO_CONNECTOR to SearchSourceConnectorType and DocumentType enums
Revision ID: 74
Revises: 73
Create Date: 2026-01-21
This migration adds the COMPOSIO_CONNECTOR enum value to both:
- searchsourceconnectortype (for connector type tracking)
- documenttype (for document type tracking)
Composio is a managed OAuth integration service that allows connecting
to various third-party services (Google Drive, Gmail, Calendar, etc.)
without requiring separate OAuth app verification.
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "74"
down_revision: str | None = "73"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# Define the ENUM type names and the new value
CONNECTOR_ENUM = "searchsourceconnectortype"
CONNECTOR_NEW_VALUE = "COMPOSIO_CONNECTOR"
DOCUMENT_ENUM = "documenttype"
DOCUMENT_NEW_VALUE = "COMPOSIO_CONNECTOR"
def upgrade() -> None:
"""Upgrade schema - add COMPOSIO_CONNECTOR to connector and document enums safely."""
# Add COMPOSIO_CONNECTOR to searchsourceconnectortype only if not exists
op.execute(
f"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = '{CONNECTOR_NEW_VALUE}'
AND enumtypid = (SELECT oid FROM pg_type WHERE typname = '{CONNECTOR_ENUM}')
) THEN
ALTER TYPE {CONNECTOR_ENUM} ADD VALUE '{CONNECTOR_NEW_VALUE}';
END IF;
END$$;
"""
)
# Add COMPOSIO_CONNECTOR to documenttype only if not exists
op.execute(
f"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = '{DOCUMENT_NEW_VALUE}'
AND enumtypid = (SELECT oid FROM pg_type WHERE typname = '{DOCUMENT_ENUM}')
) THEN
ALTER TYPE {DOCUMENT_ENUM} ADD VALUE '{DOCUMENT_NEW_VALUE}';
END IF;
END$$;
"""
)
def downgrade() -> None:
"""Downgrade schema - remove COMPOSIO_CONNECTOR from connector and document enums.
Note: PostgreSQL does not support removing enum values directly.
To properly downgrade, you would need to:
1. Delete any rows using the COMPOSIO_CONNECTOR value
2. Create new enums without COMPOSIO_CONNECTOR
3. Alter the columns to use the new enums
4. Drop the old enums
This is left as a no-op since removing enum values is complex
and typically not needed in practice.
"""
pass

29
surfsense_backend/alembic/versions/74_no_op.py Normal file
View file

@ -0,0 +1,29 @@
"""No-op migration for Composio support
Revision ID: 74
Revises: 73
Create Date: 2026-01-21
NOTE: This migration is a no-op since Composio is not supported yet.
"""
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "74"
down_revision: str | None = "73"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""No-op upgrade for Composio support."""
pass
def downgrade() -> None:
"""No-op downgrade for Composio support.
Note: PostgreSQL does not support removing enum values directly.
"""
pass

4
surfsense_backend/alembic/versions/77_add_thread_id_to_chat_comments.py
View file

@ -64,5 +64,7 @@ def upgrade() -> None:
def downgrade() -> None:
"""Remove thread_id column from chat_comments."""
op.execute("DROP INDEX IF EXISTS idx_chat_comments_thread_id")
op.execute("ALTER TABLE chat_comments DROP CONSTRAINT IF EXISTS fk_chat_comments_thread_id")
op.execute(
"ALTER TABLE chat_comments DROP CONSTRAINT IF EXISTS fk_chat_comments_thread_id"
)
op.execute("ALTER TABLE chat_comments DROP COLUMN IF EXISTS thread_id")

6
surfsense_backend/app/connectors/composio_connector.py
View file

@ -12,7 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.db import SearchSourceConnector
from app.services.composio_service import ComposioService, INDEXABLE_TOOLKITS
from app.services.composio_service import INDEXABLE_TOOLKITS, ComposioService
logger = logging.getLogger(__name__)
@ -268,7 +268,9 @@ class ComposioConnector:
from_email = header_dict.get("from", "Unknown Sender")
to_email = header_dict.get("to", "Unknown Recipient")
# Composio provides messageTimestamp directly
date_str = message.get("messageTimestamp", "") or header_dict.get("date", "Unknown Date")
date_str = message.get("messageTimestamp", "") or header_dict.get(
"date", "Unknown Date"
)
# Build markdown content
markdown_content = f"# {subject}\n\n"

46
surfsense_backend/app/connectors/github_connector.py
View file

@ -58,7 +58,9 @@ class GitHubConnector:
if self.token:
logger.info("GitHub connector initialized with authentication token.")
else:
logger.info("GitHub connector initialized without token (public repos only).")
logger.info(
"GitHub connector initialized without token (public repos only)."
)
def ingest_repository(
self,
@ -95,17 +97,27 @@ class GitHubConnector:
cmd = [
"gitingest",
repo_url,
"--output", output_path,
"--max-size", str(max_file_size),
"--output",
output_path,
"--max-size",
str(max_file_size),
# Common exclude patterns
"-e", "node_modules/*",
"-e", "vendor/*",
"-e", ".git/*",
"-e", "__pycache__/*",
"-e", "dist/*",
"-e", "build/*",
"-e", "*.lock",
"-e", "package-lock.json",
"-e",
"node_modules/*",
"-e",
"vendor/*",
"-e",
".git/*",
"-e",
"__pycache__/*",
"-e",
"dist/*",
"-e",
"build/*",
"-e",
"*.lock",
"-e",
"package-lock.json",
]
# Add branch if specified
@ -147,7 +159,9 @@ class GitHubConnector:
os.unlink(output_path)
if not full_content or not full_content.strip():
logger.warning(f"No content retrieved from repository: {repo_full_name}")
logger.warning(
f"No content retrieved from repository: {repo_full_name}"
)
return None
# Parse the gitingest output
@ -171,11 +185,11 @@ class GitHubConnector:
logger.error(f"gitingest timed out for repository: {repo_full_name}")
return None
except FileNotFoundError:
logger.error(
"gitingest CLI not found. Falling back to Python library."
)
logger.error("gitingest CLI not found. Falling back to Python library.")
# Fall back to Python library
return self._ingest_with_python_library(repo_full_name, branch, max_file_size)
return self._ingest_with_python_library(
repo_full_name, branch, max_file_size
)
except Exception as e:
logger.error(f"Failed to ingest repository {repo_full_name}: {e}")
return None

4
surfsense_backend/app/db.py
View file

@ -82,7 +82,9 @@ class SearchSourceConnectorType(str, Enum):
BOOKSTACK_CONNECTOR = "BOOKSTACK_CONNECTOR"
CIRCLEBACK_CONNECTOR = "CIRCLEBACK_CONNECTOR"
MCP_CONNECTOR = "MCP_CONNECTOR" # Model Context Protocol - User-defined API tools
COMPOSIO_CONNECTOR = "COMPOSIO_CONNECTOR" # Generic Composio integration (Google, Slack, etc.)
COMPOSIO_CONNECTOR = (
"COMPOSIO_CONNECTOR" # Generic Composio integration (Google, Slack, etc.)
)
class LiteLLMProvider(str, Enum):

30
surfsense_backend/app/routes/composio_routes.py
View file

@ -10,7 +10,6 @@ Endpoints:
- GET /auth/composio/connector/callback - Handle OAuth callback
"""
import asyncio
import logging
from uuid import UUID
@ -85,7 +84,9 @@ async def list_composio_toolkits(user: User = Depends(current_active_user)):
@router.get("/auth/composio/connector/add")
async def initiate_composio_auth(
space_id: int,
toolkit_id: str = Query(..., description="Composio toolkit ID (e.g., 'googledrive', 'gmail')"),
toolkit_id: str = Query(
..., description="Composio toolkit ID (e.g., 'googledrive', 'gmail')"
),
user: User = Depends(current_active_user),
):
"""
@ -166,7 +167,9 @@ async def initiate_composio_auth(
@router.get("/auth/composio/connector/callback")
async def composio_callback(
state: str | None = None,
connectedAccountId: str | None = None, # Composio sends camelCase
composio_connected_account_id: str | None = Query(
None, alias="connectedAccountId"
), # Composio sends camelCase
connected_account_id: str | None = None, # Fallback snake_case
error: str | None = None,
session: AsyncSession = Depends(get_async_session),
@ -233,15 +236,18 @@ async def composio_callback(
)
# Initialize Composio service
service = ComposioService()
entity_id = f"surfsense_{user_id}"
ComposioService()
# Use camelCase param if provided (Composio's format), fallback to snake_case
final_connected_account_id = connectedAccountId or connected_account_id
final_connected_account_id = (
composio_connected_account_id or connected_account_id
)
# DEBUG: Log all query parameters received
logger.info(f"DEBUG: Callback received - connectedAccountId: {connectedAccountId}, connected_account_id: {connected_account_id}, using: {final_connected_account_id}")
logger.info(
f"DEBUG: Callback received - connectedAccountId: {composio_connected_account_id}, connected_account_id: {connected_account_id}, using: {final_connected_account_id}"
)
# If we still don't have a connected_account_id, warn but continue
# (the connector will be created but indexing won't work until updated)
if not final_connected_account_id:
@ -250,7 +256,9 @@ async def composio_callback(
"The connector will be created but indexing may not work."
)
else:
logger.info(f"Successfully got connected_account_id: {final_connected_account_id}")
logger.info(
f"Successfully got connected_account_id: {final_connected_account_id}"
)
# Build connector config
connector_config = {

114
surfsense_backend/app/services/composio_service.py
View file

@ -97,7 +97,7 @@ class ComposioService:
config_toolkit = getattr(auth_config, "toolkit", None)
if config_toolkit is None:
continue
# Extract toolkit name/slug from the object
toolkit_name = None
if isinstance(config_toolkit, str):
@ -108,18 +108,22 @@ class ComposioService:
toolkit_name = config_toolkit.name
elif hasattr(config_toolkit, "id"):
toolkit_name = config_toolkit.id
# Compare case-insensitively
if toolkit_name and toolkit_name.lower() == toolkit_id.lower():
logger.info(f"Found auth config {auth_config.id} for toolkit {toolkit_id}")
logger.info(
f"Found auth config {auth_config.id} for toolkit {toolkit_id}"
)
return auth_config.id
# Log available auth configs for debugging
logger.warning(f"No auth config found for toolkit '{toolkit_id}'. Available auth configs:")
logger.warning(
f"No auth config found for toolkit '{toolkit_id}'. Available auth configs:"
)
for auth_config in auth_configs.items:
config_toolkit = getattr(auth_config, "toolkit", None)
logger.warning(f" - {auth_config.id}: toolkit={config_toolkit}")
return None
except Exception as e:
logger.error(f"Failed to list auth configs: {e!s}")
@ -148,7 +152,7 @@ class ComposioService:
try:
# First, get the auth_config_id for this toolkit
auth_config_id = self._get_auth_config_for_toolkit(toolkit_id)
if not auth_config_id:
raise ValueError(
f"No auth config found for toolkit '{toolkit_id}'. "
@ -200,7 +204,9 @@ class ComposioService:
"user_id": getattr(account, "user_id", None),
}
except Exception as e:
logger.error(f"Failed to get connected account {connected_account_id}: {e!s}")
logger.error(
f"Failed to get connected account {connected_account_id}: {e!s}"
)
return None
async def list_all_connections(self) -> list[dict[str, Any]]:
@ -212,15 +218,17 @@ class ComposioService:
"""
try:
accounts_response = self.client.connected_accounts.list()
if hasattr(accounts_response, "items"):
accounts = accounts_response.items
elif hasattr(accounts_response, "__iter__"):
accounts = accounts_response
else:
logger.warning(f"Unexpected accounts response type: {type(accounts_response)}")
logger.warning(
f"Unexpected accounts response type: {type(accounts_response)}"
)
return []
result = []
for acc in accounts:
toolkit_raw = getattr(acc, "toolkit", None)
@ -234,14 +242,16 @@ class ComposioService:
toolkit_info = toolkit_raw.name
else:
toolkit_info = str(toolkit_raw)
result.append({
"id": acc.id,
"status": getattr(acc, "status", None),
"toolkit": toolkit_info,
"user_id": getattr(acc, "user_id", None),
})
result.append(
{
"id": acc.id,
"status": getattr(acc, "status", None),
"toolkit": toolkit_info,
"user_id": getattr(acc, "user_id", None),
}
)
logger.info(f"DEBUG: Found {len(result)} TOTAL connections in Composio")
return result
except Exception as e:
@ -261,16 +271,18 @@ class ComposioService:
try:
logger.info(f"DEBUG: Calling connected_accounts.list(user_id='{user_id}')")
accounts_response = self.client.connected_accounts.list(user_id=user_id)
# Handle paginated response (may have .items attribute) or direct list
if hasattr(accounts_response, "items"):
accounts = accounts_response.items
elif hasattr(accounts_response, "__iter__"):
accounts = accounts_response
else:
logger.warning(f"Unexpected accounts response type: {type(accounts_response)}")
logger.warning(
f"Unexpected accounts response type: {type(accounts_response)}"
)
return []
result = []
for acc in accounts:
# Extract toolkit info - might be string or object
@ -285,13 +297,15 @@ class ComposioService:
toolkit_info = toolkit_raw.name
else:
toolkit_info = toolkit_raw
result.append({
"id": acc.id,
"status": getattr(acc, "status", None),
"toolkit": toolkit_info,
})
result.append(
{
"id": acc.id,
"status": getattr(acc, "status", None),
"toolkit": toolkit_info,
}
)
logger.info(f"Found {len(result)} connections for user {user_id}: {result}")
return result
except Exception as e:
@ -383,18 +397,24 @@ class ComposioService:
return [], None, result.get("error", "Unknown error")
data = result.get("data", {})
logger.info(f"DEBUG: Drive data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}")
logger.info(
f"DEBUG: Drive data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}"
)
# Handle nested response structure from Composio
files = []
next_token = None
if isinstance(data, dict):
# Try direct access first, then nested
files = data.get("files", []) or data.get("data", {}).get("files", [])
next_token = data.get("nextPageToken") or data.get("next_page_token") or data.get("data", {}).get("nextPageToken")
next_token = (
data.get("nextPageToken")
or data.get("next_page_token")
or data.get("data", {}).get("nextPageToken")
)
elif isinstance(data, list):
files = data
logger.info(f"DEBUG: Extracted {len(files)} drive files")
return files, next_token, None
@ -475,16 +495,22 @@ class ComposioService:
return [], result.get("error", "Unknown error")
data = result.get("data", {})
logger.info(f"DEBUG: Gmail data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}")
logger.info(
f"DEBUG: Gmail data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}"
)
logger.info(f"DEBUG: Gmail full data: {data}")
# Try different possible response structures
messages = []
if isinstance(data, dict):
messages = data.get("messages", []) or data.get("data", {}).get("messages", []) or data.get("emails", [])
messages = (
data.get("messages", [])
or data.get("data", {}).get("messages", [])
or data.get("emails", [])
)
elif isinstance(data, list):
messages = data
logger.info(f"DEBUG: Extracted {len(messages)} messages")
return messages, None
@ -569,16 +595,22 @@ class ComposioService:
return [], result.get("error", "Unknown error")
data = result.get("data", {})
logger.info(f"DEBUG: Calendar data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}")
logger.info(
f"DEBUG: Calendar data type: {type(data)}, keys: {data.keys() if isinstance(data, dict) else 'N/A'}"
)
logger.info(f"DEBUG: Calendar full data: {data}")
# Try different possible response structures
events = []
if isinstance(data, dict):
events = data.get("items", []) or data.get("data", {}).get("items", []) or data.get("events", [])
events = (
data.get("items", [])
or data.get("data", {}).get("items", [])
or data.get("events", [])
)
elif isinstance(data, list):
events = data
logger.info(f"DEBUG: Extracted {len(events)} calendar events")
return events, None

5
surfsense_backend/app/services/notification_service.py
View file

@ -726,7 +726,10 @@ class MentionNotificationHandler(BaseNotificationHandler):
except Exception as e:
# Handle race condition - if duplicate key error, try to fetch existing
await session.rollback()
if "duplicate key" in str(e).lower() or "unique constraint" in str(e).lower():
if (
"duplicate key" in str(e).lower()
or "unique constraint" in str(e).lower()
):
logger.warning(
f"Duplicate notification detected for mention {mention_id}, fetching existing"
)

123
surfsense_backend/app/tasks/composio_indexer.py
View file

@ -144,7 +144,9 @@ async def index_composio_connector(
# Get toolkit ID from config
toolkit_id = connector.config.get("toolkit_id")
if not toolkit_id:
error_msg = f"Composio connector {connector_id} has no toolkit_id configured"
error_msg = (
f"Composio connector {connector_id} has no toolkit_id configured"
)
await task_logger.log_task_failure(
log_entry, error_msg, {"error_type": "MissingToolkitId"}
)
@ -287,8 +289,14 @@ async def _index_composio_google_drive(
try:
# Handle both standard Google API and potential Composio variations
file_id = file_info.get("id", "") or file_info.get("fileId", "")
file_name = file_info.get("name", "") or file_info.get("fileName", "") or "Untitled"
mime_type = file_info.get("mimeType", "") or file_info.get("mime_type", "")
file_name = (
file_info.get("name", "")
or file_info.get("fileName", "")
or "Untitled"
)
mime_type = file_info.get("mimeType", "") or file_info.get(
"mime_type", ""
)
if not file_id:
documents_skipped += 1
@ -309,12 +317,15 @@ async def _index_composio_google_drive(
)
# Get file content
content, content_error = await composio_connector.get_drive_file_content(
file_id
)
(
content,
content_error,
) = await composio_connector.get_drive_file_content(file_id)
if content_error or not content:
logger.warning(f"Could not get content for file {file_name}: {content_error}")
logger.warning(
f"Could not get content for file {file_name}: {content_error}"
)
# Use metadata as content fallback
markdown_content = f"# {file_name}\n\n"
markdown_content += f"**File ID:** {file_id}\n"
@ -344,12 +355,19 @@ async def _index_composio_google_drive(
"mime_type": mime_type,
"document_type": "Google Drive File (Composio)",
}
summary_content, summary_embedding = await generate_document_summary(
(
summary_content,
summary_embedding,
) = await generate_document_summary(
markdown_content, user_llm, document_metadata
)
else:
summary_content = f"Google Drive File: {file_name}\n\nType: {mime_type}"
summary_embedding = config.embedding_model_instance.embed(summary_content)
summary_content = (
f"Google Drive File: {file_name}\n\nType: {mime_type}"
)
summary_embedding = config.embedding_model_instance.embed(
summary_content
)
chunks = await create_document_chunks(markdown_content)
@ -382,12 +400,19 @@ async def _index_composio_google_drive(
"mime_type": mime_type,
"document_type": "Google Drive File (Composio)",
}
summary_content, summary_embedding = await generate_document_summary(
(
summary_content,
summary_embedding,
) = await generate_document_summary(
markdown_content, user_llm, document_metadata
)
else:
summary_content = f"Google Drive File: {file_name}\n\nType: {mime_type}"
summary_embedding = config.embedding_model_instance.embed(summary_content)
summary_content = (
f"Google Drive File: {file_name}\n\nType: {mime_type}"
)
summary_embedding = config.embedding_model_instance.embed(
summary_content
)
chunks = await create_document_chunks(markdown_content)
@ -527,11 +552,15 @@ async def _index_composio_gmail(
date_str = value
# Format to markdown using the full message data
markdown_content = composio_connector.format_gmail_message_to_markdown(message)
markdown_content = composio_connector.format_gmail_message_to_markdown(
message
)
# Generate unique identifier
unique_identifier_hash = generate_unique_identifier_hash(
DocumentType.COMPOSIO_CONNECTOR, f"gmail_{message_id}", search_space_id
DocumentType.COMPOSIO_CONNECTOR,
f"gmail_{message_id}",
search_space_id,
)
content_hash = generate_content_hash(markdown_content, search_space_id)
@ -560,12 +589,19 @@ async def _index_composio_gmail(
"sender": sender,
"document_type": "Gmail Message (Composio)",
}
summary_content, summary_embedding = await generate_document_summary(
(
summary_content,
summary_embedding,
) = await generate_document_summary(
markdown_content, user_llm, document_metadata
)
else:
summary_content = f"Gmail: {subject}\n\nFrom: {sender}\nDate: {date_str}"
summary_embedding = config.embedding_model_instance.embed(summary_content)
summary_content = (
f"Gmail: {subject}\n\nFrom: {sender}\nDate: {date_str}"
)
summary_embedding = config.embedding_model_instance.embed(
summary_content
)
chunks = await create_document_chunks(markdown_content)
@ -600,12 +636,19 @@ async def _index_composio_gmail(
"sender": sender,
"document_type": "Gmail Message (Composio)",
}
summary_content, summary_embedding = await generate_document_summary(
(
summary_content,
summary_embedding,
) = await generate_document_summary(
markdown_content, user_llm, document_metadata
)
else:
summary_content = f"Gmail: {subject}\n\nFrom: {sender}\nDate: {date_str}"
summary_embedding = config.embedding_model_instance.embed(summary_content)
summary_content = (
f"Gmail: {subject}\n\nFrom: {sender}\nDate: {date_str}"
)
summary_embedding = config.embedding_model_instance.embed(
summary_content
)
chunks = await create_document_chunks(markdown_content)
@ -728,18 +771,24 @@ async def _index_composio_google_calendar(
try:
# Handle both standard Google API and potential Composio variations
event_id = event.get("id", "") or event.get("eventId", "")
summary = event.get("summary", "") or event.get("title", "") or "No Title"
summary = (
event.get("summary", "") or event.get("title", "") or "No Title"
)
if not event_id:
documents_skipped += 1
continue
# Format to markdown
markdown_content = composio_connector.format_calendar_event_to_markdown(event)
markdown_content = composio_connector.format_calendar_event_to_markdown(
event
)
# Generate unique identifier
unique_identifier_hash = generate_unique_identifier_hash(
DocumentType.COMPOSIO_CONNECTOR, f"calendar_{event_id}", search_space_id
DocumentType.COMPOSIO_CONNECTOR,
f"calendar_{event_id}",
search_space_id,
)
content_hash = generate_content_hash(markdown_content, search_space_id)
@ -772,14 +821,19 @@ async def _index_composio_google_calendar(
"start_time": start_time,
"document_type": "Google Calendar Event (Composio)",
}
summary_content, summary_embedding = await generate_document_summary(
(
summary_content,
summary_embedding,
) = await generate_document_summary(
markdown_content, user_llm, document_metadata
)
else:
summary_content = f"Calendar: {summary}\n\nStart: {start_time}\nEnd: {end_time}"
if location:
summary_content += f"\nLocation: {location}"
summary_embedding = config.embedding_model_instance.embed(summary_content)
summary_embedding = config.embedding_model_instance.embed(
summary_content
)
chunks = await create_document_chunks(markdown_content)
@ -814,14 +868,21 @@ async def _index_composio_google_calendar(
"start_time": start_time,
"document_type": "Google Calendar Event (Composio)",
}
summary_content, summary_embedding = await generate_document_summary(
(
summary_content,
summary_embedding,
) = await generate_document_summary(
markdown_content, user_llm, document_metadata
)
else:
summary_content = f"Calendar: {summary}\n\nStart: {start_time}\nEnd: {end_time}"
summary_content = (
f"Calendar: {summary}\n\nStart: {start_time}\nEnd: {end_time}"
)
if location:
summary_content += f"\nLocation: {location}"
summary_embedding = config.embedding_model_instance.embed(summary_content)
summary_embedding = config.embedding_model_instance.embed(
summary_content
)
chunks = await create_document_chunks(markdown_content)
@ -874,5 +935,7 @@ async def _index_composio_google_calendar(
return documents_indexed, None
except Exception as e:
logger.error(f"Failed to index Google Calendar via Composio: {e!s}", exc_info=True)
logger.error(
f"Failed to index Google Calendar via Composio: {e!s}", exc_info=True
)
return 0, f"Failed to index Google Calendar via Composio: {e!s}"

1
surfsense_backend/app/tasks/connector_indexers/__init__.py
View file

@ -26,6 +26,7 @@ Available indexers:
# Calendar and scheduling
from .airtable_indexer import index_airtable_records
from .bookstack_indexer import index_bookstack_pages
# Note: composio_indexer is imported directly in connector_tasks.py to avoid circular imports
from .clickup_indexer import index_clickup_tasks
from .confluence_indexer import index_confluence_pages

14
surfsense_backend/app/tasks/connector_indexers/github_indexer.py
View file

@ -128,7 +128,9 @@ async def index_github_repos(
if github_pat:
logger.info("Using GitHub PAT for authentication (private repos supported)")
else:
logger.info("No GitHub PAT provided - only public repositories can be indexed")
logger.info(
"No GitHub PAT provided - only public repositories can be indexed"
)
# 3. Initialize GitHub connector with gitingest backend
await task_logger.log_task_progress(
@ -308,9 +310,7 @@ async def _process_repository_digest(
if existing_document:
# Document exists - check if content has changed
if existing_document.content_hash == content_hash:
logger.info(
f"Repository {repo_full_name} unchanged. Skipping."
)
logger.info(f"Repository {repo_full_name} unchanged. Skipping.")
return 0
else:
logger.info(
@ -341,7 +341,7 @@ async def _process_repository_digest(
summary_content = (
f"# Repository: {repo_full_name}\n\n"
f"## File Structure\n\n{digest.tree}\n\n"
f"## File Contents (truncated)\n\n{digest.content[:MAX_DIGEST_CHARS - len(digest.tree) - 200]}..."
f"## File Contents (truncated)\n\n{digest.content[: MAX_DIGEST_CHARS - len(digest.tree) - 200]}..."
)
summary_text, summary_embedding = await generate_document_summary(
@ -362,9 +362,7 @@ async def _process_repository_digest(
# This preserves file-level granularity in search
chunks_data = await create_document_chunks(digest.content)
except Exception as chunk_err:
logger.error(
f"Failed to chunk repository {repo_full_name}: {chunk_err}"
)
logger.error(f"Failed to chunk repository {repo_full_name}: {chunk_err}")
# Fall back to a simpler chunking approach
chunks_data = await _simple_chunk_content(digest.content)

17
surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
View file

@ -33,7 +33,9 @@ import { DisplayImageToolUI } from "@/components/tool-ui/display-image";
import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast";
import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview";
import { ScrapeWebpageToolUI } from "@/components/tool-ui/scrape-webpage";
import { SaveMemoryToolUI, RecallMemoryToolUI } from "@/components/tool-ui/user-memory";
import { RecallMemoryToolUI, SaveMemoryToolUI } from "@/components/tool-ui/user-memory";
import { useChatSessionStateSync } from "@/hooks/use-chat-session-state";
import { useMessagesElectric } from "@/hooks/use-messages-electric";
// import { WriteTodosToolUI } from "@/components/tool-ui/write-todos";
import { getBearerToken } from "@/lib/auth-utils";
import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter";
@ -51,8 +53,6 @@ import {
type MessageRecord,
type ThreadRecord,
} from "@/lib/chat/thread-persistence";
import { useChatSessionStateSync } from "@/hooks/use-chat-session-state";
import { useMessagesElectric } from "@/hooks/use-messages-electric";
import {
trackChatCreated,
trackChatError,
@ -266,7 +266,16 @@ export default function NewChatPage() {
const { data: membersData } = useAtomValue(membersAtom);
const handleElectricMessagesUpdate = useCallback(
(electricMessages: { id: number; thread_id: number; role: string; content: unknown; author_id: string | null; created_at: string }[]) => {
(
electricMessages: {
id: number;
thread_id: number;
role: string;
content: unknown;
author_id: string | null;
created_at: string;
}[]
) => {
if (isRunning) {
return;
}

5
surfsense_web/components/assistant-ui/chat-session-status.tsx
View file

@ -1,7 +1,7 @@
"use client";
import type { FC } from "react";
import { Loader2 } from "lucide-react";
import type { FC } from "react";
import { cn } from "@/lib/utils";
interface ChatSessionStatusProps {
@ -32,7 +32,8 @@ export const ChatSessionStatus: FC<ChatSessionStatusProps> = ({
}
const respondingUser = members.find((m) => m.user_id === respondingToUserId);
const displayName = respondingUser?.user_display_name || respondingUser?.user_email || "another user";
const displayName =
respondingUser?.user_display_name || respondingUser?.user_email || "another user";
return (
<div

10
surfsense_web/components/assistant-ui/connector-popup.tsx
View file

@ -186,12 +186,10 @@ export const ConnectorIndicator: FC = () => {
) : viewingComposio && searchSpaceId ? (
<ComposioToolkitView
searchSpaceId={searchSpaceId}
connectedToolkits={
(connectors || [])
.filter((c: SearchSourceConnector) => c.connector_type === "COMPOSIO_CONNECTOR")
.map((c: SearchSourceConnector) => c.config?.toolkit_id as string)
.filter(Boolean)
}
connectedToolkits={(connectors || [])
.filter((c: SearchSourceConnector) => c.connector_type === "COMPOSIO_CONNECTOR")
.map((c: SearchSourceConnector) => c.config?.toolkit_id as string)
.filter(Boolean)}
onBack={handleBackFromComposio}
onConnectToolkit={handleConnectComposioToolkit}
isConnecting={connectingComposioToolkit !== null}

2
surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx
View file

@ -5,8 +5,8 @@ import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { BaiduSearchApiConfig } from "./components/baidu-search-api-config";
import { BookStackConfig } from "./components/bookstack-config";
import { CirclebackConfig } from "./components/circleback-config";
import { ComposioConfig } from "./components/composio-config";
import { ClickUpConfig } from "./components/clickup-config";
import { ComposioConfig } from "./components/composio-config";
import { ConfluenceConfig } from "./components/confluence-config";
import { DiscordConfig } from "./components/discord-config";
import { ElasticsearchConfig } from "./components/elasticsearch-config";

9
surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx
View file

@ -3,9 +3,14 @@
import type { FC } from "react";
import { EnumConnectorName } from "@/contracts/enums/connector";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { ConnectorCard } from "../components/connector-card";
import { ComposioConnectorCard } from "../components/composio-connector-card";
import { CRAWLERS, OAUTH_CONNECTORS, OTHER_CONNECTORS, COMPOSIO_CONNECTORS } from "../constants/connector-constants";
import { ConnectorCard } from "../components/connector-card";
import {
COMPOSIO_CONNECTORS,
CRAWLERS,
OAUTH_CONNECTORS,
OTHER_CONNECTORS,
} from "../constants/connector-constants";
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
/**

102
surfsense_web/components/assistant-ui/connector-popup/views/composio-toolkit-view.tsx
View file

@ -5,12 +5,12 @@ import {
Calendar,
Check,
ExternalLink,
FileText,
Github,
HardDrive,
Loader2,
Mail,
HardDrive,
MessageSquare,
FileText,
Zap,
} from "lucide-react";
import Image from "next/image";
@ -82,17 +82,65 @@ const getToolkitIcon = (toolkitId: string, className?: string) => {
switch (toolkitId) {
case "googledrive":
return <Image src="/connectors/google-drive.svg" alt="Google Drive" width={20} height={20} className={iconClass} />;
return (
<Image
src="/connectors/google-drive.svg"
alt="Google Drive"
width={20}
height={20}
className={iconClass}
/>
);
case "gmail":
return <Image src="/connectors/google-gmail.svg" alt="Gmail" width={20} height={20} className={iconClass} />;
return (
<Image
src="/connectors/google-gmail.svg"
alt="Gmail"
width={20}
height={20}
className={iconClass}
/>
);
case "googlecalendar":
return <Image src="/connectors/google-calendar.svg" alt="Google Calendar" width={20} height={20} className={iconClass} />;
return (
<Image
src="/connectors/google-calendar.svg"
alt="Google Calendar"
width={20}
height={20}
className={iconClass}
/>
);
case "slack":
return <Image src="/connectors/slack.svg" alt="Slack" width={20} height={20} className={iconClass} />;
return (
<Image
src="/connectors/slack.svg"
alt="Slack"
width={20}
height={20}
className={iconClass}
/>
);
case "notion":
return <Image src="/connectors/notion.svg" alt="Notion" width={20} height={20} className={iconClass} />;
return (
<Image
src="/connectors/notion.svg"
alt="Notion"
width={20}
height={20}
className={iconClass}
/>
);
case "github":
return <Image src="/connectors/github.svg" alt="GitHub" width={20} height={20} className={iconClass} />;
return (
<Image
src="/connectors/github.svg"
alt="GitHub"
width={20}
height={20}
className={iconClass}
/>
);
default:
return <Zap className={iconClass} />;
}
@ -139,9 +187,7 @@ export const ComposioToolkitView: FC<ComposioToolkitViewProps> = ({
/>
</div>
<div className="flex-1 min-w-0">
<h2 className="text-xl sm:text-2xl font-semibold tracking-tight">
Composio
</h2>
<h2 className="text-xl sm:text-2xl font-semibold tracking-tight">Composio</h2>
<p className="text-xs sm:text-sm text-muted-foreground mt-1">
Connect 100+ apps with managed OAuth - no verification needed
</p>
@ -165,12 +211,16 @@ export const ComposioToolkitView: FC<ComposioToolkitViewProps> = ({
<section className="mb-8">
<div className="flex items-center gap-2 mb-4">
<h3 className="text-sm font-semibold text-foreground">Google Services</h3>
<Badge variant="secondary" className="text-[10px] px-1.5 py-0 h-5 bg-emerald-500/10 text-emerald-600 dark:text-emerald-400 border-emerald-500/20">
<Badge
variant="secondary"
className="text-[10px] px-1.5 py-0 h-5 bg-emerald-500/10 text-emerald-600 dark:text-emerald-400 border-emerald-500/20"
>
Indexable
</Badge>
</div>
<p className="text-xs text-muted-foreground mb-4">
Connect Google services via Composio&apos;s verified OAuth app. Your data will be indexed and searchable.
Connect Google services via Composio&apos;s verified OAuth app. Your data will be
indexed and searchable.
</p>
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-3">
{indexableToolkits.map((toolkit) => {
@ -201,16 +251,17 @@ export const ComposioToolkitView: FC<ComposioToolkitViewProps> = ({
{getToolkitIcon(toolkit.id, "size-5")}
</div>
{isConnected && (
<Badge variant="secondary" className="text-[10px] px-1.5 py-0 h-5 bg-emerald-500/10 text-emerald-600 dark:text-emerald-400 border-emerald-500/20">
<Badge
variant="secondary"
className="text-[10px] px-1.5 py-0 h-5 bg-emerald-500/10 text-emerald-600 dark:text-emerald-400 border-emerald-500/20"
>
<Check className="size-3 mr-0.5" />
Connected
</Badge>
)}
</div>
<h4 className="text-sm font-medium mb-1">{toolkit.name}</h4>
<p className="text-xs text-muted-foreground mb-4 flex-1">
{toolkit.description}
</p>
<p className="text-xs text-muted-foreground mb-4 flex-1">{toolkit.description}</p>
<Button
size="sm"
variant={isConnected ? "secondary" : "default"}
@ -242,12 +293,16 @@ export const ComposioToolkitView: FC<ComposioToolkitViewProps> = ({
<section>
<div className="flex items-center gap-2 mb-4">
<h3 className="text-sm font-semibold text-foreground">More Integrations</h3>
<Badge variant="secondary" className="text-[10px] px-1.5 py-0 h-5 bg-amber-500/10 text-amber-600 dark:text-amber-400 border-amber-500/20">
<Badge
variant="secondary"
className="text-[10px] px-1.5 py-0 h-5 bg-amber-500/10 text-amber-600 dark:text-amber-400 border-amber-500/20"
>
Coming Soon
</Badge>
</div>
<p className="text-xs text-muted-foreground mb-4">
Connect these services for future indexing support. Currently available for connection only.
Connect these services for future indexing support. Currently available for connection
only.
</p>
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-3 opacity-60">
{nonIndexableToolkits.map((toolkit) => (
@ -264,9 +319,7 @@ export const ComposioToolkitView: FC<ComposioToolkitViewProps> = ({
</Badge>
</div>
<h4 className="text-sm font-medium mb-1">{toolkit.name}</h4>
<p className="text-xs text-muted-foreground mb-4 flex-1">
{toolkit.description}
</p>
<p className="text-xs text-muted-foreground mb-4 flex-1">{toolkit.description}</p>
<Button
size="sm"
variant="outline"
@ -289,8 +342,9 @@ export const ComposioToolkitView: FC<ComposioToolkitViewProps> = ({
<div>
<h4 className="text-sm font-medium mb-1">Why use Composio?</h4>
<p className="text-xs text-muted-foreground leading-relaxed">
Composio provides pre-verified OAuth apps, so you don&apos;t need to wait for Google app verification.
Your data is securely processed through Composio&apos;s managed authentication.
Composio provides pre-verified OAuth apps, so you don&apos;t need to wait for Google
app verification. Your data is securely processed through Composio&apos;s managed
authentication.
</p>
</div>
</div>

2
surfsense_web/components/assistant-ui/thread.tsx
View file

@ -26,6 +26,7 @@ import {
import { useParams } from "next/navigation";
import { type FC, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react";
import { createPortal } from "react-dom";
import { chatSessionStateAtom } from "@/atoms/chat/chat-session-state.atom";
import { showCommentsGutterAtom } from "@/atoms/chat/current-thread.atom";
import {
mentionedDocumentIdsAtom,
@ -61,7 +62,6 @@ import {
import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking";
import { Button } from "@/components/ui/button";
import type { Document } from "@/contracts/types/document.types";
import { chatSessionStateAtom } from "@/atoms/chat/chat-session-state.atom";
import { useCommentsElectric } from "@/hooks/use-comments-electric";
import { cn } from "@/lib/utils";

6
surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx
View file

@ -41,14 +41,14 @@ import { Spinner } from "@/components/ui/spinner";
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import type { InboxItem } from "@/hooks/use-inbox";
import { useMediaQuery } from "@/hooks/use-media-query";
import {
type ConnectorIndexingMetadata,
type NewMentionMetadata,
isConnectorIndexingMetadata,
isNewMentionMetadata,
type NewMentionMetadata,
} from "@/contracts/types/inbox.types";
import type { InboxItem } from "@/hooks/use-inbox";
import { useMediaQuery } from "@/hooks/use-media-query";
import { cn } from "@/lib/utils";
/**

2
surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx
View file

@ -1,6 +1,6 @@
"use client";
import { ChevronUp, Laptop, Languages, LogOut, Moon, Settings, Sun } from "lucide-react";
import { ChevronUp, Languages, Laptop, LogOut, Moon, Settings, Sun } from "lucide-react";
import { useTranslations } from "next-intl";
import {
DropdownMenu,

2
surfsense_web/components/ui/drawer.tsx
View file

@ -1,6 +1,6 @@
"use client";
import * as React from "react";
import type * as React from "react";
import { Drawer as DrawerPrimitive } from "vaul";
import { cn } from "@/lib/utils";

32
surfsense_web/hooks/use-comments-electric.ts
View file

@ -5,11 +5,7 @@ import { useAtomValue } from "jotai";
import { useCallback, useEffect, useMemo, useRef } from "react";
import { membersAtom, myAccessAtom } from "@/atoms/members/members-query.atoms";
import { currentUserAtom } from "@/atoms/user/user-query.atoms";
import type {
Comment,
CommentReply,
Author,
} from "@/contracts/types/chat-comments.types";
import type { Author, Comment, CommentReply } from "@/contracts/types/chat-comments.types";
import type { Membership } from "@/contracts/types/members.types";
import type { SyncHandle } from "@/lib/electric/client";
import { useElectricClient } from "@/lib/electric/context";
@ -123,7 +119,10 @@ function transformComments(
isOwner: boolean
): Map<number, Comment[]> {
// Group comments by message_id
const byMessage = new Map<number, { topLevel: RawCommentRow[]; replies: Map<number, RawCommentRow[]> }>();
const byMessage = new Map<
number,
{ topLevel: RawCommentRow[]; replies: Map<number, RawCommentRow[]> }
>();
for (const raw of rawComments) {
if (!byMessage.has(raw.message_id)) {
@ -176,10 +175,10 @@ function transformComments(
/**
* Hook for syncing comments with Electric SQL real-time sync.
*
*
* Syncs ALL comments for a thread in ONE subscription, then updates
* React Query cache for each message. This avoids N subscriptions for N messages.
*
*
* @param threadId - The thread ID to sync comments for
*/
export function useCommentsElectric(threadId: number | null) {
@ -247,12 +246,21 @@ export function useCommentsElectric(threadId: number | null) {
let mounted = true;
syncKeyRef.current = syncKey;
async function startSync() {
async function startSync() {
try {
const handle = await client.syncShape({
table: "chat_comments",
where: `thread_id = ${threadId}`,
columns: ["id", "message_id", "thread_id", "parent_id", "author_id", "content", "created_at", "updated_at"],
columns: [
"id",
"message_id",
"thread_id",
"parent_id",
"author_id",
"content",
"created_at",
"updated_at",
],
primaryKey: ["id"],
});
@ -283,7 +291,9 @@ export function useCommentsElectric(threadId: number | null) {
// Subscribe to the sync stream for real-time updates from Electric SQL
// This ensures we catch updates even if PGlite live query misses them
if (handle.stream) {
const stream = handle.stream as { subscribe?: (callback: (messages: unknown[]) => void) => void };
const stream = handle.stream as {
subscribe?: (callback: (messages: unknown[]) => void) => void;
};
if (typeof stream.subscribe === "function") {
stream.subscribe((messages: unknown[]) => {
if (!mounted) return;

6
surfsense_web/lib/apis/notifications-api.service.ts
View file

@ -2,12 +2,12 @@ import {
type GetNotificationsRequest,
type GetNotificationsResponse,
type GetUnreadCountResponse,
type MarkAllNotificationsReadResponse,
type MarkNotificationReadRequest,
type MarkNotificationReadResponse,
getNotificationsRequest,
getNotificationsResponse,
getUnreadCountResponse,
type MarkAllNotificationsReadResponse,
type MarkNotificationReadRequest,
type MarkNotificationReadResponse,
markAllNotificationsReadResponse,
markNotificationReadRequest,
markNotificationReadResponse,