apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-23 19:05:16 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: refactor composio connectors for modularity

Browse source
This commit is contained in:
Anish Sarkar 2026-01-23 19:56:19 +05:30
parent 8d8f69545e
commit 1343fabeee
17 changed files with 3128 additions and 2612 deletions
Download patch file Download diff file Expand all files Collapse all files

366
surfsense_backend/app/connectors/composio_connector.py
View file

@ -1,7 +1,7 @@
"""
Composio Connector Module.
Composio Connector Base Module.
Provides a unified interface for interacting with various services via Composio,
Provides a base class for interacting with various services via Composio,
primarily used during indexing operations.
"""
@ -19,10 +19,10 @@ logger = logging.getLogger(__name__)
class ComposioConnector:
"""
Generic Composio connector for data retrieval.
Base Composio connector for data retrieval.
Wraps the ComposioService to provide toolkit-specific data access
for indexing operations.
for indexing operations. Subclasses implement toolkit-specific methods.
"""
def __init__(
@ -89,354 +89,12 @@ class ComposioConnector:
toolkit_id = await self.get_toolkit_id()
return toolkit_id in INDEXABLE_TOOLKITS
# ===== Google Drive Methods =====
@property
def session(self) -> AsyncSession:
"""Get the database session."""
return self._session
async def list_drive_files(
self,
folder_id: str | None = None,
page_token: str | None = None,
page_size: int = 100,
) -> tuple[list[dict[str, Any]], str | None, str | None]:
"""
List files from Google Drive via Composio.
Args:
folder_id: Optional folder ID to list contents of.
page_token: Pagination token.
page_size: Number of files per page.
Returns:
Tuple of (files list, next_page_token, error message).
"""
connected_account_id = await self.get_connected_account_id()
if not connected_account_id:
return [], None, "No connected account ID found"
entity_id = await self.get_entity_id()
service = await self._get_service()
return await service.get_drive_files(
connected_account_id=connected_account_id,
entity_id=entity_id,
folder_id=folder_id,
page_token=page_token,
page_size=page_size,
)
async def get_drive_file_content(
self, file_id: str
) -> tuple[bytes | None, str | None]:
"""
Download file content from Google Drive via Composio.
Args:
file_id: Google Drive file ID.
Returns:
Tuple of (file content bytes, error message).
"""
connected_account_id = await self.get_connected_account_id()
if not connected_account_id:
return None, "No connected account ID found"
entity_id = await self.get_entity_id()
service = await self._get_service()
return await service.get_drive_file_content(
connected_account_id=connected_account_id,
entity_id=entity_id,
file_id=file_id,
)
async def get_drive_start_page_token(self) -> tuple[str | None, str | None]:
"""
Get the starting page token for Google Drive change tracking.
Returns:
Tuple of (start_page_token, error message).
"""
connected_account_id = await self.get_connected_account_id()
if not connected_account_id:
return None, "No connected account ID found"
entity_id = await self.get_entity_id()
service = await self._get_service()
return await service.get_drive_start_page_token(
connected_account_id=connected_account_id,
entity_id=entity_id,
)
async def list_drive_changes(
self,
page_token: str | None = None,
page_size: int = 100,
include_removed: bool = True,
) -> tuple[list[dict[str, Any]], str | None, str | None]:
"""
List changes in Google Drive since the given page token.
Args:
page_token: Page token from previous sync (optional).
page_size: Number of changes per page.
include_removed: Whether to include removed items.
Returns:
Tuple of (changes list, new_start_page_token, error message).
"""
connected_account_id = await self.get_connected_account_id()
if not connected_account_id:
return [], None, "No connected account ID found"
entity_id = await self.get_entity_id()
service = await self._get_service()
return await service.list_drive_changes(
connected_account_id=connected_account_id,
entity_id=entity_id,
page_token=page_token,
page_size=page_size,
include_removed=include_removed,
)
# ===== Gmail Methods =====
async def list_gmail_messages(
self,
query: str = "",
max_results: int = 50,
page_token: str | None = None,
) -> tuple[list[dict[str, Any]], str | None, int | None, str | None]:
"""
List Gmail messages via Composio with pagination support.
Args:
query: Gmail search query.
max_results: Maximum number of messages per page (default: 50).
page_token: Optional pagination token for next page.
Returns:
Tuple of (messages list, next_page_token, result_size_estimate, error message).
"""
connected_account_id = await self.get_connected_account_id()
if not connected_account_id:
return [], None, None, "No connected account ID found"
entity_id = await self.get_entity_id()
service = await self._get_service()
return await service.get_gmail_messages(
connected_account_id=connected_account_id,
entity_id=entity_id,
query=query,
max_results=max_results,
page_token=page_token,
)
async def get_gmail_message_detail(
self, message_id: str
) -> tuple[dict[str, Any] | None, str | None]:
"""
Get full details of a Gmail message via Composio.
Args:
message_id: Gmail message ID.
Returns:
Tuple of (message details, error message).
"""
connected_account_id = await self.get_connected_account_id()
if not connected_account_id:
return None, "No connected account ID found"
entity_id = await self.get_entity_id()
service = await self._get_service()
return await service.get_gmail_message_detail(
connected_account_id=connected_account_id,
entity_id=entity_id,
message_id=message_id,
)
# ===== Google Calendar Methods =====
async def list_calendar_events(
self,
time_min: str | None = None,
time_max: str | None = None,
max_results: int = 250,
) -> tuple[list[dict[str, Any]], str | None]:
"""
List Google Calendar events via Composio.
Args:
time_min: Start time (RFC3339 format).
time_max: End time (RFC3339 format).
max_results: Maximum number of events.
Returns:
Tuple of (events list, error message).
"""
connected_account_id = await self.get_connected_account_id()
if not connected_account_id:
return [], "No connected account ID found"
entity_id = await self.get_entity_id()
service = await self._get_service()
return await service.get_calendar_events(
connected_account_id=connected_account_id,
entity_id=entity_id,
time_min=time_min,
time_max=time_max,
max_results=max_results,
)
# ===== Utility Methods =====
def format_gmail_message_to_markdown(self, message: dict[str, Any]) -> str:
"""
Format a Gmail message to markdown.
Args:
message: Message object from Composio's GMAIL_FETCH_EMAILS response.
Composio structure: messageId, messageText, messageTimestamp,
payload.headers, labelIds, attachmentList
Returns:
Formatted markdown string.
"""
try:
# Composio uses 'messageId' (camelCase)
message_id = message.get("messageId", "") or message.get("id", "")
label_ids = message.get("labelIds", [])
# Extract headers from payload
payload = message.get("payload", {})
headers = payload.get("headers", [])
# Parse headers into a dict
header_dict = {}
for header in headers:
name = header.get("name", "").lower()
value = header.get("value", "")
header_dict[name] = value
# Extract key information
subject = header_dict.get("subject", "No Subject")
from_email = header_dict.get("from", "Unknown Sender")
to_email = header_dict.get("to", "Unknown Recipient")
# Composio provides messageTimestamp directly
date_str = message.get("messageTimestamp", "") or header_dict.get(
"date", "Unknown Date"
)
# Build markdown content
markdown_content = f"# {subject}\n\n"
markdown_content += f"**From:** {from_email}\n"
markdown_content += f"**To:** {to_email}\n"
markdown_content += f"**Date:** {date_str}\n"
if label_ids:
markdown_content += f"**Labels:** {', '.join(label_ids)}\n"
markdown_content += "\n---\n\n"
# Composio provides full message text in 'messageText'
message_text = message.get("messageText", "")
if message_text:
markdown_content += f"## Content\n\n{message_text}\n\n"
else:
# Fallback to snippet if no messageText
snippet = message.get("snippet", "")
if snippet:
markdown_content += f"## Preview\n\n{snippet}\n\n"
# Add attachment info if present
attachments = message.get("attachmentList", [])
if attachments:
markdown_content += "## Attachments\n\n"
for att in attachments:
att_name = att.get("filename", att.get("name", "Unknown"))
markdown_content += f"- {att_name}\n"
markdown_content += "\n"
# Add message metadata
markdown_content += "## Message Details\n\n"
markdown_content += f"- **Message ID:** {message_id}\n"
return markdown_content
except Exception as e:
return f"Error formatting message to markdown: {e!s}"
def format_calendar_event_to_markdown(self, event: dict[str, Any]) -> str:
"""
Format a Google Calendar event to markdown.
Args:
event: Event object from Google Calendar API.
Returns:
Formatted markdown string.
"""
from datetime import datetime
try:
# Extract basic event information
summary = event.get("summary", "No Title")
description = event.get("description", "")
location = event.get("location", "")
# Extract start and end times
start = event.get("start", {})
end = event.get("end", {})
start_time = start.get("dateTime") or start.get("date", "")
end_time = end.get("dateTime") or end.get("date", "")
# Format times for display
def format_time(time_str: str) -> str:
if not time_str:
return "Unknown"
try:
if "T" in time_str:
dt = datetime.fromisoformat(time_str.replace("Z", "+00:00"))
return dt.strftime("%Y-%m-%d %H:%M")
return time_str
except Exception:
return time_str
start_formatted = format_time(start_time)
end_formatted = format_time(end_time)
# Extract attendees
attendees = event.get("attendees", [])
attendee_list = []
for attendee in attendees:
email = attendee.get("email", "")
display_name = attendee.get("displayName", email)
response_status = attendee.get("responseStatus", "")
attendee_list.append(f"- {display_name} ({response_status})")
# Build markdown content
markdown_content = f"# {summary}\n\n"
markdown_content += f"**Start:** {start_formatted}\n"
markdown_content += f"**End:** {end_formatted}\n"
if location:
markdown_content += f"**Location:** {location}\n"
markdown_content += "\n"
if description:
markdown_content += f"## Description\n\n{description}\n\n"
if attendee_list:
markdown_content += "## Attendees\n\n"
markdown_content += "\n".join(attendee_list)
markdown_content += "\n\n"
# Add event metadata
markdown_content += "## Event Details\n\n"
markdown_content += f"- **Event ID:** {event.get('id', 'Unknown')}\n"
markdown_content += f"- **Created:** {event.get('created', 'Unknown')}\n"
markdown_content += f"- **Updated:** {event.get('updated', 'Unknown')}\n"
return markdown_content
except Exception as e:
return f"Error formatting event to markdown: {e!s}"
@property
def connector_id(self) -> int:
"""Get the connector ID."""
return self._connector_id

614
surfsense_backend/app/connectors/composio_gmail_connector.py Normal file
View file

@ -0,0 +1,614 @@
"""
Composio Gmail Connector Module.
Provides Gmail specific methods for data retrieval and indexing via Composio.
"""
import logging
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from sqlalchemy.orm import selectinload
from app.config import config
from app.connectors.composio_connector import ComposioConnector
from app.db import Document, DocumentType
from app.services.composio_service import TOOLKIT_TO_DOCUMENT_TYPE
from app.services.llm_service import get_user_long_context_llm
from app.services.task_logging_service import TaskLoggingService
from app.tasks.connector_indexers.base import calculate_date_range
from app.utils.document_converters import (
create_document_chunks,
generate_content_hash,
generate_document_summary,
generate_unique_identifier_hash,
)
logger = logging.getLogger(__name__)
def get_current_timestamp() -> datetime:
"""Get the current timestamp with timezone for updated_at field."""
return datetime.now(UTC)
async def check_document_by_unique_identifier(
session: AsyncSession, unique_identifier_hash: str
) -> Document | None:
"""Check if a document with the given unique identifier hash already exists."""
existing_doc_result = await session.execute(
select(Document)
.options(selectinload(Document.chunks))
.where(Document.unique_identifier_hash == unique_identifier_hash)
)
return existing_doc_result.scalars().first()
async def update_connector_last_indexed(
session: AsyncSession,
connector,
update_last_indexed: bool = True,
) -> None:
"""Update the last_indexed_at timestamp for a connector."""
if update_last_indexed:
connector.last_indexed_at = datetime.now(UTC)
logger.info(f"Updated last_indexed_at to {connector.last_indexed_at}")
class ComposioGmailConnector(ComposioConnector):
"""
Gmail specific Composio connector.
Provides methods for listing messages, getting message details, and formatting
Gmail messages from Gmail via Composio.
"""
async def list_gmail_messages(
self,
query: str = "",
max_results: int = 50,
page_token: str | None = None,
) -> tuple[list[dict[str, Any]], str | None, int | None, str | None]:
"""
List Gmail messages via Composio with pagination support.
Args:
query: Gmail search query.
max_results: Maximum number of messages per page (default: 50).
page_token: Optional pagination token for next page.
Returns:
Tuple of (messages list, next_page_token, result_size_estimate, error message).
"""
connected_account_id = await self.get_connected_account_id()
if not connected_account_id:
return [], None, None, "No connected account ID found"
entity_id = await self.get_entity_id()
service = await self._get_service()
return await service.get_gmail_messages(
connected_account_id=connected_account_id,
entity_id=entity_id,
query=query,
max_results=max_results,
page_token=page_token,
)
async def get_gmail_message_detail(
self, message_id: str
) -> tuple[dict[str, Any] | None, str | None]:
"""
Get full details of a Gmail message via Composio.
Args:
message_id: Gmail message ID.
Returns:
Tuple of (message details, error message).
"""
connected_account_id = await self.get_connected_account_id()
if not connected_account_id:
return None, "No connected account ID found"
entity_id = await self.get_entity_id()
service = await self._get_service()
return await service.get_gmail_message_detail(
connected_account_id=connected_account_id,
entity_id=entity_id,
message_id=message_id,
)
def format_gmail_message_to_markdown(self, message: dict[str, Any]) -> str:
"""
Format a Gmail message to markdown.
Args:
message: Message object from Composio's GMAIL_FETCH_EMAILS response.
Composio structure: messageId, messageText, messageTimestamp,
payload.headers, labelIds, attachmentList
Returns:
Formatted markdown string.
"""
try:
# Composio uses 'messageId' (camelCase)
message_id = message.get("messageId", "") or message.get("id", "")
label_ids = message.get("labelIds", [])
# Extract headers from payload
payload = message.get("payload", {})
headers = payload.get("headers", [])
# Parse headers into a dict
header_dict = {}
for header in headers:
name = header.get("name", "").lower()
value = header.get("value", "")
header_dict[name] = value
# Extract key information
subject = header_dict.get("subject", "No Subject")
from_email = header_dict.get("from", "Unknown Sender")
to_email = header_dict.get("to", "Unknown Recipient")
# Composio provides messageTimestamp directly
date_str = message.get("messageTimestamp", "") or header_dict.get(
"date", "Unknown Date"
)
# Build markdown content
markdown_content = f"# {subject}\n\n"
markdown_content += f"**From:** {from_email}\n"
markdown_content += f"**To:** {to_email}\n"
markdown_content += f"**Date:** {date_str}\n"
if label_ids:
markdown_content += f"**Labels:** {', '.join(label_ids)}\n"
markdown_content += "\n---\n\n"
# Composio provides full message text in 'messageText'
message_text = message.get("messageText", "")
if message_text:
markdown_content += f"## Content\n\n{message_text}\n\n"
else:
# Fallback to snippet if no messageText
snippet = message.get("snippet", "")
if snippet:
markdown_content += f"## Preview\n\n{snippet}\n\n"
# Add attachment info if present
attachments = message.get("attachmentList", [])
if attachments:
markdown_content += "## Attachments\n\n"
for att in attachments:
att_name = att.get("filename", att.get("name", "Unknown"))
markdown_content += f"- {att_name}\n"
markdown_content += "\n"
# Add message metadata
markdown_content += "## Message Details\n\n"
markdown_content += f"- **Message ID:** {message_id}\n"
return markdown_content
except Exception as e:
return f"Error formatting message to markdown: {e!s}"
# ============ Indexer Functions ============
async def _process_gmail_message_batch(
session: AsyncSession,
messages: list[dict[str, Any]],
composio_connector: ComposioGmailConnector,
connector_id: int,
search_space_id: int,
user_id: str,
total_documents_indexed: int = 0,
) -> tuple[int, int]:
"""
Process a batch of Gmail messages and index them.
Args:
total_documents_indexed: Running total of documents indexed so far (for batch commits).
Returns:
Tuple of (documents_indexed, documents_skipped)
"""
documents_indexed = 0
documents_skipped = 0
for message in messages:
try:
# Composio uses 'messageId' (camelCase), not 'id'
message_id = message.get("messageId", "") or message.get("id", "")
if not message_id:
documents_skipped += 1
continue
# Composio's GMAIL_FETCH_EMAILS already returns full message content
# No need for a separate detail API call
# Extract message info from Composio response
# Composio structure: messageId, messageText, messageTimestamp, payload.headers, labelIds
payload = message.get("payload", {})
headers = payload.get("headers", [])
subject = "No Subject"
sender = "Unknown Sender"
date_str = message.get("messageTimestamp", "Unknown Date")
for header in headers:
name = header.get("name", "").lower()
value = header.get("value", "")
if name == "subject":
subject = value
elif name == "from":
sender = value
elif name == "date":
date_str = value
# Format to markdown using the full message data
markdown_content = composio_connector.format_gmail_message_to_markdown(
message
)
# Check for empty content (defensive parsing per Composio best practices)
if not markdown_content.strip():
logger.warning(f"Skipping Gmail message with no content: {subject}")
documents_skipped += 1
continue
# Generate unique identifier
document_type = DocumentType(TOOLKIT_TO_DOCUMENT_TYPE["gmail"])
unique_identifier_hash = generate_unique_identifier_hash(
document_type, f"gmail_{message_id}", search_space_id
)
content_hash = generate_content_hash(markdown_content, search_space_id)
existing_document = await check_document_by_unique_identifier(
session, unique_identifier_hash
)
# Get label IDs from Composio response
label_ids = message.get("labelIds", [])
# Extract thread_id if available (for consistency with non-Composio implementation)
thread_id = message.get("threadId", "") or message.get("thread_id", "")
if existing_document:
if existing_document.content_hash == content_hash:
documents_skipped += 1
continue
# Update existing
user_llm = await get_user_long_context_llm(
session, user_id, search_space_id
)
if user_llm:
document_metadata = {
"message_id": message_id,
"thread_id": thread_id,
"subject": subject,
"sender": sender,
"document_type": "Gmail Message (Composio)",
}
(
summary_content,
summary_embedding,
) = await generate_document_summary(
markdown_content, user_llm, document_metadata
)
else:
summary_content = (
f"Gmail: {subject}\n\nFrom: {sender}\nDate: {date_str}"
)
summary_embedding = config.embedding_model_instance.embed(
summary_content
)
chunks = await create_document_chunks(markdown_content)
existing_document.title = f"Gmail: {subject}"
existing_document.content = summary_content
existing_document.content_hash = content_hash
existing_document.embedding = summary_embedding
existing_document.document_metadata = {
"message_id": message_id,
"thread_id": thread_id,
"subject": subject,
"sender": sender,
"date": date_str,
"labels": label_ids,
"connector_id": connector_id,
"source": "composio",
}
existing_document.chunks = chunks
existing_document.updated_at = get_current_timestamp()
documents_indexed += 1
# Batch commit every 10 documents
current_total = total_documents_indexed + documents_indexed
if current_total % 10 == 0:
logger.info(
f"Committing batch: {current_total} Gmail messages processed so far"
)
await session.commit()
continue
# Create new document
user_llm = await get_user_long_context_llm(
session, user_id, search_space_id
)
if user_llm:
document_metadata = {
"message_id": message_id,
"thread_id": thread_id,
"subject": subject,
"sender": sender,
"document_type": "Gmail Message (Composio)",
}
summary_content, summary_embedding = await generate_document_summary(
markdown_content, user_llm, document_metadata
)
else:
summary_content = (
f"Gmail: {subject}\n\nFrom: {sender}\nDate: {date_str}"
)
summary_embedding = config.embedding_model_instance.embed(
summary_content
)
chunks = await create_document_chunks(markdown_content)
document = Document(
search_space_id=search_space_id,
title=f"Gmail: {subject}",
document_type=DocumentType(TOOLKIT_TO_DOCUMENT_TYPE["gmail"]),
document_metadata={
"message_id": message_id,
"thread_id": thread_id,
"subject": subject,
"sender": sender,
"date": date_str,
"labels": label_ids,
"connector_id": connector_id,
"toolkit_id": "gmail",
"source": "composio",
},
content=summary_content,
content_hash=content_hash,
unique_identifier_hash=unique_identifier_hash,
embedding=summary_embedding,
chunks=chunks,
updated_at=get_current_timestamp(),
)
session.add(document)
documents_indexed += 1
# Batch commit every 10 documents
current_total = total_documents_indexed + documents_indexed
if current_total % 10 == 0:
logger.info(
f"Committing batch: {current_total} Gmail messages processed so far"
)
await session.commit()
except Exception as e:
logger.error(f"Error processing Gmail message: {e!s}", exc_info=True)
documents_skipped += 1
# Rollback on error to avoid partial state (per Composio best practices)
try:
await session.rollback()
except Exception as rollback_error:
logger.error(
f"Error during rollback: {rollback_error!s}", exc_info=True
)
continue
return documents_indexed, documents_skipped
async def index_composio_gmail(
session: AsyncSession,
connector,
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str | None,
end_date: str | None,
task_logger: TaskLoggingService,
log_entry,
update_last_indexed: bool = True,
max_items: int = 1000,
) -> tuple[int, str]:
"""Index Gmail messages via Composio with pagination and incremental processing."""
try:
composio_connector = ComposioGmailConnector(session, connector_id)
# Normalize date values - handle "undefined" strings from frontend
if start_date == "undefined" or start_date == "":
start_date = None
if end_date == "undefined" or end_date == "":
end_date = None
# Use provided dates directly if both are provided, otherwise calculate from last_indexed_at
# This ensures user-selected dates are respected (matching non-Composio Gmail connector behavior)
if start_date is not None and end_date is not None:
# User provided both dates - use them directly
start_date_str = start_date
end_date_str = end_date
else:
# Calculate date range with defaults (uses last_indexed_at or 365 days back)
# This ensures indexing works even when user doesn't specify dates
start_date_str, end_date_str = calculate_date_range(
connector, start_date, end_date, default_days_back=365
)
# Build query with date range
query_parts = []
if start_date_str:
query_parts.append(f"after:{start_date_str.replace('-', '/')}")
if end_date_str:
query_parts.append(f"before:{end_date_str.replace('-', '/')}")
query = " ".join(query_parts) if query_parts else ""
logger.info(
f"Gmail query for connector {connector_id}: '{query}' "
f"(start_date={start_date_str}, end_date={end_date_str})"
)
# Use smaller batch size to avoid 413 payload too large errors
batch_size = 50
page_token = None
total_documents_indexed = 0
total_documents_skipped = 0
total_messages_fetched = 0
result_size_estimate = None # Will be set from first API response
while total_messages_fetched < max_items:
# Calculate how many messages to fetch in this batch
remaining = max_items - total_messages_fetched
current_batch_size = min(batch_size, remaining)
# Use result_size_estimate if available, otherwise fall back to max_items
estimated_total = (
result_size_estimate if result_size_estimate is not None else max_items
)
# Cap estimated_total at max_items to avoid showing misleading progress
estimated_total = min(estimated_total, max_items)
await task_logger.log_task_progress(
log_entry,
f"Fetching Gmail messages batch via Composio for connector {connector_id} "
f"({total_messages_fetched}/{estimated_total} fetched, {total_documents_indexed} indexed)",
{
"stage": "fetching_messages",
"batch_size": current_batch_size,
"total_fetched": total_messages_fetched,
"total_indexed": total_documents_indexed,
"estimated_total": estimated_total,
},
)
# Fetch batch of messages
(
messages,
next_token,
result_size_estimate_batch,
error,
) = await composio_connector.list_gmail_messages(
query=query,
max_results=current_batch_size,
page_token=page_token,
)
if error:
await task_logger.log_task_failure(
log_entry, f"Failed to fetch Gmail messages: {error}", {}
)
return 0, f"Failed to fetch Gmail messages: {error}"
if not messages:
# No more messages available
break
# Update result_size_estimate from first response (Gmail provides this estimate)
if result_size_estimate is None and result_size_estimate_batch is not None:
result_size_estimate = result_size_estimate_batch
logger.info(
f"Gmail API estimated {result_size_estimate} total messages for query: '{query}'"
)
total_messages_fetched += len(messages)
# Recalculate estimated_total after potentially updating result_size_estimate
estimated_total = (
result_size_estimate if result_size_estimate is not None else max_items
)
estimated_total = min(estimated_total, max_items)
logger.info(
f"Fetched batch of {len(messages)} Gmail messages "
f"(total: {total_messages_fetched}/{estimated_total})"
)
# Process batch incrementally
batch_indexed, batch_skipped = await _process_gmail_message_batch(
session=session,
messages=messages,
composio_connector=composio_connector,
connector_id=connector_id,
search_space_id=search_space_id,
user_id=user_id,
total_documents_indexed=total_documents_indexed,
)
total_documents_indexed += batch_indexed
total_documents_skipped += batch_skipped
logger.info(
f"Processed batch: {batch_indexed} indexed, {batch_skipped} skipped "
f"(total: {total_documents_indexed} indexed, {total_documents_skipped} skipped)"
)
# Batch commits happen in _process_gmail_message_batch every 10 documents
# This ensures progress is saved incrementally, preventing data loss on crashes
# Check if we should continue
if not next_token:
# No more pages available
break
if len(messages) < current_batch_size:
# Last page had fewer items than requested, we're done
break
# Continue with next page
page_token = next_token
if total_messages_fetched == 0:
success_msg = "No Gmail messages found in the specified date range"
await task_logger.log_task_success(
log_entry, success_msg, {"messages_count": 0}
)
# CRITICAL: Update timestamp even when no messages found so Electric SQL syncs and UI shows indexed status
await update_connector_last_indexed(session, connector, update_last_indexed)
await session.commit()
return 0, None # Return None (not error) when no items found
# CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs
# This ensures the UI shows "Last indexed" instead of "Never indexed"
await update_connector_last_indexed(session, connector, update_last_indexed)
# Final commit to ensure all documents are persisted (safety net)
# This matches the pattern used in non-Composio Gmail indexer
logger.info(
f"Final commit: Total {total_documents_indexed} Gmail messages processed"
)
await session.commit()
logger.info(
"Successfully committed all Composio Gmail document changes to database"
)
await task_logger.log_task_success(
log_entry,
f"Successfully completed Gmail indexing via Composio for connector {connector_id}",
{
"documents_indexed": total_documents_indexed,
"documents_skipped": total_documents_skipped,
"messages_fetched": total_messages_fetched,
},
)
return total_documents_indexed, None
except Exception as e:
logger.error(f"Failed to index Gmail via Composio: {e!s}", exc_info=True)
return 0, f"Failed to index Gmail via Composio: {e!s}"

453
surfsense_backend/app/connectors/composio_google_calendar_connector.py Normal file
View file

@ -0,0 +1,453 @@
"""
Composio Google Calendar Connector Module.
Provides Google Calendar specific methods for data retrieval and indexing via Composio.
"""
import logging
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from sqlalchemy.orm import selectinload
from app.config import config
from app.connectors.composio_connector import ComposioConnector
from app.db import Document, DocumentType
from app.services.composio_service import TOOLKIT_TO_DOCUMENT_TYPE
from app.services.llm_service import get_user_long_context_llm
from app.services.task_logging_service import TaskLoggingService
from app.tasks.connector_indexers.base import calculate_date_range
from app.utils.document_converters import (
create_document_chunks,
generate_content_hash,
generate_document_summary,
generate_unique_identifier_hash,
)
logger = logging.getLogger(__name__)
def get_current_timestamp() -> datetime:
"""Get the current timestamp with timezone for updated_at field."""
return datetime.now(UTC)
async def check_document_by_unique_identifier(
session: AsyncSession, unique_identifier_hash: str
) -> Document | None:
"""Check if a document with the given unique identifier hash already exists."""
existing_doc_result = await session.execute(
select(Document)
.options(selectinload(Document.chunks))
.where(Document.unique_identifier_hash == unique_identifier_hash)
)
return existing_doc_result.scalars().first()
async def update_connector_last_indexed(
session: AsyncSession,
connector,
update_last_indexed: bool = True,
) -> None:
"""Update the last_indexed_at timestamp for a connector."""
if update_last_indexed:
connector.last_indexed_at = datetime.now(UTC)
logger.info(f"Updated last_indexed_at to {connector.last_indexed_at}")
class ComposioGoogleCalendarConnector(ComposioConnector):
"""
Google Calendar specific Composio connector.
Provides methods for listing calendar events and formatting them from
Google Calendar via Composio.
"""
async def list_calendar_events(
self,
time_min: str | None = None,
time_max: str | None = None,
max_results: int = 250,
) -> tuple[list[dict[str, Any]], str | None]:
"""
List Google Calendar events via Composio.
Args:
time_min: Start time (RFC3339 format).
time_max: End time (RFC3339 format).
max_results: Maximum number of events.
Returns:
Tuple of (events list, error message).
"""
connected_account_id = await self.get_connected_account_id()
if not connected_account_id:
return [], "No connected account ID found"
entity_id = await self.get_entity_id()
service = await self._get_service()
return await service.get_calendar_events(
connected_account_id=connected_account_id,
entity_id=entity_id,
time_min=time_min,
time_max=time_max,
max_results=max_results,
)
def format_calendar_event_to_markdown(self, event: dict[str, Any]) -> str:
"""
Format a Google Calendar event to markdown.
Args:
event: Event object from Google Calendar API.
Returns:
Formatted markdown string.
"""
try:
# Extract basic event information
summary = event.get("summary", "No Title")
description = event.get("description", "")
location = event.get("location", "")
# Extract start and end times
start = event.get("start", {})
end = event.get("end", {})
start_time = start.get("dateTime") or start.get("date", "")
end_time = end.get("dateTime") or end.get("date", "")
# Format times for display
def format_time(time_str: str) -> str:
if not time_str:
return "Unknown"
try:
if "T" in time_str:
dt = datetime.fromisoformat(time_str.replace("Z", "+00:00"))
return dt.strftime("%Y-%m-%d %H:%M")
return time_str
except Exception:
return time_str
start_formatted = format_time(start_time)
end_formatted = format_time(end_time)
# Extract attendees
attendees = event.get("attendees", [])
attendee_list = []
for attendee in attendees:
email = attendee.get("email", "")
display_name = attendee.get("displayName", email)
response_status = attendee.get("responseStatus", "")
attendee_list.append(f"- {display_name} ({response_status})")
# Build markdown content
markdown_content = f"# {summary}\n\n"
markdown_content += f"**Start:** {start_formatted}\n"
markdown_content += f"**End:** {end_formatted}\n"
if location:
markdown_content += f"**Location:** {location}\n"
markdown_content += "\n"
if description:
markdown_content += f"## Description\n\n{description}\n\n"
if attendee_list:
markdown_content += "## Attendees\n\n"
markdown_content += "\n".join(attendee_list)
markdown_content += "\n\n"
# Add event metadata
markdown_content += "## Event Details\n\n"
markdown_content += f"- **Event ID:** {event.get('id', 'Unknown')}\n"
markdown_content += f"- **Created:** {event.get('created', 'Unknown')}\n"
markdown_content += f"- **Updated:** {event.get('updated', 'Unknown')}\n"
return markdown_content
except Exception as e:
return f"Error formatting event to markdown: {e!s}"
# ============ Indexer Functions ============
async def index_composio_google_calendar(
session: AsyncSession,
connector,
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str | None,
end_date: str | None,
task_logger: TaskLoggingService,
log_entry,
update_last_indexed: bool = True,
max_items: int = 2500,
) -> tuple[int, str]:
"""Index Google Calendar events via Composio."""
try:
composio_connector = ComposioGoogleCalendarConnector(session, connector_id)
await task_logger.log_task_progress(
log_entry,
f"Fetching Google Calendar events via Composio for connector {connector_id}",
{"stage": "fetching_events"},
)
# Normalize date values - handle "undefined" strings from frontend
if start_date == "undefined" or start_date == "":
start_date = None
if end_date == "undefined" or end_date == "":
end_date = None
# Use provided dates directly if both are provided, otherwise calculate from last_indexed_at
# This ensures user-selected dates are respected (matching non-Composio Calendar connector behavior)
if start_date is not None and end_date is not None:
# User provided both dates - use them directly
start_date_str = start_date
end_date_str = end_date
else:
# Calculate date range with defaults (uses last_indexed_at or 365 days back)
# This ensures indexing works even when user doesn't specify dates
start_date_str, end_date_str = calculate_date_range(
connector, start_date, end_date, default_days_back=365
)
# Build time range for API call
time_min = f"{start_date_str}T00:00:00Z"
time_max = f"{end_date_str}T23:59:59Z"
logger.info(
f"Google Calendar query for connector {connector_id}: "
f"(start_date={start_date_str}, end_date={end_date_str})"
)
events, error = await composio_connector.list_calendar_events(
time_min=time_min,
time_max=time_max,
max_results=max_items,
)
if error:
await task_logger.log_task_failure(
log_entry, f"Failed to fetch Calendar events: {error}", {}
)
return 0, f"Failed to fetch Calendar events: {error}"
if not events:
success_msg = "No Google Calendar events found in the specified date range"
await task_logger.log_task_success(
log_entry, success_msg, {"events_count": 0}
)
# CRITICAL: Update timestamp even when no events found so Electric SQL syncs and UI shows indexed status
await update_connector_last_indexed(session, connector, update_last_indexed)
await session.commit()
return (
0,
None,
) # Return None (not error) when no items found - this is success with 0 items
logger.info(f"Found {len(events)} Google Calendar events to index via Composio")
documents_indexed = 0
documents_skipped = 0
for event in events:
try:
# Handle both standard Google API and potential Composio variations
event_id = event.get("id", "") or event.get("eventId", "")
summary = (
event.get("summary", "") or event.get("title", "") or "No Title"
)
if not event_id:
documents_skipped += 1
continue
# Format to markdown
markdown_content = composio_connector.format_calendar_event_to_markdown(
event
)
# Generate unique identifier
document_type = DocumentType(TOOLKIT_TO_DOCUMENT_TYPE["googlecalendar"])
unique_identifier_hash = generate_unique_identifier_hash(
document_type, f"calendar_{event_id}", search_space_id
)
content_hash = generate_content_hash(markdown_content, search_space_id)
existing_document = await check_document_by_unique_identifier(
session, unique_identifier_hash
)
# Extract event times
start = event.get("start", {})
end = event.get("end", {})
start_time = start.get("dateTime") or start.get("date", "")
end_time = end.get("dateTime") or end.get("date", "")
location = event.get("location", "")
if existing_document:
if existing_document.content_hash == content_hash:
documents_skipped += 1
continue
# Update existing
user_llm = await get_user_long_context_llm(
session, user_id, search_space_id
)
if user_llm:
document_metadata = {
"event_id": event_id,
"summary": summary,
"start_time": start_time,
"document_type": "Google Calendar Event (Composio)",
}
(
summary_content,
summary_embedding,
) = await generate_document_summary(
markdown_content, user_llm, document_metadata
)
else:
summary_content = f"Calendar: {summary}\n\nStart: {start_time}\nEnd: {end_time}"
if location:
summary_content += f"\nLocation: {location}"
summary_embedding = config.embedding_model_instance.embed(
summary_content
)
chunks = await create_document_chunks(markdown_content)
existing_document.title = f"Calendar: {summary}"
existing_document.content = summary_content
existing_document.content_hash = content_hash
existing_document.embedding = summary_embedding
existing_document.document_metadata = {
"event_id": event_id,
"summary": summary,
"start_time": start_time,
"end_time": end_time,
"location": location,
"connector_id": connector_id,
"source": "composio",
}
existing_document.chunks = chunks
existing_document.updated_at = get_current_timestamp()
documents_indexed += 1
# Batch commit every 10 documents
if documents_indexed % 10 == 0:
logger.info(
f"Committing batch: {documents_indexed} Google Calendar events processed so far"
)
await session.commit()
continue
# Create new document
user_llm = await get_user_long_context_llm(
session, user_id, search_space_id
)
if user_llm:
document_metadata = {
"event_id": event_id,
"summary": summary,
"start_time": start_time,
"document_type": "Google Calendar Event (Composio)",
}
(
summary_content,
summary_embedding,
) = await generate_document_summary(
markdown_content, user_llm, document_metadata
)
else:
summary_content = (
f"Calendar: {summary}\n\nStart: {start_time}\nEnd: {end_time}"
)
if location:
summary_content += f"\nLocation: {location}"
summary_embedding = config.embedding_model_instance.embed(
summary_content
)
chunks = await create_document_chunks(markdown_content)
document = Document(
search_space_id=search_space_id,
title=f"Calendar: {summary}",
document_type=DocumentType(
TOOLKIT_TO_DOCUMENT_TYPE["googlecalendar"]
),
document_metadata={
"event_id": event_id,
"summary": summary,
"start_time": start_time,
"end_time": end_time,
"location": location,
"connector_id": connector_id,
"toolkit_id": "googlecalendar",
"source": "composio",
},
content=summary_content,
content_hash=content_hash,
unique_identifier_hash=unique_identifier_hash,
embedding=summary_embedding,
chunks=chunks,
updated_at=get_current_timestamp(),
)
session.add(document)
documents_indexed += 1
# Batch commit every 10 documents
if documents_indexed % 10 == 0:
logger.info(
f"Committing batch: {documents_indexed} Google Calendar events processed so far"
)
await session.commit()
except Exception as e:
logger.error(f"Error processing Calendar event: {e!s}", exc_info=True)
documents_skipped += 1
continue
# CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs
# This ensures the UI shows "Last indexed" instead of "Never indexed"
await update_connector_last_indexed(session, connector, update_last_indexed)
# Final commit to ensure all documents are persisted (safety net)
# This matches the pattern used in non-Composio Gmail indexer
logger.info(
f"Final commit: Total {documents_indexed} Google Calendar events processed"
)
await session.commit()
logger.info(
"Successfully committed all Composio Google Calendar document changes to database"
)
await task_logger.log_task_success(
log_entry,
f"Successfully completed Google Calendar indexing via Composio for connector {connector_id}",
{
"documents_indexed": documents_indexed,
"documents_skipped": documents_skipped,
},
)
return documents_indexed, None
except Exception as e:
logger.error(
f"Failed to index Google Calendar via Composio: {e!s}", exc_info=True
)
return 0, f"Failed to index Google Calendar via Composio: {e!s}"

1162
surfsense_backend/app/connectors/composio_google_drive_connector.py Normal file
View file

File diff suppressed because it is too large Load diff

19
surfsense_backend/app/routes/composio_routes.py
View file

@ -46,6 +46,13 @@ logger = logging.getLogger(__name__)
router = APIRouter()
# Map toolkit_id to frontend connector ID
TOOLKIT_TO_FRONTEND_CONNECTOR_ID = {
"googledrive": "composio-googledrive",
"gmail": "composio-gmail",
"googlecalendar": "composio-googlecalendar",
}
# Initialize security utilities
_state_manager = None
@ -327,8 +334,12 @@ async def composio_callback(
await session.commit()
await session.refresh(existing_connector)
# Get the frontend connector ID based on toolkit_id
frontend_connector_id = TOOLKIT_TO_FRONTEND_CONNECTOR_ID.get(
toolkit_id, "composio-connector"
)
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=composio-connector&connectorId={existing_connector.id}"
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector={frontend_connector_id}&connectorId={existing_connector.id}"
)
try:
@ -358,8 +369,12 @@ async def composio_callback(
f"Successfully created Composio connector {db_connector.id} for user {user_id}, toolkit {toolkit_id}"
)
# Get the frontend connector ID based on toolkit_id
frontend_connector_id = TOOLKIT_TO_FRONTEND_CONNECTOR_ID.get(
toolkit_id, "composio-connector"
)
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=composio-connector&connectorId={db_connector.id}"
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector={frontend_connector_id}&connectorId={db_connector.id}"
)
except IntegrityError as e:

21
surfsense_backend/app/services/composio_service.py
View file

@ -53,6 +53,27 @@ TOOLKIT_TO_DOCUMENT_TYPE = {
"googlecalendar": "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR",
}
# Mapping of toolkit IDs to their indexer functions
# Format: toolkit_id -> (module_path, function_name, supports_date_filter)
# supports_date_filter: True if the indexer accepts start_date/end_date params
TOOLKIT_TO_INDEXER = {
"googledrive": (
"app.connectors.composio_google_drive_connector",
"index_composio_google_drive",
False, # Google Drive doesn't use date filtering
),
"gmail": (
"app.connectors.composio_gmail_connector",
"index_composio_gmail",
True, # Gmail uses date filtering
),
"googlecalendar": (
"app.connectors.composio_google_calendar_connector",
"index_composio_google_calendar",
True, # Calendar uses date filtering
),
}
class ComposioService:
"""Service for interacting with Composio API."""

1819
surfsense_backend/app/tasks/composio_indexer.py
View file

File diff suppressed because it is too large Load diff

78
surfsense_web/components/assistant-ui/connector-popup/components/composio-connector-card.tsx
View file

@ -1,78 +0,0 @@
"use client";
import { Zap } from "lucide-react";
import Image from "next/image";
import type { FC } from "react";
import { Button } from "@/components/ui/button";
import { cn } from "@/lib/utils";
interface ComposioConnectorCardProps {
id: string;
title: string;
description: string;
connectorCount?: number;
onConnect: () => void;
}
export const ComposioConnectorCard: FC<ComposioConnectorCardProps> = ({
id,
title,
description,
connectorCount = 0,
onConnect,
}) => {
const hasConnections = connectorCount > 0;
return (
<div
className={cn(
"group relative flex items-center gap-4 p-4 rounded-xl text-left transition-all duration-200 w-full border",
"border-violet-500/20 bg-gradient-to-br from-violet-500/5 to-purple-500/5",
"hover:border-violet-500/40 hover:from-violet-500/10 hover:to-purple-500/10"
)}
>
<div
className={cn(
"flex h-12 w-12 items-center justify-center rounded-lg transition-colors shrink-0 border",
"bg-gradient-to-br from-violet-500/10 to-purple-500/10 border-violet-500/20"
)}
>
<Image
src="/connectors/composio.svg"
alt="Composio"
width={24}
height={24}
className="size-6"
/>
</div>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-1.5">
<span className="text-[14px] font-semibold leading-tight truncate">{title}</span>
<Zap className="size-3.5 text-violet-500" />
</div>
{hasConnections ? (
<p className="text-[10px] text-muted-foreground mt-1 flex items-center gap-1.5">
<span>
{connectorCount} {connectorCount === 1 ? "connection" : "connections"}
</span>
</p>
) : (
<p className="text-[10px] text-muted-foreground mt-1">{description}</p>
)}
</div>
<Button
size="sm"
variant={hasConnections ? "secondary" : "default"}
className={cn(
"h-8 text-[11px] px-3 rounded-lg shrink-0 font-medium shadow-xs",
!hasConnections && "bg-violet-600 hover:bg-violet-700 text-white",
hasConnections &&
"bg-white text-slate-700 hover:bg-slate-50 border-0 dark:bg-secondary dark:text-secondary-foreground dark:hover:bg-secondary/80"
)}
onClick={onConnect}
>
{hasConnections ? "Manage" : "Browse"}
</Button>
</div>
);
};

220
surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/composio-calendar-config.tsx Normal file
View file

@ -0,0 +1,220 @@
"use client";
import { Calendar, Clock } from "lucide-react";
import type { FC } from "react";
import { useEffect, useState } from "react";
import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
interface ComposioCalendarConfigProps {
connector: SearchSourceConnector;
onConfigChange?: (config: Record<string, unknown>) => void;
onNameChange?: (name: string) => void;
}
interface CalendarIndexingOptions {
max_events: number;
include_recurring: boolean;
include_past_events: boolean;
days_ahead: number;
}
const DEFAULT_CALENDAR_OPTIONS: CalendarIndexingOptions = {
max_events: 500,
include_recurring: true,
include_past_events: true,
days_ahead: 365,
};
export const ComposioCalendarConfig: FC<ComposioCalendarConfigProps> = ({ connector, onConfigChange }) => {
const isIndexable = connector.config?.is_indexable as boolean;
// Initialize with existing options from connector config
const existingOptions =
(connector.config?.calendar_options as CalendarIndexingOptions | undefined) || DEFAULT_CALENDAR_OPTIONS;
const [calendarOptions, setCalendarOptions] = useState<CalendarIndexingOptions>(existingOptions);
// Update options when connector config changes
useEffect(() => {
const options =
(connector.config?.calendar_options as CalendarIndexingOptions | undefined) ||
DEFAULT_CALENDAR_OPTIONS;
setCalendarOptions(options);
}, [connector.config]);
const updateConfig = (options: CalendarIndexingOptions) => {
if (onConfigChange) {
onConfigChange({
...connector.config,
calendar_options: options,
});
}
};
const handleOptionChange = (key: keyof CalendarIndexingOptions, value: number | boolean) => {
const newOptions = { ...calendarOptions, [key]: value };
setCalendarOptions(newOptions);
updateConfig(newOptions);
};
// Only show configuration if the connector is indexable
if (!isIndexable) {
return <div className="space-y-6" />;
}
return (
<div className="space-y-6">
{/* Calendar Indexing Options */}
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-4">
<div className="space-y-1 sm:space-y-2">
<div className="flex items-center gap-2">
<Calendar className="size-4 text-blue-500" />
<h3 className="font-medium text-sm sm:text-base">Calendar Indexing Options</h3>
</div>
<p className="text-xs sm:text-sm text-muted-foreground">
Configure how events are indexed from your Google Calendar.
</p>
</div>
{/* Max events to index */}
<div className="space-y-2">
<div className="flex items-center justify-between">
<div className="space-y-0.5">
<Label htmlFor="max-events" className="text-sm font-medium">
Max events to index
</Label>
<p className="text-xs text-muted-foreground">
Maximum number of events to index per sync
</p>
</div>
<Select
value={calendarOptions.max_events.toString()}
onValueChange={(value) =>
handleOptionChange("max_events", parseInt(value, 10))
}
>
<SelectTrigger
id="max-events"
className="w-[140px] bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
>
<SelectValue placeholder="Select limit" />
</SelectTrigger>
<SelectContent className="z-[100]">
<SelectItem value="100" className="text-xs sm:text-sm">
100 events
</SelectItem>
<SelectItem value="250" className="text-xs sm:text-sm">
250 events
</SelectItem>
<SelectItem value="500" className="text-xs sm:text-sm">
500 events
</SelectItem>
<SelectItem value="1000" className="text-xs sm:text-sm">
1000 events
</SelectItem>
<SelectItem value="2500" className="text-xs sm:text-sm">
2500 events
</SelectItem>
</SelectContent>
</Select>
</div>
</div>
{/* Days ahead */}
<div className="space-y-2 pt-2 border-t border-slate-400/20">
<div className="flex items-center justify-between">
<div className="space-y-0.5">
<div className="flex items-center gap-1.5">
<Clock className="size-3.5 text-muted-foreground" />
<Label htmlFor="days-ahead" className="text-sm font-medium">
Future events range
</Label>
</div>
<p className="text-xs text-muted-foreground">
How far ahead to index future events
</p>
</div>
<Select
value={calendarOptions.days_ahead.toString()}
onValueChange={(value) =>
handleOptionChange("days_ahead", parseInt(value, 10))
}
>
<SelectTrigger
id="days-ahead"
className="w-[140px] bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
>
<SelectValue placeholder="Select range" />
</SelectTrigger>
<SelectContent className="z-[100]">
<SelectItem value="30" className="text-xs sm:text-sm">
30 days
</SelectItem>
<SelectItem value="90" className="text-xs sm:text-sm">
90 days
</SelectItem>
<SelectItem value="180" className="text-xs sm:text-sm">
180 days
</SelectItem>
<SelectItem value="365" className="text-xs sm:text-sm">
1 year
</SelectItem>
<SelectItem value="730" className="text-xs sm:text-sm">
2 years
</SelectItem>
</SelectContent>
</Select>
</div>
</div>
{/* Include recurring events toggle */}
<div className="flex items-center justify-between pt-2 border-t border-slate-400/20">
<div className="space-y-0.5">
<Label htmlFor="include-recurring" className="text-sm font-medium">
Include recurring events
</Label>
<p className="text-xs text-muted-foreground">
Index individual instances of recurring events
</p>
</div>
<Switch
id="include-recurring"
checked={calendarOptions.include_recurring}
onCheckedChange={(checked) =>
handleOptionChange("include_recurring", checked)
}
/>
</div>
{/* Include past events toggle */}
<div className="flex items-center justify-between pt-2 border-t border-slate-400/20">
<div className="space-y-0.5">
<Label htmlFor="include-past" className="text-sm font-medium">
Include past events
</Label>
<p className="text-xs text-muted-foreground">
Index events from before the selected date range
</p>
</div>
<Switch
id="include-past"
checked={calendarOptions.include_past_events}
onCheckedChange={(checked) =>
handleOptionChange("include_past_events", checked)
}
/>
</div>
</div>
</div>
);
};

353
surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/composio-config.tsx
View file

@ -1,353 +0,0 @@
"use client";
import { File, FileSpreadsheet, FileText, FolderClosed, Image, Presentation } from "lucide-react";
import type { FC } from "react";
import { useEffect, useState } from "react";
import { ComposioDriveFolderTree } from "@/components/connectors/composio-drive-folder-tree";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { cn } from "@/lib/utils";
interface ComposioConfigProps {
connector: SearchSourceConnector;
onConfigChange?: (config: Record<string, unknown>) => void;
onNameChange?: (name: string) => void;
}
interface SelectedFolder {
id: string;
name: string;
}
interface IndexingOptions {
max_files_per_folder: number;
incremental_sync: boolean;
include_subfolders: boolean;
}
const DEFAULT_INDEXING_OPTIONS: IndexingOptions = {
max_files_per_folder: 100,
incremental_sync: true,
include_subfolders: true,
};
// Helper to get appropriate icon for file type based on file name
function getFileIconFromName(fileName: string, className: string = "size-3.5 shrink-0") {
const lowerName = fileName.toLowerCase();
// Spreadsheets
if (
lowerName.endsWith(".xlsx") ||
lowerName.endsWith(".xls") ||
lowerName.endsWith(".csv") ||
lowerName.includes("spreadsheet")
) {
return <FileSpreadsheet className={`${className} text-green-500`} />;
}
// Presentations
if (
lowerName.endsWith(".pptx") ||
lowerName.endsWith(".ppt") ||
lowerName.includes("presentation")
) {
return <Presentation className={`${className} text-orange-500`} />;
}
// Documents (word, text only - not PDF)
if (
lowerName.endsWith(".docx") ||
lowerName.endsWith(".doc") ||
lowerName.endsWith(".txt") ||
lowerName.includes("document") ||
lowerName.includes("word") ||
lowerName.includes("text")
) {
return <FileText className={`${className} text-gray-500`} />;
}
// Images
if (
lowerName.endsWith(".png") ||
lowerName.endsWith(".jpg") ||
lowerName.endsWith(".jpeg") ||
lowerName.endsWith(".gif") ||
lowerName.endsWith(".webp") ||
lowerName.endsWith(".svg")
) {
return <Image className={`${className} text-purple-500`} />;
}
// Default (including PDF)
return <File className={`${className} text-gray-500`} />;
}
export const ComposioConfig: FC<ComposioConfigProps> = ({ connector, onConfigChange }) => {
const toolkitId = connector.config?.toolkit_id as string;
const isIndexable = connector.config?.is_indexable as boolean;
const composioAccountId = connector.config?.composio_connected_account_id as string;
// Check if this is a Google Drive Composio connector
const isGoogleDrive = toolkitId === "googledrive";
// Initialize with existing selected folders and files from connector config
const existingFolders =
(connector.config?.selected_folders as SelectedFolder[] | undefined) || [];
const existingFiles = (connector.config?.selected_files as SelectedFolder[] | undefined) || [];
const existingIndexingOptions =
(connector.config?.indexing_options as IndexingOptions | undefined) || DEFAULT_INDEXING_OPTIONS;
const [selectedFolders, setSelectedFolders] = useState<SelectedFolder[]>(existingFolders);
const [selectedFiles, setSelectedFiles] = useState<SelectedFolder[]>(existingFiles);
const [showFolderSelector, setShowFolderSelector] = useState(false);
const [indexingOptions, setIndexingOptions] = useState<IndexingOptions>(existingIndexingOptions);
// Update selected folders and files when connector config changes
useEffect(() => {
const folders = (connector.config?.selected_folders as SelectedFolder[] | undefined) || [];
const files = (connector.config?.selected_files as SelectedFolder[] | undefined) || [];
const options =
(connector.config?.indexing_options as IndexingOptions | undefined) ||
DEFAULT_INDEXING_OPTIONS;
setSelectedFolders(folders);
setSelectedFiles(files);
setIndexingOptions(options);
}, [connector.config]);
const updateConfig = (
folders: SelectedFolder[],
files: SelectedFolder[],
options: IndexingOptions
) => {
if (onConfigChange) {
onConfigChange({
...connector.config,
selected_folders: folders,
selected_files: files,
indexing_options: options,
});
}
};
const handleSelectFolders = (folders: SelectedFolder[]) => {
setSelectedFolders(folders);
updateConfig(folders, selectedFiles, indexingOptions);
};
const handleSelectFiles = (files: SelectedFolder[]) => {
setSelectedFiles(files);
updateConfig(selectedFolders, files, indexingOptions);
};
const handleIndexingOptionChange = (key: keyof IndexingOptions, value: number | boolean) => {
const newOptions = { ...indexingOptions, [key]: value };
setIndexingOptions(newOptions);
updateConfig(selectedFolders, selectedFiles, newOptions);
};
const totalSelected = selectedFolders.length + selectedFiles.length;
return (
<div className="space-y-6">
{/* Connection Details */}
<div className="space-y-3">
<h4 className="text-xs font-medium text-muted-foreground uppercase tracking-wider">
Connection Details
</h4>
<div className="space-y-2">
<div className="flex items-center justify-between py-2 px-3 rounded-lg bg-muted/50">
<span className="text-xs text-muted-foreground">Toolkit</span>
<span className="text-xs font-medium">{toolkitId}</span>
</div>
<div className="flex items-center justify-between py-2 px-3 rounded-lg bg-muted/50">
<span className="text-xs text-muted-foreground">Indexing Supported</span>
<Badge
variant={isIndexable ? "default" : "secondary"}
className={cn(
"text-[10px] px-1.5 py-0 h-5",
isIndexable
? "bg-emerald-500/10 text-emerald-600 dark:text-emerald-400 border-emerald-500/20"
: "bg-amber-500/10 text-amber-600 dark:text-amber-400 border-amber-500/20"
)}
>
{isIndexable ? "Yes" : "Coming Soon"}
</Badge>
</div>
{composioAccountId && (
<div className="flex items-center justify-between py-2 px-3 rounded-lg bg-muted/50">
<span className="text-xs text-muted-foreground">Account ID</span>
<span className="text-xs font-mono text-muted-foreground truncate max-w-[150px]">
{composioAccountId}
</span>
</div>
)}
</div>
</div>
{/* Google Drive specific: Folder & File Selection */}
{isGoogleDrive && isIndexable && (
<>
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-3 sm:space-y-4">
<div className="space-y-1 sm:space-y-2">
<h3 className="font-medium text-sm sm:text-base">Folder & File Selection</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Select specific folders and/or individual files to index.
</p>
</div>
{totalSelected > 0 && (
<div className="p-2 sm:p-3 bg-muted rounded-lg text-xs sm:text-sm space-y-1 sm:space-y-2">
<p className="font-medium">
Selected {totalSelected} item{totalSelected > 1 ? "s" : ""}: {(() => {
const parts: string[] = [];
if (selectedFolders.length > 0) {
parts.push(
`${selectedFolders.length} folder${selectedFolders.length > 1 ? "s" : ""}`
);
}
if (selectedFiles.length > 0) {
parts.push(
`${selectedFiles.length} file${selectedFiles.length > 1 ? "s" : ""}`
);
}
return parts.length > 0 ? `(${parts.join(" ")})` : "";
})()}
</p>
<div className="max-h-20 sm:max-h-24 overflow-y-auto space-y-1">
{selectedFolders.map((folder) => (
<p
key={folder.id}
className="text-xs sm:text-sm text-muted-foreground truncate flex items-center gap-1.5"
title={folder.name}
>
<FolderClosed className="size-3.5 shrink-0 text-gray-500" />
{folder.name}
</p>
))}
{selectedFiles.map((file) => (
<p
key={file.id}
className="text-xs sm:text-sm text-muted-foreground truncate flex items-center gap-1.5"
title={file.name}
>
{getFileIconFromName(file.name)}
{file.name}
</p>
))}
</div>
</div>
)}
{showFolderSelector ? (
<div className="space-y-2 sm:space-y-3">
<ComposioDriveFolderTree
connectorId={connector.id}
selectedFolders={selectedFolders}
onSelectFolders={handleSelectFolders}
selectedFiles={selectedFiles}
onSelectFiles={handleSelectFiles}
/>
<Button
type="button"
variant="outline"
size="sm"
onClick={() => setShowFolderSelector(false)}
className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-white/10 text-xs sm:text-sm h-8 sm:h-9"
>
Done Selecting
</Button>
</div>
) : (
<Button
type="button"
variant="outline"
onClick={() => setShowFolderSelector(true)}
className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-white/10 text-xs sm:text-sm h-8 sm:h-9"
>
{totalSelected > 0 ? "Change Selection" : "Select Folders & Files"}
</Button>
)}
</div>
{/* Indexing Options */}
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-4">
<div className="space-y-1 sm:space-y-2">
<h3 className="font-medium text-sm sm:text-base">Indexing Options</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Configure how files are indexed from your Google Drive.
</p>
</div>
{/* Max files per folder */}
<div className="space-y-2">
<div className="flex items-center justify-between">
<div className="space-y-0.5">
<Label htmlFor="max-files" className="text-sm font-medium">
Max files per folder
</Label>
<p className="text-xs text-muted-foreground">
Maximum number of files to index from each folder
</p>
</div>
<Select
value={indexingOptions.max_files_per_folder.toString()}
onValueChange={(value) =>
handleIndexingOptionChange("max_files_per_folder", parseInt(value, 10))
}
>
<SelectTrigger
id="max-files"
className="w-[140px] bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
>
<SelectValue placeholder="Select limit" />
</SelectTrigger>
<SelectContent className="z-[100]">
<SelectItem value="50" className="text-xs sm:text-sm">
50 files
</SelectItem>
<SelectItem value="100" className="text-xs sm:text-sm">
100 files
</SelectItem>
<SelectItem value="250" className="text-xs sm:text-sm">
250 files
</SelectItem>
<SelectItem value="500" className="text-xs sm:text-sm">
500 files
</SelectItem>
<SelectItem value="1000" className="text-xs sm:text-sm">
1000 files
</SelectItem>
</SelectContent>
</Select>
</div>
</div>
{/* Include subfolders toggle */}
<div className="flex items-center justify-between pt-2 border-t border-slate-400/20">
<div className="space-y-0.5">
<Label htmlFor="include-subfolders" className="text-sm font-medium">
Include subfolders
</Label>
<p className="text-xs text-muted-foreground">
Recursively index files in subfolders of selected folders
</p>
</div>
<Switch
id="include-subfolders"
checked={indexingOptions.include_subfolders}
onCheckedChange={(checked) =>
handleIndexingOptionChange("include_subfolders", checked)
}
/>
</div>
</div>
</>
)}
</div>
);
};

313
surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/composio-drive-config.tsx Normal file
View file

@ -0,0 +1,313 @@
"use client";
import { File, FileSpreadsheet, FileText, FolderClosed, Image, Presentation } from "lucide-react";
import type { FC } from "react";
import { useEffect, useState } from "react";
import { ComposioDriveFolderTree } from "@/components/connectors/composio-drive-folder-tree";
import { Button } from "@/components/ui/button";
import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
interface ComposioDriveConfigProps {
connector: SearchSourceConnector;
onConfigChange?: (config: Record<string, unknown>) => void;
onNameChange?: (name: string) => void;
}
interface SelectedFolder {
id: string;
name: string;
}
interface IndexingOptions {
max_files_per_folder: number;
incremental_sync: boolean;
include_subfolders: boolean;
}
const DEFAULT_INDEXING_OPTIONS: IndexingOptions = {
max_files_per_folder: 100,
incremental_sync: true,
include_subfolders: true,
};
// Helper to get appropriate icon for file type based on file name
function getFileIconFromName(fileName: string, className: string = "size-3.5 shrink-0") {
const lowerName = fileName.toLowerCase();
// Spreadsheets
if (
lowerName.endsWith(".xlsx") ||
lowerName.endsWith(".xls") ||
lowerName.endsWith(".csv") ||
lowerName.includes("spreadsheet")
) {
return <FileSpreadsheet className={`${className} text-green-500`} />;
}
// Presentations
if (
lowerName.endsWith(".pptx") ||
lowerName.endsWith(".ppt") ||
lowerName.includes("presentation")
) {
return <Presentation className={`${className} text-orange-500`} />;
}
// Documents (word, text only - not PDF)
if (
lowerName.endsWith(".docx") ||
lowerName.endsWith(".doc") ||
lowerName.endsWith(".txt") ||
lowerName.includes("document") ||
lowerName.includes("word") ||
lowerName.includes("text")
) {
return <FileText className={`${className} text-gray-500`} />;
}
// Images
if (
lowerName.endsWith(".png") ||
lowerName.endsWith(".jpg") ||
lowerName.endsWith(".jpeg") ||
lowerName.endsWith(".gif") ||
lowerName.endsWith(".webp") ||
lowerName.endsWith(".svg")
) {
return <Image className={`${className} text-purple-500`} />;
}
// Default (including PDF)
return <File className={`${className} text-gray-500`} />;
}
export const ComposioDriveConfig: FC<ComposioDriveConfigProps> = ({ connector, onConfigChange }) => {
const isIndexable = connector.config?.is_indexable as boolean;
// Initialize with existing selected folders and files from connector config
const existingFolders =
(connector.config?.selected_folders as SelectedFolder[] | undefined) || [];
const existingFiles = (connector.config?.selected_files as SelectedFolder[] | undefined) || [];
const existingIndexingOptions =
(connector.config?.indexing_options as IndexingOptions | undefined) || DEFAULT_INDEXING_OPTIONS;
const [selectedFolders, setSelectedFolders] = useState<SelectedFolder[]>(existingFolders);
const [selectedFiles, setSelectedFiles] = useState<SelectedFolder[]>(existingFiles);
const [showFolderSelector, setShowFolderSelector] = useState(false);
const [indexingOptions, setIndexingOptions] = useState<IndexingOptions>(existingIndexingOptions);
// Update selected folders and files when connector config changes
useEffect(() => {
const folders = (connector.config?.selected_folders as SelectedFolder[] | undefined) || [];
const files = (connector.config?.selected_files as SelectedFolder[] | undefined) || [];
const options =
(connector.config?.indexing_options as IndexingOptions | undefined) ||
DEFAULT_INDEXING_OPTIONS;
setSelectedFolders(folders);
setSelectedFiles(files);
setIndexingOptions(options);
}, [connector.config]);
const updateConfig = (
folders: SelectedFolder[],
files: SelectedFolder[],
options: IndexingOptions
) => {
if (onConfigChange) {
onConfigChange({
...connector.config,
selected_folders: folders,
selected_files: files,
indexing_options: options,
});
}
};
const handleSelectFolders = (folders: SelectedFolder[]) => {
setSelectedFolders(folders);
updateConfig(folders, selectedFiles, indexingOptions);
};
const handleSelectFiles = (files: SelectedFolder[]) => {
setSelectedFiles(files);
updateConfig(selectedFolders, files, indexingOptions);
};
const handleIndexingOptionChange = (key: keyof IndexingOptions, value: number | boolean) => {
const newOptions = { ...indexingOptions, [key]: value };
setIndexingOptions(newOptions);
updateConfig(selectedFolders, selectedFiles, newOptions);
};
const totalSelected = selectedFolders.length + selectedFiles.length;
// Only show configuration if the connector is indexable
if (!isIndexable) {
return <div className="space-y-6" />;
}
return (
<div className="space-y-6">
{/* Folder & File Selection */}
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-3 sm:space-y-4">
<div className="space-y-1 sm:space-y-2">
<h3 className="font-medium text-sm sm:text-base">Folder & File Selection</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Select specific folders and/or individual files to index from your Google Drive.
</p>
</div>
{totalSelected > 0 && (
<div className="p-2 sm:p-3 bg-muted rounded-lg text-xs sm:text-sm space-y-1 sm:space-y-2">
<p className="font-medium">
Selected {totalSelected} item{totalSelected > 1 ? "s" : ""}: {(() => {
const parts: string[] = [];
if (selectedFolders.length > 0) {
parts.push(
`${selectedFolders.length} folder${selectedFolders.length > 1 ? "s" : ""}`
);
}
if (selectedFiles.length > 0) {
parts.push(
`${selectedFiles.length} file${selectedFiles.length > 1 ? "s" : ""}`
);
}
return parts.length > 0 ? `(${parts.join(" ")})` : "";
})()}
</p>
<div className="max-h-20 sm:max-h-24 overflow-y-auto space-y-1">
{selectedFolders.map((folder) => (
<p
key={folder.id}
className="text-xs sm:text-sm text-muted-foreground truncate flex items-center gap-1.5"
title={folder.name}
>
<FolderClosed className="size-3.5 shrink-0 text-gray-500" />
{folder.name}
</p>
))}
{selectedFiles.map((file) => (
<p
key={file.id}
className="text-xs sm:text-sm text-muted-foreground truncate flex items-center gap-1.5"
title={file.name}
>
{getFileIconFromName(file.name)}
{file.name}
</p>
))}
</div>
</div>
)}
{showFolderSelector ? (
<div className="space-y-2 sm:space-y-3">
<ComposioDriveFolderTree
connectorId={connector.id}
selectedFolders={selectedFolders}
onSelectFolders={handleSelectFolders}
selectedFiles={selectedFiles}
onSelectFiles={handleSelectFiles}
/>
<Button
type="button"
variant="outline"
size="sm"
onClick={() => setShowFolderSelector(false)}
className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-white/10 text-xs sm:text-sm h-8 sm:h-9"
>
Done Selecting
</Button>
</div>
) : (
<Button
type="button"
variant="outline"
onClick={() => setShowFolderSelector(true)}
className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-white/10 text-xs sm:text-sm h-8 sm:h-9"
>
{totalSelected > 0 ? "Change Selection" : "Select Folders & Files"}
</Button>
)}
</div>
{/* Indexing Options */}
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-4">
<div className="space-y-1 sm:space-y-2">
<h3 className="font-medium text-sm sm:text-base">Indexing Options</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Configure how files are indexed from your Google Drive.
</p>
</div>
{/* Max files per folder */}
<div className="space-y-2">
<div className="flex items-center justify-between">
<div className="space-y-0.5">
<Label htmlFor="max-files" className="text-sm font-medium">
Max files per folder
</Label>
<p className="text-xs text-muted-foreground">
Maximum number of files to index from each folder
</p>
</div>
<Select
value={indexingOptions.max_files_per_folder.toString()}
onValueChange={(value) =>
handleIndexingOptionChange("max_files_per_folder", parseInt(value, 10))
}
>
<SelectTrigger
id="max-files"
className="w-[140px] bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
>
<SelectValue placeholder="Select limit" />
</SelectTrigger>
<SelectContent className="z-[100]">
<SelectItem value="50" className="text-xs sm:text-sm">
50 files
</SelectItem>
<SelectItem value="100" className="text-xs sm:text-sm">
100 files
</SelectItem>
<SelectItem value="250" className="text-xs sm:text-sm">
250 files
</SelectItem>
<SelectItem value="500" className="text-xs sm:text-sm">
500 files
</SelectItem>
<SelectItem value="1000" className="text-xs sm:text-sm">
1000 files
</SelectItem>
</SelectContent>
</Select>
</div>
</div>
{/* Include subfolders toggle */}
<div className="flex items-center justify-between pt-2 border-t border-slate-400/20">
<div className="space-y-0.5">
<Label htmlFor="include-subfolders" className="text-sm font-medium">
Include subfolders
</Label>
<p className="text-xs text-muted-foreground">
Recursively index files in subfolders of selected folders
</p>
</div>
<Switch
id="include-subfolders"
checked={indexingOptions.include_subfolders}
onCheckedChange={(checked) =>
handleIndexingOptionChange("include_subfolders", checked)
}
/>
</div>
</div>
</div>
);
};

174
surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/composio-gmail-config.tsx Normal file
View file

@ -0,0 +1,174 @@
"use client";
import { Mail, Tag } from "lucide-react";
import type { FC } from "react";
import { useEffect, useState } from "react";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
interface ComposioGmailConfigProps {
connector: SearchSourceConnector;
onConfigChange?: (config: Record<string, unknown>) => void;
onNameChange?: (name: string) => void;
}
interface GmailIndexingOptions {
max_emails: number;
label_filter: string;
search_query: string;
}
const DEFAULT_GMAIL_OPTIONS: GmailIndexingOptions = {
max_emails: 500,
label_filter: "",
search_query: "",
};
export const ComposioGmailConfig: FC<ComposioGmailConfigProps> = ({ connector, onConfigChange }) => {
const isIndexable = connector.config?.is_indexable as boolean;
// Initialize with existing options from connector config
const existingOptions =
(connector.config?.gmail_options as GmailIndexingOptions | undefined) || DEFAULT_GMAIL_OPTIONS;
const [gmailOptions, setGmailOptions] = useState<GmailIndexingOptions>(existingOptions);
// Update options when connector config changes
useEffect(() => {
const options =
(connector.config?.gmail_options as GmailIndexingOptions | undefined) ||
DEFAULT_GMAIL_OPTIONS;
setGmailOptions(options);
}, [connector.config]);
const updateConfig = (options: GmailIndexingOptions) => {
if (onConfigChange) {
onConfigChange({
...connector.config,
gmail_options: options,
});
}
};
const handleOptionChange = (key: keyof GmailIndexingOptions, value: number | string) => {
const newOptions = { ...gmailOptions, [key]: value };
setGmailOptions(newOptions);
updateConfig(newOptions);
};
// Only show configuration if the connector is indexable
if (!isIndexable) {
return <div className="space-y-6" />;
}
return (
<div className="space-y-6">
{/* Gmail Indexing Options */}
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-4">
<div className="space-y-1 sm:space-y-2">
<div className="flex items-center gap-2">
<Mail className="size-4 text-red-500" />
<h3 className="font-medium text-sm sm:text-base">Gmail Indexing Options</h3>
</div>
<p className="text-xs sm:text-sm text-muted-foreground">
Configure how emails are indexed from your Gmail account.
</p>
</div>
{/* Max emails to index */}
<div className="space-y-2">
<div className="flex items-center justify-between">
<div className="space-y-0.5">
<Label htmlFor="max-emails" className="text-sm font-medium">
Max emails to index
</Label>
<p className="text-xs text-muted-foreground">
Maximum number of emails to index per sync
</p>
</div>
<Select
value={gmailOptions.max_emails.toString()}
onValueChange={(value) =>
handleOptionChange("max_emails", parseInt(value, 10))
}
>
<SelectTrigger
id="max-emails"
className="w-[140px] bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
>
<SelectValue placeholder="Select limit" />
</SelectTrigger>
<SelectContent className="z-[100]">
<SelectItem value="100" className="text-xs sm:text-sm">
100 emails
</SelectItem>
<SelectItem value="250" className="text-xs sm:text-sm">
250 emails
</SelectItem>
<SelectItem value="500" className="text-xs sm:text-sm">
500 emails
</SelectItem>
<SelectItem value="1000" className="text-xs sm:text-sm">
1000 emails
</SelectItem>
<SelectItem value="2500" className="text-xs sm:text-sm">
2500 emails
</SelectItem>
</SelectContent>
</Select>
</div>
</div>
{/* Label filter */}
<div className="space-y-2 pt-2 border-t border-slate-400/20">
<div className="space-y-0.5">
<div className="flex items-center gap-1.5">
<Tag className="size-3.5 text-muted-foreground" />
<Label htmlFor="label-filter" className="text-sm font-medium">
Label filter (optional)
</Label>
</div>
<p className="text-xs text-muted-foreground">
Only index emails with this label (e.g., "INBOX", "IMPORTANT", "work")
</p>
</div>
<Input
id="label-filter"
value={gmailOptions.label_filter}
onChange={(e) => handleOptionChange("label_filter", e.target.value)}
placeholder="Enter label name..."
className="bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
/>
</div>
{/* Search query */}
<div className="space-y-2 pt-2 border-t border-slate-400/20">
<div className="space-y-0.5">
<Label htmlFor="search-query" className="text-sm font-medium">
Search query (optional)
</Label>
<p className="text-xs text-muted-foreground">
Gmail search query to filter emails (e.g., "from:boss@company.com", "has:attachment")
</p>
</div>
<Input
id="search-query"
value={gmailOptions.search_query}
onChange={(e) => handleOptionChange("search_query", e.target.value)}
placeholder="Enter Gmail search query..."
className="bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
/>
</div>
</div>
</div>
);
};

8
surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx
View file

@ -6,7 +6,9 @@ import { BaiduSearchApiConfig } from "./components/baidu-search-api-config";
import { BookStackConfig } from "./components/bookstack-config";
import { CirclebackConfig } from "./components/circleback-config";
import { ClickUpConfig } from "./components/clickup-config";
import { ComposioConfig } from "./components/composio-config";
import { ComposioCalendarConfig } from "./components/composio-calendar-config";
import { ComposioDriveConfig } from "./components/composio-drive-config";
import { ComposioGmailConfig } from "./components/composio-gmail-config";
import { ConfluenceConfig } from "./components/confluence-config";
import { DiscordConfig } from "./components/discord-config";
import { ElasticsearchConfig } from "./components/elasticsearch-config";
@ -78,9 +80,11 @@ export function getConnectorConfigComponent(
case "OBSIDIAN_CONNECTOR":
return ObsidianConfig;
case "COMPOSIO_GOOGLE_DRIVE_CONNECTOR":
return ComposioDriveConfig;
case "COMPOSIO_GMAIL_CONNECTOR":
return ComposioGmailConfig;
case "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR":
return ComposioConfig;
return ComposioCalendarConfig;
// OAuth connectors (Gmail, Calendar, Airtable, Notion) and others don't need special config UI
default:
return null;

4
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx
View file

@ -206,8 +206,9 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
{/* Date range selector and periodic sync - only shown for indexable connectors */}
{connector.is_indexable && (
<>
{/* Date range selector - not shown for Google Drive, Webcrawler, or GitHub (indexes full repo snapshots) */}
{/* Date range selector - not shown for Google Drive (regular and Composio), Webcrawler, or GitHub (indexes full repo snapshots) */}
{connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" &&
connector.connector_type !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" &&
connector.connector_type !== "WEBCRAWLER_CONNECTOR" &&
connector.connector_type !== "GITHUB_CONNECTOR" && (
<DateRangeSelector
@ -217,6 +218,7 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
onEndDateChange={onEndDateChange}
allowFutureDates={
connector.connector_type === "GOOGLE_CALENDAR_CONNECTOR" ||
connector.connector_type === "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR" ||
connector.connector_type === "LUMA_CONNECTOR"
}
/>

20
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx
View file

@ -9,11 +9,7 @@ import { getConnectorTypeDisplay } from "@/lib/connectors/utils";
import { cn } from "@/lib/utils";
import { DateRangeSelector } from "../../components/date-range-selector";
import { PeriodicSyncConfig } from "../../components/periodic-sync-config";
import {
COMPOSIO_CONNECTORS,
type IndexingConfigState,
OAUTH_CONNECTORS,
} from "../../constants/connector-constants";
import type { IndexingConfigState } from "../../constants/connector-constants";
import { getConnectorDisplayName } from "../../tabs/all-connectors-tab";
import { getConnectorConfigComponent } from "../index";
@ -95,11 +91,6 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
};
}, [checkScrollState]);
// Check both OAUTH_CONNECTORS and COMPOSIO_CONNECTORS
const authConnector =
OAUTH_CONNECTORS.find((c) => c.connectorType === connector?.connector_type) ||
COMPOSIO_CONNECTORS.find((c) => c.connectorType === connector?.connector_type);
return (
<div className="flex-1 flex flex-col min-h-0 overflow-hidden">
{/* Fixed Header */}
@ -158,8 +149,9 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
{/* Date range selector and periodic sync - only shown for indexable connectors */}
{connector?.is_indexable && (
<>
{/* Date range selector - not shown for Google Drive, Webcrawler, or GitHub (indexes full repo snapshots) */}
{/* Date range selector - not shown for Google Drive (regular and Composio), Webcrawler, or GitHub (indexes full repo snapshots) */}
{config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" &&
config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" &&
config.connectorType !== "WEBCRAWLER_CONNECTOR" &&
config.connectorType !== "GITHUB_CONNECTOR" && (
<DateRangeSelector
@ -169,13 +161,15 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
onEndDateChange={onEndDateChange}
allowFutureDates={
config.connectorType === "GOOGLE_CALENDAR_CONNECTOR" ||
config.connectorType === "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR" ||
config.connectorType === "LUMA_CONNECTOR"
}
/>
)}
{/* Periodic sync - not shown for Google Drive */}
{config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && (
{/* Periodic sync - not shown for Google Drive (regular and Composio) */}
{config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" &&
config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && (
<PeriodicSyncConfig
enabled={periodicEnabled}
frequencyMinutes={frequencyMinutes}

113
surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts
View file

@ -330,56 +330,78 @@ export const useConnectorDialog = () => {
if (
params.success === "true" &&
params.connector &&
searchSpaceId &&
params.modal === "connectors"
) {
const oauthConnector = OAUTH_CONNECTORS.find((c) => c.id === params.connector);
if (oauthConnector) {
refetchAllConnectors().then((result) => {
if (!result.data) return;
refetchAllConnectors().then((result) => {
if (!result.data) return;
let newConnector: SearchSourceConnector | undefined;
if (params.connectorId) {
const connectorId = parseInt(params.connectorId, 10);
newConnector = result.data.find((c: SearchSourceConnector) => c.id === connectorId);
} else {
let newConnector: SearchSourceConnector | undefined;
let oauthConnector:
| (typeof OAUTH_CONNECTORS)[number]
| (typeof COMPOSIO_CONNECTORS)[number]
| undefined;
// First, try to find connector by connectorId if provided
if (params.connectorId) {
const connectorId = parseInt(params.connectorId, 10);
newConnector = result.data.find((c: SearchSourceConnector) => c.id === connectorId);
// If we found the connector, find the matching OAuth/Composio connector by type
if (newConnector) {
oauthConnector =
OAUTH_CONNECTORS.find(
(c) => c.connectorType === newConnector!.connector_type
) ||
COMPOSIO_CONNECTORS.find(
(c) => c.connectorType === newConnector!.connector_type
);
}
}
// If we don't have a connector yet, try to find by connector param
if (!newConnector && params.connector) {
oauthConnector =
OAUTH_CONNECTORS.find((c) => c.id === params.connector) ||
COMPOSIO_CONNECTORS.find((c) => c.id === params.connector);
if (oauthConnector) {
newConnector = result.data.find(
(c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType
(c: SearchSourceConnector) => c.connector_type === oauthConnector!.connectorType
);
}
}
if (newConnector) {
const connectorValidation = searchSourceConnector.safeParse(newConnector);
if (connectorValidation.success) {
// Track connector connected event for OAuth connectors
trackConnectorConnected(
Number(searchSpaceId),
oauthConnector.connectorType,
newConnector.id
);
if (newConnector && oauthConnector) {
const connectorValidation = searchSourceConnector.safeParse(newConnector);
if (connectorValidation.success) {
// Track connector connected event for OAuth/Composio connectors
trackConnectorConnected(
Number(searchSpaceId),
oauthConnector.connectorType,
newConnector.id
);
const config = validateIndexingConfigState({
connectorType: oauthConnector.connectorType,
connectorId: newConnector.id,
connectorTitle: oauthConnector.title,
});
setIndexingConfig(config);
setIndexingConnector(newConnector);
setIndexingConnectorConfig(newConnector.config);
setIsOpen(true);
const url = new URL(window.location.href);
url.searchParams.delete("success");
url.searchParams.set("connectorId", newConnector.id.toString());
url.searchParams.set("view", "configure");
window.history.replaceState({}, "", url.toString());
} else {
console.warn("Invalid connector data after OAuth:", connectorValidation.error);
toast.error("Failed to validate connector data");
}
const config = validateIndexingConfigState({
connectorType: oauthConnector.connectorType,
connectorId: newConnector.id,
connectorTitle: oauthConnector.title,
});
setIndexingConfig(config);
setIndexingConnector(newConnector);
setIndexingConnectorConfig(newConnector.config);
setIsOpen(true);
const url = new URL(window.location.href);
url.searchParams.delete("success");
url.searchParams.set("connectorId", newConnector.id.toString());
url.searchParams.set("view", "configure");
window.history.replaceState({}, "", url.toString());
} else {
console.warn("Invalid connector data after OAuth:", connectorValidation.error);
toast.error("Failed to validate connector data");
}
});
}
}
});
}
} catch (error) {
// Invalid query params - log but don't crash
@ -863,9 +885,10 @@ export const useConnectorDialog = () => {
async (refreshConnectors: () => void) => {
if (!indexingConfig || !searchSpaceId) return;
// Validate date range (skip for Google Drive and Webcrawler)
// Validate date range (skip for Google Drive, Composio Drive, and Webcrawler)
if (
indexingConfig.connectorType !== "GOOGLE_DRIVE_CONNECTOR" &&
indexingConfig.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" &&
indexingConfig.connectorType !== "WEBCRAWLER_CONNECTOR"
) {
const dateRangeValidation = dateRangeSchema.safeParse({ startDate, endDate });
@ -910,8 +933,12 @@ export const useConnectorDialog = () => {
});
}
// Handle Google Drive folder selection
if (indexingConfig.connectorType === "GOOGLE_DRIVE_CONNECTOR" && indexingConnectorConfig) {
// Handle Google Drive folder selection (regular and Composio)
if (
(indexingConfig.connectorType === "GOOGLE_DRIVE_CONNECTOR" ||
indexingConfig.connectorType === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR") &&
indexingConnectorConfig
) {
const selectedFolders = indexingConnectorConfig.selected_folders as
| Array<{ id: string; name: string }>
| undefined;

3
surfsense_web/lib/connectors/utils.ts
View file

@ -16,6 +16,9 @@ export const getConnectorTypeDisplay = (type: string): string => {
GOOGLE_CALENDAR_CONNECTOR: "Google Calendar",
GOOGLE_GMAIL_CONNECTOR: "Google Gmail",
GOOGLE_DRIVE_CONNECTOR: "Google Drive",
COMPOSIO_GOOGLE_DRIVE_CONNECTOR: "Google Drive",
COMPOSIO_GMAIL_CONNECTOR: "Gmail",
COMPOSIO_GOOGLE_CALENDAR_CONNECTOR: "Google Calendar",
AIRTABLE_CONNECTOR: "Airtable",
LUMA_CONNECTOR: "Luma",
ELASTICSEARCH_CONNECTOR: "Elasticsearch",