apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-04-25 00:36:31 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix: treat all Gmail/Calendar as live connectors, hide indexing UI

Browse source
This commit is contained in:
CREDO23 2026-04-22 22:07:55 +02:00
parent 9977f9b641
commit b6c506abef
10 changed files with 115 additions and 126 deletions
Download patch file Download diff file Expand all files Collapse all files

31
surfsense_backend/app/agents/new_chat/tools/gmail/search_emails.py
View file

@ -15,10 +15,30 @@ _GMAIL_TYPES = [
SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR,
]
_token_encryption_cache: object | None = None
def _get_token_encryption():
global _token_encryption_cache
if _token_encryption_cache is None:
from app.config import config
from app.utils.oauth_security import TokenEncryption
if not config.SECRET_KEY:
raise RuntimeError("SECRET_KEY not configured for token decryption.")
_token_encryption_cache = TokenEncryption(config.SECRET_KEY)
return _token_encryption_cache
def _build_credentials(connector: SearchSourceConnector):
"""Build Google OAuth Credentials from a Gmail connector's config."""
if connector.connector_type == SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR:
"""Build Google OAuth Credentials from a connector's stored config.
Handles both native OAuth connectors (with encrypted tokens) and
Composio-backed connectors. Shared by Gmail and Calendar tools.
"""
from app.utils.google_credentials import COMPOSIO_GOOGLE_CONNECTOR_TYPES
if connector.connector_type in COMPOSIO_GOOGLE_CONNECTOR_TYPES:
from app.utils.google_credentials import build_composio_credentials
cca_id = connector.config.get("composio_connected_account_id")
@ -28,12 +48,9 @@ def _build_credentials(connector: SearchSourceConnector):
from google.oauth2.credentials import Credentials
from app.config import config
from app.utils.oauth_security import TokenEncryption
cfg = dict(connector.config)
if cfg.get("_token_encrypted") and config.SECRET_KEY:
enc = TokenEncryption(config.SECRET_KEY)
if cfg.get("_token_encrypted"):
enc = _get_token_encryption()
for key in ("token", "refresh_token", "client_secret"):
if cfg.get(key):
cfg[key] = enc.decrypt_token(cfg[key])

36
surfsense_backend/app/agents/new_chat/tools/google_calendar/search_events.py
View file

@ -1,11 +1,11 @@
import logging
from datetime import datetime
from typing import Any
from langchain_core.tools import tool
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.agents.new_chat.tools.gmail.search_emails import _build_credentials
from app.db import SearchSourceConnector, SearchSourceConnectorType
logger = logging.getLogger(__name__)
@ -16,40 +16,6 @@ _CALENDAR_TYPES = [
]
def _build_credentials(connector: SearchSourceConnector):
"""Build Google OAuth Credentials from a Calendar connector's config."""
if connector.connector_type == SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR:
from app.utils.google_credentials import build_composio_credentials
cca_id = connector.config.get("composio_connected_account_id")
if not cca_id:
raise ValueError("Composio connected account ID not found.")
return build_composio_credentials(cca_id)
from google.oauth2.credentials import Credentials
from app.config import config
from app.utils.oauth_security import TokenEncryption
cfg = dict(connector.config)
if cfg.get("_token_encrypted") and config.SECRET_KEY:
enc = TokenEncryption(config.SECRET_KEY)
for key in ("token", "refresh_token", "client_secret"):
if cfg.get(key):
cfg[key] = enc.decrypt_token(cfg[key])
exp = (cfg.get("expiry") or "").replace("Z", "")
return Credentials(
token=cfg.get("token"),
refresh_token=cfg.get("refresh_token"),
token_uri=cfg.get("token_uri"),
client_id=cfg.get("client_id"),
client_secret=cfg.get("client_secret"),
scopes=cfg.get("scopes", []),
expiry=datetime.fromisoformat(exp) if exp else None,
)
def create_search_calendar_events_tool(
db_session: AsyncSession | None = None,
search_space_id: int | None = None,

16
surfsense_backend/app/routes/search_source_connectors_routes.py
View file

@ -777,19 +777,9 @@ async def index_connector_content(
# For non-calendar connectors, cap at today
indexing_to = end_date if end_date else today_str
_LIVE_CONNECTOR_TYPES = {
SearchSourceConnectorType.SLACK_CONNECTOR,
SearchSourceConnectorType.TEAMS_CONNECTOR,
SearchSourceConnectorType.LINEAR_CONNECTOR,
SearchSourceConnectorType.JIRA_CONNECTOR,
SearchSourceConnectorType.CLICKUP_CONNECTOR,
SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR,
SearchSourceConnectorType.AIRTABLE_CONNECTOR,
SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR,
SearchSourceConnectorType.DISCORD_CONNECTOR,
SearchSourceConnectorType.LUMA_CONNECTOR,
}
if connector.connector_type in _LIVE_CONNECTOR_TYPES:
from app.services.mcp_oauth.registry import LIVE_CONNECTOR_TYPES
if connector.connector_type in LIVE_CONNECTOR_TYPES:
return {
"message": (
f"{connector.connector_type.value} uses real-time agent tools; "

2
surfsense_backend/app/services/composio_service.py
View file

@ -26,7 +26,7 @@ COMPOSIO_TOOLKIT_NAMES = {
}
# Toolkits that support indexing (Phase 1: Google services only)
INDEXABLE_TOOLKITS = {"googledrive", "gmail", "googlecalendar"}
INDEXABLE_TOOLKITS = {"googledrive"}
# Mapping of toolkit IDs to connector types
TOOLKIT_TO_CONNECTOR_TYPE = {

17
surfsense_backend/app/services/mcp_oauth/registry.py
View file

@ -16,6 +16,8 @@ from __future__ import annotations
from dataclasses import dataclass, field
from app.db import SearchSourceConnectorType
@dataclass(frozen=True)
class MCPServiceConfig:
@ -134,6 +136,21 @@ _CONNECTOR_TYPE_TO_SERVICE: dict[str, MCPServiceConfig] = {
svc.connector_type: svc for svc in MCP_SERVICES.values()
}
LIVE_CONNECTOR_TYPES: frozenset[SearchSourceConnectorType] = frozenset({
SearchSourceConnectorType.SLACK_CONNECTOR,
SearchSourceConnectorType.TEAMS_CONNECTOR,
SearchSourceConnectorType.LINEAR_CONNECTOR,
SearchSourceConnectorType.JIRA_CONNECTOR,
SearchSourceConnectorType.CLICKUP_CONNECTOR,
SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR,
SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR,
SearchSourceConnectorType.AIRTABLE_CONNECTOR,
SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR,
SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR,
SearchSourceConnectorType.DISCORD_CONNECTOR,
SearchSourceConnectorType.LUMA_CONNECTOR,
})
def get_service(key: str) -> MCPServiceConfig | None:
return MCP_SERVICES.get(key)

43
surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py
View file

@ -59,9 +59,7 @@ async def _check_and_trigger_schedules():
index_crawled_urls_task,
index_elasticsearch_documents_task,
index_github_repos_task,
index_google_calendar_events_task,
index_google_drive_files_task,
index_google_gmail_messages_task,
index_notion_pages_task,
)
@ -73,34 +71,29 @@ async def _check_and_trigger_schedules():
SearchSourceConnectorType.WEBCRAWLER_CONNECTOR: index_crawled_urls_task,
SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR: index_google_drive_files_task,
SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR: index_google_drive_files_task,
SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR: index_google_gmail_messages_task,
SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR: index_google_calendar_events_task,
}
_LIVE_CONNECTOR_TYPES = {
SearchSourceConnectorType.SLACK_CONNECTOR,
SearchSourceConnectorType.TEAMS_CONNECTOR,
SearchSourceConnectorType.LINEAR_CONNECTOR,
SearchSourceConnectorType.JIRA_CONNECTOR,
SearchSourceConnectorType.CLICKUP_CONNECTOR,
SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR,
SearchSourceConnectorType.AIRTABLE_CONNECTOR,
SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR,
SearchSourceConnectorType.DISCORD_CONNECTOR,
SearchSourceConnectorType.LUMA_CONNECTOR,
}
from app.services.mcp_oauth.registry import LIVE_CONNECTOR_TYPES
# Disable obsolete periodic indexing for live connectors in one batch.
live_disabled = []
for connector in due_connectors:
if connector.connector_type in LIVE_CONNECTOR_TYPES:
connector.periodic_indexing_enabled = False
connector.next_scheduled_at = None
live_disabled.append(connector)
if live_disabled:
await session.commit()
for c in live_disabled:
logger.info(
"Disabled obsolete periodic indexing for live connector %s (%s)",
c.id,
c.connector_type.value,
)
# Trigger indexing for each due connector
for connector in due_connectors:
if connector.connector_type in _LIVE_CONNECTOR_TYPES:
connector.periodic_indexing_enabled = False
connector.next_scheduled_at = None
await session.commit()
logger.info(
"Disabled obsolete periodic indexing for live connector %s (%s)",
connector.id,
connector.connector_type.value,
)
if connector in live_disabled:
continue
# Primary guard: Redis lock indicates a task is currently running.

14
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx
View file

@ -236,8 +236,8 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
</p>
</div>
</div>
{/* Quick Index Button - hidden when auth is expired */}
{connector.is_indexable && onQuickIndex && !isAuthExpired && (
{/* Quick Index Button - hidden for live connectors and when auth is expired */}
{connector.is_indexable && !isLive && onQuickIndex && !isAuthExpired && (
<Button
variant="secondary"
size="sm"
@ -280,8 +280,8 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
/>
)}
{/* Summary and sync settings - only shown for indexable connectors */}
{connector.is_indexable && (
{/* Summary and sync settings - hidden for live connectors */}
{connector.is_indexable && !isLive && (
<>
{/* AI Summary toggle */}
<SummaryConfig enabled={enableSummary} onEnabledChange={onEnableSummaryChange} />
@ -352,8 +352,8 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
</>
)}
{/* Info box - only shown for indexable connectors */}
{connector.is_indexable && (
{/* Info box - hidden for live connectors */}
{connector.is_indexable && !isLive && (
<div className="rounded-xl border border-border bg-primary/5 p-4 flex items-start gap-3">
<div className="flex h-8 w-8 items-center justify-center rounded-lg bg-primary/10 shrink-0 mt-0.5">
<Info className="size-4" />
@ -432,7 +432,7 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
<RefreshCw className={cn("size-3.5", reauthing && "animate-spin")} />
Re-authenticate
</Button>
) : !isMCPBacked ? (
) : !isLive ? (
<Button
onClick={onSave}
disabled={isSaving || isDisconnecting}

38
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx
View file

@ -11,7 +11,7 @@ import { DateRangeSelector } from "../../components/date-range-selector";
import { PeriodicSyncConfig } from "../../components/periodic-sync-config";
import { SummaryConfig } from "../../components/summary-config";
import { VisionLLMConfig } from "../../components/vision-llm-config";
import type { IndexingConfigState } from "../../constants/connector-constants";
import { LIVE_CONNECTOR_TYPES, type IndexingConfigState } from "../../constants/connector-constants";
import { getConnectorDisplayName } from "../../tabs/all-connectors-tab";
import { getConnectorConfigComponent } from "../index";
@ -58,6 +58,8 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
onStartIndexing,
onSkip,
}) => {
const isLive = LIVE_CONNECTOR_TYPES.has(config.connectorType);
// Get connector-specific config component
const ConnectorConfigComponent = useMemo(
() => (connector ? getConnectorConfigComponent(connector.connector_type) : null),
@ -138,7 +140,9 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
)}
</div>
<p className="text-xs sm:text-base text-muted-foreground mt-1">
Configure when to start syncing your data
{isLive
? "Your account is ready to use"
: "Configure when to start syncing your data"}
</p>
</div>
</div>
@ -157,8 +161,8 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
<ConnectorConfigComponent connector={connector} onConfigChange={onConfigChange} />
)}
{/* Summary and sync settings - only shown for indexable connectors */}
{connector?.is_indexable && (
{/* Summary and sync settings - hidden for live connectors */}
{connector?.is_indexable && !isLive && (
<>
{/* AI Summary toggle */}
<SummaryConfig enabled={enableSummary} onEnabledChange={onEnableSummaryChange} />
@ -209,8 +213,8 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
</>
)}
{/* Info box - only shown for indexable connectors */}
{connector?.is_indexable && (
{/* Info box - hidden for live connectors */}
{connector?.is_indexable && !isLive && (
<div className="rounded-xl border border-border bg-primary/5 p-4 flex items-start gap-3">
<div className="flex h-8 w-8 items-center justify-center rounded-lg bg-primary/10 shrink-0 mt-0.5">
<Info className="size-4" />
@ -238,14 +242,20 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
{/* Fixed Footer - Action buttons */}
<div className="flex-shrink-0 flex items-center justify-end px-6 sm:px-12 py-6 bg-muted">
<Button
onClick={onStartIndexing}
disabled={isStartingIndexing}
className="text-xs sm:text-sm relative"
>
<span className={isStartingIndexing ? "opacity-0" : ""}>Start Indexing</span>
{isStartingIndexing && <Spinner size="sm" className="absolute" />}
</Button>
{isLive ? (
<Button onClick={onSkip} className="text-xs sm:text-sm">
Done
</Button>
) : (
<Button
onClick={onStartIndexing}
disabled={isStartingIndexing}
className="text-xs sm:text-sm relative"
>
<span className={isStartingIndexing ? "opacity-0" : ""}>Start Indexing</span>
{isStartingIndexing && <Spinner size="sm" className="absolute" />}
</Button>
)}
</div>
</div>
);

36
surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts
View file

@ -13,7 +13,9 @@ export const LIVE_CONNECTOR_TYPES = new Set<string>([
EnumConnectorName.DISCORD_CONNECTOR,
EnumConnectorName.TEAMS_CONNECTOR,
EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR,
EnumConnectorName.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR,
EnumConnectorName.GOOGLE_GMAIL_CONNECTOR,
EnumConnectorName.COMPOSIO_GMAIL_CONNECTOR,
EnumConnectorName.LUMA_CONNECTOR,
]);
@ -30,7 +32,7 @@ export const OAUTH_CONNECTORS = [
{
id: "google-gmail-connector",
title: "Gmail",
description: "Search and read your emails",
description: "Search, read, draft, and send emails",
connectorType: EnumConnectorName.GOOGLE_GMAIL_CONNECTOR,
authEndpoint: "/api/v1/auth/google/gmail/connector/add/",
selfHostedOnly: true,
@ -46,7 +48,7 @@ export const OAUTH_CONNECTORS = [
{
id: "airtable-connector",
title: "Airtable",
description: "Search, read, and manage records",
description: "Browse bases, tables, and records",
connectorType: EnumConnectorName.AIRTABLE_CONNECTOR,
authEndpoint: "/api/v1/auth/mcp/airtable/connector/add/",
},
@ -67,7 +69,7 @@ export const OAUTH_CONNECTORS = [
{
id: "slack-connector",
title: "Slack",
description: "Search, read, and send messages",
description: "Search and read channels and threads",
connectorType: EnumConnectorName.SLACK_CONNECTOR,
authEndpoint: "/api/v1/auth/mcp/slack/connector/add/",
},
@ -116,7 +118,7 @@ export const OAUTH_CONNECTORS = [
{
id: "clickup-connector",
title: "ClickUp",
description: "Search, read, and manage tasks",
description: "Search and read tasks",
connectorType: EnumConnectorName.CLICKUP_CONNECTOR,
authEndpoint: "/api/v1/auth/mcp/clickup/connector/add/",
},
@ -155,7 +157,7 @@ export const OTHER_CONNECTORS = [
{
id: "luma-connector",
title: "Luma",
description: "Search and manage events",
description: "Browse, read, and create events",
connectorType: EnumConnectorName.LUMA_CONNECTOR,
},
{
@ -214,14 +216,14 @@ export const COMPOSIO_CONNECTORS = [
{
id: "composio-gmail",
title: "Gmail",
description: "Search through your emails via Composio",
description: "Search, read, draft, and send emails via Composio",
connectorType: EnumConnectorName.COMPOSIO_GMAIL_CONNECTOR,
authEndpoint: "/api/v1/auth/composio/connector/add/?toolkit_id=gmail",
},
{
id: "composio-googlecalendar",
title: "Google Calendar",
description: "Search through your events via Composio",
description: "Search and manage your events via Composio",
connectorType: EnumConnectorName.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR,
authEndpoint: "/api/v1/auth/composio/connector/add/?toolkit_id=googlecalendar",
},
@ -238,14 +240,14 @@ export const COMPOSIO_TOOLKITS = [
{
id: "gmail",
name: "Gmail",
description: "Search through your emails",
isIndexable: true,
description: "Search, read, draft, and send emails",
isIndexable: false,
},
{
id: "googlecalendar",
name: "Google Calendar",
description: "Search through your events",
isIndexable: true,
description: "Search and manage your events",
isIndexable: false,
},
{
id: "slack",
@ -275,18 +277,6 @@ export interface AutoIndexConfig {
}
export const AUTO_INDEX_DEFAULTS: Record<string, AutoIndexConfig> = {
[EnumConnectorName.COMPOSIO_GMAIL_CONNECTOR]: {
daysBack: 30,
daysForward: 0,
frequencyMinutes: 1440,
syncDescription: "Syncing your last 30 days of emails.",
},
[EnumConnectorName.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR]: {
daysBack: 90,
daysForward: 90,
frequencyMinutes: 1440,
syncDescription: "Syncing 90 days of past and upcoming events.",
},
[EnumConnectorName.NOTION_CONNECTOR]: {
daysBack: 365,
daysForward: 0,

8
surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts
View file

@ -38,6 +38,7 @@ import {
AUTO_INDEX_CONNECTOR_TYPES,
AUTO_INDEX_DEFAULTS,
COMPOSIO_CONNECTORS,
LIVE_CONNECTOR_TYPES,
OAUTH_CONNECTORS,
OTHER_CONNECTORS,
} from "../constants/connector-constants";
@ -317,7 +318,12 @@ export const useConnectorDialog = () => {
newConnector.id
);
if (
const isLiveConnector = LIVE_CONNECTOR_TYPES.has(oauthConnector.connectorType);
if (isLiveConnector) {
toast.success(`${oauthConnector.title} connected successfully!`);
await refetchAllConnectors();
} else if (
newConnector.is_indexable &&
AUTO_INDEX_CONNECTOR_TYPES.has(oauthConnector.connectorType)
) {