From 662cdf6692aad82ef3f3f594988eaa828a17ebf0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Nov 2025 23:25:37 +0000 Subject: [PATCH] Consolidate connector metadata into NamedTuple and use helper functions - Create ConnectorMetadata NamedTuple to consolidate emoji, result_name, and friendly_name - Replace three separate dictionaries with single CONNECTOR_METADATA dict - Update helper functions to use consolidated structure - Replace direct dict access in nodes.py with helper functions - Remove unused CONNECTOR_EMOJIS and CONNECTOR_RESULT_NAMES imports - Add default metadata for unknown connectors Addresses PR #16 review feedback for unused imports and data consolidation --- .../app/agents/researcher/nodes.py | 6 +- .../app/agents/researcher/utils.py | 111 ++++++------------ 2 files changed, 41 insertions(+), 76 deletions(-) diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index 12ca18467..5d01eecd9 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -21,8 +21,6 @@ from .prompts import get_further_questions_system_prompt from .qna_agent.graph import graph as qna_agent_graph from .state import State from .utils import ( - CONNECTOR_EMOJIS, - CONNECTOR_RESULT_NAMES, get_connector_emoji, get_connector_friendly_name, get_connector_result_name, @@ -843,8 +841,8 @@ async def fetch_relevant_documents( # Stream found document count if streaming_service and writer and chunks: - emoji = CONNECTOR_EMOJIS.get(connector, "📄") - result_name = CONNECTOR_RESULT_NAMES.get(connector, "chunks") + emoji = get_connector_emoji(connector) + result_name = get_connector_result_name(connector) writer( { "yield_value": streaming_service.format_terminal_info_delta( diff --git a/surfsense_backend/app/agents/researcher/utils.py b/surfsense_backend/app/agents/researcher/utils.py index 9fc303318..16ff0b0a7 100644 --- a/surfsense_backend/app/agents/researcher/utils.py +++ b/surfsense_backend/app/agents/researcher/utils.py @@ -14,93 +14,60 @@ class DocumentTokenInfo(NamedTuple): token_count: int +class ConnectorMetadata(NamedTuple): + """Consolidated metadata for a connector type.""" + + emoji: str + result_name: str + friendly_name: str + + # Centralized connector metadata - single source of truth -CONNECTOR_EMOJIS = { - "YOUTUBE_VIDEO": "📹", - "EXTENSION": "🧩", - "CRAWLED_URL": "🌐", - "FILE": "📄", - "SLACK_CONNECTOR": "💬", - "NOTION_CONNECTOR": "📘", - "GITHUB_CONNECTOR": "🐙", - "LINEAR_CONNECTOR": "📊", - "TAVILY_API": "🔍", - "SEARXNG_API": "🌐", - "LINKUP_API": "🔗", - "BAIDU_SEARCH_API": "🇨🇳", - "DISCORD_CONNECTOR": "🗨️", - "JIRA_CONNECTOR": "🎫", - "GOOGLE_CALENDAR_CONNECTOR": "📅", - "AIRTABLE_CONNECTOR": "🗃️", - "GOOGLE_GMAIL_CONNECTOR": "📧", - "CONFLUENCE_CONNECTOR": "📚", - "CLICKUP_CONNECTOR": "📋", - "LUMA_CONNECTOR": "🎯", - "ELASTICSEARCH_CONNECTOR": "🔎", +# All connector information is consolidated in one place for easier maintenance +CONNECTOR_METADATA: dict[str, ConnectorMetadata] = { + "YOUTUBE_VIDEO": ConnectorMetadata("📹", "YouTube chunks", "YouTube"), + "EXTENSION": ConnectorMetadata("🧩", "Browser Extension chunks", "Browser Extension"), + "CRAWLED_URL": ConnectorMetadata("🌐", "Web Pages chunks", "Web Pages"), + "FILE": ConnectorMetadata("📄", "Files chunks", "Files"), + "SLACK_CONNECTOR": ConnectorMetadata("💬", "Slack messages", "Slack"), + "NOTION_CONNECTOR": ConnectorMetadata("📘", "Notion pages/blocks", "Notion"), + "GITHUB_CONNECTOR": ConnectorMetadata("🐙", "GitHub files/issues", "GitHub"), + "LINEAR_CONNECTOR": ConnectorMetadata("📊", "Linear issues", "Linear"), + "TAVILY_API": ConnectorMetadata("🔍", "Web Search results", "Tavily Search"), + "SEARXNG_API": ConnectorMetadata("🌐", "SearxNG results", "SearxNG Search"), + "LINKUP_API": ConnectorMetadata("🔗", "Linkup results", "Linkup Search"), + "BAIDU_SEARCH_API": ConnectorMetadata("🇨🇳", "Baidu Search results", "Baidu Search"), + "DISCORD_CONNECTOR": ConnectorMetadata("🗨️", "Discord messages", "Discord"), + "JIRA_CONNECTOR": ConnectorMetadata("🎫", "Jira issues", "Jira"), + "GOOGLE_CALENDAR_CONNECTOR": ConnectorMetadata("📅", "calendar events", "Google Calendar"), + "AIRTABLE_CONNECTOR": ConnectorMetadata("🗃️", "Airtable records", "Airtable"), + "GOOGLE_GMAIL_CONNECTOR": ConnectorMetadata("📧", "Gmail messages", "Gmail"), + "CONFLUENCE_CONNECTOR": ConnectorMetadata("📚", "Confluence pages", "Confluence"), + "CLICKUP_CONNECTOR": ConnectorMetadata("📋", "ClickUp tasks", "ClickUp"), + "LUMA_CONNECTOR": ConnectorMetadata("🎯", "Luma events", "Luma"), + "ELASTICSEARCH_CONNECTOR": ConnectorMetadata("🔎", "Elasticsearch chunks", "Elasticsearch"), } -CONNECTOR_RESULT_NAMES = { - "YOUTUBE_VIDEO": "YouTube chunks", - "EXTENSION": "Browser Extension chunks", - "CRAWLED_URL": "Web Pages chunks", - "FILE": "Files chunks", - "SLACK_CONNECTOR": "Slack messages", - "NOTION_CONNECTOR": "Notion pages/blocks", - "GITHUB_CONNECTOR": "GitHub files/issues", - "LINEAR_CONNECTOR": "Linear issues", - "TAVILY_API": "Web Search results", - "SEARXNG_API": "SearxNG results", - "LINKUP_API": "Linkup results", - "BAIDU_SEARCH_API": "Baidu Search results", - "DISCORD_CONNECTOR": "Discord messages", - "JIRA_CONNECTOR": "Jira issues", - "GOOGLE_CALENDAR_CONNECTOR": "calendar events", - "AIRTABLE_CONNECTOR": "Airtable records", - "GOOGLE_GMAIL_CONNECTOR": "Gmail messages", - "CONFLUENCE_CONNECTOR": "Confluence pages", - "CLICKUP_CONNECTOR": "ClickUp tasks", - "LUMA_CONNECTOR": "Luma events", - "ELASTICSEARCH_CONNECTOR": "Elasticsearch chunks", -} - -CONNECTOR_FRIENDLY_NAMES = { - "YOUTUBE_VIDEO": "YouTube", - "EXTENSION": "Browser Extension", - "CRAWLED_URL": "Web Pages", - "FILE": "Files", - "SLACK_CONNECTOR": "Slack", - "NOTION_CONNECTOR": "Notion", - "GITHUB_CONNECTOR": "GitHub", - "LINEAR_CONNECTOR": "Linear", - "JIRA_CONNECTOR": "Jira", - "CONFLUENCE_CONNECTOR": "Confluence", - "GOOGLE_CALENDAR_CONNECTOR": "Google Calendar", - "DISCORD_CONNECTOR": "Discord", - "TAVILY_API": "Tavily Search", - "SEARXNG_API": "SearxNG Search", - "LINKUP_API": "Linkup Search", - "BAIDU_SEARCH_API": "Baidu Search", - "AIRTABLE_CONNECTOR": "Airtable", - "GOOGLE_GMAIL_CONNECTOR": "Gmail", - "CLICKUP_CONNECTOR": "ClickUp", - "LUMA_CONNECTOR": "Luma", - "ELASTICSEARCH_CONNECTOR": "Elasticsearch", -} +# Default metadata for unknown connectors +_DEFAULT_METADATA = ConnectorMetadata("🔎", "chunks", "Unknown") def get_connector_emoji(connector_name: str) -> str: """Get an appropriate emoji for a connector type.""" - return CONNECTOR_EMOJIS.get(connector_name, "🔎") + metadata = CONNECTOR_METADATA.get(connector_name) + return metadata.emoji if metadata else _DEFAULT_METADATA.emoji def get_connector_friendly_name(connector_name: str) -> str: """Convert technical connector IDs to user-friendly names.""" - return CONNECTOR_FRIENDLY_NAMES.get(connector_name, connector_name) + metadata = CONNECTOR_METADATA.get(connector_name) + return metadata.friendly_name if metadata else connector_name def get_connector_result_name(connector_name: str) -> str: """Get the result type name for a connector (e.g., 'Slack messages').""" - return CONNECTOR_RESULT_NAMES.get(connector_name, "chunks") + metadata = CONNECTOR_METADATA.get(connector_name) + return metadata.result_name if metadata else _DEFAULT_METADATA.result_name def convert_langchain_messages_to_dict(