mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-22 23:31:12 +02:00
Consolidate connector metadata into NamedTuple and use helper functions
- Create ConnectorMetadata NamedTuple to consolidate emoji, result_name, and friendly_name - Replace three separate dictionaries with single CONNECTOR_METADATA dict - Update helper functions to use consolidated structure - Replace direct dict access in nodes.py with helper functions - Remove unused CONNECTOR_EMOJIS and CONNECTOR_RESULT_NAMES imports - Add default metadata for unknown connectors Addresses PR #16 review feedback for unused imports and data consolidation
This commit is contained in:
parent
0372fecdfa
commit
662cdf6692
2 changed files with 41 additions and 76 deletions
|
|
@ -21,8 +21,6 @@ from .prompts import get_further_questions_system_prompt
|
|||
from .qna_agent.graph import graph as qna_agent_graph
|
||||
from .state import State
|
||||
from .utils import (
|
||||
CONNECTOR_EMOJIS,
|
||||
CONNECTOR_RESULT_NAMES,
|
||||
get_connector_emoji,
|
||||
get_connector_friendly_name,
|
||||
get_connector_result_name,
|
||||
|
|
@ -843,8 +841,8 @@ async def fetch_relevant_documents(
|
|||
|
||||
# Stream found document count
|
||||
if streaming_service and writer and chunks:
|
||||
emoji = CONNECTOR_EMOJIS.get(connector, "📄")
|
||||
result_name = CONNECTOR_RESULT_NAMES.get(connector, "chunks")
|
||||
emoji = get_connector_emoji(connector)
|
||||
result_name = get_connector_result_name(connector)
|
||||
writer(
|
||||
{
|
||||
"yield_value": streaming_service.format_terminal_info_delta(
|
||||
|
|
|
|||
|
|
@ -14,93 +14,60 @@ class DocumentTokenInfo(NamedTuple):
|
|||
token_count: int
|
||||
|
||||
|
||||
class ConnectorMetadata(NamedTuple):
|
||||
"""Consolidated metadata for a connector type."""
|
||||
|
||||
emoji: str
|
||||
result_name: str
|
||||
friendly_name: str
|
||||
|
||||
|
||||
# Centralized connector metadata - single source of truth
|
||||
CONNECTOR_EMOJIS = {
|
||||
"YOUTUBE_VIDEO": "📹",
|
||||
"EXTENSION": "🧩",
|
||||
"CRAWLED_URL": "🌐",
|
||||
"FILE": "📄",
|
||||
"SLACK_CONNECTOR": "💬",
|
||||
"NOTION_CONNECTOR": "📘",
|
||||
"GITHUB_CONNECTOR": "🐙",
|
||||
"LINEAR_CONNECTOR": "📊",
|
||||
"TAVILY_API": "🔍",
|
||||
"SEARXNG_API": "🌐",
|
||||
"LINKUP_API": "🔗",
|
||||
"BAIDU_SEARCH_API": "🇨🇳",
|
||||
"DISCORD_CONNECTOR": "🗨️",
|
||||
"JIRA_CONNECTOR": "🎫",
|
||||
"GOOGLE_CALENDAR_CONNECTOR": "📅",
|
||||
"AIRTABLE_CONNECTOR": "🗃️",
|
||||
"GOOGLE_GMAIL_CONNECTOR": "📧",
|
||||
"CONFLUENCE_CONNECTOR": "📚",
|
||||
"CLICKUP_CONNECTOR": "📋",
|
||||
"LUMA_CONNECTOR": "🎯",
|
||||
"ELASTICSEARCH_CONNECTOR": "🔎",
|
||||
# All connector information is consolidated in one place for easier maintenance
|
||||
CONNECTOR_METADATA: dict[str, ConnectorMetadata] = {
|
||||
"YOUTUBE_VIDEO": ConnectorMetadata("📹", "YouTube chunks", "YouTube"),
|
||||
"EXTENSION": ConnectorMetadata("🧩", "Browser Extension chunks", "Browser Extension"),
|
||||
"CRAWLED_URL": ConnectorMetadata("🌐", "Web Pages chunks", "Web Pages"),
|
||||
"FILE": ConnectorMetadata("📄", "Files chunks", "Files"),
|
||||
"SLACK_CONNECTOR": ConnectorMetadata("💬", "Slack messages", "Slack"),
|
||||
"NOTION_CONNECTOR": ConnectorMetadata("📘", "Notion pages/blocks", "Notion"),
|
||||
"GITHUB_CONNECTOR": ConnectorMetadata("🐙", "GitHub files/issues", "GitHub"),
|
||||
"LINEAR_CONNECTOR": ConnectorMetadata("📊", "Linear issues", "Linear"),
|
||||
"TAVILY_API": ConnectorMetadata("🔍", "Web Search results", "Tavily Search"),
|
||||
"SEARXNG_API": ConnectorMetadata("🌐", "SearxNG results", "SearxNG Search"),
|
||||
"LINKUP_API": ConnectorMetadata("🔗", "Linkup results", "Linkup Search"),
|
||||
"BAIDU_SEARCH_API": ConnectorMetadata("🇨🇳", "Baidu Search results", "Baidu Search"),
|
||||
"DISCORD_CONNECTOR": ConnectorMetadata("🗨️", "Discord messages", "Discord"),
|
||||
"JIRA_CONNECTOR": ConnectorMetadata("🎫", "Jira issues", "Jira"),
|
||||
"GOOGLE_CALENDAR_CONNECTOR": ConnectorMetadata("📅", "calendar events", "Google Calendar"),
|
||||
"AIRTABLE_CONNECTOR": ConnectorMetadata("🗃️", "Airtable records", "Airtable"),
|
||||
"GOOGLE_GMAIL_CONNECTOR": ConnectorMetadata("📧", "Gmail messages", "Gmail"),
|
||||
"CONFLUENCE_CONNECTOR": ConnectorMetadata("📚", "Confluence pages", "Confluence"),
|
||||
"CLICKUP_CONNECTOR": ConnectorMetadata("📋", "ClickUp tasks", "ClickUp"),
|
||||
"LUMA_CONNECTOR": ConnectorMetadata("🎯", "Luma events", "Luma"),
|
||||
"ELASTICSEARCH_CONNECTOR": ConnectorMetadata("🔎", "Elasticsearch chunks", "Elasticsearch"),
|
||||
}
|
||||
|
||||
CONNECTOR_RESULT_NAMES = {
|
||||
"YOUTUBE_VIDEO": "YouTube chunks",
|
||||
"EXTENSION": "Browser Extension chunks",
|
||||
"CRAWLED_URL": "Web Pages chunks",
|
||||
"FILE": "Files chunks",
|
||||
"SLACK_CONNECTOR": "Slack messages",
|
||||
"NOTION_CONNECTOR": "Notion pages/blocks",
|
||||
"GITHUB_CONNECTOR": "GitHub files/issues",
|
||||
"LINEAR_CONNECTOR": "Linear issues",
|
||||
"TAVILY_API": "Web Search results",
|
||||
"SEARXNG_API": "SearxNG results",
|
||||
"LINKUP_API": "Linkup results",
|
||||
"BAIDU_SEARCH_API": "Baidu Search results",
|
||||
"DISCORD_CONNECTOR": "Discord messages",
|
||||
"JIRA_CONNECTOR": "Jira issues",
|
||||
"GOOGLE_CALENDAR_CONNECTOR": "calendar events",
|
||||
"AIRTABLE_CONNECTOR": "Airtable records",
|
||||
"GOOGLE_GMAIL_CONNECTOR": "Gmail messages",
|
||||
"CONFLUENCE_CONNECTOR": "Confluence pages",
|
||||
"CLICKUP_CONNECTOR": "ClickUp tasks",
|
||||
"LUMA_CONNECTOR": "Luma events",
|
||||
"ELASTICSEARCH_CONNECTOR": "Elasticsearch chunks",
|
||||
}
|
||||
|
||||
CONNECTOR_FRIENDLY_NAMES = {
|
||||
"YOUTUBE_VIDEO": "YouTube",
|
||||
"EXTENSION": "Browser Extension",
|
||||
"CRAWLED_URL": "Web Pages",
|
||||
"FILE": "Files",
|
||||
"SLACK_CONNECTOR": "Slack",
|
||||
"NOTION_CONNECTOR": "Notion",
|
||||
"GITHUB_CONNECTOR": "GitHub",
|
||||
"LINEAR_CONNECTOR": "Linear",
|
||||
"JIRA_CONNECTOR": "Jira",
|
||||
"CONFLUENCE_CONNECTOR": "Confluence",
|
||||
"GOOGLE_CALENDAR_CONNECTOR": "Google Calendar",
|
||||
"DISCORD_CONNECTOR": "Discord",
|
||||
"TAVILY_API": "Tavily Search",
|
||||
"SEARXNG_API": "SearxNG Search",
|
||||
"LINKUP_API": "Linkup Search",
|
||||
"BAIDU_SEARCH_API": "Baidu Search",
|
||||
"AIRTABLE_CONNECTOR": "Airtable",
|
||||
"GOOGLE_GMAIL_CONNECTOR": "Gmail",
|
||||
"CLICKUP_CONNECTOR": "ClickUp",
|
||||
"LUMA_CONNECTOR": "Luma",
|
||||
"ELASTICSEARCH_CONNECTOR": "Elasticsearch",
|
||||
}
|
||||
# Default metadata for unknown connectors
|
||||
_DEFAULT_METADATA = ConnectorMetadata("🔎", "chunks", "Unknown")
|
||||
|
||||
|
||||
def get_connector_emoji(connector_name: str) -> str:
|
||||
"""Get an appropriate emoji for a connector type."""
|
||||
return CONNECTOR_EMOJIS.get(connector_name, "🔎")
|
||||
metadata = CONNECTOR_METADATA.get(connector_name)
|
||||
return metadata.emoji if metadata else _DEFAULT_METADATA.emoji
|
||||
|
||||
|
||||
def get_connector_friendly_name(connector_name: str) -> str:
|
||||
"""Convert technical connector IDs to user-friendly names."""
|
||||
return CONNECTOR_FRIENDLY_NAMES.get(connector_name, connector_name)
|
||||
metadata = CONNECTOR_METADATA.get(connector_name)
|
||||
return metadata.friendly_name if metadata else connector_name
|
||||
|
||||
|
||||
def get_connector_result_name(connector_name: str) -> str:
|
||||
"""Get the result type name for a connector (e.g., 'Slack messages')."""
|
||||
return CONNECTOR_RESULT_NAMES.get(connector_name, "chunks")
|
||||
metadata = CONNECTOR_METADATA.get(connector_name)
|
||||
return metadata.result_name if metadata else _DEFAULT_METADATA.result_name
|
||||
|
||||
|
||||
def convert_langchain_messages_to_dict(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue