apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-03 04:42:39 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Webcrawler connector draft

Browse source
This commit is contained in:
samkul-swe 2025-11-21 20:45:59 -08:00
parent 419f94e8ee
commit 896e410e2a
26 changed files with 1225 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

4
surfsense_backend/app/agents/researcher/nodes.py
View file

@ -667,7 +667,7 @@ async def fetch_relevant_documents(
}
)
elif connector == "CRAWLED_URL":
elif connector == "WEBCRAWLER_CONNECTOR":
(
source_object,
crawled_urls_chunks,
@ -689,7 +689,7 @@ async def fetch_relevant_documents(
writer(
{
"yield_value": streaming_service.format_terminal_info_delta(
f"🌐 Found {len(crawled_urls_chunks)} Web Pages chunks related to your query"
f"🌐 Found {len(crawled_urls_chunks)} Web Page chunks related to your query"
)
}
)

2
surfsense_backend/app/agents/researcher/qna_agent/default_prompts.py
View file

@ -17,7 +17,6 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel
{chat_history_section}
<knowledge_sources>
- EXTENSION: "Web content saved via SurfSense browser extension" (personal browsing history)
- CRAWLED_URL: "Webpages indexed by SurfSense web crawler" (personally selected websites)
- FILE: "User-uploaded documents (PDFs, Word, etc.)" (personal files)
- SLACK_CONNECTOR: "Slack conversations and shared content" (personal workspace communications)
- NOTION_CONNECTOR: "Notion workspace pages and databases" (personal knowledge management)
@ -35,6 +34,7 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel
- TAVILY_API: "Tavily search API results" (personalized search results)
- LINKUP_API: "Linkup search API results" (personalized search results)
- LUMA_CONNECTOR: "Luma events"
- WEBCRAWLER_CONNECTOR: "Webpages indexed by SurfSense web crawler" (personally selected websites)
</knowledge_sources>
<instructions>

4
surfsense_backend/app/agents/researcher/utils.py
View file

@ -19,7 +19,6 @@ def get_connector_emoji(connector_name: str) -> str:
connector_emojis = {
"YOUTUBE_VIDEO": "📹",
"EXTENSION": "🧩",
"CRAWLED_URL": "🌐",
"FILE": "📄",
"SLACK_CONNECTOR": "💬",
"NOTION_CONNECTOR": "📘",
@ -34,6 +33,7 @@ def get_connector_emoji(connector_name: str) -> str:
"AIRTABLE_CONNECTOR": "🗃️",
"LUMA_CONNECTOR": "✨",
"ELASTICSEARCH_CONNECTOR": "⚡",
"WEBCRAWLER_CONNECTOR": "🌐",
}
return connector_emojis.get(connector_name, "🔎")
@ -43,7 +43,6 @@ def get_connector_friendly_name(connector_name: str) -> str:
connector_friendly_names = {
"YOUTUBE_VIDEO": "YouTube",
"EXTENSION": "Browser Extension",
"CRAWLED_URL": "Web Pages",
"FILE": "Files",
"SLACK_CONNECTOR": "Slack",
"NOTION_CONNECTOR": "Notion",
@ -59,6 +58,7 @@ def get_connector_friendly_name(connector_name: str) -> str:
"AIRTABLE_CONNECTOR": "Airtable",
"LUMA_CONNECTOR": "Luma",
"ELASTICSEARCH_CONNECTOR": "Elasticsearch",
"WEBCRAWLER_CONNECTOR": "Web Pages",
}
return connector_friendly_names.get(connector_name, connector_name)