apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-02 20:32:39 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

chore: merge upstream with local feature additions

Browse source 
- Merged dexscreener connector, composio connectors, crypto realtime tools from upstream
- Kept local additions: dropbox/onedrive connectors, memory routes, model_list routes, RefreshToken model
- Resolved frontend conflicts: kept tool UIs from both sides
- Accepted upstream lock files (uv.lock, pnpm-lock.yaml)
This commit is contained in:
Vonic 2026-04-13 23:31:52 +07:00
commit 6e86cd7e8a
803 changed files with 152168 additions and 14005 deletions
Download patch file Download diff file Expand all files Collapse all files

2
surfsense_backend/app/agents/new_chat/chat_deepagent.py
View file

@ -92,6 +92,8 @@ _CONNECTOR_TYPE_TO_SEARCHABLE: dict[str, str] = {
"COMPOSIO_GOOGLE_DRIVE_CONNECTOR": "GOOGLE_DRIVE_FILE",
"COMPOSIO_GMAIL_CONNECTOR": "GOOGLE_GMAIL_CONNECTOR",
"COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": "GOOGLE_CALENDAR_CONNECTOR",
# Cryptocurrency data
"DEXSCREENER_CONNECTOR": "DEXSCREENER_CONNECTOR",
}
# Document types that don't come from SearchSourceConnector but should always be searchable

14
surfsense_backend/app/agents/new_chat/tools/__init__.py
View file

@ -11,10 +11,19 @@ Available tools:
- generate_image: Generate images from text descriptions using AI models
- scrape_webpage: Extract content from webpages
- update_memory: Update the user's / team's memory document
- save_memory: Store facts/preferences about the user
- recall_memory: Retrieve relevant user memories
- get_live_token_price: Get real-time crypto price from DexScreener
- get_live_token_data: Get comprehensive real-time crypto market data
"""
# Registry exports
# Tool factory exports (for direct use)
from .crypto_realtime import (
create_get_live_token_data_tool,
create_get_live_token_price_tool,
)
from .display_image import create_display_image_tool
from .generate_image import create_generate_image_tool
from .knowledge_base import (
CONNECTOR_DESCRIPTIONS,
@ -46,6 +55,11 @@ __all__ = [
"create_generate_image_tool",
"create_generate_podcast_tool",
"create_generate_video_presentation_tool",
"create_get_live_token_data_tool",
"create_get_live_token_price_tool",
"create_link_preview_tool",
"create_recall_memory_tool",
"create_save_memory_tool",
"create_scrape_webpage_tool",
"create_search_surfsense_docs_tool",
"create_update_memory_tool",

322
surfsense_backend/app/agents/new_chat/tools/crypto_realtime.py Normal file
View file

@ -0,0 +1,322 @@
"""
Real-time cryptocurrency data tools for the SurfSense agent.
This module provides tools for fetching LIVE crypto data directly from DexScreener API.
These tools complement the RAG-based search_knowledge_base tool:
- RAG (search_knowledge_base): Historical context, trends, analysis from indexed data
- Real-time tools: Current prices, live market data
The AI agent decides which to use based on the query:
- "What's the current price of BULLA?" → get_live_token_price (real-time)
- "How has BULLA performed this week?" → search_knowledge_base (RAG)
- "Analyze BULLA for me" → Both (RAG for context + real-time for current data)
"""
import hashlib
import logging
from typing import Any
from langchain_core.tools import tool
from app.connectors.dexscreener_connector import DexScreenerConnector
logger = logging.getLogger(__name__)
def generate_token_id(chain: str, address: str) -> str:
"""Generate a unique ID for a token query."""
hash_val = hashlib.md5(f"{chain}:{address}".encode()).hexdigest()[:12]
return f"token-{hash_val}"
def create_get_live_token_price_tool():
"""
Factory function to create the get_live_token_price tool.
This tool fetches REAL-TIME price data directly from DexScreener API.
Use this when users ask for current/live prices.
Returns:
A configured tool function for fetching live token prices.
"""
@tool
async def get_live_token_price(
chain: str,
token_address: str,
token_symbol: str | None = None,
) -> dict[str, Any]:
"""
Get the LIVE/CURRENT price of a cryptocurrency token from DexScreener.
Use this tool when the user asks for:
- Current price: "What's the price of BULLA right now?"
- Live data: "Show me live price for SOL"
- Real-time info: "What's WETH trading at?"
DO NOT use this for historical analysis - use search_knowledge_base instead.
Args:
chain: Blockchain network (e.g., 'solana', 'ethereum', 'base', 'bsc')
token_address: The token's contract address
token_symbol: Optional token symbol for display (e.g., 'BULLA', 'SOL')
Returns:
Dictionary with live price data including:
- price_usd: Current price in USD
- price_change_24h: 24-hour price change percentage
- price_change_1h: 1-hour price change percentage
- volume_24h: 24-hour trading volume
- liquidity_usd: Total liquidity in USD
- market_cap: Market capitalization
- dex: DEX where the best liquidity is found
- pair_url: Link to DexScreener chart
"""
token_id = generate_token_id(chain, token_address)
try:
# Initialize DexScreener connector
connector = DexScreenerConnector()
# Fetch live data from API
pairs, error = await connector.get_token_pairs(chain, token_address)
if error:
logger.warning(f"[get_live_token_price] Error: {error}")
return {
"id": token_id,
"kind": "live_token_price",
"chain": chain,
"token_address": token_address,
"token_symbol": token_symbol,
"error": error,
}
if not pairs:
return {
"id": token_id,
"kind": "live_token_price",
"chain": chain,
"token_address": token_address,
"token_symbol": token_symbol,
"error": f"No trading pairs found for {token_symbol or token_address} on {chain}",
}
# Get the best pair (highest liquidity)
best_pair = max(pairs, key=lambda p: float(p.get("liquidity", {}).get("usd", 0) or 0))
# Extract data from best pair
base_token = best_pair.get("baseToken", {})
price_change = best_pair.get("priceChange", {})
volume = best_pair.get("volume", {})
liquidity = best_pair.get("liquidity", {})
return {
"id": token_id,
"kind": "live_token_price",
"chain": chain,
"token_address": token_address,
"token_symbol": token_symbol or base_token.get("symbol", "Unknown"),
"token_name": base_token.get("name", "Unknown"),
"price_usd": best_pair.get("priceUsd", "N/A"),
"price_native": best_pair.get("priceNative", "N/A"),
"price_change_5m": price_change.get("m5", 0),
"price_change_1h": price_change.get("h1", 0),
"price_change_6h": price_change.get("h6", 0),
"price_change_24h": price_change.get("h24", 0),
"volume_24h": volume.get("h24", 0),
"volume_6h": volume.get("h6", 0),
"volume_1h": volume.get("h1", 0),
"liquidity_usd": liquidity.get("usd", 0),
"market_cap": best_pair.get("marketCap", 0),
"fdv": best_pair.get("fdv", 0),
"dex": best_pair.get("dexId", "Unknown"),
"pair_address": best_pair.get("pairAddress", ""),
"pair_url": best_pair.get("url", ""),
"total_pairs": len(pairs),
"data_source": "DexScreener API (Real-time)",
}
except Exception as e:
error_message = str(e)
logger.error(f"[get_live_token_price] Error fetching {chain}/{token_address}: {error_message}")
return {
"id": token_id,
"kind": "live_token_price",
"chain": chain,
"token_address": token_address,
"token_symbol": token_symbol,
"error": f"Failed to fetch live price: {error_message[:100]}",
}
return get_live_token_price
def create_get_live_token_data_tool():
"""
Factory function to create the get_live_token_data tool.
This tool fetches comprehensive REAL-TIME market data from DexScreener API.
Use this when users want detailed current market information.
Returns:
A configured tool function for fetching live token market data.
"""
@tool
async def get_live_token_data(
chain: str,
token_address: str,
token_symbol: str | None = None,
include_all_pairs: bool = False,
) -> dict[str, Any]:
"""
Get comprehensive LIVE market data for a cryptocurrency token.
Use this tool when the user asks for:
- Detailed market info: "Show me full market data for BULLA"
- Trading activity: "What's the trading volume for SOL?"
- Liquidity info: "How much liquidity does WETH have?"
- Transaction counts: "How many buys/sells for this token?"
This returns more detailed data than get_live_token_price.
For historical trends and analysis, use search_knowledge_base instead.
Args:
chain: Blockchain network (e.g., 'solana', 'ethereum', 'base', 'bsc')
token_address: The token's contract address
token_symbol: Optional token symbol for display
include_all_pairs: If True, include data from all trading pairs
Returns:
Dictionary with comprehensive market data including:
- All price data from get_live_token_price
- Transaction counts (buys/sells in 24h, 6h, 1h)
- All trading pairs (if include_all_pairs=True)
- Aggregated volume across all pairs
"""
token_id = generate_token_id(chain, token_address)
try:
# Initialize DexScreener connector
connector = DexScreenerConnector()
# Fetch live data from API
pairs, error = await connector.get_token_pairs(chain, token_address)
if error:
logger.warning(f"[get_live_token_data] Error: {error}")
return {
"id": token_id,
"kind": "live_token_data",
"chain": chain,
"token_address": token_address,
"token_symbol": token_symbol,
"error": error,
}
if not pairs:
return {
"id": token_id,
"kind": "live_token_data",
"chain": chain,
"token_address": token_address,
"token_symbol": token_symbol,
"error": f"No trading pairs found for {token_symbol or token_address} on {chain}",
}
# Get the best pair (highest liquidity)
best_pair = max(pairs, key=lambda p: float(p.get("liquidity", {}).get("usd", 0) or 0))
# Extract data from best pair
base_token = best_pair.get("baseToken", {})
price_change = best_pair.get("priceChange", {})
volume = best_pair.get("volume", {})
liquidity = best_pair.get("liquidity", {})
txns = best_pair.get("txns", {})
# Calculate aggregated stats across all pairs
total_volume_24h = sum(float(p.get("volume", {}).get("h24", 0) or 0) for p in pairs)
total_liquidity = sum(float(p.get("liquidity", {}).get("usd", 0) or 0) for p in pairs)
total_buys_24h = sum(p.get("txns", {}).get("h24", {}).get("buys", 0) or 0 for p in pairs)
total_sells_24h = sum(p.get("txns", {}).get("h24", {}).get("sells", 0) or 0 for p in pairs)
result = {
"id": token_id,
"kind": "live_token_data",
"chain": chain,
"token_address": token_address,
"token_symbol": token_symbol or base_token.get("symbol", "Unknown"),
"token_name": base_token.get("name", "Unknown"),
# Price data
"price_usd": best_pair.get("priceUsd", "N/A"),
"price_native": best_pair.get("priceNative", "N/A"),
"price_change_5m": price_change.get("m5", 0),
"price_change_1h": price_change.get("h1", 0),
"price_change_6h": price_change.get("h6", 0),
"price_change_24h": price_change.get("h24", 0),
# Volume data (best pair)
"volume_24h": volume.get("h24", 0),
"volume_6h": volume.get("h6", 0),
"volume_1h": volume.get("h1", 0),
"volume_5m": volume.get("m5", 0),
# Liquidity
"liquidity_usd": liquidity.get("usd", 0),
"liquidity_base": liquidity.get("base", 0),
"liquidity_quote": liquidity.get("quote", 0),
# Market metrics
"market_cap": best_pair.get("marketCap", 0),
"fdv": best_pair.get("fdv", 0),
# Transaction counts (best pair)
"txns_24h_buys": txns.get("h24", {}).get("buys", 0),
"txns_24h_sells": txns.get("h24", {}).get("sells", 0),
"txns_6h_buys": txns.get("h6", {}).get("buys", 0),
"txns_6h_sells": txns.get("h6", {}).get("sells", 0),
"txns_1h_buys": txns.get("h1", {}).get("buys", 0),
"txns_1h_sells": txns.get("h1", {}).get("sells", 0),
# Aggregated stats (all pairs)
"total_volume_24h_all_pairs": total_volume_24h,
"total_liquidity_all_pairs": total_liquidity,
"total_buys_24h_all_pairs": total_buys_24h,
"total_sells_24h_all_pairs": total_sells_24h,
# DEX info
"dex": best_pair.get("dexId", "Unknown"),
"pair_address": best_pair.get("pairAddress", ""),
"pair_url": best_pair.get("url", ""),
"pair_created_at": best_pair.get("pairCreatedAt"),
# Metadata
"total_pairs": len(pairs),
"data_source": "DexScreener API (Real-time)",
}
# Include all pairs if requested
if include_all_pairs and len(pairs) > 1:
result["all_pairs"] = [
{
"dex": p.get("dexId"),
"pair_address": p.get("pairAddress"),
"quote_symbol": p.get("quoteToken", {}).get("symbol"),
"price_usd": p.get("priceUsd"),
"liquidity_usd": p.get("liquidity", {}).get("usd", 0),
"volume_24h": p.get("volume", {}).get("h24", 0),
"url": p.get("url"),
}
for p in sorted(pairs, key=lambda x: float(x.get("liquidity", {}).get("usd", 0) or 0), reverse=True)[:10]
]
return result
except Exception as e:
error_message = str(e)
logger.error(f"[get_live_token_data] Error fetching {chain}/{token_address}: {error_message}")
return {
"id": token_id,
"kind": "live_token_data",
"chain": chain,
"token_address": token_address,
"token_symbol": token_symbol,
"error": f"Failed to fetch live data: {error_message[:100]}",
}
return get_live_token_data

274
surfsense_backend/app/agents/new_chat/tools/knowledge_base.py
View file

@ -203,6 +203,11 @@ _ALL_CONNECTORS: list[str] = [
"OBSIDIAN_CONNECTOR",
"ONEDRIVE_FILE",
"DROPBOX_FILE",
"DEXSCREENER_CONNECTOR",
# Composio connectors
"COMPOSIO_GOOGLE_DRIVE_CONNECTOR",
"COMPOSIO_GMAIL_CONNECTOR",
"COMPOSIO_GOOGLE_CALENDAR_CONNECTOR",
]
# Human-readable descriptions for each connector type
@ -234,6 +239,11 @@ CONNECTOR_DESCRIPTIONS: dict[str, str] = {
"OBSIDIAN_CONNECTOR": "Obsidian vault notes and markdown files (personal notes)",
"ONEDRIVE_FILE": "Microsoft OneDrive files and documents (personal cloud storage)",
"DROPBOX_FILE": "Dropbox files and documents (cloud storage)",
"DEXSCREENER_CONNECTOR": "DexScreener real-time cryptocurrency trading pair data and market information",
# Composio connectors
"COMPOSIO_GOOGLE_DRIVE_CONNECTOR": "Google Drive files via Composio (personal cloud storage)",
"COMPOSIO_GMAIL_CONNECTOR": "Gmail emails via Composio (personal emails)",
"COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": "Google Calendar events via Composio (personal calendar)",
}
@ -771,6 +781,270 @@ async def search_knowledge_base_raw_async(
for docs in connector_results:
all_documents.extend(docs)
elif connector == "TEAMS_CONNECTOR":
_, chunks = await connector_service.search_teams(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "NOTION_CONNECTOR":
_, chunks = await connector_service.search_notion(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "GITHUB_CONNECTOR":
_, chunks = await connector_service.search_github(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "LINEAR_CONNECTOR":
_, chunks = await connector_service.search_linear(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "TAVILY_API":
_, chunks = await connector_service.search_tavily(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
)
all_documents.extend(chunks)
elif connector == "SEARXNG_API":
_, chunks = await connector_service.search_searxng(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
)
all_documents.extend(chunks)
elif connector == "LINKUP_API":
# Keep behavior aligned with researcher: default "standard"
_, chunks = await connector_service.search_linkup(
user_query=query,
search_space_id=search_space_id,
mode="standard",
)
all_documents.extend(chunks)
elif connector == "BAIDU_SEARCH_API":
_, chunks = await connector_service.search_baidu(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
)
all_documents.extend(chunks)
elif connector == "DISCORD_CONNECTOR":
_, chunks = await connector_service.search_discord(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "JIRA_CONNECTOR":
_, chunks = await connector_service.search_jira(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "GOOGLE_CALENDAR_CONNECTOR":
_, chunks = await connector_service.search_google_calendar(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "AIRTABLE_CONNECTOR":
_, chunks = await connector_service.search_airtable(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "GOOGLE_GMAIL_CONNECTOR":
_, chunks = await connector_service.search_google_gmail(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "GOOGLE_DRIVE_FILE":
_, chunks = await connector_service.search_google_drive(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "CONFLUENCE_CONNECTOR":
_, chunks = await connector_service.search_confluence(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "CLICKUP_CONNECTOR":
_, chunks = await connector_service.search_clickup(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "LUMA_CONNECTOR":
_, chunks = await connector_service.search_luma(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "ELASTICSEARCH_CONNECTOR":
_, chunks = await connector_service.search_elasticsearch(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "NOTE":
_, chunks = await connector_service.search_notes(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "BOOKSTACK_CONNECTOR":
_, chunks = await connector_service.search_bookstack(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "CIRCLEBACK":
_, chunks = await connector_service.search_circleback(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "OBSIDIAN_CONNECTOR":
_, chunks = await connector_service.search_obsidian(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "DEXSCREENER_CONNECTOR":
_, chunks = await connector_service.search_dexscreener(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
print(f"[DEBUG] DexScreener search returned {len(chunks)} chunks")
if chunks:
print(f"[DEBUG] First chunk metadata: {chunks[0].get('document', {}).get('metadata', {})}")
all_documents.extend(chunks)
# =========================================================
# Composio Connectors
# =========================================================
elif connector == "COMPOSIO_GOOGLE_DRIVE_CONNECTOR":
_, chunks = await connector_service.search_composio_google_drive(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "COMPOSIO_GMAIL_CONNECTOR":
_, chunks = await connector_service.search_composio_gmail(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
elif connector == "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR":
_, chunks = await connector_service.search_composio_google_calendar(
user_query=query,
search_space_id=search_space_id,
top_k=top_k,
start_date=resolved_start_date,
end_date=resolved_end_date,
)
all_documents.extend(chunks)
except Exception as e:
print(f"Error searching connector {connector}: {e}")
continue
# Deduplicate by content hash
seen_doc_ids: set[Any] = set()
seen_content_hashes: set[int] = set()
deduplicated: list[dict[str, Any]] = []

27
surfsense_backend/app/agents/new_chat/tools/registry.py
View file

@ -50,6 +50,11 @@ from .confluence import (
create_delete_confluence_page_tool,
create_update_confluence_page_tool,
)
from .crypto_realtime import (
create_get_live_token_data_tool,
create_get_live_token_price_tool,
)
from .display_image import create_display_image_tool
from .dropbox import (
create_create_dropbox_file_tool,
create_delete_dropbox_file_tool,
@ -80,6 +85,8 @@ from .linear import (
create_delete_linear_issue_tool,
create_update_linear_issue_tool,
)
from .knowledge_base import create_search_knowledge_base_tool
from .link_preview import create_link_preview_tool
from .mcp_tool import load_mcp_tools
from .notion import (
create_create_notion_page_tool,
@ -522,6 +529,26 @@ BUILTIN_TOOLS: list[ToolDefinition] = [
),
requires=["db_session", "search_space_id", "user_id"],
),
# =========================================================================
# CRYPTO REAL-TIME TOOLS - Hybrid approach (RAG + Real-time)
# =========================================================================
# These tools fetch LIVE data directly from DexScreener API.
# Use alongside search_knowledge_base for comprehensive crypto analysis:
# - search_knowledge_base: Historical context, trends (from indexed data)
# - get_live_token_price: Current price (real-time API call)
# - get_live_token_data: Full market data (real-time API call)
ToolDefinition(
name="get_live_token_price",
description="Get LIVE/CURRENT cryptocurrency price from DexScreener API. Use for real-time price queries.",
factory=lambda deps: create_get_live_token_price_tool(),
requires=[],
),
ToolDefinition(
name="get_live_token_data",
description="Get comprehensive LIVE market data (price, volume, liquidity, transactions) from DexScreener API.",
factory=lambda deps: create_get_live_token_data_tool(),
requires=[],
),
]