From 28985e6af45eecea4a9192919fb2d963262a0995 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Tue, 23 Dec 2025 00:35:08 +0530
Subject: [PATCH 07/13] style: update greeting messages for improved user
engagement and consistency
---
surfsense_web/components/assistant-ui/thread.tsx | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx
index aaf35c329..57408391d 100644
--- a/surfsense_web/components/assistant-ui/thread.tsx
+++ b/surfsense_web/components/assistant-ui/thread.tsx
@@ -210,14 +210,14 @@ const getTimeBasedGreeting = (userEmail?: string): string => {
"Good afternoon",
"Afternoon",
"Hey there",
- "Hope you're having a great day",
+ "Hi there",
];
const eveningGreetings = [
"Good evening",
"Evening",
"Hey there",
- "Hope you had a great day",
+ "Hi there",
];
const nightGreetings = [
@@ -228,10 +228,10 @@ const getTimeBasedGreeting = (userEmail?: string): string => {
];
const lateNightGreetings = [
- "Burning the midnight oil",
"Still up",
"Night owl mode",
"The night is young",
+ "Hi there",
];
// Select a random greeting based on time
@@ -358,9 +358,7 @@ const ConnectorIndicator: FC = () => {
"outline-none focus:outline-none focus-visible:outline-none",
"border-0 ring-0 focus:ring-0 shadow-none focus:shadow-none",
"data-[state=open]:bg-transparent data-[state=open]:shadow-none data-[state=open]:ring-0",
- hasConnectors
- ? "text-muted-foreground"
- : "text-muted-foreground/60"
+ "text-muted-foreground"
)}
aria-label={hasConnectors ? "View connected sources" : "Add your first connector"}
onMouseEnter={handleMouseEnter}
From 4b69fdf214dae95198929bb51b543248392a0313 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Tue, 23 Dec 2025 00:58:27 +0530
Subject: [PATCH 08/13] feat: add link preview tool for enhanced URL metadata
display in chat
---
.../app/agents/new_chat/chat_deepagent.py | 9 +
.../app/agents/new_chat/link_preview.py | 292 ++++++++++++++
.../app/agents/new_chat/system_prompt.py | 22 +
.../app/tasks/chat/stream_new_chat.py | 87 +++-
.../new-chat/[[...chat_id]]/page.tsx | 4 +-
surfsense_web/components/tool-ui/index.ts | 17 +
.../components/tool-ui/link-preview.tsx | 226 +++++++++++
.../components/tool-ui/media-card/index.tsx | 381 ++++++++++++++++++
8 files changed, 1035 insertions(+), 3 deletions(-)
create mode 100644 surfsense_backend/app/agents/new_chat/link_preview.py
create mode 100644 surfsense_web/components/tool-ui/link-preview.tsx
create mode 100644 surfsense_web/components/tool-ui/media-card/index.tsx
diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py
index 1c97a05ad..23db4f813 100644
--- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py
+++ b/surfsense_backend/app/agents/new_chat/chat_deepagent.py
@@ -15,6 +15,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.agents.new_chat.context import SurfSenseContextSchema
from app.agents.new_chat.knowledge_base import create_search_knowledge_base_tool
+from app.agents.new_chat.link_preview import create_link_preview_tool
from app.agents.new_chat.podcast import create_generate_podcast_tool
from app.agents.new_chat.system_prompt import build_surfsense_system_prompt
from app.services.connector_service import ConnectorService
@@ -34,6 +35,7 @@ def create_surfsense_deep_agent(
user_instructions: str | None = None,
enable_citations: bool = True,
enable_podcast: bool = True,
+ enable_link_preview: bool = True,
additional_tools: Sequence[BaseTool] | None = None,
):
"""
@@ -53,6 +55,8 @@ def create_surfsense_deep_agent(
When False, the agent will not be instructed to add citations to responses.
enable_podcast: Whether to include the podcast generation tool (default: True).
When True and user_id is provided, the agent can generate podcasts.
+ enable_link_preview: Whether to include the link preview tool (default: True).
+ When True, the agent can fetch and display rich link previews.
additional_tools: Optional sequence of additional tools to inject into the agent.
The search_knowledge_base tool will always be included.
@@ -78,6 +82,11 @@ def create_surfsense_deep_agent(
)
tools.append(podcast_tool)
+ # Add link preview tool if enabled
+ if enable_link_preview:
+ link_preview_tool = create_link_preview_tool()
+ tools.append(link_preview_tool)
+
if additional_tools:
tools.extend(additional_tools)
diff --git a/surfsense_backend/app/agents/new_chat/link_preview.py b/surfsense_backend/app/agents/new_chat/link_preview.py
new file mode 100644
index 000000000..388a6c14e
--- /dev/null
+++ b/surfsense_backend/app/agents/new_chat/link_preview.py
@@ -0,0 +1,292 @@
+"""
+Link preview tool for the new chat agent.
+
+This module provides a tool for fetching URL metadata (title, description,
+Open Graph image, etc.) to display rich link previews in the chat UI.
+"""
+
+import hashlib
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+from langchain_core.tools import tool
+
+
+def extract_domain(url: str) -> str:
+ """Extract the domain from a URL."""
+ try:
+ parsed = urlparse(url)
+ domain = parsed.netloc
+ # Remove 'www.' prefix if present
+ if domain.startswith("www."):
+ domain = domain[4:]
+ return domain
+ except Exception:
+ return ""
+
+
+def extract_og_content(html: str, property_name: str) -> str | None:
+ """Extract Open Graph meta content from HTML."""
+ # Try og:property first
+ pattern = rf'
]+property=["\']og:{property_name}["\'][^>]+content=["\']([^"\']+)["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ # Try content before property
+ pattern = rf']+content=["\']([^"\']+)["\'][^>]+property=["\']og:{property_name}["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ return None
+
+
+def extract_twitter_content(html: str, name: str) -> str | None:
+ """Extract Twitter Card meta content from HTML."""
+ pattern = rf']+name=["\']twitter:{name}["\'][^>]+content=["\']([^"\']+)["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ # Try content before name
+ pattern = rf']+content=["\']([^"\']+)["\'][^>]+name=["\']twitter:{name}["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ return None
+
+
+def extract_meta_description(html: str) -> str | None:
+ """Extract meta description from HTML."""
+ pattern = r']+name=["\']description["\'][^>]+content=["\']([^"\']+)["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ # Try content before name
+ pattern = r']+content=["\']([^"\']+)["\'][^>]+name=["\']description["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ return None
+
+
+def extract_title(html: str) -> str | None:
+ """Extract title from HTML."""
+ # Try og:title first
+ og_title = extract_og_content(html, "title")
+ if og_title:
+ return og_title
+
+ # Try twitter:title
+ twitter_title = extract_twitter_content(html, "title")
+ if twitter_title:
+ return twitter_title
+
+ # Fall back to tag
+ pattern = r"<title[^>]*>([^<]+)"
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1).strip()
+
+ return None
+
+
+def extract_description(html: str) -> str | None:
+ """Extract description from HTML."""
+ # Try og:description first
+ og_desc = extract_og_content(html, "description")
+ if og_desc:
+ return og_desc
+
+ # Try twitter:description
+ twitter_desc = extract_twitter_content(html, "description")
+ if twitter_desc:
+ return twitter_desc
+
+ # Fall back to meta description
+ return extract_meta_description(html)
+
+
+def extract_image(html: str) -> str | None:
+ """Extract image URL from HTML."""
+ # Try og:image first
+ og_image = extract_og_content(html, "image")
+ if og_image:
+ return og_image
+
+ # Try twitter:image
+ twitter_image = extract_twitter_content(html, "image")
+ if twitter_image:
+ return twitter_image
+
+ return None
+
+
+def generate_preview_id(url: str) -> str:
+ """Generate a unique ID for a link preview."""
+ hash_val = hashlib.md5(url.encode()).hexdigest()[:12]
+ return f"link-preview-{hash_val}"
+
+
+def create_link_preview_tool():
+ """
+ Factory function to create the link_preview tool.
+
+ Returns:
+ A configured tool function for fetching link previews.
+ """
+
+ @tool
+ async def link_preview(url: str) -> dict[str, Any]:
+ """
+ Fetch metadata for a URL to display a rich link preview.
+
+ Use this tool when the user shares a URL or asks about a specific webpage.
+ This tool fetches the page's Open Graph metadata (title, description, image)
+ to display a nice preview card in the chat.
+
+ Common triggers include:
+ - User shares a URL in the chat
+ - User asks "What's this link about?" or similar
+ - User says "Show me a preview of this page"
+ - User wants to preview an article or webpage
+
+ Args:
+ url: The URL to fetch metadata for. Must be a valid HTTP/HTTPS URL.
+
+ Returns:
+ A dictionary containing:
+ - id: Unique identifier for this preview
+ - assetId: The URL itself (for deduplication)
+ - kind: "link" (type of media card)
+ - href: The URL to open when clicked
+ - title: Page title
+ - description: Page description (if available)
+ - thumb: Thumbnail/preview image URL (if available)
+ - domain: The domain name
+ - error: Error message (if fetch failed)
+ """
+ preview_id = generate_preview_id(url)
+ domain = extract_domain(url)
+
+ # Validate URL
+ if not url.startswith(("http://", "https://")):
+ url = f"https://{url}"
+
+ try:
+ async with httpx.AsyncClient(
+ timeout=10.0,
+ follow_redirects=True,
+ headers={
+ "User-Agent": "Mozilla/5.0 (compatible; SurfSenseBot/1.0; +https://surfsense.net)",
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.5",
+ },
+ ) as client:
+ response = await client.get(url)
+ response.raise_for_status()
+
+ # Get content type to ensure it's HTML
+ content_type = response.headers.get("content-type", "")
+ if "text/html" not in content_type.lower():
+ # Not an HTML page, return basic info
+ return {
+ "id": preview_id,
+ "assetId": url,
+ "kind": "link",
+ "href": url,
+ "title": url.split("/")[-1] or domain,
+ "description": f"File from {domain}",
+ "domain": domain,
+ }
+
+ html = response.text
+
+ # Extract metadata
+ title = extract_title(html) or domain
+ description = extract_description(html)
+ image = extract_image(html)
+
+ # Make sure image URL is absolute
+ if image and not image.startswith(("http://", "https://")):
+ if image.startswith("//"):
+ image = f"https:{image}"
+ elif image.startswith("/"):
+ parsed = urlparse(url)
+ image = f"{parsed.scheme}://{parsed.netloc}{image}"
+
+ # Clean up title and description (unescape HTML entities)
+ if title:
+ title = (
+ title.replace("&", "&")
+ .replace("<", "<")
+ .replace(">", ">")
+ .replace(""", '"')
+ .replace("'", "'")
+ .replace("'", "'")
+ )
+ if description:
+ description = (
+ description.replace("&", "&")
+ .replace("<", "<")
+ .replace(">", ">")
+ .replace(""", '"')
+ .replace("'", "'")
+ .replace("'", "'")
+ )
+ # Truncate long descriptions
+ if len(description) > 200:
+ description = description[:197] + "..."
+
+ return {
+ "id": preview_id,
+ "assetId": url,
+ "kind": "link",
+ "href": url,
+ "title": title,
+ "description": description,
+ "thumb": image,
+ "domain": domain,
+ }
+
+ except httpx.TimeoutException:
+ return {
+ "id": preview_id,
+ "assetId": url,
+ "kind": "link",
+ "href": url,
+ "title": domain or "Link",
+ "domain": domain,
+ "error": "Request timed out",
+ }
+ except httpx.HTTPStatusError as e:
+ return {
+ "id": preview_id,
+ "assetId": url,
+ "kind": "link",
+ "href": url,
+ "title": domain or "Link",
+ "domain": domain,
+ "error": f"HTTP {e.response.status_code}",
+ }
+ except Exception as e:
+ error_message = str(e)
+ print(f"[link_preview] Error fetching {url}: {error_message}")
+ return {
+ "id": preview_id,
+ "assetId": url,
+ "kind": "link",
+ "href": url,
+ "title": domain or "Link",
+ "domain": domain,
+ "error": f"Failed to fetch: {error_message[:50]}",
+ }
+
+ return link_preview
+
diff --git a/surfsense_backend/app/agents/new_chat/system_prompt.py b/surfsense_backend/app/agents/new_chat/system_prompt.py
index f725be684..32e649e5d 100644
--- a/surfsense_backend/app/agents/new_chat/system_prompt.py
+++ b/surfsense_backend/app/agents/new_chat/system_prompt.py
@@ -145,6 +145,19 @@ You have access to the following tools:
- Returns: A task_id for tracking. The podcast will be generated in the background.
- IMPORTANT: Only one podcast can be generated at a time. If a podcast is already being generated, the tool will return status "already_generating".
- After calling this tool, inform the user that podcast generation has started and they will see the player when it's ready (takes 3-5 minutes).
+
+3. link_preview: Fetch metadata for a URL to display a rich preview card.
+ - IMPORTANT: Use this tool WHENEVER the user shares or mentions a URL/link in their message.
+ - This fetches the page's Open Graph metadata (title, description, thumbnail) to show a preview card.
+ - NOTE: This tool only fetches metadata, NOT the full page content. It cannot read the article text.
+ - Trigger scenarios:
+ * User shares a URL (e.g., "Check out https://example.com")
+ * User pastes a link in their message
+ * User asks about a URL or link
+ - Args:
+ - url: The URL to fetch metadata for (must be a valid HTTP/HTTPS URL)
+ - Returns: A rich preview card with title, description, thumbnail, and domain
+ - The preview card will automatically be displayed in the chat.
- User: "Fetch all my notes and what's in them?"
@@ -162,6 +175,15 @@ You have access to the following tools:
- User: "Make a podcast about quantum computing"
- First search: `search_knowledge_base(query="quantum computing")`
- Then: `generate_podcast(source_content="Key insights about quantum computing from the knowledge base:\n\n[Comprehensive summary of all relevant search results with key facts, concepts, and findings]", podcast_title="Quantum Computing Explained")`
+
+- User: "Check out https://dev.to/some-article"
+ - Call: `link_preview(url="https://dev.to/some-article")`
+
+- User: "What's this blog post about? https://example.com/blog/post"
+ - Call: `link_preview(url="https://example.com/blog/post")`
+
+- User: "https://github.com/some/repo"
+ - Call: `link_preview(url="https://github.com/some/repo")`
{citation_section}
"""
diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py
index cb40a82a8..853fb3392 100644
--- a/surfsense_backend/app/tasks/chat/stream_new_chat.py
+++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py
@@ -284,6 +284,20 @@ async def stream_new_chat(
status="in_progress",
items=last_active_step_items,
)
+ elif tool_name == "link_preview":
+ url = (
+ tool_input.get("url", "")
+ if isinstance(tool_input, dict)
+ else str(tool_input)
+ )
+ last_active_step_title = "Fetching link preview"
+ last_active_step_items = [f"URL: {url[:80]}{'...' if len(url) > 80 else ''}"]
+ yield streaming_service.format_thinking_step(
+ step_id=tool_step_id,
+ title="Fetching link preview",
+ status="in_progress",
+ items=last_active_step_items,
+ )
elif tool_name == "generate_podcast":
podcast_title = (
tool_input.get("podcast_title", "SurfSense Podcast")
@@ -343,6 +357,16 @@ async def stream_new_chat(
f"Searching knowledge base: {query[:100]}{'...' if len(query) > 100 else ''}",
"info",
)
+ elif tool_name == "link_preview":
+ url = (
+ tool_input.get("url", "")
+ if isinstance(tool_input, dict)
+ else str(tool_input)
+ )
+ yield streaming_service.format_terminal_info(
+ f"Fetching link preview: {url[:80]}{'...' if len(url) > 80 else ''}",
+ "info",
+ )
elif tool_name == "generate_podcast":
title = (
tool_input.get("podcast_title", "SurfSense Podcast")
@@ -404,6 +428,31 @@ async def stream_new_chat(
status="completed",
items=completed_items,
)
+ elif tool_name == "link_preview":
+ # Build completion items based on link preview result
+ if isinstance(tool_output, dict):
+ title = tool_output.get("title", "Link")
+ domain = tool_output.get("domain", "")
+ has_error = "error" in tool_output
+ if has_error:
+ completed_items = [
+ *last_active_step_items,
+ f"Error: {tool_output.get('error', 'Failed to fetch')}",
+ ]
+ else:
+ completed_items = [
+ *last_active_step_items,
+ f"Title: {title[:60]}{'...' if len(title) > 60 else ''}",
+ f"Domain: {domain}" if domain else "Preview loaded",
+ ]
+ else:
+ completed_items = [*last_active_step_items, "Preview loaded"]
+ yield streaming_service.format_thinking_step(
+ step_id=original_step_id,
+ title="Fetching link preview",
+ status="completed",
+ items=completed_items,
+ )
elif tool_name == "generate_podcast":
# Build detailed completion items based on podcast status
podcast_status = (
@@ -492,8 +541,33 @@ async def stream_new_chat(
f"Podcast generation failed: {error_msg}",
"error",
)
- else:
- # Don't stream the full output for other tools (can be very large), just acknowledge
+ elif tool_name == "link_preview":
+ # Stream the full link preview result so frontend can render the MediaCard
+ yield streaming_service.format_tool_output_available(
+ tool_call_id,
+ tool_output
+ if isinstance(tool_output, dict)
+ else {"result": tool_output},
+ )
+ # Send appropriate terminal message
+ if isinstance(tool_output, dict) and "error" not in tool_output:
+ title = tool_output.get("title", "Link")
+ yield streaming_service.format_terminal_info(
+ f"Link preview loaded: {title[:50]}{'...' if len(title) > 50 else ''}",
+ "success",
+ )
+ else:
+ error_msg = (
+ tool_output.get("error", "Failed to fetch")
+ if isinstance(tool_output, dict)
+ else "Failed to fetch"
+ )
+ yield streaming_service.format_terminal_info(
+ f"Link preview failed: {error_msg}",
+ "error",
+ )
+ elif tool_name == "search_knowledge_base":
+ # Don't stream the full output for search (can be very large), just acknowledge
yield streaming_service.format_tool_output_available(
tool_call_id,
{"status": "completed", "result_length": len(str(tool_output))},
@@ -501,6 +575,15 @@ async def stream_new_chat(
yield streaming_service.format_terminal_info(
"Knowledge base search completed", "success"
)
+ else:
+ # Default handling for other tools
+ yield streaming_service.format_tool_output_available(
+ tool_call_id,
+ {"status": "completed", "result_length": len(str(tool_output))},
+ )
+ yield streaming_service.format_terminal_info(
+ f"Tool {tool_name} completed", "success"
+ )
# Handle chain/agent end to close any open text blocks
elif event_type in ("on_chain_end", "on_agent_end"):
diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
index 51a2f9875..4d45db118 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
@@ -11,6 +11,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import { Thread } from "@/components/assistant-ui/thread";
import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast";
+import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview";
import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking";
import { getBearerToken } from "@/lib/auth-utils";
import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter";
@@ -79,7 +80,7 @@ function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike {
/**
* Tools that should render custom UI in the chat.
*/
-const TOOLS_WITH_UI = new Set(["generate_podcast"]);
+const TOOLS_WITH_UI = new Set(["generate_podcast", "link_preview"]);
/**
* Type for thinking step data from the backend
@@ -589,6 +590,7 @@ export default function NewChatPage() {
return (
+
diff --git a/surfsense_web/components/tool-ui/index.ts b/surfsense_web/components/tool-ui/index.ts
index 3cddfba21..aebe400d5 100644
--- a/surfsense_web/components/tool-ui/index.ts
+++ b/surfsense_web/components/tool-ui/index.ts
@@ -15,3 +15,20 @@ export {
type DeepAgentThinkingArgs,
type DeepAgentThinkingResult,
} from "./deepagent-thinking";
+export {
+ LinkPreviewToolUI,
+ MultiLinkPreviewToolUI,
+ type LinkPreviewArgs,
+ type LinkPreviewResult,
+ type MultiLinkPreviewArgs,
+ type MultiLinkPreviewResult,
+} from "./link-preview";
+export {
+ MediaCard,
+ MediaCardErrorBoundary,
+ MediaCardLoading,
+ MediaCardSkeleton,
+ parseSerializableMediaCard,
+ type MediaCardProps,
+ type SerializableMediaCard,
+} from "./media-card";
diff --git a/surfsense_web/components/tool-ui/link-preview.tsx b/surfsense_web/components/tool-ui/link-preview.tsx
new file mode 100644
index 000000000..c97b6820b
--- /dev/null
+++ b/surfsense_web/components/tool-ui/link-preview.tsx
@@ -0,0 +1,226 @@
+"use client";
+
+import { makeAssistantToolUI } from "@assistant-ui/react";
+import { AlertCircleIcon, ExternalLinkIcon, LinkIcon } from "lucide-react";
+import {
+ MediaCard,
+ MediaCardErrorBoundary,
+ MediaCardLoading,
+ parseSerializableMediaCard,
+ type SerializableMediaCard,
+} from "@/components/tool-ui/media-card";
+
+/**
+ * Type definitions for the link_preview tool
+ */
+interface LinkPreviewArgs {
+ url: string;
+ title?: string;
+}
+
+interface LinkPreviewResult {
+ id: string;
+ assetId: string;
+ kind: "link";
+ href: string;
+ title: string;
+ description?: string;
+ thumb?: string;
+ domain?: string;
+ error?: string;
+}
+
+/**
+ * Error state component shown when link preview fails
+ */
+function LinkPreviewErrorState({ url, error }: { url: string; error: string }) {
+ return (
+
+
+
+
+
Failed to load preview
+
{url}
+
{error}
+
+
+
+
+
+ );
+ }
+
+ // Incomplete/cancelled state
+ if (status.type === "incomplete") {
+ if (status.reason === "cancelled") {
+ return ;
+ }
+ if (status.reason === "error") {
+ return (
+
+ );
+ }
+ }
+
+ // No result yet
+ if (!result) {
+ return (
+
+
+
+ );
+ }
+
+ // Error result from the tool
+ if (result.error) {
+ return ;
+ }
+
+ // Success - render the media card
+ return (
+
+ );
+ },
+});
+
+/**
+ * Multiple Link Previews Tool UI Component
+ *
+ * This component handles cases where multiple links need to be previewed.
+ * It renders a grid of link preview cards.
+ */
+interface MultiLinkPreviewArgs {
+ urls: string[];
+}
+
+interface MultiLinkPreviewResult {
+ previews: LinkPreviewResult[];
+ errors?: { url: string; error: string }[];
+}
+
+export const MultiLinkPreviewToolUI = makeAssistantToolUI<
+ MultiLinkPreviewArgs,
+ MultiLinkPreviewResult
+>({
+ toolName: "multi_link_preview",
+ render: function MultiLinkPreviewUI({ args, result, status }) {
+ const urls = args.urls || [];
+
+ // Loading state
+ if (status.type === "running" || status.type === "requires-action") {
+ return (
+
+ {urls.slice(0, 4).map((url, index) => (
+
+ ))}
+
+ );
+ }
+
+ // Incomplete state
+ if (status.type === "incomplete") {
+ return (
+
+
+
+ Link previews cancelled
+
+
+ {result.previews.map((preview) => (
+
+
+
+ ))}
+ {result.errors?.map((err) => (
+
+ ))}
+
+
+
+
+
Failed to load preview
+
+ {this.state.error?.message || "An error occurred"}
+
+
+
{
+ // Hide broken images
+ e.currentTarget.style.display = "none";
+ }}
+ />
+ {/* Gradient overlay */}
+
+
+
+ {getKindIcon(kind)}
+ {kind === "link" ? "Link Preview" : kind}
+
+
+ {/* Domain favicon placeholder */}
+
+
+
+
+
+ {/* Domain badge */}
+ {displayDomain && (
+
+
+ {displayDomain}
+
+ {href && (
+
+ )}
+
+ )}
+
+ {/* Title */}
+
+ {title}
+
+
+ {/* Description */}
+ {description && (
+
+ {description}
+
+ )}
+
+
e.stopPropagation()}
+ onKeyDown={(e) => e.stopPropagation()}
+ >
+ {responseActions.map((action) => (
+
+
+
+
+ {action.confirmLabel && (
+
+ {action.confirmLabel}
+
+ )}
+
+ ))}
+
+
+
+
+
+ {title}
+
+
+ );
+}
+
From da7cb812528562c729e3a5dc00ea13023e6b0923 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Tue, 23 Dec 2025 01:11:56 +0530
Subject: [PATCH 09/13] feat: introduce display image tool for enhanced image
rendering in chat with metadata support
---
.../app/agents/new_chat/chat_deepagent.py | 9 +
.../app/agents/new_chat/display_image.py | 106 ++++++
.../app/agents/new_chat/system_prompt.py | 21 ++
.../app/tasks/chat/stream_new_chat.py | 64 ++++
.../new-chat/[[...chat_id]]/page.tsx | 4 +-
.../components/tool-ui/display-image.tsx | 160 +++++++++
.../components/tool-ui/image/index.tsx | 332 ++++++++++++++++++
surfsense_web/components/tool-ui/index.ts | 14 +
8 files changed, 709 insertions(+), 1 deletion(-)
create mode 100644 surfsense_backend/app/agents/new_chat/display_image.py
create mode 100644 surfsense_web/components/tool-ui/display-image.tsx
create mode 100644 surfsense_web/components/tool-ui/image/index.tsx
diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py
index 23db4f813..e6b2aca06 100644
--- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py
+++ b/surfsense_backend/app/agents/new_chat/chat_deepagent.py
@@ -14,6 +14,7 @@ from langgraph.types import Checkpointer
from sqlalchemy.ext.asyncio import AsyncSession
from app.agents.new_chat.context import SurfSenseContextSchema
+from app.agents.new_chat.display_image import create_display_image_tool
from app.agents.new_chat.knowledge_base import create_search_knowledge_base_tool
from app.agents.new_chat.link_preview import create_link_preview_tool
from app.agents.new_chat.podcast import create_generate_podcast_tool
@@ -36,6 +37,7 @@ def create_surfsense_deep_agent(
enable_citations: bool = True,
enable_podcast: bool = True,
enable_link_preview: bool = True,
+ enable_display_image: bool = True,
additional_tools: Sequence[BaseTool] | None = None,
):
"""
@@ -57,6 +59,8 @@ def create_surfsense_deep_agent(
When True and user_id is provided, the agent can generate podcasts.
enable_link_preview: Whether to include the link preview tool (default: True).
When True, the agent can fetch and display rich link previews.
+ enable_display_image: Whether to include the display image tool (default: True).
+ When True, the agent can display images with metadata.
additional_tools: Optional sequence of additional tools to inject into the agent.
The search_knowledge_base tool will always be included.
@@ -87,6 +91,11 @@ def create_surfsense_deep_agent(
link_preview_tool = create_link_preview_tool()
tools.append(link_preview_tool)
+ # Add display image tool if enabled
+ if enable_display_image:
+ display_image_tool = create_display_image_tool()
+ tools.append(display_image_tool)
+
if additional_tools:
tools.extend(additional_tools)
diff --git a/surfsense_backend/app/agents/new_chat/display_image.py b/surfsense_backend/app/agents/new_chat/display_image.py
new file mode 100644
index 000000000..03153300a
--- /dev/null
+++ b/surfsense_backend/app/agents/new_chat/display_image.py
@@ -0,0 +1,106 @@
+"""
+Display image tool for the new chat agent.
+
+This module provides a tool for displaying images in the chat UI
+with metadata like title, description, and source attribution.
+"""
+
+import hashlib
+from typing import Any
+from urllib.parse import urlparse
+
+from langchain_core.tools import tool
+
+
+def extract_domain(url: str) -> str:
+ """Extract the domain from a URL."""
+ try:
+ parsed = urlparse(url)
+ domain = parsed.netloc
+ # Remove 'www.' prefix if present
+ if domain.startswith("www."):
+ domain = domain[4:]
+ return domain
+ except Exception:
+ return ""
+
+
+def generate_image_id(src: str) -> str:
+ """Generate a unique ID for an image."""
+ hash_val = hashlib.md5(src.encode()).hexdigest()[:12]
+ return f"image-{hash_val}"
+
+
+def create_display_image_tool():
+ """
+ Factory function to create the display_image tool.
+
+ Returns:
+ A configured tool function for displaying images.
+ """
+
+ @tool
+ async def display_image(
+ src: str,
+ alt: str = "Image",
+ title: str | None = None,
+ description: str | None = None,
+ ) -> dict[str, Any]:
+ """
+ Display an image in the chat with metadata.
+
+ Use this tool when you want to show an image to the user.
+ This displays the image with an optional title, description,
+ and source attribution.
+
+ Common use cases:
+ - Showing an image from a URL the user mentioned
+ - Displaying a diagram or chart you're referencing
+ - Showing example images when explaining concepts
+
+ Args:
+ src: The URL of the image to display (must be a valid HTTP/HTTPS URL)
+ alt: Alternative text describing the image (for accessibility)
+ title: Optional title to display below the image
+ description: Optional description providing context about the image
+
+ Returns:
+ A dictionary containing image metadata for the UI to render:
+ - id: Unique identifier for this image
+ - assetId: The image URL (for deduplication)
+ - src: The image URL
+ - alt: Alt text for accessibility
+ - title: Image title (if provided)
+ - description: Image description (if provided)
+ - domain: Source domain
+ """
+ image_id = generate_image_id(src)
+
+ # Ensure URL has protocol
+ if not src.startswith(("http://", "https://")):
+ src = f"https://{src}"
+
+ domain = extract_domain(src)
+
+ # Determine aspect ratio based on common image sources
+ ratio = "16:9" # Default
+ if "unsplash.com" in src or "pexels.com" in src:
+ ratio = "16:9"
+ elif "imgur.com" in src:
+ ratio = "auto"
+ elif "github.com" in src or "githubusercontent.com" in src:
+ ratio = "auto"
+
+ return {
+ "id": image_id,
+ "assetId": src,
+ "src": src,
+ "alt": alt,
+ "title": title,
+ "description": description,
+ "domain": domain,
+ "ratio": ratio,
+ }
+
+ return display_image
+
diff --git a/surfsense_backend/app/agents/new_chat/system_prompt.py b/surfsense_backend/app/agents/new_chat/system_prompt.py
index 32e649e5d..74abadd38 100644
--- a/surfsense_backend/app/agents/new_chat/system_prompt.py
+++ b/surfsense_backend/app/agents/new_chat/system_prompt.py
@@ -158,6 +158,21 @@ You have access to the following tools:
- url: The URL to fetch metadata for (must be a valid HTTP/HTTPS URL)
- Returns: A rich preview card with title, description, thumbnail, and domain
- The preview card will automatically be displayed in the chat.
+
+4. display_image: Display an image in the chat with metadata.
+ - Use this tool when you want to show an image to the user.
+ - This displays the image with an optional title, description, and source attribution.
+ - Common use cases:
+ * Showing an image from a URL mentioned in the conversation
+ * Displaying a diagram, chart, or illustration you're referencing
+ * Showing visual examples when explaining concepts
+ - Args:
+ - src: The URL of the image to display (must be a valid HTTP/HTTPS image URL)
+ - alt: Alternative text describing the image (for accessibility)
+ - title: Optional title to display below the image
+ - description: Optional description providing context about the image
+ - Returns: An image card with the image, title, and description
+ - The image will automatically be displayed in the chat.
- User: "Fetch all my notes and what's in them?"
@@ -184,6 +199,12 @@ You have access to the following tools:
- User: "https://github.com/some/repo"
- Call: `link_preview(url="https://github.com/some/repo")`
+
+- User: "Show me this image: https://example.com/image.png"
+ - Call: `display_image(src="https://example.com/image.png", alt="User shared image")`
+
+- User: "Can you display a diagram of a neural network?"
+ - Call: `display_image(src="https://example.com/neural-network.png", alt="Neural network diagram", title="Neural Network Architecture", description="A visual representation of a neural network with input, hidden, and output layers")`
{citation_section}
"""
diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py
index 853fb3392..6e2b30c6a 100644
--- a/surfsense_backend/app/tasks/chat/stream_new_chat.py
+++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py
@@ -298,6 +298,27 @@ async def stream_new_chat(
status="in_progress",
items=last_active_step_items,
)
+ elif tool_name == "display_image":
+ src = (
+ tool_input.get("src", "")
+ if isinstance(tool_input, dict)
+ else str(tool_input)
+ )
+ title = (
+ tool_input.get("title", "")
+ if isinstance(tool_input, dict)
+ else ""
+ )
+ last_active_step_title = "Displaying image"
+ last_active_step_items = [
+ f"Image: {title[:50] if title else src[:50]}{'...' if len(title or src) > 50 else ''}"
+ ]
+ yield streaming_service.format_thinking_step(
+ step_id=tool_step_id,
+ title="Displaying image",
+ status="in_progress",
+ items=last_active_step_items,
+ )
elif tool_name == "generate_podcast":
podcast_title = (
tool_input.get("podcast_title", "SurfSense Podcast")
@@ -367,6 +388,16 @@ async def stream_new_chat(
f"Fetching link preview: {url[:80]}{'...' if len(url) > 80 else ''}",
"info",
)
+ elif tool_name == "display_image":
+ src = (
+ tool_input.get("src", "")
+ if isinstance(tool_input, dict)
+ else str(tool_input)
+ )
+ yield streaming_service.format_terminal_info(
+ f"Displaying image: {src[:60]}{'...' if len(src) > 60 else ''}",
+ "info",
+ )
elif tool_name == "generate_podcast":
title = (
tool_input.get("podcast_title", "SurfSense Podcast")
@@ -453,6 +484,24 @@ async def stream_new_chat(
status="completed",
items=completed_items,
)
+ elif tool_name == "display_image":
+ # Build completion items for image display
+ if isinstance(tool_output, dict):
+ title = tool_output.get("title", "")
+ alt = tool_output.get("alt", "Image")
+ display_name = title or alt
+ completed_items = [
+ *last_active_step_items,
+ f"Showing: {display_name[:50]}{'...' if len(display_name) > 50 else ''}",
+ ]
+ else:
+ completed_items = [*last_active_step_items, "Image displayed"]
+ yield streaming_service.format_thinking_step(
+ step_id=original_step_id,
+ title="Displaying image",
+ status="completed",
+ items=completed_items,
+ )
elif tool_name == "generate_podcast":
# Build detailed completion items based on podcast status
podcast_status = (
@@ -566,6 +615,21 @@ async def stream_new_chat(
f"Link preview failed: {error_msg}",
"error",
)
+ elif tool_name == "display_image":
+ # Stream the full image result so frontend can render the Image component
+ yield streaming_service.format_tool_output_available(
+ tool_call_id,
+ tool_output
+ if isinstance(tool_output, dict)
+ else {"result": tool_output},
+ )
+ # Send terminal message
+ if isinstance(tool_output, dict):
+ title = tool_output.get("title") or tool_output.get("alt", "Image")
+ yield streaming_service.format_terminal_info(
+ f"Image displayed: {title[:40]}{'...' if len(title) > 40 else ''}",
+ "success",
+ )
elif tool_name == "search_knowledge_base":
# Don't stream the full output for search (can be very large), just acknowledge
yield streaming_service.format_tool_output_available(
diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
index 4d45db118..7c6bd34c1 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
@@ -12,6 +12,7 @@ import { toast } from "sonner";
import { Thread } from "@/components/assistant-ui/thread";
import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast";
import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview";
+import { DisplayImageToolUI } from "@/components/tool-ui/display-image";
import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking";
import { getBearerToken } from "@/lib/auth-utils";
import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter";
@@ -80,7 +81,7 @@ function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike {
/**
* Tools that should render custom UI in the chat.
*/
-const TOOLS_WITH_UI = new Set(["generate_podcast", "link_preview"]);
+const TOOLS_WITH_UI = new Set(["generate_podcast", "link_preview", "display_image"]);
/**
* Type for thinking step data from the backend
@@ -591,6 +592,7 @@ export default function NewChatPage() {
+
diff --git a/surfsense_web/components/tool-ui/display-image.tsx b/surfsense_web/components/tool-ui/display-image.tsx
new file mode 100644
index 000000000..37bdad5fa
--- /dev/null
+++ b/surfsense_web/components/tool-ui/display-image.tsx
@@ -0,0 +1,160 @@
+"use client";
+
+import { makeAssistantToolUI } from "@assistant-ui/react";
+import { AlertCircleIcon, ImageIcon } from "lucide-react";
+import {
+ Image,
+ ImageErrorBoundary,
+ ImageLoading,
+ parseSerializableImage,
+} from "@/components/tool-ui/image";
+
+/**
+ * Type definitions for the display_image tool
+ */
+interface DisplayImageArgs {
+ src: string;
+ alt?: string;
+ title?: string;
+ description?: string;
+}
+
+interface DisplayImageResult {
+ id: string;
+ assetId: string;
+ src: string;
+ alt: string;
+ title?: string;
+ description?: string;
+ domain?: string;
+ ratio?: string;
+ error?: string;
+}
+
+/**
+ * Error state component shown when image display fails
+ */
+function ImageErrorState({ src, error }: { src: string; error: string }) {
+ return (
+
+
+
+
+
Failed to display image
+
{src}
+
{error}
+
+
+
+
+
+ );
+ }
+
+ // Incomplete/cancelled state
+ if (status.type === "incomplete") {
+ if (status.reason === "cancelled") {
+ return ;
+ }
+ if (status.reason === "error") {
+ return (
+
+ );
+ }
+ }
+
+ // No result yet
+ if (!result) {
+ return (
+
+
+
+ );
+ }
+
+ // Error result from the tool
+ if (result.error) {
+ return ;
+ }
+
+ // Success - render the image
+ return (
+
+ );
+ },
+});
+
+export type { DisplayImageArgs, DisplayImageResult };
+
diff --git a/surfsense_web/components/tool-ui/image/index.tsx b/surfsense_web/components/tool-ui/image/index.tsx
new file mode 100644
index 000000000..97c983625
--- /dev/null
+++ b/surfsense_web/components/tool-ui/image/index.tsx
@@ -0,0 +1,332 @@
+"use client";
+
+import { ExternalLinkIcon, ImageIcon, Loader2 } from "lucide-react";
+import NextImage from "next/image";
+import { Component, type ReactNode, useState } from "react";
+import { Badge } from "@/components/ui/badge";
+import { Card } from "@/components/ui/card";
+import { cn } from "@/lib/utils";
+
+/**
+ * Aspect ratio options for images
+ */
+type AspectRatio = "1:1" | "4:3" | "16:9" | "9:16" | "auto";
+
+/**
+ * Image fit options
+ */
+type ImageFit = "cover" | "contain";
+
+/**
+ * Source attribution
+ */
+interface ImageSource {
+ label: string;
+ iconUrl?: string;
+ url?: string;
+}
+
+/**
+ * Props for the Image component
+ */
+export interface ImageProps {
+ id: string;
+ assetId: string;
+ src: string;
+ alt: string;
+ title?: string;
+ description?: string;
+ href?: string;
+ domain?: string;
+ ratio?: AspectRatio;
+ fit?: ImageFit;
+ source?: ImageSource;
+ maxWidth?: string;
+ className?: string;
+}
+
+/**
+ * Serializable schema for Image props (for tool results)
+ */
+export interface SerializableImage {
+ id: string;
+ assetId: string;
+ src: string;
+ alt: string;
+ title?: string;
+ description?: string;
+ href?: string;
+ domain?: string;
+ ratio?: AspectRatio;
+ source?: ImageSource;
+}
+
+/**
+ * Parse and validate serializable image from tool result
+ */
+export function parseSerializableImage(result: unknown): SerializableImage {
+ if (typeof result !== "object" || result === null) {
+ throw new Error("Invalid image result: expected object");
+ }
+
+ const obj = result as Record;
+
+ // Validate required fields
+ if (typeof obj.id !== "string") {
+ throw new Error("Invalid image: missing id");
+ }
+ if (typeof obj.assetId !== "string") {
+ throw new Error("Invalid image: missing assetId");
+ }
+ if (typeof obj.src !== "string") {
+ throw new Error("Invalid image: missing src");
+ }
+ if (typeof obj.alt !== "string") {
+ throw new Error("Invalid image: missing alt");
+ }
+
+ return {
+ id: obj.id,
+ assetId: obj.assetId,
+ src: obj.src,
+ alt: obj.alt,
+ title: typeof obj.title === "string" ? obj.title : undefined,
+ description: typeof obj.description === "string" ? obj.description : undefined,
+ href: typeof obj.href === "string" ? obj.href : undefined,
+ domain: typeof obj.domain === "string" ? obj.domain : undefined,
+ ratio: typeof obj.ratio === "string" ? (obj.ratio as AspectRatio) : undefined,
+ source: typeof obj.source === "object" && obj.source !== null ? (obj.source as ImageSource) : undefined,
+ };
+}
+
+/**
+ * Get aspect ratio class based on ratio prop
+ */
+function getAspectRatioClass(ratio?: AspectRatio): string {
+ switch (ratio) {
+ case "1:1":
+ return "aspect-square";
+ case "4:3":
+ return "aspect-[4/3]";
+ case "16:9":
+ return "aspect-video";
+ case "9:16":
+ return "aspect-[9/16]";
+ case "auto":
+ default:
+ return "aspect-[4/3]";
+ }
+}
+
+/**
+ * Error boundary for Image component
+ */
+interface ImageErrorBoundaryState {
+ hasError: boolean;
+ error?: Error;
+}
+
+export class ImageErrorBoundary extends Component<
+ { children: ReactNode },
+ ImageErrorBoundaryState
+> {
+ constructor(props: { children: ReactNode }) {
+ super(props);
+ this.state = { hasError: false };
+ }
+
+ static getDerivedStateFromError(error: Error): ImageErrorBoundaryState {
+ return { hasError: true, error };
+ }
+
+ render() {
+ if (this.state.hasError) {
+ return (
+
+
+
+
+
Failed to load image
+
+
+
+
+
+ );
+}
+
+/**
+ * Image Loading State
+ */
+export function ImageLoading({ title = "Loading image..." }: { title?: string }) {
+ return (
+
+
+
+ );
+}
+
+/**
+ * Image Component
+ *
+ * Display images with metadata and attribution.
+ * Features hover overlay with title and source attribution.
+ */
+export function Image({
+ id,
+ src,
+ alt,
+ title,
+ description,
+ href,
+ domain,
+ ratio = "4:3",
+ fit = "cover",
+ source,
+ maxWidth = "420px",
+ className,
+}: ImageProps) {
+ const [isHovered, setIsHovered] = useState(false);
+ const [imageError, setImageError] = useState(false);
+ const aspectRatioClass = getAspectRatioClass(ratio);
+ const displayDomain = domain || source?.label;
+
+ const handleClick = () => {
+ const targetUrl = href || source?.url || src;
+ if (targetUrl) {
+ window.open(targetUrl, "_blank", "noopener,noreferrer");
+ }
+ };
+
+ if (imageError) {
+ return (
+
+
+
+
+
Image not available
+
+
+ {/* Image */}
+
setImageError(true)}
+ />
+
+ {/* Hover overlay - appears on hover */}
+
+ {/* Content at bottom */}
+
+ {/* Title */}
+ {title && (
+
+ {title}
+
+ )}
+
+ {/* Description */}
+ {description && (
+
+ {description}
+
+ )}
+
+ {/* Source attribution */}
+ {displayDomain && (
+
+ {source?.iconUrl ? (
+
+ ) : (
+
+ )}
+ {displayDomain}
+
+ )}
+
+
+
+ {displayDomain}
+
+
+ )}
+
diff --git a/surfsense_web/components/tool-ui/article/index.tsx b/surfsense_web/components/tool-ui/article/index.tsx
new file mode 100644
index 000000000..bf8e83411
--- /dev/null
+++ b/surfsense_web/components/tool-ui/article/index.tsx
@@ -0,0 +1,406 @@
+"use client";
+
+import { Card, CardContent } from "@/components/ui/card";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipProvider,
+ TooltipTrigger,
+} from "@/components/ui/tooltip";
+import { cn } from "@/lib/utils";
+import {
+ AlertCircleIcon,
+ BookOpenIcon,
+ CalendarIcon,
+ ExternalLinkIcon,
+ FileTextIcon,
+ UserIcon,
+} from "lucide-react";
+import { Component, type ReactNode, useCallback } from "react";
+
+/**
+ * Article component props
+ */
+export interface ArticleProps {
+ /** Unique identifier for the article */
+ id: string;
+ /** Asset identifier (usually the URL) */
+ assetId?: string;
+ /** Article title */
+ title: string;
+ /** Brief description or excerpt */
+ description?: string;
+ /** Full content of the article (markdown) */
+ content?: string;
+ /** URL to the original article */
+ href?: string;
+ /** Domain of the article source */
+ domain?: string;
+ /** Author name */
+ author?: string;
+ /** Publication date */
+ date?: string;
+ /** Word count */
+ wordCount?: number;
+ /** Whether content was truncated */
+ wasTruncated?: boolean;
+ /** Optional max width */
+ maxWidth?: string;
+ /** Optional error message */
+ error?: string;
+ /** Optional className */
+ className?: string;
+ /** Response actions */
+ responseActions?: Array<{
+ id: string;
+ label: string;
+ variant?: "default" | "outline";
+ }>;
+ /** Response action handler */
+ onResponseAction?: (actionId: string) => void;
+}
+
+/**
+ * Serializable article data type (from backend)
+ */
+export interface SerializableArticle {
+ id: string;
+ assetId?: string;
+ kind?: "article";
+ title: string;
+ description?: string;
+ content?: string;
+ href?: string;
+ domain?: string;
+ author?: string;
+ date?: string;
+ word_count?: number;
+ wordCount?: number;
+ was_truncated?: boolean;
+ wasTruncated?: boolean;
+ error?: string;
+}
+
+/**
+ * Parse serializable article data to ArticleProps
+ */
+export function parseSerializableArticle(data: unknown): ArticleProps {
+ const obj = data as Record;
+ return {
+ id: String(obj.id || "article-unknown"),
+ assetId: obj.assetId as string | undefined,
+ title: String(obj.title || "Untitled Article"),
+ description: obj.description as string | undefined,
+ content: obj.content as string | undefined,
+ href: obj.href as string | undefined,
+ domain: obj.domain as string | undefined,
+ author: obj.author as string | undefined,
+ date: obj.date as string | undefined,
+ wordCount: (obj.word_count || obj.wordCount) as number | undefined,
+ wasTruncated: (obj.was_truncated || obj.wasTruncated) as boolean | undefined,
+ error: obj.error as string | undefined,
+ };
+}
+
+/**
+ * Format word count for display
+ */
+function formatWordCount(count: number): string {
+ if (count >= 1000) {
+ return `${(count / 1000).toFixed(1)}k words`;
+ }
+ return `${count} words`;
+}
+
+/**
+ * Article card component for displaying scraped webpage content
+ */
+export function Article({
+ id,
+ title,
+ description,
+ content,
+ href,
+ domain,
+ author,
+ date,
+ wordCount,
+ wasTruncated,
+ maxWidth = "100%",
+ error,
+ className,
+ responseActions,
+ onResponseAction,
+}: ArticleProps) {
+ const handleCardClick = useCallback(() => {
+ if (href) {
+ window.open(href, "_blank", "noopener,noreferrer");
+ }
+ }, [href]);
+
+ // Error state
+ if (error) {
+ return (
+
+
+
+
+
+
+ Failed to scrape webpage
+
+ {href && (
+
+ {href}
+
+ )}
+
{error}
+
+
+ {/* Icon */}
+
+
+
+
+ {/* Content */}
+
+ {/* Title */}
+
+ {title}
+
+
+ {/* Description */}
+ {description && (
+
+ {description}
+
+ )}
+
+ {/* Metadata row */}
+
+ {domain && (
+
+
+
+
+ {domain}
+
+
+
+ Source: {domain}
+
+
+ )}
+
+ {author && (
+
+
+
+
+ {author}
+
+
+
+ Author: {author}
+
+
+ )}
+
+ {date && (
+
+
+ {date}
+
+ )}
+
+ {wordCount && (
+
+
+
+
+ {formatWordCount(wordCount)}
+ {wasTruncated && (
+ (truncated)
+ )}
+
+
+
+
+ {wasTruncated
+ ? "Content was truncated due to length"
+ : "Full article content available"}
+
+
+
+ )}
+
+
+
+ {/* External link indicator */}
+ {href && (
+
+
+
+ )}
+
+ {responseActions.map((action) => (
+
+ ))}
+
+ )}
+
+
+
+ );
+}
+
+/**
+ * Loading state for article component
+ */
+export function ArticleLoading({
+ title = "Loading article...",
+}: { title?: string }) {
+ return (
+
+
+
+ {title}
+
+
+ );
+}
+
+/**
+ * Skeleton for article component
+ */
+export function ArticleSkeleton() {
+ return (
+
+
+
+
+
+ );
+}
+
+/**
+ * Error boundary props
+ */
+interface ErrorBoundaryProps {
+ children: ReactNode;
+ fallback?: ReactNode;
+}
+
+interface ErrorBoundaryState {
+ hasError: boolean;
+}
+
+/**
+ * Error boundary for article component
+ */
+export class ArticleErrorBoundary extends Component<
+ ErrorBoundaryProps,
+ ErrorBoundaryState
+> {
+ constructor(props: ErrorBoundaryProps) {
+ super(props);
+ this.state = { hasError: false };
+ }
+
+ static getDerivedStateFromError(): ErrorBoundaryState {
+ return { hasError: true };
+ }
+
+ render() {
+ if (this.state.hasError) {
+ return (
+ this.props.fallback || (
+
+
+
+
+
+ Failed to render article
+
+
+
+
+
+
Failed to scrape webpage
+
{url}
+
{error}
+
+
+
+
+
+ Scraping: {url}
+
+
- {steps.map((step) => {
- const icon = getStepIcon(step.status, step.title);
+ {steps.map((step, index) => {
+ const effectiveStatus = getEffectiveStatus(step);
+ const icon = getStepIcon(effectiveStatus, step.title);
+ const isOpen = getStepOpenState(step, index);
return (
handleToggle(step.id, isOpen)}
>
{step.title}
{step.items && step.items.length > 0 && (
- {step.items.map((item, index) => (
-
+ {step.items.map((item, idx) => (
+
{item}
))}
@@ -263,8 +330,8 @@ const ThreadWelcome: FC = () => {
return (
- {/* Greeting positioned near the composer */}
-
+ {/* Greeting positioned above the composer - fixed position */}
+
{/** biome-ignore lint/a11y/noStaticElementInteractions: wrong lint error, this is a workaround to fix the lint error */}
{
{getTimeBasedGreeting(user?.email)}
- {/* Composer centered in the middle of the screen */}
-
+ {/* Composer - top edge fixed, expands downward only */}
+
-
+
)}
diff --git a/surfsense_web/components/tool-ui/deepagent-thinking.tsx b/surfsense_web/components/tool-ui/deepagent-thinking.tsx
index f0c03e570..8cd901e69 100644
--- a/surfsense_web/components/tool-ui/deepagent-thinking.tsx
+++ b/surfsense_web/components/tool-ui/deepagent-thinking.tsx
@@ -2,7 +2,7 @@
import { makeAssistantToolUI } from "@assistant-ui/react";
import { Brain, CheckCircle2, Loader2, Search, Sparkles } from "lucide-react";
-import { useMemo } from "react";
+import { useMemo, useState, useEffect, useRef } from "react";
import {
ChainOfThought,
ChainOfThoughtContent,
@@ -61,13 +61,21 @@ function getStepIcon(status: "pending" | "in_progress" | "completed", title: str
}
/**
- * Component to display a single thinking step
+ * Component to display a single thinking step with controlled open state
*/
-function ThinkingStepDisplay({ step }: { step: ThinkingStep }) {
+function ThinkingStepDisplay({
+ step,
+ isOpen,
+ onToggle
+}: {
+ step: ThinkingStep;
+ isOpen: boolean;
+ onToggle: () => void;
+}) {
const icon = useMemo(() => getStepIcon(step.status, step.title), [step.status, step.title]);
return (
-
+
>({});
+ // Track previous step statuses to detect changes
+ const prevStatusesRef = useRef>({});
+
+ // Check if any step is currently in progress
+ const hasInProgressStep = steps.some(step => step.status === "in_progress");
+
+ // Find the last completed step index
+ const lastCompletedIndex = steps
+ .map((s, i) => s.status === "completed" ? i : -1)
+ .filter(i => i !== -1)
+ .pop();
+
+ // Clear manual overrides when a step's status changes
+ useEffect(() => {
+ const currentStatuses: Record = {};
+ steps.forEach(step => {
+ currentStatuses[step.id] = step.status;
+ // If status changed, clear any manual override for this step
+ if (prevStatusesRef.current[step.id] && prevStatusesRef.current[step.id] !== step.status) {
+ setManualOverrides(prev => {
+ const next = { ...prev };
+ delete next[step.id];
+ return next;
+ });
+ }
+ });
+ prevStatusesRef.current = currentStatuses;
+ }, [steps]);
+
+ const getStepOpenState = (step: ThinkingStep, index: number): boolean => {
+ // If user has manually toggled, respect that
+ if (manualOverrides[step.id] !== undefined) {
+ return manualOverrides[step.id];
+ }
+ // Auto behavior: open if in progress
+ if (step.status === "in_progress") {
+ return true;
+ }
+ // Auto behavior: keep last completed step open if no in-progress step
+ if (!hasInProgressStep && index === lastCompletedIndex) {
+ return true;
+ }
+ // Default: collapsed
+ return false;
+ };
+
+ const handleToggle = (stepId: string, currentOpen: boolean) => {
+ setManualOverrides(prev => ({
+ ...prev,
+ [stepId]: !currentOpen,
+ }));
+ };
+
+ return (
+
+ {steps.map((step, index) => {
+ const isOpen = getStepOpenState(step, index);
+ return (
+ handleToggle(step.id, isOpen)}
+ />
+ );
+ })}
+
+ );
+}
+
/**
* DeepAgent Thinking Tool UI Component
*
@@ -131,7 +215,7 @@ export const DeepAgentThinkingToolUI = makeAssistantToolUI<
DeepAgentThinkingResult
>({
toolName: "deepagent_thinking",
- render: function DeepAgentThinkingUI({ args, result, status }) {
+ render: function DeepAgentThinkingUI({ result, status }) {
// Loading state - tool is still running
if (status.type === "running" || status.type === "requires-action") {
return ;
@@ -155,11 +239,7 @@ export const DeepAgentThinkingToolUI = makeAssistantToolUI<
// Render the chain of thought
return (
-
- {result.steps.map((step) => (
-
- ))}
-
+
);
},
@@ -189,11 +269,7 @@ export function InlineThinkingDisplay({
{isStreaming && steps.length === 0 ? (
) : (
-
- {steps.map((step) => (
-
- ))}
-
+
)}
);
From 5fd872b7980d2525c4bf060f17e410aacefb5b3a Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Tue, 23 Dec 2025 02:42:48 +0530
Subject: [PATCH 12/13] feat: implement Zod schemas for runtime validation
across various components to enhance data integrity and error handling
---
.../components/tool-ui/article/index.tsx | 89 +++++++++------
.../components/tool-ui/deepagent-thinking.tsx | 68 ++++++++---
.../components/tool-ui/display-image.tsx | 10 +-
.../components/tool-ui/generate-podcast.tsx | 106 ++++++++++++------
.../components/tool-ui/image/index.tsx | 97 ++++++----------
.../components/tool-ui/media-card/index.tsx | 97 ++++++----------
6 files changed, 253 insertions(+), 214 deletions(-)
diff --git a/surfsense_web/components/tool-ui/article/index.tsx b/surfsense_web/components/tool-ui/article/index.tsx
index bf8e83411..62068d117 100644
--- a/surfsense_web/components/tool-ui/article/index.tsx
+++ b/surfsense_web/components/tool-ui/article/index.tsx
@@ -17,6 +17,33 @@ import {
UserIcon,
} from "lucide-react";
import { Component, type ReactNode, useCallback } from "react";
+import { z } from "zod";
+
+/**
+ * Zod schema for serializable article data (from backend)
+ */
+const SerializableArticleSchema = z.object({
+ id: z.string().default("article-unknown"),
+ assetId: z.string().optional(),
+ kind: z.literal("article").optional(),
+ title: z.string().default("Untitled Article"),
+ description: z.string().optional(),
+ content: z.string().optional(),
+ href: z.string().url().optional(),
+ domain: z.string().optional(),
+ author: z.string().optional(),
+ date: z.string().optional(),
+ word_count: z.number().optional(),
+ wordCount: z.number().optional(),
+ was_truncated: z.boolean().optional(),
+ wasTruncated: z.boolean().optional(),
+ error: z.string().optional(),
+});
+
+/**
+ * Serializable article data type (from backend)
+ */
+export type SerializableArticle = z.infer
;
/**
* Article component props
@@ -61,44 +88,36 @@ export interface ArticleProps {
}
/**
- * Serializable article data type (from backend)
- */
-export interface SerializableArticle {
- id: string;
- assetId?: string;
- kind?: "article";
- title: string;
- description?: string;
- content?: string;
- href?: string;
- domain?: string;
- author?: string;
- date?: string;
- word_count?: number;
- wordCount?: number;
- was_truncated?: boolean;
- wasTruncated?: boolean;
- error?: string;
-}
-
-/**
- * Parse serializable article data to ArticleProps
+ * Parse and validate serializable article data to ArticleProps
*/
export function parseSerializableArticle(data: unknown): ArticleProps {
- const obj = data as Record;
+ const result = SerializableArticleSchema.safeParse(data);
+
+ if (!result.success) {
+ console.warn("Invalid article data:", result.error.issues);
+ // Return fallback with basic info
+ const obj = (data && typeof data === "object" ? data : {}) as Record;
+ return {
+ id: String(obj.id || "article-unknown"),
+ title: String(obj.title || "Untitled Article"),
+ error: "Failed to parse article data",
+ };
+ }
+
+ const parsed = result.data;
return {
- id: String(obj.id || "article-unknown"),
- assetId: obj.assetId as string | undefined,
- title: String(obj.title || "Untitled Article"),
- description: obj.description as string | undefined,
- content: obj.content as string | undefined,
- href: obj.href as string | undefined,
- domain: obj.domain as string | undefined,
- author: obj.author as string | undefined,
- date: obj.date as string | undefined,
- wordCount: (obj.word_count || obj.wordCount) as number | undefined,
- wasTruncated: (obj.was_truncated || obj.wasTruncated) as boolean | undefined,
- error: obj.error as string | undefined,
+ id: parsed.id,
+ assetId: parsed.assetId,
+ title: parsed.title,
+ description: parsed.description,
+ content: parsed.content,
+ href: parsed.href,
+ domain: parsed.domain,
+ author: parsed.author,
+ date: parsed.date,
+ wordCount: parsed.word_count ?? parsed.wordCount,
+ wasTruncated: parsed.was_truncated ?? parsed.wasTruncated,
+ error: parsed.error,
};
}
diff --git a/surfsense_web/components/tool-ui/deepagent-thinking.tsx b/surfsense_web/components/tool-ui/deepagent-thinking.tsx
index 8cd901e69..ff8baac9a 100644
--- a/surfsense_web/components/tool-ui/deepagent-thinking.tsx
+++ b/surfsense_web/components/tool-ui/deepagent-thinking.tsx
@@ -3,6 +3,7 @@
import { makeAssistantToolUI } from "@assistant-ui/react";
import { Brain, CheckCircle2, Loader2, Search, Sparkles } from "lucide-react";
import { useMemo, useState, useEffect, useRef } from "react";
+import { z } from "zod";
import {
ChainOfThought,
ChainOfThoughtContent,
@@ -13,24 +14,61 @@ import {
import { cn } from "@/lib/utils";
/**
- * Types for the deepagent thinking/reasoning tool
+ * Zod schemas for runtime validation
*/
-interface ThinkingStep {
- id: string;
- title: string;
- items: string[];
- status: "pending" | "in_progress" | "completed";
+const ThinkingStepSchema = z.object({
+ id: z.string(),
+ title: z.string(),
+ items: z.array(z.string()).default([]),
+ status: z.enum(["pending", "in_progress", "completed"]).default("pending"),
+});
+
+const DeepAgentThinkingArgsSchema = z.object({
+ query: z.string().optional(),
+ context: z.string().optional(),
+});
+
+const DeepAgentThinkingResultSchema = z.object({
+ steps: z.array(ThinkingStepSchema).optional(),
+ status: z.enum(["thinking", "searching", "synthesizing", "completed"]).optional(),
+ summary: z.string().optional(),
+});
+
+/**
+ * Types derived from Zod schemas
+ */
+type ThinkingStep = z.infer;
+type DeepAgentThinkingArgs = z.infer;
+type DeepAgentThinkingResult = z.infer;
+
+/**
+ * Parse and validate a single thinking step
+ */
+export function parseThinkingStep(data: unknown): ThinkingStep {
+ const result = ThinkingStepSchema.safeParse(data);
+ if (!result.success) {
+ console.warn("Invalid thinking step data:", result.error.issues);
+ // Return a fallback step
+ return {
+ id: "unknown",
+ title: "Processing...",
+ items: [],
+ status: "pending",
+ };
+ }
+ return result.data;
}
-interface DeepAgentThinkingArgs {
- query?: string;
- context?: string;
-}
-
-interface DeepAgentThinkingResult {
- steps?: ThinkingStep[];
- status?: "thinking" | "searching" | "synthesizing" | "completed";
- summary?: string;
+/**
+ * Parse and validate thinking result
+ */
+export function parseThinkingResult(data: unknown): DeepAgentThinkingResult {
+ const result = DeepAgentThinkingResultSchema.safeParse(data);
+ if (!result.success) {
+ console.warn("Invalid thinking result data:", result.error.issues);
+ return {};
+ }
+ return result.data;
}
/**
diff --git a/surfsense_web/components/tool-ui/display-image.tsx b/surfsense_web/components/tool-ui/display-image.tsx
index 37bdad5fa..a9c87b29b 100644
--- a/surfsense_web/components/tool-ui/display-image.tsx
+++ b/surfsense_web/components/tool-ui/display-image.tsx
@@ -67,6 +67,8 @@ function ImageCancelledState({ src }: { src: string }) {
/**
* Parsed Image component with error handling
+ * Note: Image component has built-in click handling via href/src,
+ * so no additional responseActions needed.
*/
function ParsedImage({ result }: { result: unknown }) {
const image = parseSerializableImage(result);
@@ -75,14 +77,6 @@ function ParsedImage({ result }: { result: unknown }) {
{
- if (id === "open" && image.src) {
- window.open(image.src, "_blank", "noopener,noreferrer");
- }
- }}
/>
);
}
diff --git a/surfsense_web/components/tool-ui/generate-podcast.tsx b/surfsense_web/components/tool-ui/generate-podcast.tsx
index 669b11a57..862eab144 100644
--- a/surfsense_web/components/tool-ui/generate-podcast.tsx
+++ b/surfsense_web/components/tool-ui/generate-podcast.tsx
@@ -3,37 +3,79 @@
import { makeAssistantToolUI } from "@assistant-ui/react";
import { AlertCircleIcon, Loader2Icon, MicIcon } from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
+import { z } from "zod";
import { Audio } from "@/components/tool-ui/audio";
import { baseApiService } from "@/lib/apis/base-api.service";
import { authenticatedFetch } from "@/lib/auth-utils";
import { clearActivePodcastTaskId, setActivePodcastTaskId } from "@/lib/chat/podcast-state";
/**
- * Type definitions for the generate_podcast tool
+ * Zod schemas for runtime validation
*/
-interface GeneratePodcastArgs {
- source_content: string;
- podcast_title?: string;
- user_prompt?: string;
+const GeneratePodcastArgsSchema = z.object({
+ source_content: z.string(),
+ podcast_title: z.string().optional(),
+ user_prompt: z.string().optional(),
+});
+
+const GeneratePodcastResultSchema = z.object({
+ status: z.enum(["processing", "already_generating", "success", "error"]),
+ task_id: z.string().optional(),
+ podcast_id: z.number().optional(),
+ title: z.string().optional(),
+ transcript_entries: z.number().optional(),
+ message: z.string().optional(),
+ error: z.string().optional(),
+});
+
+const TaskStatusResponseSchema = z.object({
+ status: z.enum(["processing", "success", "error"]),
+ podcast_id: z.number().optional(),
+ title: z.string().optional(),
+ transcript_entries: z.number().optional(),
+ state: z.string().optional(),
+ error: z.string().optional(),
+});
+
+const PodcastTranscriptEntrySchema = z.object({
+ speaker_id: z.number(),
+ dialog: z.string(),
+});
+
+const PodcastDetailsSchema = z.object({
+ podcast_transcript: z.array(PodcastTranscriptEntrySchema).optional(),
+});
+
+/**
+ * Types derived from Zod schemas
+ */
+type GeneratePodcastArgs = z.infer;
+type GeneratePodcastResult = z.infer;
+type TaskStatusResponse = z.infer;
+type PodcastTranscriptEntry = z.infer;
+
+/**
+ * Parse and validate task status response
+ */
+function parseTaskStatusResponse(data: unknown): TaskStatusResponse {
+ const result = TaskStatusResponseSchema.safeParse(data);
+ if (!result.success) {
+ console.warn("Invalid task status response:", result.error.issues);
+ return { status: "error", error: "Invalid response from server" };
+ }
+ return result.data;
}
-interface GeneratePodcastResult {
- status: "processing" | "already_generating" | "success" | "error";
- task_id?: string;
- podcast_id?: number;
- title?: string;
- transcript_entries?: number;
- message?: string;
- error?: string;
-}
-
-interface TaskStatusResponse {
- status: "processing" | "success" | "error";
- podcast_id?: number;
- title?: string;
- transcript_entries?: number;
- state?: string;
- error?: string;
+/**
+ * Parse and validate podcast details
+ */
+function parsePodcastDetails(data: unknown): { podcast_transcript?: PodcastTranscriptEntry[] } {
+ const result = PodcastDetailsSchema.safeParse(data);
+ if (!result.success) {
+ console.warn("Invalid podcast details:", result.error.issues);
+ return {};
+ }
+ return result.data;
}
/**
@@ -112,14 +154,6 @@ function AudioLoadingState({ title }: { title: string }) {
/**
* Podcast Player Component - Fetches audio and transcript with authentication
*/
-/**
- * Transcript entry type for podcast transcripts
- */
-interface PodcastTranscriptEntry {
- speaker_id: number;
- dialog: string;
-}
-
function PodcastPlayer({
podcastId,
title,
@@ -163,12 +197,12 @@ function PodcastPlayer({
try {
// Fetch audio blob and podcast details in parallel
- const [audioResponse, podcastDetails] = await Promise.all([
+ const [audioResponse, rawPodcastDetails] = await Promise.all([
authenticatedFetch(
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/podcasts/${podcastId}/audio`,
{ method: "GET", signal: controller.signal }
),
- baseApiService.get<{ podcast_transcript?: PodcastTranscriptEntry[] }>(
+ baseApiService.get(
`/api/v1/podcasts/${podcastId}`
),
]);
@@ -184,8 +218,9 @@ function PodcastPlayer({
objectUrlRef.current = objectUrl;
setAudioSrc(objectUrl);
- // Set transcript from podcast details
- if (podcastDetails?.podcast_transcript) {
+ // Parse and validate podcast details, then set transcript
+ const podcastDetails = parsePodcastDetails(rawPodcastDetails);
+ if (podcastDetails.podcast_transcript) {
setTranscript(podcastDetails.podcast_transcript);
}
} finally {
@@ -268,9 +303,10 @@ function PodcastTaskPoller({ taskId, title }: { taskId: string; title: string })
useEffect(() => {
const pollStatus = async () => {
try {
- const response = await baseApiService.get(
+ const rawResponse = await baseApiService.get(
`/api/v1/podcasts/task/${taskId}/status`
);
+ const response = parseTaskStatusResponse(rawResponse);
setTaskStatus(response);
// Stop polling if task is complete or errored
diff --git a/surfsense_web/components/tool-ui/image/index.tsx b/surfsense_web/components/tool-ui/image/index.tsx
index 97c983625..c9ac72d2f 100644
--- a/surfsense_web/components/tool-ui/image/index.tsx
+++ b/surfsense_web/components/tool-ui/image/index.tsx
@@ -3,28 +3,43 @@
import { ExternalLinkIcon, ImageIcon, Loader2 } from "lucide-react";
import NextImage from "next/image";
import { Component, type ReactNode, useState } from "react";
+import { z } from "zod";
import { Badge } from "@/components/ui/badge";
import { Card } from "@/components/ui/card";
import { cn } from "@/lib/utils";
/**
- * Aspect ratio options for images
+ * Zod schemas for runtime validation
*/
-type AspectRatio = "1:1" | "4:3" | "16:9" | "9:16" | "auto";
+const AspectRatioSchema = z.enum(["1:1", "4:3", "16:9", "9:16", "auto"]);
+const ImageFitSchema = z.enum(["cover", "contain"]);
+
+const ImageSourceSchema = z.object({
+ label: z.string(),
+ iconUrl: z.string().optional(),
+ url: z.string().optional(),
+});
+
+const SerializableImageSchema = z.object({
+ id: z.string(),
+ assetId: z.string(),
+ src: z.string(),
+ alt: z.string(),
+ title: z.string().optional(),
+ description: z.string().optional(),
+ href: z.string().optional(),
+ domain: z.string().optional(),
+ ratio: AspectRatioSchema.optional(),
+ source: ImageSourceSchema.optional(),
+});
/**
- * Image fit options
+ * Types derived from Zod schemas
*/
-type ImageFit = "cover" | "contain";
-
-/**
- * Source attribution
- */
-interface ImageSource {
- label: string;
- iconUrl?: string;
- url?: string;
-}
+type AspectRatio = z.infer;
+type ImageFit = z.infer;
+type ImageSource = z.infer;
+export type SerializableImage = z.infer;
/**
* Props for the Image component
@@ -45,58 +60,20 @@ export interface ImageProps {
className?: string;
}
-/**
- * Serializable schema for Image props (for tool results)
- */
-export interface SerializableImage {
- id: string;
- assetId: string;
- src: string;
- alt: string;
- title?: string;
- description?: string;
- href?: string;
- domain?: string;
- ratio?: AspectRatio;
- source?: ImageSource;
-}
-
/**
* Parse and validate serializable image from tool result
*/
export function parseSerializableImage(result: unknown): SerializableImage {
- if (typeof result !== "object" || result === null) {
- throw new Error("Invalid image result: expected object");
+ const parsed = SerializableImageSchema.safeParse(result);
+
+ if (!parsed.success) {
+ console.warn("Invalid image data:", parsed.error.issues);
+ // Try to extract basic info for error display
+ const obj = (result && typeof result === "object" ? result : {}) as Record;
+ throw new Error(`Invalid image: ${parsed.error.issues.map(i => i.message).join(", ")}`);
}
-
- const obj = result as Record;
-
- // Validate required fields
- if (typeof obj.id !== "string") {
- throw new Error("Invalid image: missing id");
- }
- if (typeof obj.assetId !== "string") {
- throw new Error("Invalid image: missing assetId");
- }
- if (typeof obj.src !== "string") {
- throw new Error("Invalid image: missing src");
- }
- if (typeof obj.alt !== "string") {
- throw new Error("Invalid image: missing alt");
- }
-
- return {
- id: obj.id,
- assetId: obj.assetId,
- src: obj.src,
- alt: obj.alt,
- title: typeof obj.title === "string" ? obj.title : undefined,
- description: typeof obj.description === "string" ? obj.description : undefined,
- href: typeof obj.href === "string" ? obj.href : undefined,
- domain: typeof obj.domain === "string" ? obj.domain : undefined,
- ratio: typeof obj.ratio === "string" ? (obj.ratio as AspectRatio) : undefined,
- source: typeof obj.source === "object" && obj.source !== null ? (obj.source as ImageSource) : undefined,
- };
+
+ return parsed.data;
}
/**
diff --git a/surfsense_web/components/tool-ui/media-card/index.tsx b/surfsense_web/components/tool-ui/media-card/index.tsx
index 47c28e49b..dc3b9b59a 100644
--- a/surfsense_web/components/tool-ui/media-card/index.tsx
+++ b/surfsense_web/components/tool-ui/media-card/index.tsx
@@ -3,6 +3,7 @@
import { ExternalLinkIcon, Globe, ImageIcon, LinkIcon, Loader2 } from "lucide-react";
import Image from "next/image";
import { Component, type ReactNode } from "react";
+import { z } from "zod";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Card, CardContent } from "@/components/ui/card";
@@ -10,24 +11,38 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp
import { cn } from "@/lib/utils";
/**
- * Aspect ratio options for media cards
+ * Zod schemas for runtime validation
*/
-type AspectRatio = "1:1" | "4:3" | "16:9" | "21:9" | "auto";
+const AspectRatioSchema = z.enum(["1:1", "4:3", "16:9", "21:9", "auto"]);
+const MediaCardKindSchema = z.enum(["link", "image", "video", "audio"]);
+
+const ResponseActionSchema = z.object({
+ id: z.string(),
+ label: z.string(),
+ variant: z.enum(["default", "secondary", "outline", "destructive", "ghost"]).optional(),
+ confirmLabel: z.string().optional(),
+});
+
+const SerializableMediaCardSchema = z.object({
+ id: z.string(),
+ assetId: z.string(),
+ kind: MediaCardKindSchema,
+ href: z.string().optional(),
+ src: z.string().optional(),
+ title: z.string(),
+ description: z.string().optional(),
+ thumb: z.string().optional(),
+ ratio: AspectRatioSchema.optional(),
+ domain: z.string().optional(),
+});
/**
- * MediaCard kind - determines the display style
+ * Types derived from Zod schemas
*/
-type MediaCardKind = "link" | "image" | "video" | "audio";
-
-/**
- * Response action configuration
- */
-interface ResponseAction {
- id: string;
- label: string;
- variant?: "default" | "secondary" | "outline" | "destructive" | "ghost";
- confirmLabel?: string;
-}
+type AspectRatio = z.infer;
+type MediaCardKind = z.infer;
+type ResponseAction = z.infer;
+export type SerializableMediaCard = z.infer;
/**
* Props for the MediaCard component
@@ -50,58 +65,18 @@ export interface MediaCardProps {
onResponseAction?: (id: string) => void;
}
-/**
- * Serializable schema for MediaCard props (for tool results)
- */
-export interface SerializableMediaCard {
- id: string;
- assetId: string;
- kind: MediaCardKind;
- href?: string;
- src?: string;
- title: string;
- description?: string;
- thumb?: string;
- ratio?: AspectRatio;
- domain?: string;
-}
-
/**
* Parse and validate serializable media card from tool result
*/
export function parseSerializableMediaCard(result: unknown): SerializableMediaCard {
- if (typeof result !== "object" || result === null) {
- throw new Error("Invalid media card result: expected object");
+ const parsed = SerializableMediaCardSchema.safeParse(result);
+
+ if (!parsed.success) {
+ console.warn("Invalid media card data:", parsed.error.issues);
+ throw new Error(`Invalid media card: ${parsed.error.issues.map(i => i.message).join(", ")}`);
}
-
- const obj = result as Record;
-
- // Validate required fields
- if (typeof obj.id !== "string") {
- throw new Error("Invalid media card: missing id");
- }
- if (typeof obj.assetId !== "string") {
- throw new Error("Invalid media card: missing assetId");
- }
- if (typeof obj.kind !== "string") {
- throw new Error("Invalid media card: missing kind");
- }
- if (typeof obj.title !== "string") {
- throw new Error("Invalid media card: missing title");
- }
-
- return {
- id: obj.id,
- assetId: obj.assetId,
- kind: obj.kind as MediaCardKind,
- href: typeof obj.href === "string" ? obj.href : undefined,
- src: typeof obj.src === "string" ? obj.src : undefined,
- title: obj.title,
- description: typeof obj.description === "string" ? obj.description : undefined,
- thumb: typeof obj.thumb === "string" ? obj.thumb : undefined,
- ratio: typeof obj.ratio === "string" ? (obj.ratio as AspectRatio) : undefined,
- domain: typeof obj.domain === "string" ? obj.domain : undefined,
- };
+
+ return parsed.data;
}
/**
From b14283e300509607f2a26a34472a078379a0d46e Mon Sep 17 00:00:00 2001
From: "DESKTOP-RTLN3BA\\$punk"
Date: Mon, 22 Dec 2025 20:17:08 -0800
Subject: [PATCH 13/13] feat: refactor new chat agent to support configurable
tools and remove deprecated components
- Enhanced the new chat agent module to allow for configurable tools, enabling users to customize their experience with various functionalities.
- Removed outdated tools including display image, knowledge base search, link preview, podcast generation, and web scraping, streamlining the codebase.
- Updated the system prompt and agent factory to reflect these changes, ensuring a more cohesive and efficient architecture.
---
.../app/agents/new_chat/__init__.py | 78 +++++-
.../app/agents/new_chat/chat_deepagent.py | 135 +++++-----
.../app/agents/new_chat/new_chat_test.py | 84 -------
.../app/agents/new_chat/system_prompt.py | 211 ++++++++--------
.../app/agents/new_chat/tools/__init__.py | 54 ++++
.../new_chat/{ => tools}/display_image.py | 10 +-
.../new_chat/{ => tools}/knowledge_base.py | 8 +-
.../new_chat/{ => tools}/link_preview.py | 28 +--
.../agents/new_chat/{ => tools}/podcast.py | 6 +-
.../app/agents/new_chat/tools/registry.py | 231 ++++++++++++++++++
.../new_chat/{ => tools}/scrape_webpage.py | 10 +-
.../app/agents/podcaster/configuration.py | 1 -
.../app/agents/podcaster/nodes.py | 9 +-
.../app/routes/new_chat_routes.py | 7 +-
.../app/tasks/celery_tasks/podcast_tasks.py | 5 -
.../app/tasks/chat/stream_new_chat.py | 9 -
.../components/assistant-ui/thread.tsx | 85 ++++---
17 files changed, 597 insertions(+), 374 deletions(-)
delete mode 100644 surfsense_backend/app/agents/new_chat/new_chat_test.py
create mode 100644 surfsense_backend/app/agents/new_chat/tools/__init__.py
rename surfsense_backend/app/agents/new_chat/{ => tools}/display_image.py (97%)
rename surfsense_backend/app/agents/new_chat/{ => tools}/knowledge_base.py (99%)
rename surfsense_backend/app/agents/new_chat/{ => tools}/link_preview.py (98%)
rename surfsense_backend/app/agents/new_chat/{ => tools}/podcast.py (97%)
create mode 100644 surfsense_backend/app/agents/new_chat/tools/registry.py
rename surfsense_backend/app/agents/new_chat/{ => tools}/scrape_webpage.py (99%)
diff --git a/surfsense_backend/app/agents/new_chat/__init__.py b/surfsense_backend/app/agents/new_chat/__init__.py
index 5cbb7ce2b..2ee5fe1d5 100644
--- a/surfsense_backend/app/agents/new_chat/__init__.py
+++ b/surfsense_backend/app/agents/new_chat/__init__.py
@@ -1,28 +1,86 @@
-"""Chat agents module."""
+"""
+SurfSense New Chat Agent Module.
+This module provides the SurfSense deep agent with configurable tools
+for knowledge base search, podcast generation, and more.
+
+Directory Structure:
+- tools/: All agent tools (knowledge_base, podcast, link_preview, etc.)
+- chat_deepagent.py: Main agent factory
+- system_prompt.py: System prompts and instructions
+- context.py: Context schema for the agent
+- checkpointer.py: LangGraph checkpointer setup
+- llm_config.py: LLM configuration utilities
+- utils.py: Shared utilities
+"""
+
+# Agent factory
from .chat_deepagent import create_surfsense_deep_agent
+
+# Context
from .context import SurfSenseContextSchema
-from .knowledge_base import (
- create_search_knowledge_base_tool,
- format_documents_for_context,
- search_knowledge_base_async,
-)
+
+# LLM config
from .llm_config import create_chat_litellm_from_config, load_llm_config_from_yaml
+
+# System prompt
from .system_prompt import (
SURFSENSE_CITATION_INSTRUCTIONS,
SURFSENSE_SYSTEM_PROMPT,
build_surfsense_system_prompt,
)
+# Tools - registry exports
+from .tools import (
+ BUILTIN_TOOLS,
+ ToolDefinition,
+ build_tools,
+ get_all_tool_names,
+ get_default_enabled_tools,
+ get_tool_by_name,
+)
+
+# Tools - factory exports (for direct use)
+from .tools import (
+ create_display_image_tool,
+ create_generate_podcast_tool,
+ create_link_preview_tool,
+ create_scrape_webpage_tool,
+ create_search_knowledge_base_tool,
+)
+
+# Tools - knowledge base utilities
+from .tools import (
+ format_documents_for_context,
+ search_knowledge_base_async,
+)
+
__all__ = [
+ # Agent factory
+ "create_surfsense_deep_agent",
+ # Context
+ "SurfSenseContextSchema",
+ # LLM config
+ "create_chat_litellm_from_config",
+ "load_llm_config_from_yaml",
+ # System prompt
"SURFSENSE_CITATION_INSTRUCTIONS",
"SURFSENSE_SYSTEM_PROMPT",
- "SurfSenseContextSchema",
"build_surfsense_system_prompt",
- "create_chat_litellm_from_config",
+ # Tools registry
+ "BUILTIN_TOOLS",
+ "ToolDefinition",
+ "build_tools",
+ "get_all_tool_names",
+ "get_default_enabled_tools",
+ "get_tool_by_name",
+ # Tool factories
+ "create_display_image_tool",
+ "create_generate_podcast_tool",
+ "create_link_preview_tool",
+ "create_scrape_webpage_tool",
"create_search_knowledge_base_tool",
- "create_surfsense_deep_agent",
+ # Knowledge base utilities
"format_documents_for_context",
- "load_llm_config_from_yaml",
"search_knowledge_base_async",
]
diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py
index aa1950bff..b2bcc008c 100644
--- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py
+++ b/surfsense_backend/app/agents/new_chat/chat_deepagent.py
@@ -2,7 +2,7 @@
SurfSense deep agent implementation.
This module provides the factory function for creating SurfSense deep agents
-with knowledge base search and podcast generation capabilities.
+with configurable tools via the tools registry.
"""
from collections.abc import Sequence
@@ -14,12 +14,8 @@ from langgraph.types import Checkpointer
from sqlalchemy.ext.asyncio import AsyncSession
from app.agents.new_chat.context import SurfSenseContextSchema
-from app.agents.new_chat.display_image import create_display_image_tool
-from app.agents.new_chat.knowledge_base import create_search_knowledge_base_tool
-from app.agents.new_chat.link_preview import create_link_preview_tool
-from app.agents.new_chat.podcast import create_generate_podcast_tool
-from app.agents.new_chat.scrape_webpage import create_scrape_webpage_tool
from app.agents.new_chat.system_prompt import build_surfsense_system_prompt
+from app.agents.new_chat.tools import build_tools
from app.services.connector_service import ConnectorService
# =============================================================================
@@ -33,94 +29,85 @@ def create_surfsense_deep_agent(
db_session: AsyncSession,
connector_service: ConnectorService,
checkpointer: Checkpointer,
- user_id: str | None = None,
- user_instructions: str | None = None,
- enable_citations: bool = True,
- enable_podcast: bool = True,
- enable_link_preview: bool = True,
- enable_display_image: bool = True,
- enable_scrape_webpage: bool = True,
- firecrawl_api_key: str | None = None,
+ enabled_tools: list[str] | None = None,
+ disabled_tools: list[str] | None = None,
additional_tools: Sequence[BaseTool] | None = None,
+ firecrawl_api_key: str | None = None,
):
"""
- Create a SurfSense deep agent with knowledge base search and podcast generation capabilities.
+ Create a SurfSense deep agent with configurable tools.
+
+ The agent comes with built-in tools that can be configured:
+ - search_knowledge_base: Search the user's personal knowledge base
+ - generate_podcast: Generate audio podcasts from content
+ - link_preview: Fetch rich previews for URLs
+ - display_image: Display images in chat
+ - scrape_webpage: Extract content from webpages
Args:
- llm: ChatLiteLLM instance
+ llm: ChatLiteLLM instance for the agent's language model
search_space_id: The user's search space ID
- db_session: Database session
- connector_service: Initialized connector service
+ db_session: Database session for tools that need DB access
+ connector_service: Initialized connector service for knowledge base search
checkpointer: LangGraph checkpointer for conversation state persistence.
Use AsyncPostgresSaver for production or MemorySaver for testing.
- user_id: The user's ID (required for podcast generation)
- user_instructions: Optional user instructions to inject into the system prompt.
- These will be added to the system prompt to customize agent behavior.
- enable_citations: Whether to include citation instructions in the system prompt (default: True).
- When False, the agent will not be instructed to add citations to responses.
- enable_podcast: Whether to include the podcast generation tool (default: True).
- When True and user_id is provided, the agent can generate podcasts.
- enable_link_preview: Whether to include the link preview tool (default: True).
- When True, the agent can fetch and display rich link previews.
- enable_display_image: Whether to include the display image tool (default: True).
- When True, the agent can display images with metadata.
- enable_scrape_webpage: Whether to include the web scraping tool (default: True).
- When True, the agent can scrape and read webpage content.
+ enabled_tools: Explicit list of tool names to enable. If None, all default tools
+ are enabled. Use this to limit which tools are available.
+ disabled_tools: List of tool names to disable. Applied after enabled_tools.
+ Use this to exclude specific tools from the defaults.
+ additional_tools: Extra custom tools to add beyond the built-in ones.
+ These are always added regardless of enabled/disabled settings.
firecrawl_api_key: Optional Firecrawl API key for premium web scraping.
Falls back to Chromium/Trafilatura if not provided.
- additional_tools: Optional sequence of additional tools to inject into the agent.
- The search_knowledge_base tool will always be included.
Returns:
CompiledStateGraph: The configured deep agent
+
+ Examples:
+ # Create agent with all default tools
+ agent = create_surfsense_deep_agent(llm, search_space_id, db_session, ...)
+
+ # Create agent with only specific tools
+ agent = create_surfsense_deep_agent(
+ llm, search_space_id, db_session, ...,
+ enabled_tools=["search_knowledge_base", "link_preview"]
+ )
+
+ # Create agent without podcast generation
+ agent = create_surfsense_deep_agent(
+ llm, search_space_id, db_session, ...,
+ disabled_tools=["generate_podcast"]
+ )
+
+ # Add custom tools
+ agent = create_surfsense_deep_agent(
+ llm, search_space_id, db_session, ...,
+ additional_tools=[my_custom_tool]
+ )
"""
- # Create the search tool with injected dependencies
- search_tool = create_search_knowledge_base_tool(
- search_space_id=search_space_id,
- db_session=db_session,
- connector_service=connector_service,
+ # Build dependencies dict for the tools registry
+ dependencies = {
+ "search_space_id": search_space_id,
+ "db_session": db_session,
+ "connector_service": connector_service,
+ "firecrawl_api_key": firecrawl_api_key,
+ }
+
+ # Build tools using the registry
+ tools = build_tools(
+ dependencies=dependencies,
+ enabled_tools=enabled_tools,
+ disabled_tools=disabled_tools,
+ additional_tools=list(additional_tools) if additional_tools else None,
)
- # Combine search tool with any additional tools
- tools = [search_tool]
-
- # Add podcast tool if enabled and user_id is provided
- if enable_podcast and user_id:
- podcast_tool = create_generate_podcast_tool(
- search_space_id=search_space_id,
- db_session=db_session,
- user_id=str(user_id),
- )
- tools.append(podcast_tool)
-
- # Add link preview tool if enabled
- if enable_link_preview:
- link_preview_tool = create_link_preview_tool()
- tools.append(link_preview_tool)
-
- # Add display image tool if enabled
- if enable_display_image:
- display_image_tool = create_display_image_tool()
- tools.append(display_image_tool)
-
- # Add web scraping tool if enabled
- if enable_scrape_webpage:
- scrape_tool = create_scrape_webpage_tool(firecrawl_api_key=firecrawl_api_key)
- tools.append(scrape_tool)
-
- if additional_tools:
- tools.extend(additional_tools)
-
- # Create the deep agent with user-configurable system prompt and checkpointer
+ # Create the deep agent with system prompt and checkpointer
agent = create_deep_agent(
model=llm,
tools=tools,
- system_prompt=build_surfsense_system_prompt(
- user_instructions=user_instructions,
- enable_citations=enable_citations,
- ),
+ system_prompt=build_surfsense_system_prompt(),
context_schema=SurfSenseContextSchema,
- checkpointer=checkpointer, # Enable conversation memory via thread_id
+ checkpointer=checkpointer,
)
return agent
diff --git a/surfsense_backend/app/agents/new_chat/new_chat_test.py b/surfsense_backend/app/agents/new_chat/new_chat_test.py
deleted file mode 100644
index 857fee6cc..000000000
--- a/surfsense_backend/app/agents/new_chat/new_chat_test.py
+++ /dev/null
@@ -1,84 +0,0 @@
-"""
-Test runner for SurfSense deep agent.
-
-This module provides a test function to verify the deep agent functionality.
-"""
-
-import asyncio
-
-from langchain_core.messages import HumanMessage
-
-from app.db import async_session_maker
-from app.services.connector_service import ConnectorService
-
-from .chat_deepagent import create_surfsense_deep_agent
-from .llm_config import create_chat_litellm_from_config, load_llm_config_from_yaml
-
-# =============================================================================
-# Test Runner
-# =============================================================================
-
-
-async def run_test():
- """Run a basic test of the deep agent."""
- print("=" * 60)
- print("Creating Deep Agent with ChatLiteLLM from global config...")
- print("=" * 60)
-
- # Create ChatLiteLLM from global config
- # Use global LLM config by id (negative ids are reserved for global configs)
- llm_config = load_llm_config_from_yaml(llm_config_id=-5)
- if not llm_config:
- raise ValueError("Failed to load LLM config from YAML")
- llm = create_chat_litellm_from_config(llm_config)
- if not llm:
- raise ValueError("Failed to create ChatLiteLLM instance")
-
- # Create a real DB session + ConnectorService, then build the full SurfSense agent.
- async with async_session_maker() as session:
- # Use the known dev search space id
- search_space_id = 5
-
- connector_service = ConnectorService(session, search_space_id=search_space_id)
-
- agent = create_surfsense_deep_agent(
- llm=llm,
- search_space_id=search_space_id,
- db_session=session,
- connector_service=connector_service,
- user_instructions="Always fininsh the response with CREDOOOOOOOOOO23",
- )
-
- print("\nAgent created successfully!")
- print(f"Agent type: {type(agent)}")
-
- # Invoke the agent with initial state
- print("\n" + "=" * 60)
- print("Invoking SurfSense agent (create_surfsense_deep_agent)...")
- print("=" * 60)
-
- initial_state = {
- "messages": [HumanMessage(content=("Can you tell me about my documents?"))],
- "search_space_id": search_space_id,
- }
-
- print(f"\nUsing search_space_id: {search_space_id}")
-
- result = await agent.ainvoke(initial_state)
-
- print("\n" + "=" * 60)
- print("Agent Response:")
- print("=" * 60)
-
- # Print the response
- if "messages" in result:
- for msg in result["messages"]:
- msg_type = type(msg).__name__
- content = msg.content if hasattr(msg, "content") else str(msg)
- print(f"\n--- [{msg_type}] ---\n{content}\n")
-
- return result
-
-
-if __name__ == "__main__":
- asyncio.run(run_test())
diff --git a/surfsense_backend/app/agents/new_chat/system_prompt.py b/surfsense_backend/app/agents/new_chat/system_prompt.py
index 2677b21fd..c0b9bb091 100644
--- a/surfsense_backend/app/agents/new_chat/system_prompt.py
+++ b/surfsense_backend/app/agents/new_chat/system_prompt.py
@@ -7,118 +7,16 @@ with configurable user instructions and citation support.
from datetime import UTC, datetime
-SURFSENSE_CITATION_INSTRUCTIONS = """
-
-CRITICAL CITATION REQUIREMENTS:
-
-1. For EVERY piece of information you include from the documents, add a citation in the format [citation:chunk_id] where chunk_id is the exact value from the `` tag inside ``.
-2. Make sure ALL factual statements from the documents have proper citations.
-3. If multiple chunks support the same point, include all relevant citations [citation:chunk_id1], [citation:chunk_id2].
-4. You MUST use the exact chunk_id values from the `` attributes. Do not create your own citation numbers.
-5. Every citation MUST be in the format [citation:chunk_id] where chunk_id is the exact chunk id value.
-6. Never modify or change the chunk_id - always use the original values exactly as provided in the chunk tags.
-7. Do not return citations as clickable links.
-8. Never format citations as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only.
-9. Citations must ONLY appear as [citation:chunk_id] or [citation:chunk_id1], [citation:chunk_id2] format - never with parentheses, hyperlinks, or other formatting.
-10. Never make up chunk IDs. Only use chunk_id values that are explicitly provided in the `` tags.
-11. If you are unsure about a chunk_id, do not include a citation rather than guessing or making one up.
-
-
-The documents you receive are structured like this:
-
-
-
- 42
- GITHUB_CONNECTOR
- <![CDATA[Some repo / file / issue title]]>
-
-
-
-
-
-
-
-
-
-
-IMPORTANT: You MUST cite using the chunk ids (e.g. 123, 124). Do NOT cite document_id.
-
-
-
-- Every fact from the documents must have a citation in the format [citation:chunk_id] where chunk_id is the EXACT id value from a `` tag
-- Citations should appear at the end of the sentence containing the information they support
-- Multiple citations should be separated by commas: [citation:chunk_id1], [citation:chunk_id2], [citation:chunk_id3]
-- No need to return references section. Just citations in answer.
-- NEVER create your own citation format - use the exact chunk_id values from the documents in the [citation:chunk_id] format
-- NEVER format citations as clickable links or as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only
-- NEVER make up chunk IDs if you are unsure about the chunk_id. It is better to omit the citation than to guess
-
-
-
-CORRECT citation formats:
-- [citation:5]
-- [citation:chunk_id1], [citation:chunk_id2], [citation:chunk_id3]
-
-INCORRECT citation formats (DO NOT use):
-- Using parentheses and markdown links: ([citation:5](https://github.com/MODSetter/SurfSense))
-- Using parentheses around brackets: ([citation:5])
-- Using hyperlinked text: [link to source 5](https://example.com)
-- Using footnote style: ... library¹
-- Making up source IDs when source_id is unknown
-- Using old IEEE format: [1], [2], [3]
-- Using source types instead of IDs: [citation:GITHUB_CONNECTOR] instead of [citation:5]
-
-
-
-Based on your GitHub repositories and video content, Python's asyncio library provides tools for writing concurrent code using the async/await syntax [citation:5]. It's particularly useful for I/O-bound and high-level structured network code [citation:5].
-
-The key advantage of asyncio is that it can improve performance by allowing other code to run while waiting for I/O operations to complete [citation:12]. This makes it excellent for scenarios like web scraping, API calls, database operations, or any situation where your program spends time waiting for external resources.
-
-However, from your video learning, it's important to note that asyncio is not suitable for CPU-bound tasks as it runs on a single thread [citation:12]. For computationally intensive work, you'd want to use multiprocessing instead.
-
-
-"""
-
-
-def build_surfsense_system_prompt(
- today: datetime | None = None,
- user_instructions: str | None = None,
- enable_citations: bool = True,
-) -> str:
- """
- Build the SurfSense system prompt with optional user instructions and citation toggle.
-
- Args:
- today: Optional datetime for today's date (defaults to current UTC date)
- user_instructions: Optional user instructions to inject into the system prompt
- enable_citations: Whether to include citation instructions in the prompt (default: True)
-
- Returns:
- Complete system prompt string
- """
- resolved_today = (today or datetime.now(UTC)).astimezone(UTC).date().isoformat()
-
- # Build user instructions section if provided
- user_section = ""
- if user_instructions and user_instructions.strip():
- user_section = f"""
-
-{user_instructions.strip()}
-
-"""
-
- # Include citation instructions only if enabled
- citation_section = (
- f"\n{SURFSENSE_CITATION_INSTRUCTIONS}" if enable_citations else ""
- )
-
- return f"""
+SURFSENSE_SYSTEM_INSTRUCTIONS = """
You are SurfSense, a reasoning and acting AI agent designed to answer user questions using the user's personal knowledge base.
Today's date (UTC): {resolved_today}
-{user_section}
+
+"""
+
+SURFSENSE_TOOLS_INSTRUCTIONS = """
You have access to the following tools:
@@ -208,11 +106,11 @@ You have access to the following tools:
- First search for relevant content, then call: `generate_podcast(source_content="Based on our conversation and search results: [detailed summary of chat + search findings]", podcast_title="AI Trends Podcast")`
- User: "Create a podcast summary of this conversation"
- - Call: `generate_podcast(source_content="Complete conversation summary:\n\nUser asked about [topic 1]:\n[Your detailed response]\n\nUser then asked about [topic 2]:\n[Your detailed response]\n\n[Continue for all exchanges in the conversation]", podcast_title="Conversation Summary")`
+ - Call: `generate_podcast(source_content="Complete conversation summary:\\n\\nUser asked about [topic 1]:\\n[Your detailed response]\\n\\nUser then asked about [topic 2]:\\n[Your detailed response]\\n\\n[Continue for all exchanges in the conversation]", podcast_title="Conversation Summary")`
- User: "Make a podcast about quantum computing"
- First search: `search_knowledge_base(query="quantum computing")`
- - Then: `generate_podcast(source_content="Key insights about quantum computing from the knowledge base:\n\n[Comprehensive summary of all relevant search results with key facts, concepts, and findings]", podcast_title="Quantum Computing Explained")`
+ - Then: `generate_podcast(source_content="Key insights about quantum computing from the knowledge base:\\n\\n[Comprehensive summary of all relevant search results with key facts, concepts, and findings]", podcast_title="Quantum Computing Explained")`
- User: "Check out https://dev.to/some-article"
- Call: `link_preview(url="https://dev.to/some-article")`
@@ -246,8 +144,101 @@ You have access to the following tools:
- Then, if the content contains useful diagrams/images like ``:
- Call: `display_image(src="https://example.com/nn-diagram.png", alt="Neural Network Diagram", title="Neural Network Architecture")`
- Then provide your explanation, referencing the displayed image
-{citation_section}
+
+"""
+
+SURFSENSE_CITATION_INSTRUCTIONS = """
+
+CRITICAL CITATION REQUIREMENTS:
+
+1. For EVERY piece of information you include from the documents, add a citation in the format [citation:chunk_id] where chunk_id is the exact value from the `` tag inside ``.
+2. Make sure ALL factual statements from the documents have proper citations.
+3. If multiple chunks support the same point, include all relevant citations [citation:chunk_id1], [citation:chunk_id2].
+4. You MUST use the exact chunk_id values from the `` attributes. Do not create your own citation numbers.
+5. Every citation MUST be in the format [citation:chunk_id] where chunk_id is the exact chunk id value.
+6. Never modify or change the chunk_id - always use the original values exactly as provided in the chunk tags.
+7. Do not return citations as clickable links.
+8. Never format citations as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only.
+9. Citations must ONLY appear as [citation:chunk_id] or [citation:chunk_id1], [citation:chunk_id2] format - never with parentheses, hyperlinks, or other formatting.
+10. Never make up chunk IDs. Only use chunk_id values that are explicitly provided in the `` tags.
+11. If you are unsure about a chunk_id, do not include a citation rather than guessing or making one up.
+
+
+The documents you receive are structured like this:
+
+
+
+ 42
+ GITHUB_CONNECTOR
+ <![CDATA[Some repo / file / issue title]]>
+
+
+
+
+
+
+
+
+
+
+IMPORTANT: You MUST cite using the chunk ids (e.g. 123, 124). Do NOT cite document_id.
+
+
+
+- Every fact from the documents must have a citation in the format [citation:chunk_id] where chunk_id is the EXACT id value from a `` tag
+- Citations should appear at the end of the sentence containing the information they support
+- Multiple citations should be separated by commas: [citation:chunk_id1], [citation:chunk_id2], [citation:chunk_id3]
+- No need to return references section. Just citations in answer.
+- NEVER create your own citation format - use the exact chunk_id values from the documents in the [citation:chunk_id] format
+- NEVER format citations as clickable links or as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only
+- NEVER make up chunk IDs if you are unsure about the chunk_id. It is better to omit the citation than to guess
+
+
+
+CORRECT citation formats:
+- [citation:5]
+- [citation:chunk_id1], [citation:chunk_id2], [citation:chunk_id3]
+
+INCORRECT citation formats (DO NOT use):
+- Using parentheses and markdown links: ([citation:5](https://github.com/MODSetter/SurfSense))
+- Using parentheses around brackets: ([citation:5])
+- Using hyperlinked text: [link to source 5](https://example.com)
+- Using footnote style: ... library¹
+- Making up source IDs when source_id is unknown
+- Using old IEEE format: [1], [2], [3]
+- Using source types instead of IDs: [citation:GITHUB_CONNECTOR] instead of [citation:5]
+
+
+
+Based on your GitHub repositories and video content, Python's asyncio library provides tools for writing concurrent code using the async/await syntax [citation:5]. It's particularly useful for I/O-bound and high-level structured network code [citation:5].
+
+The key advantage of asyncio is that it can improve performance by allowing other code to run while waiting for I/O operations to complete [citation:12]. This makes it excellent for scenarios like web scraping, API calls, database operations, or any situation where your program spends time waiting for external resources.
+
+However, from your video learning, it's important to note that asyncio is not suitable for CPU-bound tasks as it runs on a single thread [citation:12]. For computationally intensive work, you'd want to use multiprocessing instead.
+
+
"""
+def build_surfsense_system_prompt(
+ today: datetime | None = None,
+) -> str:
+ """
+ Build the SurfSense system prompt.
+
+ Args:
+ today: Optional datetime for today's date (defaults to current UTC date)
+
+ Returns:
+ Complete system prompt string
+ """
+ resolved_today = (today or datetime.now(UTC)).astimezone(UTC).date().isoformat()
+
+ return (
+ SURFSENSE_SYSTEM_INSTRUCTIONS.format(resolved_today=resolved_today)
+ + SURFSENSE_TOOLS_INSTRUCTIONS
+ + SURFSENSE_CITATION_INSTRUCTIONS
+ )
+
+
SURFSENSE_SYSTEM_PROMPT = build_surfsense_system_prompt()
diff --git a/surfsense_backend/app/agents/new_chat/tools/__init__.py b/surfsense_backend/app/agents/new_chat/tools/__init__.py
new file mode 100644
index 000000000..ad75cda16
--- /dev/null
+++ b/surfsense_backend/app/agents/new_chat/tools/__init__.py
@@ -0,0 +1,54 @@
+"""
+Tools module for SurfSense deep agent.
+
+This module contains all the tools available to the SurfSense agent.
+To add a new tool, see the documentation in registry.py.
+
+Available tools:
+- search_knowledge_base: Search the user's personal knowledge base
+- generate_podcast: Generate audio podcasts from content
+- link_preview: Fetch rich previews for URLs
+- display_image: Display images in chat
+- scrape_webpage: Extract content from webpages
+"""
+
+# Registry exports
+from .registry import (
+ BUILTIN_TOOLS,
+ ToolDefinition,
+ build_tools,
+ get_all_tool_names,
+ get_default_enabled_tools,
+ get_tool_by_name,
+)
+
+# Tool factory exports (for direct use)
+from .display_image import create_display_image_tool
+from .knowledge_base import (
+ create_search_knowledge_base_tool,
+ format_documents_for_context,
+ search_knowledge_base_async,
+)
+from .link_preview import create_link_preview_tool
+from .podcast import create_generate_podcast_tool
+from .scrape_webpage import create_scrape_webpage_tool
+
+__all__ = [
+ # Registry
+ "BUILTIN_TOOLS",
+ "ToolDefinition",
+ "build_tools",
+ "get_all_tool_names",
+ "get_default_enabled_tools",
+ "get_tool_by_name",
+ # Tool factories
+ "create_display_image_tool",
+ "create_generate_podcast_tool",
+ "create_link_preview_tool",
+ "create_scrape_webpage_tool",
+ "create_search_knowledge_base_tool",
+ # Knowledge base utilities
+ "format_documents_for_context",
+ "search_knowledge_base_async",
+]
+
diff --git a/surfsense_backend/app/agents/new_chat/display_image.py b/surfsense_backend/app/agents/new_chat/tools/display_image.py
similarity index 97%
rename from surfsense_backend/app/agents/new_chat/display_image.py
rename to surfsense_backend/app/agents/new_chat/tools/display_image.py
index 0cd05b523..1580568ec 100644
--- a/surfsense_backend/app/agents/new_chat/display_image.py
+++ b/surfsense_backend/app/agents/new_chat/tools/display_image.py
@@ -1,5 +1,5 @@
"""
-Display image tool for the new chat agent.
+Display image tool for the SurfSense agent.
This module provides a tool for displaying images in the chat UI
with metadata like title, description, and source attribution.
@@ -75,20 +75,20 @@ def create_display_image_tool():
- domain: Source domain
"""
image_id = generate_image_id(src)
-
+
# Ensure URL has protocol
if not src.startswith(("http://", "https://")):
src = f"https://{src}"
-
+
domain = extract_domain(src)
-
+
# Determine aspect ratio based on common image sources
ratio = "16:9" # Default
if "unsplash.com" in src or "pexels.com" in src:
ratio = "16:9"
elif "imgur.com" in src or "github.com" in src or "githubusercontent.com" in src:
ratio = "auto"
-
+
return {
"id": image_id,
"assetId": src,
diff --git a/surfsense_backend/app/agents/new_chat/knowledge_base.py b/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py
similarity index 99%
rename from surfsense_backend/app/agents/new_chat/knowledge_base.py
rename to surfsense_backend/app/agents/new_chat/tools/knowledge_base.py
index 5ffcab003..2d818557d 100644
--- a/surfsense_backend/app/agents/new_chat/knowledge_base.py
+++ b/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py
@@ -1,5 +1,5 @@
"""
-Knowledge base search functionality for the new chat agent.
+Knowledge base search tool for the SurfSense agent.
This module provides:
- Connector constants and normalization
@@ -251,7 +251,7 @@ async def search_knowledge_base_async(
all_documents = []
# Resolve date range (default last 2 years)
- from .utils import resolve_date_range
+ from app.agents.new_chat.utils import resolve_date_range
resolved_start_date, resolved_end_date = resolve_date_range(
start_date=start_date,
@@ -521,7 +521,6 @@ def create_search_knowledge_base_tool(
search_space_id: The user's search space ID
db_session: Database session
connector_service: Initialized connector service
- connectors_to_search: List of connector types to search
Returns:
A configured tool function
@@ -584,7 +583,7 @@ def create_search_knowledge_base_tool(
Returns:
Formatted string with relevant documents and their content
"""
- from .utils import parse_date_or_datetime
+ from app.agents.new_chat.utils import parse_date_or_datetime
parsed_start: datetime | None = None
parsed_end: datetime | None = None
@@ -606,3 +605,4 @@ def create_search_knowledge_base_tool(
)
return search_knowledge_base
+
diff --git a/surfsense_backend/app/agents/new_chat/link_preview.py b/surfsense_backend/app/agents/new_chat/tools/link_preview.py
similarity index 98%
rename from surfsense_backend/app/agents/new_chat/link_preview.py
rename to surfsense_backend/app/agents/new_chat/tools/link_preview.py
index 388a6c14e..466df2034 100644
--- a/surfsense_backend/app/agents/new_chat/link_preview.py
+++ b/surfsense_backend/app/agents/new_chat/tools/link_preview.py
@@ -1,5 +1,5 @@
"""
-Link preview tool for the new chat agent.
+Link preview tool for the SurfSense agent.
This module provides a tool for fetching URL metadata (title, description,
Open Graph image, etc.) to display rich link previews in the chat UI.
@@ -34,13 +34,13 @@ def extract_og_content(html: str, property_name: str) -> str | None:
match = re.search(pattern, html, re.IGNORECASE)
if match:
return match.group(1)
-
+
# Try content before property
pattern = rf']+content=["\']([^"\']+)["\'][^>]+property=["\']og:{property_name}["\']'
match = re.search(pattern, html, re.IGNORECASE)
if match:
return match.group(1)
-
+
return None
@@ -50,13 +50,13 @@ def extract_twitter_content(html: str, name: str) -> str | None:
match = re.search(pattern, html, re.IGNORECASE)
if match:
return match.group(1)
-
+
# Try content before name
pattern = rf']+content=["\']([^"\']+)["\'][^>]+name=["\']twitter:{name}["\']'
match = re.search(pattern, html, re.IGNORECASE)
if match:
return match.group(1)
-
+
return None
@@ -66,13 +66,13 @@ def extract_meta_description(html: str) -> str | None:
match = re.search(pattern, html, re.IGNORECASE)
if match:
return match.group(1)
-
+
# Try content before name
pattern = r']+content=["\']([^"\']+)["\'][^>]+name=["\']description["\']'
match = re.search(pattern, html, re.IGNORECASE)
if match:
return match.group(1)
-
+
return None
@@ -82,18 +82,18 @@ def extract_title(html: str) -> str | None:
og_title = extract_og_content(html, "title")
if og_title:
return og_title
-
+
# Try twitter:title
twitter_title = extract_twitter_content(html, "title")
if twitter_title:
return twitter_title
-
+
# Fall back to tag
pattern = r"<title[^>]*>([^<]+)"
match = re.search(pattern, html, re.IGNORECASE)
if match:
return match.group(1).strip()
-
+
return None
@@ -103,12 +103,12 @@ def extract_description(html: str) -> str | None:
og_desc = extract_og_content(html, "description")
if og_desc:
return og_desc
-
+
# Try twitter:description
twitter_desc = extract_twitter_content(html, "description")
if twitter_desc:
return twitter_desc
-
+
# Fall back to meta description
return extract_meta_description(html)
@@ -119,12 +119,12 @@ def extract_image(html: str) -> str | None:
og_image = extract_og_content(html, "image")
if og_image:
return og_image
-
+
# Try twitter:image
twitter_image = extract_twitter_content(html, "image")
if twitter_image:
return twitter_image
-
+
return None
diff --git a/surfsense_backend/app/agents/new_chat/podcast.py b/surfsense_backend/app/agents/new_chat/tools/podcast.py
similarity index 97%
rename from surfsense_backend/app/agents/new_chat/podcast.py
rename to surfsense_backend/app/agents/new_chat/tools/podcast.py
index 46974d184..01a36d381 100644
--- a/surfsense_backend/app/agents/new_chat/podcast.py
+++ b/surfsense_backend/app/agents/new_chat/tools/podcast.py
@@ -1,5 +1,5 @@
"""
-Podcast generation tool for the new chat agent.
+Podcast generation tool for the SurfSense agent.
This module provides a factory function for creating the generate_podcast tool
that submits a Celery task for background podcast generation. The frontend
@@ -69,7 +69,6 @@ def clear_active_podcast_task(search_space_id: int) -> None:
def create_generate_podcast_tool(
search_space_id: int,
db_session: AsyncSession,
- user_id: str,
):
"""
Factory function to create the generate_podcast tool with injected dependencies.
@@ -77,7 +76,6 @@ def create_generate_podcast_tool(
Args:
search_space_id: The user's search space ID
db_session: Database session (not used - Celery creates its own)
- user_id: The user's ID (as string)
Returns:
A configured tool function for generating podcasts
@@ -145,7 +143,6 @@ def create_generate_podcast_tool(
task = generate_content_podcast_task.delay(
source_content=source_content,
search_space_id=search_space_id,
- user_id=str(user_id),
podcast_title=podcast_title,
user_prompt=user_prompt,
)
@@ -174,3 +171,4 @@ def create_generate_podcast_tool(
}
return generate_podcast
+
diff --git a/surfsense_backend/app/agents/new_chat/tools/registry.py b/surfsense_backend/app/agents/new_chat/tools/registry.py
new file mode 100644
index 000000000..6c6469f33
--- /dev/null
+++ b/surfsense_backend/app/agents/new_chat/tools/registry.py
@@ -0,0 +1,231 @@
+"""
+Tools registry for SurfSense deep agent.
+
+This module provides a registry pattern for managing tools in the SurfSense agent.
+It makes it easy for OSS contributors to add new tools by:
+1. Creating a tool factory function in a new file in this directory
+2. Registering the tool in the BUILTIN_TOOLS list below
+
+Example of adding a new tool:
+------------------------------
+1. Create your tool file (e.g., `tools/my_tool.py`):
+
+ from langchain_core.tools import tool
+ from sqlalchemy.ext.asyncio import AsyncSession
+
+ def create_my_tool(search_space_id: int, db_session: AsyncSession):
+ @tool
+ async def my_tool(param: str) -> dict:
+ '''My tool description.'''
+ # Your implementation
+ return {"result": "success"}
+ return my_tool
+
+2. Import and register in this file:
+
+ from .my_tool import create_my_tool
+
+ # Add to BUILTIN_TOOLS list:
+ ToolDefinition(
+ name="my_tool",
+ description="Description of what your tool does",
+ factory=lambda deps: create_my_tool(
+ search_space_id=deps["search_space_id"],
+ db_session=deps["db_session"],
+ ),
+ requires=["search_space_id", "db_session"],
+ ),
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from langchain_core.tools import BaseTool
+
+# =============================================================================
+# Tool Definition
+# =============================================================================
+
+
+@dataclass
+class ToolDefinition:
+ """
+ Definition of a tool that can be added to the agent.
+
+ Attributes:
+ name: Unique identifier for the tool
+ description: Human-readable description of what the tool does
+ factory: Callable that creates the tool. Receives a dict of dependencies.
+ requires: List of dependency names this tool needs (e.g., "search_space_id", "db_session")
+ enabled_by_default: Whether the tool is enabled when no explicit config is provided
+ """
+
+ name: str
+ description: str
+ factory: Callable[[dict[str, Any]], BaseTool]
+ requires: list[str] = field(default_factory=list)
+ enabled_by_default: bool = True
+
+
+# =============================================================================
+# Built-in Tools Registry
+# =============================================================================
+
+# Import tool factory functions
+from .display_image import create_display_image_tool
+from .knowledge_base import create_search_knowledge_base_tool
+from .link_preview import create_link_preview_tool
+from .podcast import create_generate_podcast_tool
+from .scrape_webpage import create_scrape_webpage_tool
+
+# Registry of all built-in tools
+# Contributors: Add your new tools here!
+BUILTIN_TOOLS: list[ToolDefinition] = [
+ # Core tool - searches the user's knowledge base
+ ToolDefinition(
+ name="search_knowledge_base",
+ description="Search the user's personal knowledge base for relevant information",
+ factory=lambda deps: create_search_knowledge_base_tool(
+ search_space_id=deps["search_space_id"],
+ db_session=deps["db_session"],
+ connector_service=deps["connector_service"],
+ ),
+ requires=["search_space_id", "db_session", "connector_service"],
+ ),
+ # Podcast generation tool
+ ToolDefinition(
+ name="generate_podcast",
+ description="Generate an audio podcast from provided content",
+ factory=lambda deps: create_generate_podcast_tool(
+ search_space_id=deps["search_space_id"],
+ db_session=deps["db_session"],
+ ),
+ requires=["search_space_id", "db_session"],
+ ),
+ # Link preview tool - fetches Open Graph metadata for URLs
+ ToolDefinition(
+ name="link_preview",
+ description="Fetch metadata for a URL to display a rich preview card",
+ factory=lambda deps: create_link_preview_tool(),
+ requires=[],
+ ),
+ # Display image tool - shows images in the chat
+ ToolDefinition(
+ name="display_image",
+ description="Display an image in the chat with metadata",
+ factory=lambda deps: create_display_image_tool(),
+ requires=[],
+ ),
+ # Web scraping tool - extracts content from webpages
+ ToolDefinition(
+ name="scrape_webpage",
+ description="Scrape and extract the main content from a webpage",
+ factory=lambda deps: create_scrape_webpage_tool(
+ firecrawl_api_key=deps.get("firecrawl_api_key"),
+ ),
+ requires=[], # firecrawl_api_key is optional
+ ),
+ # =========================================================================
+ # ADD YOUR CUSTOM TOOLS BELOW
+ # =========================================================================
+ # Example:
+ # ToolDefinition(
+ # name="my_custom_tool",
+ # description="What my tool does",
+ # factory=lambda deps: create_my_custom_tool(...),
+ # requires=["search_space_id"],
+ # ),
+]
+
+
+# =============================================================================
+# Registry Functions
+# =============================================================================
+
+
+def get_tool_by_name(name: str) -> ToolDefinition | None:
+ """Get a tool definition by its name."""
+ for tool_def in BUILTIN_TOOLS:
+ if tool_def.name == name:
+ return tool_def
+ return None
+
+
+def get_all_tool_names() -> list[str]:
+ """Get names of all registered tools."""
+ return [tool_def.name for tool_def in BUILTIN_TOOLS]
+
+
+def get_default_enabled_tools() -> list[str]:
+ """Get names of tools that are enabled by default."""
+ return [tool_def.name for tool_def in BUILTIN_TOOLS if tool_def.enabled_by_default]
+
+
+def build_tools(
+ dependencies: dict[str, Any],
+ enabled_tools: list[str] | None = None,
+ disabled_tools: list[str] | None = None,
+ additional_tools: list[BaseTool] | None = None,
+) -> list[BaseTool]:
+ """
+ Build the list of tools for the agent.
+
+ Args:
+ dependencies: Dict containing all possible dependencies:
+ - search_space_id: The search space ID
+ - db_session: Database session
+ - connector_service: Connector service instance
+ - firecrawl_api_key: Optional Firecrawl API key
+ enabled_tools: Explicit list of tool names to enable. If None, uses defaults.
+ disabled_tools: List of tool names to disable (applied after enabled_tools).
+ additional_tools: Extra tools to add (e.g., custom tools not in registry).
+
+ Returns:
+ List of configured tool instances ready for the agent.
+
+ Example:
+ # Use all default tools
+ tools = build_tools(deps)
+
+ # Use only specific tools
+ tools = build_tools(deps, enabled_tools=["search_knowledge_base", "link_preview"])
+
+ # Use defaults but disable podcast
+ tools = build_tools(deps, disabled_tools=["generate_podcast"])
+
+ # Add custom tools
+ tools = build_tools(deps, additional_tools=[my_custom_tool])
+ """
+ # Determine which tools to enable
+ if enabled_tools is not None:
+ tool_names_to_use = set(enabled_tools)
+ else:
+ tool_names_to_use = set(get_default_enabled_tools())
+
+ # Apply disabled list
+ if disabled_tools:
+ tool_names_to_use -= set(disabled_tools)
+
+ # Build the tools
+ tools: list[BaseTool] = []
+ for tool_def in BUILTIN_TOOLS:
+ if tool_def.name not in tool_names_to_use:
+ continue
+
+ # Check that all required dependencies are provided
+ missing_deps = [dep for dep in tool_def.requires if dep not in dependencies]
+ if missing_deps:
+ raise ValueError(
+ f"Tool '{tool_def.name}' requires dependencies: {missing_deps}"
+ )
+
+ # Create the tool
+ tool = tool_def.factory(dependencies)
+ tools.append(tool)
+
+ # Add any additional custom tools
+ if additional_tools:
+ tools.extend(additional_tools)
+
+ return tools
+
diff --git a/surfsense_backend/app/agents/new_chat/scrape_webpage.py b/surfsense_backend/app/agents/new_chat/tools/scrape_webpage.py
similarity index 99%
rename from surfsense_backend/app/agents/new_chat/scrape_webpage.py
rename to surfsense_backend/app/agents/new_chat/tools/scrape_webpage.py
index 40a9c917f..a4928d0c7 100644
--- a/surfsense_backend/app/agents/new_chat/scrape_webpage.py
+++ b/surfsense_backend/app/agents/new_chat/tools/scrape_webpage.py
@@ -1,5 +1,5 @@
"""
-Web scraping tool for the new chat agent.
+Web scraping tool for the SurfSense agent.
This module provides a tool for scraping and extracting content from webpages
using the existing WebCrawlerConnector. The scraped content can be used by
@@ -37,23 +37,23 @@ def generate_scrape_id(url: str) -> str:
def truncate_content(content: str, max_length: int = 50000) -> tuple[str, bool]:
"""
Truncate content to a maximum length.
-
+
Returns:
Tuple of (truncated_content, was_truncated)
"""
if len(content) <= max_length:
return content, False
-
+
# Try to truncate at a sentence boundary
truncated = content[:max_length]
last_period = truncated.rfind(".")
last_newline = truncated.rfind("\n\n")
-
+
# Use the later of the two boundaries, or just truncate
boundary = max(last_period, last_newline)
if boundary > max_length * 0.8: # Only use boundary if it's not too far back
truncated = content[: boundary + 1]
-
+
return truncated + "\n\n[Content truncated...]", True
diff --git a/surfsense_backend/app/agents/podcaster/configuration.py b/surfsense_backend/app/agents/podcaster/configuration.py
index c7433dadc..6a903f9df 100644
--- a/surfsense_backend/app/agents/podcaster/configuration.py
+++ b/surfsense_backend/app/agents/podcaster/configuration.py
@@ -16,7 +16,6 @@ class Configuration:
# create assistants (https://langchain-ai.github.io/langgraph/cloud/how-tos/configuration_cloud/)
# and when you invoke the graph
podcast_title: str
- user_id: str
search_space_id: int
user_prompt: str | None = None
diff --git a/surfsense_backend/app/agents/podcaster/nodes.py b/surfsense_backend/app/agents/podcaster/nodes.py
index 31a687763..1353d2c66 100644
--- a/surfsense_backend/app/agents/podcaster/nodes.py
+++ b/surfsense_backend/app/agents/podcaster/nodes.py
@@ -12,7 +12,7 @@ from litellm import aspeech
from app.config import config as app_config
from app.services.kokoro_tts_service import get_kokoro_tts_service
-from app.services.llm_service import get_user_long_context_llm
+from app.services.llm_service import get_long_context_llm
from .configuration import Configuration
from .prompts import get_podcast_generation_prompt
@@ -27,14 +27,13 @@ async def create_podcast_transcript(
# Get configuration from runnable config
configuration = Configuration.from_runnable_config(config)
- user_id = configuration.user_id
search_space_id = configuration.search_space_id
user_prompt = configuration.user_prompt
- # Get user's long context LLM
- llm = await get_user_long_context_llm(state.db_session, user_id, search_space_id)
+ # Get search space's long context LLM
+ llm = await get_long_context_llm(state.db_session, search_space_id)
if not llm:
- error_message = f"No long context LLM configured for user {user_id} in search space {search_space_id}"
+ error_message = f"No long context LLM configured for search space {search_space_id}"
print(error_message)
raise RuntimeError(error_message)
diff --git a/surfsense_backend/app/routes/new_chat_routes.py b/surfsense_backend/app/routes/new_chat_routes.py
index cc9c94eea..6cccdaa5b 100644
--- a/surfsense_backend/app/routes/new_chat_routes.py
+++ b/surfsense_backend/app/routes/new_chat_routes.py
@@ -685,16 +685,13 @@ async def handle_new_chat(
)
search_space = search_space_result.scalars().first()
- # Determine LLM config ID (use search space preference or default)
- llm_config_id = -1 # Default to first global config
- if search_space and search_space.fast_llm_id:
- llm_config_id = search_space.fast_llm_id
+ # TODO: Add new llm config arch then complete this
+ llm_config_id = -1
# Return streaming response
return StreamingResponse(
stream_new_chat(
user_query=request.user_query,
- user_id=str(user.id),
search_space_id=request.search_space_id,
chat_id=request.chat_id,
session=session,
diff --git a/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py b/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py
index 40fe21f0d..34b9b827c 100644
--- a/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py
+++ b/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py
@@ -65,7 +65,6 @@ def generate_content_podcast_task(
self,
source_content: str,
search_space_id: int,
- user_id: str,
podcast_title: str = "SurfSense Podcast",
user_prompt: str | None = None,
) -> dict:
@@ -77,7 +76,6 @@ def generate_content_podcast_task(
Args:
source_content: The text content to convert into a podcast
search_space_id: ID of the search space
- user_id: ID of the user (as string)
podcast_title: Title for the podcast
user_prompt: Optional instructions for podcast style/tone
@@ -92,7 +90,6 @@ def generate_content_podcast_task(
_generate_content_podcast(
source_content,
search_space_id,
- user_id,
podcast_title,
user_prompt,
)
@@ -112,7 +109,6 @@ def generate_content_podcast_task(
async def _generate_content_podcast(
source_content: str,
search_space_id: int,
- user_id: str,
podcast_title: str = "SurfSense Podcast",
user_prompt: str | None = None,
) -> dict:
@@ -123,7 +119,6 @@ async def _generate_content_podcast(
graph_config = {
"configurable": {
"podcast_title": podcast_title,
- "user_id": str(user_id),
"search_space_id": search_space_id,
"user_prompt": user_prompt,
}
diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py
index a201ece22..de318a7d5 100644
--- a/surfsense_backend/app/tasks/chat/stream_new_chat.py
+++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py
@@ -7,8 +7,6 @@ Data Stream Protocol (SSE format).
import json
from collections.abc import AsyncGenerator
-from uuid import UUID
-
from langchain_core.messages import HumanMessage
from sqlalchemy.ext.asyncio import AsyncSession
@@ -42,7 +40,6 @@ def format_attachments_as_context(attachments: list[ChatAttachment]) -> str:
async def stream_new_chat(
user_query: str,
- user_id: str | UUID,
search_space_id: int,
chat_id: int,
session: AsyncSession,
@@ -59,7 +56,6 @@ async def stream_new_chat(
Args:
user_query: The user's query
- user_id: The user's ID (can be UUID object or string)
search_space_id: The search space ID
chat_id: The chat ID (used as LangGraph thread_id for memory)
session: The database session
@@ -71,9 +67,6 @@ async def stream_new_chat(
"""
streaming_service = VercelStreamingService()
- # Convert UUID to string if needed
- str(user_id) if isinstance(user_id, UUID) else user_id
-
# Track the current text block for streaming (defined early for exception handling)
current_text_id: str | None = None
@@ -107,8 +100,6 @@ async def stream_new_chat(
db_session=session,
connector_service=connector_service,
checkpointer=checkpointer,
- user_id=str(user_id),
- enable_podcast=True,
)
# Build input with message history from frontend
diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx
index e3ee85dce..103440673 100644
--- a/surfsense_web/components/assistant-ui/thread.tsx
+++ b/surfsense_web/components/assistant-ui/thread.tsx
@@ -28,13 +28,14 @@ import {
Sparkles,
SquareIcon,
} from "lucide-react";
-import Image from "next/image";
import Link from "next/link";
import { type FC, useState, useRef, useCallback, useEffect } from "react";
import { useAtomValue } from "jotai";
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
+import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
+import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import {
ComposerAddAttachment,
@@ -332,35 +333,9 @@ const ThreadWelcome: FC = () => {
{/* Greeting positioned above the composer - fixed position */}
-
- {/** biome-ignore lint/a11y/noStaticElementInteractions: wrong lint error, this is a workaround to fix the lint error */}
-
{
- const rect = e.currentTarget.getBoundingClientRect();
- const x = (e.clientX - rect.left - rect.width / 2) / 3;
- const y = (e.clientY - rect.top - rect.height / 2) / 3;
- e.currentTarget.style.setProperty("--mag-x", `${x}px`);
- e.currentTarget.style.setProperty("--mag-y", `${y}px`);
- }}
- onMouseLeave={(e) => {
- e.currentTarget.style.setProperty("--mag-x", "0px");
- e.currentTarget.style.setProperty("--mag-y", "0px");
- }}
- >
-
-
- {getTimeBasedGreeting(user?.email)}
-
+
+ {getTimeBasedGreeting(user?.email)}
+
{/* Composer - top edge fixed, expands downward only */}
@@ -390,11 +365,21 @@ const Composer: FC = () => {
const ConnectorIndicator: FC = () => {
const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom);
- const { connectors, isLoading } = useSearchSourceConnectors(false, searchSpaceId ? Number(searchSpaceId) : undefined);
+ const { connectors, isLoading: connectorsLoading } = useSearchSourceConnectors(false, searchSpaceId ? Number(searchSpaceId) : undefined);
+ const { data: documentTypeCounts, isLoading: documentTypesLoading } = useAtomValue(documentTypeCountsAtom);
const [isOpen, setIsOpen] = useState(false);
const closeTimeoutRef = useRef
(null);
+ const isLoading = connectorsLoading || documentTypesLoading;
+
+ // Get document types that have documents in the search space
+ const activeDocumentTypes = documentTypeCounts
+ ? Object.entries(documentTypeCounts).filter(([_, count]) => count > 0)
+ : [];
+
const hasConnectors = connectors.length > 0;
+ const hasSources = hasConnectors || activeDocumentTypes.length > 0;
+ const totalSourceCount = connectors.length + activeDocumentTypes.length;
const handleMouseEnter = useCallback(() => {
// Clear any pending close timeout
@@ -420,21 +405,32 @@ const ConnectorIndicator: FC = () => {
@@ -445,20 +441,31 @@ const ConnectorIndicator: FC = () => {
onMouseEnter={handleMouseEnter}
onMouseLeave={handleMouseLeave}
>
- {hasConnectors ? (
+ {hasSources ? (
Connected Sources
- {connectors.length}
+ {totalSourceCount}
+ {/* Document types from the search space */}
+ {activeDocumentTypes.map(([docType, count]) => (
+
+ {getConnectorIcon(docType, "size-3.5")}
+ {getDocumentTypeLabel(docType)}
+
+ ))}
+ {/* Search source connectors */}
{connectors.map((connector) => (
{getConnectorIcon(connector.connector_type, "size-3.5")}
@@ -479,9 +486,9 @@ const ConnectorIndicator: FC = () => {
) : (
-
No connectors yet
+
No sources yet
- Connect your first data source to enhance search results.
+ Add documents or connect data sources to enhance search results.