From 28985e6af45eecea4a9192919fb2d963262a0995 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Tue, 23 Dec 2025 00:35:08 +0530
Subject: [PATCH 07/12] style: update greeting messages for improved user
engagement and consistency
---
surfsense_web/components/assistant-ui/thread.tsx | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx
index aaf35c329..57408391d 100644
--- a/surfsense_web/components/assistant-ui/thread.tsx
+++ b/surfsense_web/components/assistant-ui/thread.tsx
@@ -210,14 +210,14 @@ const getTimeBasedGreeting = (userEmail?: string): string => {
"Good afternoon",
"Afternoon",
"Hey there",
- "Hope you're having a great day",
+ "Hi there",
];
const eveningGreetings = [
"Good evening",
"Evening",
"Hey there",
- "Hope you had a great day",
+ "Hi there",
];
const nightGreetings = [
@@ -228,10 +228,10 @@ const getTimeBasedGreeting = (userEmail?: string): string => {
];
const lateNightGreetings = [
- "Burning the midnight oil",
"Still up",
"Night owl mode",
"The night is young",
+ "Hi there",
];
// Select a random greeting based on time
@@ -358,9 +358,7 @@ const ConnectorIndicator: FC = () => {
"outline-none focus:outline-none focus-visible:outline-none",
"border-0 ring-0 focus:ring-0 shadow-none focus:shadow-none",
"data-[state=open]:bg-transparent data-[state=open]:shadow-none data-[state=open]:ring-0",
- hasConnectors
- ? "text-muted-foreground"
- : "text-muted-foreground/60"
+ "text-muted-foreground"
)}
aria-label={hasConnectors ? "View connected sources" : "Add your first connector"}
onMouseEnter={handleMouseEnter}
From 4b69fdf214dae95198929bb51b543248392a0313 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Tue, 23 Dec 2025 00:58:27 +0530
Subject: [PATCH 08/12] feat: add link preview tool for enhanced URL metadata
display in chat
---
.../app/agents/new_chat/chat_deepagent.py | 9 +
.../app/agents/new_chat/link_preview.py | 292 ++++++++++++++
.../app/agents/new_chat/system_prompt.py | 22 +
.../app/tasks/chat/stream_new_chat.py | 87 +++-
.../new-chat/[[...chat_id]]/page.tsx | 4 +-
surfsense_web/components/tool-ui/index.ts | 17 +
.../components/tool-ui/link-preview.tsx | 226 +++++++++++
.../components/tool-ui/media-card/index.tsx | 381 ++++++++++++++++++
8 files changed, 1035 insertions(+), 3 deletions(-)
create mode 100644 surfsense_backend/app/agents/new_chat/link_preview.py
create mode 100644 surfsense_web/components/tool-ui/link-preview.tsx
create mode 100644 surfsense_web/components/tool-ui/media-card/index.tsx
diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py
index 1c97a05ad..23db4f813 100644
--- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py
+++ b/surfsense_backend/app/agents/new_chat/chat_deepagent.py
@@ -15,6 +15,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.agents.new_chat.context import SurfSenseContextSchema
from app.agents.new_chat.knowledge_base import create_search_knowledge_base_tool
+from app.agents.new_chat.link_preview import create_link_preview_tool
from app.agents.new_chat.podcast import create_generate_podcast_tool
from app.agents.new_chat.system_prompt import build_surfsense_system_prompt
from app.services.connector_service import ConnectorService
@@ -34,6 +35,7 @@ def create_surfsense_deep_agent(
user_instructions: str | None = None,
enable_citations: bool = True,
enable_podcast: bool = True,
+ enable_link_preview: bool = True,
additional_tools: Sequence[BaseTool] | None = None,
):
"""
@@ -53,6 +55,8 @@ def create_surfsense_deep_agent(
When False, the agent will not be instructed to add citations to responses.
enable_podcast: Whether to include the podcast generation tool (default: True).
When True and user_id is provided, the agent can generate podcasts.
+ enable_link_preview: Whether to include the link preview tool (default: True).
+ When True, the agent can fetch and display rich link previews.
additional_tools: Optional sequence of additional tools to inject into the agent.
The search_knowledge_base tool will always be included.
@@ -78,6 +82,11 @@ def create_surfsense_deep_agent(
)
tools.append(podcast_tool)
+ # Add link preview tool if enabled
+ if enable_link_preview:
+ link_preview_tool = create_link_preview_tool()
+ tools.append(link_preview_tool)
+
if additional_tools:
tools.extend(additional_tools)
diff --git a/surfsense_backend/app/agents/new_chat/link_preview.py b/surfsense_backend/app/agents/new_chat/link_preview.py
new file mode 100644
index 000000000..388a6c14e
--- /dev/null
+++ b/surfsense_backend/app/agents/new_chat/link_preview.py
@@ -0,0 +1,292 @@
+"""
+Link preview tool for the new chat agent.
+
+This module provides a tool for fetching URL metadata (title, description,
+Open Graph image, etc.) to display rich link previews in the chat UI.
+"""
+
+import hashlib
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+from langchain_core.tools import tool
+
+
+def extract_domain(url: str) -> str:
+ """Extract the domain from a URL."""
+ try:
+ parsed = urlparse(url)
+ domain = parsed.netloc
+ # Remove 'www.' prefix if present
+ if domain.startswith("www."):
+ domain = domain[4:]
+ return domain
+ except Exception:
+ return ""
+
+
+def extract_og_content(html: str, property_name: str) -> str | None:
+ """Extract Open Graph meta content from HTML."""
+ # Try og:property first
+ pattern = rf'
]+property=["\']og:{property_name}["\'][^>]+content=["\']([^"\']+)["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ # Try content before property
+ pattern = rf']+content=["\']([^"\']+)["\'][^>]+property=["\']og:{property_name}["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ return None
+
+
+def extract_twitter_content(html: str, name: str) -> str | None:
+ """Extract Twitter Card meta content from HTML."""
+ pattern = rf']+name=["\']twitter:{name}["\'][^>]+content=["\']([^"\']+)["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ # Try content before name
+ pattern = rf']+content=["\']([^"\']+)["\'][^>]+name=["\']twitter:{name}["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ return None
+
+
+def extract_meta_description(html: str) -> str | None:
+ """Extract meta description from HTML."""
+ pattern = r']+name=["\']description["\'][^>]+content=["\']([^"\']+)["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ # Try content before name
+ pattern = r']+content=["\']([^"\']+)["\'][^>]+name=["\']description["\']'
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1)
+
+ return None
+
+
+def extract_title(html: str) -> str | None:
+ """Extract title from HTML."""
+ # Try og:title first
+ og_title = extract_og_content(html, "title")
+ if og_title:
+ return og_title
+
+ # Try twitter:title
+ twitter_title = extract_twitter_content(html, "title")
+ if twitter_title:
+ return twitter_title
+
+ # Fall back to tag
+ pattern = r"<title[^>]*>([^<]+)"
+ match = re.search(pattern, html, re.IGNORECASE)
+ if match:
+ return match.group(1).strip()
+
+ return None
+
+
+def extract_description(html: str) -> str | None:
+ """Extract description from HTML."""
+ # Try og:description first
+ og_desc = extract_og_content(html, "description")
+ if og_desc:
+ return og_desc
+
+ # Try twitter:description
+ twitter_desc = extract_twitter_content(html, "description")
+ if twitter_desc:
+ return twitter_desc
+
+ # Fall back to meta description
+ return extract_meta_description(html)
+
+
+def extract_image(html: str) -> str | None:
+ """Extract image URL from HTML."""
+ # Try og:image first
+ og_image = extract_og_content(html, "image")
+ if og_image:
+ return og_image
+
+ # Try twitter:image
+ twitter_image = extract_twitter_content(html, "image")
+ if twitter_image:
+ return twitter_image
+
+ return None
+
+
+def generate_preview_id(url: str) -> str:
+ """Generate a unique ID for a link preview."""
+ hash_val = hashlib.md5(url.encode()).hexdigest()[:12]
+ return f"link-preview-{hash_val}"
+
+
+def create_link_preview_tool():
+ """
+ Factory function to create the link_preview tool.
+
+ Returns:
+ A configured tool function for fetching link previews.
+ """
+
+ @tool
+ async def link_preview(url: str) -> dict[str, Any]:
+ """
+ Fetch metadata for a URL to display a rich link preview.
+
+ Use this tool when the user shares a URL or asks about a specific webpage.
+ This tool fetches the page's Open Graph metadata (title, description, image)
+ to display a nice preview card in the chat.
+
+ Common triggers include:
+ - User shares a URL in the chat
+ - User asks "What's this link about?" or similar
+ - User says "Show me a preview of this page"
+ - User wants to preview an article or webpage
+
+ Args:
+ url: The URL to fetch metadata for. Must be a valid HTTP/HTTPS URL.
+
+ Returns:
+ A dictionary containing:
+ - id: Unique identifier for this preview
+ - assetId: The URL itself (for deduplication)
+ - kind: "link" (type of media card)
+ - href: The URL to open when clicked
+ - title: Page title
+ - description: Page description (if available)
+ - thumb: Thumbnail/preview image URL (if available)
+ - domain: The domain name
+ - error: Error message (if fetch failed)
+ """
+ preview_id = generate_preview_id(url)
+ domain = extract_domain(url)
+
+ # Validate URL
+ if not url.startswith(("http://", "https://")):
+ url = f"https://{url}"
+
+ try:
+ async with httpx.AsyncClient(
+ timeout=10.0,
+ follow_redirects=True,
+ headers={
+ "User-Agent": "Mozilla/5.0 (compatible; SurfSenseBot/1.0; +https://surfsense.net)",
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.5",
+ },
+ ) as client:
+ response = await client.get(url)
+ response.raise_for_status()
+
+ # Get content type to ensure it's HTML
+ content_type = response.headers.get("content-type", "")
+ if "text/html" not in content_type.lower():
+ # Not an HTML page, return basic info
+ return {
+ "id": preview_id,
+ "assetId": url,
+ "kind": "link",
+ "href": url,
+ "title": url.split("/")[-1] or domain,
+ "description": f"File from {domain}",
+ "domain": domain,
+ }
+
+ html = response.text
+
+ # Extract metadata
+ title = extract_title(html) or domain
+ description = extract_description(html)
+ image = extract_image(html)
+
+ # Make sure image URL is absolute
+ if image and not image.startswith(("http://", "https://")):
+ if image.startswith("//"):
+ image = f"https:{image}"
+ elif image.startswith("/"):
+ parsed = urlparse(url)
+ image = f"{parsed.scheme}://{parsed.netloc}{image}"
+
+ # Clean up title and description (unescape HTML entities)
+ if title:
+ title = (
+ title.replace("&", "&")
+ .replace("<", "<")
+ .replace(">", ">")
+ .replace(""", '"')
+ .replace("'", "'")
+ .replace("'", "'")
+ )
+ if description:
+ description = (
+ description.replace("&", "&")
+ .replace("<", "<")
+ .replace(">", ">")
+ .replace(""", '"')
+ .replace("'", "'")
+ .replace("'", "'")
+ )
+ # Truncate long descriptions
+ if len(description) > 200:
+ description = description[:197] + "..."
+
+ return {
+ "id": preview_id,
+ "assetId": url,
+ "kind": "link",
+ "href": url,
+ "title": title,
+ "description": description,
+ "thumb": image,
+ "domain": domain,
+ }
+
+ except httpx.TimeoutException:
+ return {
+ "id": preview_id,
+ "assetId": url,
+ "kind": "link",
+ "href": url,
+ "title": domain or "Link",
+ "domain": domain,
+ "error": "Request timed out",
+ }
+ except httpx.HTTPStatusError as e:
+ return {
+ "id": preview_id,
+ "assetId": url,
+ "kind": "link",
+ "href": url,
+ "title": domain or "Link",
+ "domain": domain,
+ "error": f"HTTP {e.response.status_code}",
+ }
+ except Exception as e:
+ error_message = str(e)
+ print(f"[link_preview] Error fetching {url}: {error_message}")
+ return {
+ "id": preview_id,
+ "assetId": url,
+ "kind": "link",
+ "href": url,
+ "title": domain or "Link",
+ "domain": domain,
+ "error": f"Failed to fetch: {error_message[:50]}",
+ }
+
+ return link_preview
+
diff --git a/surfsense_backend/app/agents/new_chat/system_prompt.py b/surfsense_backend/app/agents/new_chat/system_prompt.py
index f725be684..32e649e5d 100644
--- a/surfsense_backend/app/agents/new_chat/system_prompt.py
+++ b/surfsense_backend/app/agents/new_chat/system_prompt.py
@@ -145,6 +145,19 @@ You have access to the following tools:
- Returns: A task_id for tracking. The podcast will be generated in the background.
- IMPORTANT: Only one podcast can be generated at a time. If a podcast is already being generated, the tool will return status "already_generating".
- After calling this tool, inform the user that podcast generation has started and they will see the player when it's ready (takes 3-5 minutes).
+
+3. link_preview: Fetch metadata for a URL to display a rich preview card.
+ - IMPORTANT: Use this tool WHENEVER the user shares or mentions a URL/link in their message.
+ - This fetches the page's Open Graph metadata (title, description, thumbnail) to show a preview card.
+ - NOTE: This tool only fetches metadata, NOT the full page content. It cannot read the article text.
+ - Trigger scenarios:
+ * User shares a URL (e.g., "Check out https://example.com")
+ * User pastes a link in their message
+ * User asks about a URL or link
+ - Args:
+ - url: The URL to fetch metadata for (must be a valid HTTP/HTTPS URL)
+ - Returns: A rich preview card with title, description, thumbnail, and domain
+ - The preview card will automatically be displayed in the chat.
- User: "Fetch all my notes and what's in them?"
@@ -162,6 +175,15 @@ You have access to the following tools:
- User: "Make a podcast about quantum computing"
- First search: `search_knowledge_base(query="quantum computing")`
- Then: `generate_podcast(source_content="Key insights about quantum computing from the knowledge base:\n\n[Comprehensive summary of all relevant search results with key facts, concepts, and findings]", podcast_title="Quantum Computing Explained")`
+
+- User: "Check out https://dev.to/some-article"
+ - Call: `link_preview(url="https://dev.to/some-article")`
+
+- User: "What's this blog post about? https://example.com/blog/post"
+ - Call: `link_preview(url="https://example.com/blog/post")`
+
+- User: "https://github.com/some/repo"
+ - Call: `link_preview(url="https://github.com/some/repo")`
{citation_section}
"""
diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py
index cb40a82a8..853fb3392 100644
--- a/surfsense_backend/app/tasks/chat/stream_new_chat.py
+++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py
@@ -284,6 +284,20 @@ async def stream_new_chat(
status="in_progress",
items=last_active_step_items,
)
+ elif tool_name == "link_preview":
+ url = (
+ tool_input.get("url", "")
+ if isinstance(tool_input, dict)
+ else str(tool_input)
+ )
+ last_active_step_title = "Fetching link preview"
+ last_active_step_items = [f"URL: {url[:80]}{'...' if len(url) > 80 else ''}"]
+ yield streaming_service.format_thinking_step(
+ step_id=tool_step_id,
+ title="Fetching link preview",
+ status="in_progress",
+ items=last_active_step_items,
+ )
elif tool_name == "generate_podcast":
podcast_title = (
tool_input.get("podcast_title", "SurfSense Podcast")
@@ -343,6 +357,16 @@ async def stream_new_chat(
f"Searching knowledge base: {query[:100]}{'...' if len(query) > 100 else ''}",
"info",
)
+ elif tool_name == "link_preview":
+ url = (
+ tool_input.get("url", "")
+ if isinstance(tool_input, dict)
+ else str(tool_input)
+ )
+ yield streaming_service.format_terminal_info(
+ f"Fetching link preview: {url[:80]}{'...' if len(url) > 80 else ''}",
+ "info",
+ )
elif tool_name == "generate_podcast":
title = (
tool_input.get("podcast_title", "SurfSense Podcast")
@@ -404,6 +428,31 @@ async def stream_new_chat(
status="completed",
items=completed_items,
)
+ elif tool_name == "link_preview":
+ # Build completion items based on link preview result
+ if isinstance(tool_output, dict):
+ title = tool_output.get("title", "Link")
+ domain = tool_output.get("domain", "")
+ has_error = "error" in tool_output
+ if has_error:
+ completed_items = [
+ *last_active_step_items,
+ f"Error: {tool_output.get('error', 'Failed to fetch')}",
+ ]
+ else:
+ completed_items = [
+ *last_active_step_items,
+ f"Title: {title[:60]}{'...' if len(title) > 60 else ''}",
+ f"Domain: {domain}" if domain else "Preview loaded",
+ ]
+ else:
+ completed_items = [*last_active_step_items, "Preview loaded"]
+ yield streaming_service.format_thinking_step(
+ step_id=original_step_id,
+ title="Fetching link preview",
+ status="completed",
+ items=completed_items,
+ )
elif tool_name == "generate_podcast":
# Build detailed completion items based on podcast status
podcast_status = (
@@ -492,8 +541,33 @@ async def stream_new_chat(
f"Podcast generation failed: {error_msg}",
"error",
)
- else:
- # Don't stream the full output for other tools (can be very large), just acknowledge
+ elif tool_name == "link_preview":
+ # Stream the full link preview result so frontend can render the MediaCard
+ yield streaming_service.format_tool_output_available(
+ tool_call_id,
+ tool_output
+ if isinstance(tool_output, dict)
+ else {"result": tool_output},
+ )
+ # Send appropriate terminal message
+ if isinstance(tool_output, dict) and "error" not in tool_output:
+ title = tool_output.get("title", "Link")
+ yield streaming_service.format_terminal_info(
+ f"Link preview loaded: {title[:50]}{'...' if len(title) > 50 else ''}",
+ "success",
+ )
+ else:
+ error_msg = (
+ tool_output.get("error", "Failed to fetch")
+ if isinstance(tool_output, dict)
+ else "Failed to fetch"
+ )
+ yield streaming_service.format_terminal_info(
+ f"Link preview failed: {error_msg}",
+ "error",
+ )
+ elif tool_name == "search_knowledge_base":
+ # Don't stream the full output for search (can be very large), just acknowledge
yield streaming_service.format_tool_output_available(
tool_call_id,
{"status": "completed", "result_length": len(str(tool_output))},
@@ -501,6 +575,15 @@ async def stream_new_chat(
yield streaming_service.format_terminal_info(
"Knowledge base search completed", "success"
)
+ else:
+ # Default handling for other tools
+ yield streaming_service.format_tool_output_available(
+ tool_call_id,
+ {"status": "completed", "result_length": len(str(tool_output))},
+ )
+ yield streaming_service.format_terminal_info(
+ f"Tool {tool_name} completed", "success"
+ )
# Handle chain/agent end to close any open text blocks
elif event_type in ("on_chain_end", "on_agent_end"):
diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
index 51a2f9875..4d45db118 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
@@ -11,6 +11,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import { Thread } from "@/components/assistant-ui/thread";
import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast";
+import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview";
import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking";
import { getBearerToken } from "@/lib/auth-utils";
import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter";
@@ -79,7 +80,7 @@ function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike {
/**
* Tools that should render custom UI in the chat.
*/
-const TOOLS_WITH_UI = new Set(["generate_podcast"]);
+const TOOLS_WITH_UI = new Set(["generate_podcast", "link_preview"]);
/**
* Type for thinking step data from the backend
@@ -589,6 +590,7 @@ export default function NewChatPage() {
return (
+
diff --git a/surfsense_web/components/tool-ui/index.ts b/surfsense_web/components/tool-ui/index.ts
index 3cddfba21..aebe400d5 100644
--- a/surfsense_web/components/tool-ui/index.ts
+++ b/surfsense_web/components/tool-ui/index.ts
@@ -15,3 +15,20 @@ export {
type DeepAgentThinkingArgs,
type DeepAgentThinkingResult,
} from "./deepagent-thinking";
+export {
+ LinkPreviewToolUI,
+ MultiLinkPreviewToolUI,
+ type LinkPreviewArgs,
+ type LinkPreviewResult,
+ type MultiLinkPreviewArgs,
+ type MultiLinkPreviewResult,
+} from "./link-preview";
+export {
+ MediaCard,
+ MediaCardErrorBoundary,
+ MediaCardLoading,
+ MediaCardSkeleton,
+ parseSerializableMediaCard,
+ type MediaCardProps,
+ type SerializableMediaCard,
+} from "./media-card";
diff --git a/surfsense_web/components/tool-ui/link-preview.tsx b/surfsense_web/components/tool-ui/link-preview.tsx
new file mode 100644
index 000000000..c97b6820b
--- /dev/null
+++ b/surfsense_web/components/tool-ui/link-preview.tsx
@@ -0,0 +1,226 @@
+"use client";
+
+import { makeAssistantToolUI } from "@assistant-ui/react";
+import { AlertCircleIcon, ExternalLinkIcon, LinkIcon } from "lucide-react";
+import {
+ MediaCard,
+ MediaCardErrorBoundary,
+ MediaCardLoading,
+ parseSerializableMediaCard,
+ type SerializableMediaCard,
+} from "@/components/tool-ui/media-card";
+
+/**
+ * Type definitions for the link_preview tool
+ */
+interface LinkPreviewArgs {
+ url: string;
+ title?: string;
+}
+
+interface LinkPreviewResult {
+ id: string;
+ assetId: string;
+ kind: "link";
+ href: string;
+ title: string;
+ description?: string;
+ thumb?: string;
+ domain?: string;
+ error?: string;
+}
+
+/**
+ * Error state component shown when link preview fails
+ */
+function LinkPreviewErrorState({ url, error }: { url: string; error: string }) {
+ return (
+
+
+
+
+
Failed to load preview
+
{url}
+
{error}
+
+
+
+
+
+ );
+ }
+
+ // Incomplete/cancelled state
+ if (status.type === "incomplete") {
+ if (status.reason === "cancelled") {
+ return ;
+ }
+ if (status.reason === "error") {
+ return (
+
+ );
+ }
+ }
+
+ // No result yet
+ if (!result) {
+ return (
+
+
+
+ );
+ }
+
+ // Error result from the tool
+ if (result.error) {
+ return ;
+ }
+
+ // Success - render the media card
+ return (
+
+ );
+ },
+});
+
+/**
+ * Multiple Link Previews Tool UI Component
+ *
+ * This component handles cases where multiple links need to be previewed.
+ * It renders a grid of link preview cards.
+ */
+interface MultiLinkPreviewArgs {
+ urls: string[];
+}
+
+interface MultiLinkPreviewResult {
+ previews: LinkPreviewResult[];
+ errors?: { url: string; error: string }[];
+}
+
+export const MultiLinkPreviewToolUI = makeAssistantToolUI<
+ MultiLinkPreviewArgs,
+ MultiLinkPreviewResult
+>({
+ toolName: "multi_link_preview",
+ render: function MultiLinkPreviewUI({ args, result, status }) {
+ const urls = args.urls || [];
+
+ // Loading state
+ if (status.type === "running" || status.type === "requires-action") {
+ return (
+
+ {urls.slice(0, 4).map((url, index) => (
+
+ ))}
+
+ );
+ }
+
+ // Incomplete state
+ if (status.type === "incomplete") {
+ return (
+
+
+
+ Link previews cancelled
+
+
+ {result.previews.map((preview) => (
+
+
+
+ ))}
+ {result.errors?.map((err) => (
+
+ ))}
+
+
+
+
+
Failed to load preview
+
+ {this.state.error?.message || "An error occurred"}
+
+
+
{
+ // Hide broken images
+ e.currentTarget.style.display = "none";
+ }}
+ />
+ {/* Gradient overlay */}
+
+
+
+ {getKindIcon(kind)}
+ {kind === "link" ? "Link Preview" : kind}
+
+
+ {/* Domain favicon placeholder */}
+
+
+
+
+
+ {/* Domain badge */}
+ {displayDomain && (
+
+
+ {displayDomain}
+
+ {href && (
+
+ )}
+
+ )}
+
+ {/* Title */}
+
+ {title}
+
+
+ {/* Description */}
+ {description && (
+
+ {description}
+
+ )}
+
+
e.stopPropagation()}
+ onKeyDown={(e) => e.stopPropagation()}
+ >
+ {responseActions.map((action) => (
+
+
+
+
+ {action.confirmLabel && (
+
+ {action.confirmLabel}
+
+ )}
+
+ ))}
+
+
+
+
+
+ {title}
+
+
+ );
+}
+
From da7cb812528562c729e3a5dc00ea13023e6b0923 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Tue, 23 Dec 2025 01:11:56 +0530
Subject: [PATCH 09/12] feat: introduce display image tool for enhanced image
rendering in chat with metadata support
---
.../app/agents/new_chat/chat_deepagent.py | 9 +
.../app/agents/new_chat/display_image.py | 106 ++++++
.../app/agents/new_chat/system_prompt.py | 21 ++
.../app/tasks/chat/stream_new_chat.py | 64 ++++
.../new-chat/[[...chat_id]]/page.tsx | 4 +-
.../components/tool-ui/display-image.tsx | 160 +++++++++
.../components/tool-ui/image/index.tsx | 332 ++++++++++++++++++
surfsense_web/components/tool-ui/index.ts | 14 +
8 files changed, 709 insertions(+), 1 deletion(-)
create mode 100644 surfsense_backend/app/agents/new_chat/display_image.py
create mode 100644 surfsense_web/components/tool-ui/display-image.tsx
create mode 100644 surfsense_web/components/tool-ui/image/index.tsx
diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py
index 23db4f813..e6b2aca06 100644
--- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py
+++ b/surfsense_backend/app/agents/new_chat/chat_deepagent.py
@@ -14,6 +14,7 @@ from langgraph.types import Checkpointer
from sqlalchemy.ext.asyncio import AsyncSession
from app.agents.new_chat.context import SurfSenseContextSchema
+from app.agents.new_chat.display_image import create_display_image_tool
from app.agents.new_chat.knowledge_base import create_search_knowledge_base_tool
from app.agents.new_chat.link_preview import create_link_preview_tool
from app.agents.new_chat.podcast import create_generate_podcast_tool
@@ -36,6 +37,7 @@ def create_surfsense_deep_agent(
enable_citations: bool = True,
enable_podcast: bool = True,
enable_link_preview: bool = True,
+ enable_display_image: bool = True,
additional_tools: Sequence[BaseTool] | None = None,
):
"""
@@ -57,6 +59,8 @@ def create_surfsense_deep_agent(
When True and user_id is provided, the agent can generate podcasts.
enable_link_preview: Whether to include the link preview tool (default: True).
When True, the agent can fetch and display rich link previews.
+ enable_display_image: Whether to include the display image tool (default: True).
+ When True, the agent can display images with metadata.
additional_tools: Optional sequence of additional tools to inject into the agent.
The search_knowledge_base tool will always be included.
@@ -87,6 +91,11 @@ def create_surfsense_deep_agent(
link_preview_tool = create_link_preview_tool()
tools.append(link_preview_tool)
+ # Add display image tool if enabled
+ if enable_display_image:
+ display_image_tool = create_display_image_tool()
+ tools.append(display_image_tool)
+
if additional_tools:
tools.extend(additional_tools)
diff --git a/surfsense_backend/app/agents/new_chat/display_image.py b/surfsense_backend/app/agents/new_chat/display_image.py
new file mode 100644
index 000000000..03153300a
--- /dev/null
+++ b/surfsense_backend/app/agents/new_chat/display_image.py
@@ -0,0 +1,106 @@
+"""
+Display image tool for the new chat agent.
+
+This module provides a tool for displaying images in the chat UI
+with metadata like title, description, and source attribution.
+"""
+
+import hashlib
+from typing import Any
+from urllib.parse import urlparse
+
+from langchain_core.tools import tool
+
+
+def extract_domain(url: str) -> str:
+ """Extract the domain from a URL."""
+ try:
+ parsed = urlparse(url)
+ domain = parsed.netloc
+ # Remove 'www.' prefix if present
+ if domain.startswith("www."):
+ domain = domain[4:]
+ return domain
+ except Exception:
+ return ""
+
+
+def generate_image_id(src: str) -> str:
+ """Generate a unique ID for an image."""
+ hash_val = hashlib.md5(src.encode()).hexdigest()[:12]
+ return f"image-{hash_val}"
+
+
+def create_display_image_tool():
+ """
+ Factory function to create the display_image tool.
+
+ Returns:
+ A configured tool function for displaying images.
+ """
+
+ @tool
+ async def display_image(
+ src: str,
+ alt: str = "Image",
+ title: str | None = None,
+ description: str | None = None,
+ ) -> dict[str, Any]:
+ """
+ Display an image in the chat with metadata.
+
+ Use this tool when you want to show an image to the user.
+ This displays the image with an optional title, description,
+ and source attribution.
+
+ Common use cases:
+ - Showing an image from a URL the user mentioned
+ - Displaying a diagram or chart you're referencing
+ - Showing example images when explaining concepts
+
+ Args:
+ src: The URL of the image to display (must be a valid HTTP/HTTPS URL)
+ alt: Alternative text describing the image (for accessibility)
+ title: Optional title to display below the image
+ description: Optional description providing context about the image
+
+ Returns:
+ A dictionary containing image metadata for the UI to render:
+ - id: Unique identifier for this image
+ - assetId: The image URL (for deduplication)
+ - src: The image URL
+ - alt: Alt text for accessibility
+ - title: Image title (if provided)
+ - description: Image description (if provided)
+ - domain: Source domain
+ """
+ image_id = generate_image_id(src)
+
+ # Ensure URL has protocol
+ if not src.startswith(("http://", "https://")):
+ src = f"https://{src}"
+
+ domain = extract_domain(src)
+
+ # Determine aspect ratio based on common image sources
+ ratio = "16:9" # Default
+ if "unsplash.com" in src or "pexels.com" in src:
+ ratio = "16:9"
+ elif "imgur.com" in src:
+ ratio = "auto"
+ elif "github.com" in src or "githubusercontent.com" in src:
+ ratio = "auto"
+
+ return {
+ "id": image_id,
+ "assetId": src,
+ "src": src,
+ "alt": alt,
+ "title": title,
+ "description": description,
+ "domain": domain,
+ "ratio": ratio,
+ }
+
+ return display_image
+
diff --git a/surfsense_backend/app/agents/new_chat/system_prompt.py b/surfsense_backend/app/agents/new_chat/system_prompt.py
index 32e649e5d..74abadd38 100644
--- a/surfsense_backend/app/agents/new_chat/system_prompt.py
+++ b/surfsense_backend/app/agents/new_chat/system_prompt.py
@@ -158,6 +158,21 @@ You have access to the following tools:
- url: The URL to fetch metadata for (must be a valid HTTP/HTTPS URL)
- Returns: A rich preview card with title, description, thumbnail, and domain
- The preview card will automatically be displayed in the chat.
+
+4. display_image: Display an image in the chat with metadata.
+ - Use this tool when you want to show an image to the user.
+ - This displays the image with an optional title, description, and source attribution.
+ - Common use cases:
+ * Showing an image from a URL mentioned in the conversation
+ * Displaying a diagram, chart, or illustration you're referencing
+ * Showing visual examples when explaining concepts
+ - Args:
+ - src: The URL of the image to display (must be a valid HTTP/HTTPS image URL)
+ - alt: Alternative text describing the image (for accessibility)
+ - title: Optional title to display below the image
+ - description: Optional description providing context about the image
+ - Returns: An image card with the image, title, and description
+ - The image will automatically be displayed in the chat.
- User: "Fetch all my notes and what's in them?"
@@ -184,6 +199,12 @@ You have access to the following tools:
- User: "https://github.com/some/repo"
- Call: `link_preview(url="https://github.com/some/repo")`
+
+- User: "Show me this image: https://example.com/image.png"
+ - Call: `display_image(src="https://example.com/image.png", alt="User shared image")`
+
+- User: "Can you display a diagram of a neural network?"
+ - Call: `display_image(src="https://example.com/neural-network.png", alt="Neural network diagram", title="Neural Network Architecture", description="A visual representation of a neural network with input, hidden, and output layers")`
{citation_section}
"""
diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py
index 853fb3392..6e2b30c6a 100644
--- a/surfsense_backend/app/tasks/chat/stream_new_chat.py
+++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py
@@ -298,6 +298,27 @@ async def stream_new_chat(
status="in_progress",
items=last_active_step_items,
)
+ elif tool_name == "display_image":
+ src = (
+ tool_input.get("src", "")
+ if isinstance(tool_input, dict)
+ else str(tool_input)
+ )
+ title = (
+ tool_input.get("title", "")
+ if isinstance(tool_input, dict)
+ else ""
+ )
+ last_active_step_title = "Displaying image"
+ last_active_step_items = [
+ f"Image: {title[:50] if title else src[:50]}{'...' if len(title or src) > 50 else ''}"
+ ]
+ yield streaming_service.format_thinking_step(
+ step_id=tool_step_id,
+ title="Displaying image",
+ status="in_progress",
+ items=last_active_step_items,
+ )
elif tool_name == "generate_podcast":
podcast_title = (
tool_input.get("podcast_title", "SurfSense Podcast")
@@ -367,6 +388,16 @@ async def stream_new_chat(
f"Fetching link preview: {url[:80]}{'...' if len(url) > 80 else ''}",
"info",
)
+ elif tool_name == "display_image":
+ src = (
+ tool_input.get("src", "")
+ if isinstance(tool_input, dict)
+ else str(tool_input)
+ )
+ yield streaming_service.format_terminal_info(
+ f"Displaying image: {src[:60]}{'...' if len(src) > 60 else ''}",
+ "info",
+ )
elif tool_name == "generate_podcast":
title = (
tool_input.get("podcast_title", "SurfSense Podcast")
@@ -453,6 +484,24 @@ async def stream_new_chat(
status="completed",
items=completed_items,
)
+ elif tool_name == "display_image":
+ # Build completion items for image display
+ if isinstance(tool_output, dict):
+ title = tool_output.get("title", "")
+ alt = tool_output.get("alt", "Image")
+ display_name = title or alt
+ completed_items = [
+ *last_active_step_items,
+ f"Showing: {display_name[:50]}{'...' if len(display_name) > 50 else ''}",
+ ]
+ else:
+ completed_items = [*last_active_step_items, "Image displayed"]
+ yield streaming_service.format_thinking_step(
+ step_id=original_step_id,
+ title="Displaying image",
+ status="completed",
+ items=completed_items,
+ )
elif tool_name == "generate_podcast":
# Build detailed completion items based on podcast status
podcast_status = (
@@ -566,6 +615,21 @@ async def stream_new_chat(
f"Link preview failed: {error_msg}",
"error",
)
+ elif tool_name == "display_image":
+ # Stream the full image result so frontend can render the Image component
+ yield streaming_service.format_tool_output_available(
+ tool_call_id,
+ tool_output
+ if isinstance(tool_output, dict)
+ else {"result": tool_output},
+ )
+ # Send terminal message
+ if isinstance(tool_output, dict):
+ title = tool_output.get("title") or tool_output.get("alt", "Image")
+ yield streaming_service.format_terminal_info(
+ f"Image displayed: {title[:40]}{'...' if len(title) > 40 else ''}",
+ "success",
+ )
elif tool_name == "search_knowledge_base":
# Don't stream the full output for search (can be very large), just acknowledge
yield streaming_service.format_tool_output_available(
diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
index 4d45db118..7c6bd34c1 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
@@ -12,6 +12,7 @@ import { toast } from "sonner";
import { Thread } from "@/components/assistant-ui/thread";
import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast";
import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview";
+import { DisplayImageToolUI } from "@/components/tool-ui/display-image";
import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking";
import { getBearerToken } from "@/lib/auth-utils";
import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter";
@@ -80,7 +81,7 @@ function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike {
/**
* Tools that should render custom UI in the chat.
*/
-const TOOLS_WITH_UI = new Set(["generate_podcast", "link_preview"]);
+const TOOLS_WITH_UI = new Set(["generate_podcast", "link_preview", "display_image"]);
/**
* Type for thinking step data from the backend
@@ -591,6 +592,7 @@ export default function NewChatPage() {
+
diff --git a/surfsense_web/components/tool-ui/display-image.tsx b/surfsense_web/components/tool-ui/display-image.tsx
new file mode 100644
index 000000000..37bdad5fa
--- /dev/null
+++ b/surfsense_web/components/tool-ui/display-image.tsx
@@ -0,0 +1,160 @@
+"use client";
+
+import { makeAssistantToolUI } from "@assistant-ui/react";
+import { AlertCircleIcon, ImageIcon } from "lucide-react";
+import {
+ Image,
+ ImageErrorBoundary,
+ ImageLoading,
+ parseSerializableImage,
+} from "@/components/tool-ui/image";
+
+/**
+ * Type definitions for the display_image tool
+ */
+interface DisplayImageArgs {
+ src: string;
+ alt?: string;
+ title?: string;
+ description?: string;
+}
+
+interface DisplayImageResult {
+ id: string;
+ assetId: string;
+ src: string;
+ alt: string;
+ title?: string;
+ description?: string;
+ domain?: string;
+ ratio?: string;
+ error?: string;
+}
+
+/**
+ * Error state component shown when image display fails
+ */
+function ImageErrorState({ src, error }: { src: string; error: string }) {
+ return (
+
+
+
+
+
Failed to display image
+
{src}
+
{error}
+
+
+
+
+
+ );
+ }
+
+ // Incomplete/cancelled state
+ if (status.type === "incomplete") {
+ if (status.reason === "cancelled") {
+ return ;
+ }
+ if (status.reason === "error") {
+ return (
+
+ );
+ }
+ }
+
+ // No result yet
+ if (!result) {
+ return (
+
+
+
+ );
+ }
+
+ // Error result from the tool
+ if (result.error) {
+ return ;
+ }
+
+ // Success - render the image
+ return (
+
+ );
+ },
+});
+
+export type { DisplayImageArgs, DisplayImageResult };
+
diff --git a/surfsense_web/components/tool-ui/image/index.tsx b/surfsense_web/components/tool-ui/image/index.tsx
new file mode 100644
index 000000000..97c983625
--- /dev/null
+++ b/surfsense_web/components/tool-ui/image/index.tsx
@@ -0,0 +1,332 @@
+"use client";
+
+import { ExternalLinkIcon, ImageIcon, Loader2 } from "lucide-react";
+import NextImage from "next/image";
+import { Component, type ReactNode, useState } from "react";
+import { Badge } from "@/components/ui/badge";
+import { Card } from "@/components/ui/card";
+import { cn } from "@/lib/utils";
+
+/**
+ * Aspect ratio options for images
+ */
+type AspectRatio = "1:1" | "4:3" | "16:9" | "9:16" | "auto";
+
+/**
+ * Image fit options
+ */
+type ImageFit = "cover" | "contain";
+
+/**
+ * Source attribution
+ */
+interface ImageSource {
+ label: string;
+ iconUrl?: string;
+ url?: string;
+}
+
+/**
+ * Props for the Image component
+ */
+export interface ImageProps {
+ id: string;
+ assetId: string;
+ src: string;
+ alt: string;
+ title?: string;
+ description?: string;
+ href?: string;
+ domain?: string;
+ ratio?: AspectRatio;
+ fit?: ImageFit;
+ source?: ImageSource;
+ maxWidth?: string;
+ className?: string;
+}
+
+/**
+ * Serializable schema for Image props (for tool results)
+ */
+export interface SerializableImage {
+ id: string;
+ assetId: string;
+ src: string;
+ alt: string;
+ title?: string;
+ description?: string;
+ href?: string;
+ domain?: string;
+ ratio?: AspectRatio;
+ source?: ImageSource;
+}
+
+/**
+ * Parse and validate serializable image from tool result
+ */
+export function parseSerializableImage(result: unknown): SerializableImage {
+ if (typeof result !== "object" || result === null) {
+ throw new Error("Invalid image result: expected object");
+ }
+
+ const obj = result as Record;
+
+ // Validate required fields
+ if (typeof obj.id !== "string") {
+ throw new Error("Invalid image: missing id");
+ }
+ if (typeof obj.assetId !== "string") {
+ throw new Error("Invalid image: missing assetId");
+ }
+ if (typeof obj.src !== "string") {
+ throw new Error("Invalid image: missing src");
+ }
+ if (typeof obj.alt !== "string") {
+ throw new Error("Invalid image: missing alt");
+ }
+
+ return {
+ id: obj.id,
+ assetId: obj.assetId,
+ src: obj.src,
+ alt: obj.alt,
+ title: typeof obj.title === "string" ? obj.title : undefined,
+ description: typeof obj.description === "string" ? obj.description : undefined,
+ href: typeof obj.href === "string" ? obj.href : undefined,
+ domain: typeof obj.domain === "string" ? obj.domain : undefined,
+ ratio: typeof obj.ratio === "string" ? (obj.ratio as AspectRatio) : undefined,
+ source: typeof obj.source === "object" && obj.source !== null ? (obj.source as ImageSource) : undefined,
+ };
+}
+
+/**
+ * Get aspect ratio class based on ratio prop
+ */
+function getAspectRatioClass(ratio?: AspectRatio): string {
+ switch (ratio) {
+ case "1:1":
+ return "aspect-square";
+ case "4:3":
+ return "aspect-[4/3]";
+ case "16:9":
+ return "aspect-video";
+ case "9:16":
+ return "aspect-[9/16]";
+ case "auto":
+ default:
+ return "aspect-[4/3]";
+ }
+}
+
+/**
+ * Error boundary for Image component
+ */
+interface ImageErrorBoundaryState {
+ hasError: boolean;
+ error?: Error;
+}
+
+export class ImageErrorBoundary extends Component<
+ { children: ReactNode },
+ ImageErrorBoundaryState
+> {
+ constructor(props: { children: ReactNode }) {
+ super(props);
+ this.state = { hasError: false };
+ }
+
+ static getDerivedStateFromError(error: Error): ImageErrorBoundaryState {
+ return { hasError: true, error };
+ }
+
+ render() {
+ if (this.state.hasError) {
+ return (
+
+
+
+
+
Failed to load image
+
+
+
+
+
+ );
+}
+
+/**
+ * Image Loading State
+ */
+export function ImageLoading({ title = "Loading image..." }: { title?: string }) {
+ return (
+
+
+
+ );
+}
+
+/**
+ * Image Component
+ *
+ * Display images with metadata and attribution.
+ * Features hover overlay with title and source attribution.
+ */
+export function Image({
+ id,
+ src,
+ alt,
+ title,
+ description,
+ href,
+ domain,
+ ratio = "4:3",
+ fit = "cover",
+ source,
+ maxWidth = "420px",
+ className,
+}: ImageProps) {
+ const [isHovered, setIsHovered] = useState(false);
+ const [imageError, setImageError] = useState(false);
+ const aspectRatioClass = getAspectRatioClass(ratio);
+ const displayDomain = domain || source?.label;
+
+ const handleClick = () => {
+ const targetUrl = href || source?.url || src;
+ if (targetUrl) {
+ window.open(targetUrl, "_blank", "noopener,noreferrer");
+ }
+ };
+
+ if (imageError) {
+ return (
+
+
+
+
+
Image not available
+
+
+ {/* Image */}
+
setImageError(true)}
+ />
+
+ {/* Hover overlay - appears on hover */}
+
+ {/* Content at bottom */}
+
+ {/* Title */}
+ {title && (
+
+ {title}
+
+ )}
+
+ {/* Description */}
+ {description && (
+
+ {description}
+
+ )}
+
+ {/* Source attribution */}
+ {displayDomain && (
+
+ {source?.iconUrl ? (
+
+ ) : (
+
+ )}
+ {displayDomain}
+
+ )}
+
+
+
+ {displayDomain}
+
+
+ )}
+
diff --git a/surfsense_web/components/tool-ui/article/index.tsx b/surfsense_web/components/tool-ui/article/index.tsx
new file mode 100644
index 000000000..bf8e83411
--- /dev/null
+++ b/surfsense_web/components/tool-ui/article/index.tsx
@@ -0,0 +1,406 @@
+"use client";
+
+import { Card, CardContent } from "@/components/ui/card";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipProvider,
+ TooltipTrigger,
+} from "@/components/ui/tooltip";
+import { cn } from "@/lib/utils";
+import {
+ AlertCircleIcon,
+ BookOpenIcon,
+ CalendarIcon,
+ ExternalLinkIcon,
+ FileTextIcon,
+ UserIcon,
+} from "lucide-react";
+import { Component, type ReactNode, useCallback } from "react";
+
+/**
+ * Article component props
+ */
+export interface ArticleProps {
+ /** Unique identifier for the article */
+ id: string;
+ /** Asset identifier (usually the URL) */
+ assetId?: string;
+ /** Article title */
+ title: string;
+ /** Brief description or excerpt */
+ description?: string;
+ /** Full content of the article (markdown) */
+ content?: string;
+ /** URL to the original article */
+ href?: string;
+ /** Domain of the article source */
+ domain?: string;
+ /** Author name */
+ author?: string;
+ /** Publication date */
+ date?: string;
+ /** Word count */
+ wordCount?: number;
+ /** Whether content was truncated */
+ wasTruncated?: boolean;
+ /** Optional max width */
+ maxWidth?: string;
+ /** Optional error message */
+ error?: string;
+ /** Optional className */
+ className?: string;
+ /** Response actions */
+ responseActions?: Array<{
+ id: string;
+ label: string;
+ variant?: "default" | "outline";
+ }>;
+ /** Response action handler */
+ onResponseAction?: (actionId: string) => void;
+}
+
+/**
+ * Serializable article data type (from backend)
+ */
+export interface SerializableArticle {
+ id: string;
+ assetId?: string;
+ kind?: "article";
+ title: string;
+ description?: string;
+ content?: string;
+ href?: string;
+ domain?: string;
+ author?: string;
+ date?: string;
+ word_count?: number;
+ wordCount?: number;
+ was_truncated?: boolean;
+ wasTruncated?: boolean;
+ error?: string;
+}
+
+/**
+ * Parse serializable article data to ArticleProps
+ */
+export function parseSerializableArticle(data: unknown): ArticleProps {
+ const obj = data as Record;
+ return {
+ id: String(obj.id || "article-unknown"),
+ assetId: obj.assetId as string | undefined,
+ title: String(obj.title || "Untitled Article"),
+ description: obj.description as string | undefined,
+ content: obj.content as string | undefined,
+ href: obj.href as string | undefined,
+ domain: obj.domain as string | undefined,
+ author: obj.author as string | undefined,
+ date: obj.date as string | undefined,
+ wordCount: (obj.word_count || obj.wordCount) as number | undefined,
+ wasTruncated: (obj.was_truncated || obj.wasTruncated) as boolean | undefined,
+ error: obj.error as string | undefined,
+ };
+}
+
+/**
+ * Format word count for display
+ */
+function formatWordCount(count: number): string {
+ if (count >= 1000) {
+ return `${(count / 1000).toFixed(1)}k words`;
+ }
+ return `${count} words`;
+}
+
+/**
+ * Article card component for displaying scraped webpage content
+ */
+export function Article({
+ id,
+ title,
+ description,
+ content,
+ href,
+ domain,
+ author,
+ date,
+ wordCount,
+ wasTruncated,
+ maxWidth = "100%",
+ error,
+ className,
+ responseActions,
+ onResponseAction,
+}: ArticleProps) {
+ const handleCardClick = useCallback(() => {
+ if (href) {
+ window.open(href, "_blank", "noopener,noreferrer");
+ }
+ }, [href]);
+
+ // Error state
+ if (error) {
+ return (
+
+
+
+
+
+
+ Failed to scrape webpage
+
+ {href && (
+
+ {href}
+
+ )}
+
{error}
+
+
+ {/* Icon */}
+
+
+
+
+ {/* Content */}
+
+ {/* Title */}
+
+ {title}
+
+
+ {/* Description */}
+ {description && (
+
+ {description}
+
+ )}
+
+ {/* Metadata row */}
+
+ {domain && (
+
+
+
+
+ {domain}
+
+
+
+ Source: {domain}
+
+
+ )}
+
+ {author && (
+
+
+
+
+ {author}
+
+
+
+ Author: {author}
+
+
+ )}
+
+ {date && (
+
+
+ {date}
+
+ )}
+
+ {wordCount && (
+
+
+
+
+ {formatWordCount(wordCount)}
+ {wasTruncated && (
+ (truncated)
+ )}
+
+
+
+
+ {wasTruncated
+ ? "Content was truncated due to length"
+ : "Full article content available"}
+
+
+
+ )}
+
+
+
+ {/* External link indicator */}
+ {href && (
+
+
+
+ )}
+
+ {responseActions.map((action) => (
+
+ ))}
+
+ )}
+
+
+
+ );
+}
+
+/**
+ * Loading state for article component
+ */
+export function ArticleLoading({
+ title = "Loading article...",
+}: { title?: string }) {
+ return (
+
+
+
+ {title}
+
+
+ );
+}
+
+/**
+ * Skeleton for article component
+ */
+export function ArticleSkeleton() {
+ return (
+
+
+
+
+
+ );
+}
+
+/**
+ * Error boundary props
+ */
+interface ErrorBoundaryProps {
+ children: ReactNode;
+ fallback?: ReactNode;
+}
+
+interface ErrorBoundaryState {
+ hasError: boolean;
+}
+
+/**
+ * Error boundary for article component
+ */
+export class ArticleErrorBoundary extends Component<
+ ErrorBoundaryProps,
+ ErrorBoundaryState
+> {
+ constructor(props: ErrorBoundaryProps) {
+ super(props);
+ this.state = { hasError: false };
+ }
+
+ static getDerivedStateFromError(): ErrorBoundaryState {
+ return { hasError: true };
+ }
+
+ render() {
+ if (this.state.hasError) {
+ return (
+ this.props.fallback || (
+
+
+
+
+
+ Failed to render article
+
+
+
+
+
+
Failed to scrape webpage
+
{url}
+
{error}
+
+
+
+
+
+ Scraping: {url}
+
+
- {steps.map((step) => {
- const icon = getStepIcon(step.status, step.title);
+ {steps.map((step, index) => {
+ const effectiveStatus = getEffectiveStatus(step);
+ const icon = getStepIcon(effectiveStatus, step.title);
+ const isOpen = getStepOpenState(step, index);
return (
handleToggle(step.id, isOpen)}
>
{step.title}
{step.items && step.items.length > 0 && (
- {step.items.map((item, index) => (
-
+ {step.items.map((item, idx) => (
+
{item}
))}
@@ -263,8 +330,8 @@ const ThreadWelcome: FC = () => {
return (
- {/* Greeting positioned near the composer */}
-
+ {/* Greeting positioned above the composer - fixed position */}
+
{/** biome-ignore lint/a11y/noStaticElementInteractions: wrong lint error, this is a workaround to fix the lint error */}
{
{getTimeBasedGreeting(user?.email)}
- {/* Composer centered in the middle of the screen */}
-
+ {/* Composer - top edge fixed, expands downward only */}
+
-
+
)}
diff --git a/surfsense_web/components/tool-ui/deepagent-thinking.tsx b/surfsense_web/components/tool-ui/deepagent-thinking.tsx
index f0c03e570..8cd901e69 100644
--- a/surfsense_web/components/tool-ui/deepagent-thinking.tsx
+++ b/surfsense_web/components/tool-ui/deepagent-thinking.tsx
@@ -2,7 +2,7 @@
import { makeAssistantToolUI } from "@assistant-ui/react";
import { Brain, CheckCircle2, Loader2, Search, Sparkles } from "lucide-react";
-import { useMemo } from "react";
+import { useMemo, useState, useEffect, useRef } from "react";
import {
ChainOfThought,
ChainOfThoughtContent,
@@ -61,13 +61,21 @@ function getStepIcon(status: "pending" | "in_progress" | "completed", title: str
}
/**
- * Component to display a single thinking step
+ * Component to display a single thinking step with controlled open state
*/
-function ThinkingStepDisplay({ step }: { step: ThinkingStep }) {
+function ThinkingStepDisplay({
+ step,
+ isOpen,
+ onToggle
+}: {
+ step: ThinkingStep;
+ isOpen: boolean;
+ onToggle: () => void;
+}) {
const icon = useMemo(() => getStepIcon(step.status, step.title), [step.status, step.title]);
return (
-
+
>({});
+ // Track previous step statuses to detect changes
+ const prevStatusesRef = useRef>({});
+
+ // Check if any step is currently in progress
+ const hasInProgressStep = steps.some(step => step.status === "in_progress");
+
+ // Find the last completed step index
+ const lastCompletedIndex = steps
+ .map((s, i) => s.status === "completed" ? i : -1)
+ .filter(i => i !== -1)
+ .pop();
+
+ // Clear manual overrides when a step's status changes
+ useEffect(() => {
+ const currentStatuses: Record = {};
+ steps.forEach(step => {
+ currentStatuses[step.id] = step.status;
+ // If status changed, clear any manual override for this step
+ if (prevStatusesRef.current[step.id] && prevStatusesRef.current[step.id] !== step.status) {
+ setManualOverrides(prev => {
+ const next = { ...prev };
+ delete next[step.id];
+ return next;
+ });
+ }
+ });
+ prevStatusesRef.current = currentStatuses;
+ }, [steps]);
+
+ const getStepOpenState = (step: ThinkingStep, index: number): boolean => {
+ // If user has manually toggled, respect that
+ if (manualOverrides[step.id] !== undefined) {
+ return manualOverrides[step.id];
+ }
+ // Auto behavior: open if in progress
+ if (step.status === "in_progress") {
+ return true;
+ }
+ // Auto behavior: keep last completed step open if no in-progress step
+ if (!hasInProgressStep && index === lastCompletedIndex) {
+ return true;
+ }
+ // Default: collapsed
+ return false;
+ };
+
+ const handleToggle = (stepId: string, currentOpen: boolean) => {
+ setManualOverrides(prev => ({
+ ...prev,
+ [stepId]: !currentOpen,
+ }));
+ };
+
+ return (
+
+ {steps.map((step, index) => {
+ const isOpen = getStepOpenState(step, index);
+ return (
+ handleToggle(step.id, isOpen)}
+ />
+ );
+ })}
+
+ );
+}
+
/**
* DeepAgent Thinking Tool UI Component
*
@@ -131,7 +215,7 @@ export const DeepAgentThinkingToolUI = makeAssistantToolUI<
DeepAgentThinkingResult
>({
toolName: "deepagent_thinking",
- render: function DeepAgentThinkingUI({ args, result, status }) {
+ render: function DeepAgentThinkingUI({ result, status }) {
// Loading state - tool is still running
if (status.type === "running" || status.type === "requires-action") {
return ;
@@ -155,11 +239,7 @@ export const DeepAgentThinkingToolUI = makeAssistantToolUI<
// Render the chain of thought
return (
-
- {result.steps.map((step) => (
-
- ))}
-
+
);
},
@@ -189,11 +269,7 @@ export function InlineThinkingDisplay({
{isStreaming && steps.length === 0 ? (
) : (
-
- {steps.map((step) => (
-
- ))}
-
+
)}
);
From 5fd872b7980d2525c4bf060f17e410aacefb5b3a Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Tue, 23 Dec 2025 02:42:48 +0530
Subject: [PATCH 12/12] feat: implement Zod schemas for runtime validation
across various components to enhance data integrity and error handling
---
.../components/tool-ui/article/index.tsx | 89 +++++++++------
.../components/tool-ui/deepagent-thinking.tsx | 68 ++++++++---
.../components/tool-ui/display-image.tsx | 10 +-
.../components/tool-ui/generate-podcast.tsx | 106 ++++++++++++------
.../components/tool-ui/image/index.tsx | 97 ++++++----------
.../components/tool-ui/media-card/index.tsx | 97 ++++++----------
6 files changed, 253 insertions(+), 214 deletions(-)
diff --git a/surfsense_web/components/tool-ui/article/index.tsx b/surfsense_web/components/tool-ui/article/index.tsx
index bf8e83411..62068d117 100644
--- a/surfsense_web/components/tool-ui/article/index.tsx
+++ b/surfsense_web/components/tool-ui/article/index.tsx
@@ -17,6 +17,33 @@ import {
UserIcon,
} from "lucide-react";
import { Component, type ReactNode, useCallback } from "react";
+import { z } from "zod";
+
+/**
+ * Zod schema for serializable article data (from backend)
+ */
+const SerializableArticleSchema = z.object({
+ id: z.string().default("article-unknown"),
+ assetId: z.string().optional(),
+ kind: z.literal("article").optional(),
+ title: z.string().default("Untitled Article"),
+ description: z.string().optional(),
+ content: z.string().optional(),
+ href: z.string().url().optional(),
+ domain: z.string().optional(),
+ author: z.string().optional(),
+ date: z.string().optional(),
+ word_count: z.number().optional(),
+ wordCount: z.number().optional(),
+ was_truncated: z.boolean().optional(),
+ wasTruncated: z.boolean().optional(),
+ error: z.string().optional(),
+});
+
+/**
+ * Serializable article data type (from backend)
+ */
+export type SerializableArticle = z.infer
;
/**
* Article component props
@@ -61,44 +88,36 @@ export interface ArticleProps {
}
/**
- * Serializable article data type (from backend)
- */
-export interface SerializableArticle {
- id: string;
- assetId?: string;
- kind?: "article";
- title: string;
- description?: string;
- content?: string;
- href?: string;
- domain?: string;
- author?: string;
- date?: string;
- word_count?: number;
- wordCount?: number;
- was_truncated?: boolean;
- wasTruncated?: boolean;
- error?: string;
-}
-
-/**
- * Parse serializable article data to ArticleProps
+ * Parse and validate serializable article data to ArticleProps
*/
export function parseSerializableArticle(data: unknown): ArticleProps {
- const obj = data as Record;
+ const result = SerializableArticleSchema.safeParse(data);
+
+ if (!result.success) {
+ console.warn("Invalid article data:", result.error.issues);
+ // Return fallback with basic info
+ const obj = (data && typeof data === "object" ? data : {}) as Record;
+ return {
+ id: String(obj.id || "article-unknown"),
+ title: String(obj.title || "Untitled Article"),
+ error: "Failed to parse article data",
+ };
+ }
+
+ const parsed = result.data;
return {
- id: String(obj.id || "article-unknown"),
- assetId: obj.assetId as string | undefined,
- title: String(obj.title || "Untitled Article"),
- description: obj.description as string | undefined,
- content: obj.content as string | undefined,
- href: obj.href as string | undefined,
- domain: obj.domain as string | undefined,
- author: obj.author as string | undefined,
- date: obj.date as string | undefined,
- wordCount: (obj.word_count || obj.wordCount) as number | undefined,
- wasTruncated: (obj.was_truncated || obj.wasTruncated) as boolean | undefined,
- error: obj.error as string | undefined,
+ id: parsed.id,
+ assetId: parsed.assetId,
+ title: parsed.title,
+ description: parsed.description,
+ content: parsed.content,
+ href: parsed.href,
+ domain: parsed.domain,
+ author: parsed.author,
+ date: parsed.date,
+ wordCount: parsed.word_count ?? parsed.wordCount,
+ wasTruncated: parsed.was_truncated ?? parsed.wasTruncated,
+ error: parsed.error,
};
}
diff --git a/surfsense_web/components/tool-ui/deepagent-thinking.tsx b/surfsense_web/components/tool-ui/deepagent-thinking.tsx
index 8cd901e69..ff8baac9a 100644
--- a/surfsense_web/components/tool-ui/deepagent-thinking.tsx
+++ b/surfsense_web/components/tool-ui/deepagent-thinking.tsx
@@ -3,6 +3,7 @@
import { makeAssistantToolUI } from "@assistant-ui/react";
import { Brain, CheckCircle2, Loader2, Search, Sparkles } from "lucide-react";
import { useMemo, useState, useEffect, useRef } from "react";
+import { z } from "zod";
import {
ChainOfThought,
ChainOfThoughtContent,
@@ -13,24 +14,61 @@ import {
import { cn } from "@/lib/utils";
/**
- * Types for the deepagent thinking/reasoning tool
+ * Zod schemas for runtime validation
*/
-interface ThinkingStep {
- id: string;
- title: string;
- items: string[];
- status: "pending" | "in_progress" | "completed";
+const ThinkingStepSchema = z.object({
+ id: z.string(),
+ title: z.string(),
+ items: z.array(z.string()).default([]),
+ status: z.enum(["pending", "in_progress", "completed"]).default("pending"),
+});
+
+const DeepAgentThinkingArgsSchema = z.object({
+ query: z.string().optional(),
+ context: z.string().optional(),
+});
+
+const DeepAgentThinkingResultSchema = z.object({
+ steps: z.array(ThinkingStepSchema).optional(),
+ status: z.enum(["thinking", "searching", "synthesizing", "completed"]).optional(),
+ summary: z.string().optional(),
+});
+
+/**
+ * Types derived from Zod schemas
+ */
+type ThinkingStep = z.infer;
+type DeepAgentThinkingArgs = z.infer;
+type DeepAgentThinkingResult = z.infer;
+
+/**
+ * Parse and validate a single thinking step
+ */
+export function parseThinkingStep(data: unknown): ThinkingStep {
+ const result = ThinkingStepSchema.safeParse(data);
+ if (!result.success) {
+ console.warn("Invalid thinking step data:", result.error.issues);
+ // Return a fallback step
+ return {
+ id: "unknown",
+ title: "Processing...",
+ items: [],
+ status: "pending",
+ };
+ }
+ return result.data;
}
-interface DeepAgentThinkingArgs {
- query?: string;
- context?: string;
-}
-
-interface DeepAgentThinkingResult {
- steps?: ThinkingStep[];
- status?: "thinking" | "searching" | "synthesizing" | "completed";
- summary?: string;
+/**
+ * Parse and validate thinking result
+ */
+export function parseThinkingResult(data: unknown): DeepAgentThinkingResult {
+ const result = DeepAgentThinkingResultSchema.safeParse(data);
+ if (!result.success) {
+ console.warn("Invalid thinking result data:", result.error.issues);
+ return {};
+ }
+ return result.data;
}
/**
diff --git a/surfsense_web/components/tool-ui/display-image.tsx b/surfsense_web/components/tool-ui/display-image.tsx
index 37bdad5fa..a9c87b29b 100644
--- a/surfsense_web/components/tool-ui/display-image.tsx
+++ b/surfsense_web/components/tool-ui/display-image.tsx
@@ -67,6 +67,8 @@ function ImageCancelledState({ src }: { src: string }) {
/**
* Parsed Image component with error handling
+ * Note: Image component has built-in click handling via href/src,
+ * so no additional responseActions needed.
*/
function ParsedImage({ result }: { result: unknown }) {
const image = parseSerializableImage(result);
@@ -75,14 +77,6 @@ function ParsedImage({ result }: { result: unknown }) {
{
- if (id === "open" && image.src) {
- window.open(image.src, "_blank", "noopener,noreferrer");
- }
- }}
/>
);
}
diff --git a/surfsense_web/components/tool-ui/generate-podcast.tsx b/surfsense_web/components/tool-ui/generate-podcast.tsx
index 669b11a57..862eab144 100644
--- a/surfsense_web/components/tool-ui/generate-podcast.tsx
+++ b/surfsense_web/components/tool-ui/generate-podcast.tsx
@@ -3,37 +3,79 @@
import { makeAssistantToolUI } from "@assistant-ui/react";
import { AlertCircleIcon, Loader2Icon, MicIcon } from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
+import { z } from "zod";
import { Audio } from "@/components/tool-ui/audio";
import { baseApiService } from "@/lib/apis/base-api.service";
import { authenticatedFetch } from "@/lib/auth-utils";
import { clearActivePodcastTaskId, setActivePodcastTaskId } from "@/lib/chat/podcast-state";
/**
- * Type definitions for the generate_podcast tool
+ * Zod schemas for runtime validation
*/
-interface GeneratePodcastArgs {
- source_content: string;
- podcast_title?: string;
- user_prompt?: string;
+const GeneratePodcastArgsSchema = z.object({
+ source_content: z.string(),
+ podcast_title: z.string().optional(),
+ user_prompt: z.string().optional(),
+});
+
+const GeneratePodcastResultSchema = z.object({
+ status: z.enum(["processing", "already_generating", "success", "error"]),
+ task_id: z.string().optional(),
+ podcast_id: z.number().optional(),
+ title: z.string().optional(),
+ transcript_entries: z.number().optional(),
+ message: z.string().optional(),
+ error: z.string().optional(),
+});
+
+const TaskStatusResponseSchema = z.object({
+ status: z.enum(["processing", "success", "error"]),
+ podcast_id: z.number().optional(),
+ title: z.string().optional(),
+ transcript_entries: z.number().optional(),
+ state: z.string().optional(),
+ error: z.string().optional(),
+});
+
+const PodcastTranscriptEntrySchema = z.object({
+ speaker_id: z.number(),
+ dialog: z.string(),
+});
+
+const PodcastDetailsSchema = z.object({
+ podcast_transcript: z.array(PodcastTranscriptEntrySchema).optional(),
+});
+
+/**
+ * Types derived from Zod schemas
+ */
+type GeneratePodcastArgs = z.infer;
+type GeneratePodcastResult = z.infer;
+type TaskStatusResponse = z.infer;
+type PodcastTranscriptEntry = z.infer;
+
+/**
+ * Parse and validate task status response
+ */
+function parseTaskStatusResponse(data: unknown): TaskStatusResponse {
+ const result = TaskStatusResponseSchema.safeParse(data);
+ if (!result.success) {
+ console.warn("Invalid task status response:", result.error.issues);
+ return { status: "error", error: "Invalid response from server" };
+ }
+ return result.data;
}
-interface GeneratePodcastResult {
- status: "processing" | "already_generating" | "success" | "error";
- task_id?: string;
- podcast_id?: number;
- title?: string;
- transcript_entries?: number;
- message?: string;
- error?: string;
-}
-
-interface TaskStatusResponse {
- status: "processing" | "success" | "error";
- podcast_id?: number;
- title?: string;
- transcript_entries?: number;
- state?: string;
- error?: string;
+/**
+ * Parse and validate podcast details
+ */
+function parsePodcastDetails(data: unknown): { podcast_transcript?: PodcastTranscriptEntry[] } {
+ const result = PodcastDetailsSchema.safeParse(data);
+ if (!result.success) {
+ console.warn("Invalid podcast details:", result.error.issues);
+ return {};
+ }
+ return result.data;
}
/**
@@ -112,14 +154,6 @@ function AudioLoadingState({ title }: { title: string }) {
/**
* Podcast Player Component - Fetches audio and transcript with authentication
*/
-/**
- * Transcript entry type for podcast transcripts
- */
-interface PodcastTranscriptEntry {
- speaker_id: number;
- dialog: string;
-}
-
function PodcastPlayer({
podcastId,
title,
@@ -163,12 +197,12 @@ function PodcastPlayer({
try {
// Fetch audio blob and podcast details in parallel
- const [audioResponse, podcastDetails] = await Promise.all([
+ const [audioResponse, rawPodcastDetails] = await Promise.all([
authenticatedFetch(
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/podcasts/${podcastId}/audio`,
{ method: "GET", signal: controller.signal }
),
- baseApiService.get<{ podcast_transcript?: PodcastTranscriptEntry[] }>(
+ baseApiService.get(
`/api/v1/podcasts/${podcastId}`
),
]);
@@ -184,8 +218,9 @@ function PodcastPlayer({
objectUrlRef.current = objectUrl;
setAudioSrc(objectUrl);
- // Set transcript from podcast details
- if (podcastDetails?.podcast_transcript) {
+ // Parse and validate podcast details, then set transcript
+ const podcastDetails = parsePodcastDetails(rawPodcastDetails);
+ if (podcastDetails.podcast_transcript) {
setTranscript(podcastDetails.podcast_transcript);
}
} finally {
@@ -268,9 +303,10 @@ function PodcastTaskPoller({ taskId, title }: { taskId: string; title: string })
useEffect(() => {
const pollStatus = async () => {
try {
- const response = await baseApiService.get(
+ const rawResponse = await baseApiService.get(
`/api/v1/podcasts/task/${taskId}/status`
);
+ const response = parseTaskStatusResponse(rawResponse);
setTaskStatus(response);
// Stop polling if task is complete or errored
diff --git a/surfsense_web/components/tool-ui/image/index.tsx b/surfsense_web/components/tool-ui/image/index.tsx
index 97c983625..c9ac72d2f 100644
--- a/surfsense_web/components/tool-ui/image/index.tsx
+++ b/surfsense_web/components/tool-ui/image/index.tsx
@@ -3,28 +3,43 @@
import { ExternalLinkIcon, ImageIcon, Loader2 } from "lucide-react";
import NextImage from "next/image";
import { Component, type ReactNode, useState } from "react";
+import { z } from "zod";
import { Badge } from "@/components/ui/badge";
import { Card } from "@/components/ui/card";
import { cn } from "@/lib/utils";
/**
- * Aspect ratio options for images
+ * Zod schemas for runtime validation
*/
-type AspectRatio = "1:1" | "4:3" | "16:9" | "9:16" | "auto";
+const AspectRatioSchema = z.enum(["1:1", "4:3", "16:9", "9:16", "auto"]);
+const ImageFitSchema = z.enum(["cover", "contain"]);
+
+const ImageSourceSchema = z.object({
+ label: z.string(),
+ iconUrl: z.string().optional(),
+ url: z.string().optional(),
+});
+
+const SerializableImageSchema = z.object({
+ id: z.string(),
+ assetId: z.string(),
+ src: z.string(),
+ alt: z.string(),
+ title: z.string().optional(),
+ description: z.string().optional(),
+ href: z.string().optional(),
+ domain: z.string().optional(),
+ ratio: AspectRatioSchema.optional(),
+ source: ImageSourceSchema.optional(),
+});
/**
- * Image fit options
+ * Types derived from Zod schemas
*/
-type ImageFit = "cover" | "contain";
-
-/**
- * Source attribution
- */
-interface ImageSource {
- label: string;
- iconUrl?: string;
- url?: string;
-}
+type AspectRatio = z.infer;
+type ImageFit = z.infer;
+type ImageSource = z.infer;
+export type SerializableImage = z.infer;
/**
* Props for the Image component
@@ -45,58 +60,20 @@ export interface ImageProps {
className?: string;
}
-/**
- * Serializable schema for Image props (for tool results)
- */
-export interface SerializableImage {
- id: string;
- assetId: string;
- src: string;
- alt: string;
- title?: string;
- description?: string;
- href?: string;
- domain?: string;
- ratio?: AspectRatio;
- source?: ImageSource;
-}
-
/**
* Parse and validate serializable image from tool result
*/
export function parseSerializableImage(result: unknown): SerializableImage {
- if (typeof result !== "object" || result === null) {
- throw new Error("Invalid image result: expected object");
+ const parsed = SerializableImageSchema.safeParse(result);
+
+ if (!parsed.success) {
+ console.warn("Invalid image data:", parsed.error.issues);
+ // Try to extract basic info for error display
+ const obj = (result && typeof result === "object" ? result : {}) as Record;
+ throw new Error(`Invalid image: ${parsed.error.issues.map(i => i.message).join(", ")}`);
}
-
- const obj = result as Record;
-
- // Validate required fields
- if (typeof obj.id !== "string") {
- throw new Error("Invalid image: missing id");
- }
- if (typeof obj.assetId !== "string") {
- throw new Error("Invalid image: missing assetId");
- }
- if (typeof obj.src !== "string") {
- throw new Error("Invalid image: missing src");
- }
- if (typeof obj.alt !== "string") {
- throw new Error("Invalid image: missing alt");
- }
-
- return {
- id: obj.id,
- assetId: obj.assetId,
- src: obj.src,
- alt: obj.alt,
- title: typeof obj.title === "string" ? obj.title : undefined,
- description: typeof obj.description === "string" ? obj.description : undefined,
- href: typeof obj.href === "string" ? obj.href : undefined,
- domain: typeof obj.domain === "string" ? obj.domain : undefined,
- ratio: typeof obj.ratio === "string" ? (obj.ratio as AspectRatio) : undefined,
- source: typeof obj.source === "object" && obj.source !== null ? (obj.source as ImageSource) : undefined,
- };
+
+ return parsed.data;
}
/**
diff --git a/surfsense_web/components/tool-ui/media-card/index.tsx b/surfsense_web/components/tool-ui/media-card/index.tsx
index 47c28e49b..dc3b9b59a 100644
--- a/surfsense_web/components/tool-ui/media-card/index.tsx
+++ b/surfsense_web/components/tool-ui/media-card/index.tsx
@@ -3,6 +3,7 @@
import { ExternalLinkIcon, Globe, ImageIcon, LinkIcon, Loader2 } from "lucide-react";
import Image from "next/image";
import { Component, type ReactNode } from "react";
+import { z } from "zod";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Card, CardContent } from "@/components/ui/card";
@@ -10,24 +11,38 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp
import { cn } from "@/lib/utils";
/**
- * Aspect ratio options for media cards
+ * Zod schemas for runtime validation
*/
-type AspectRatio = "1:1" | "4:3" | "16:9" | "21:9" | "auto";
+const AspectRatioSchema = z.enum(["1:1", "4:3", "16:9", "21:9", "auto"]);
+const MediaCardKindSchema = z.enum(["link", "image", "video", "audio"]);
+
+const ResponseActionSchema = z.object({
+ id: z.string(),
+ label: z.string(),
+ variant: z.enum(["default", "secondary", "outline", "destructive", "ghost"]).optional(),
+ confirmLabel: z.string().optional(),
+});
+
+const SerializableMediaCardSchema = z.object({
+ id: z.string(),
+ assetId: z.string(),
+ kind: MediaCardKindSchema,
+ href: z.string().optional(),
+ src: z.string().optional(),
+ title: z.string(),
+ description: z.string().optional(),
+ thumb: z.string().optional(),
+ ratio: AspectRatioSchema.optional(),
+ domain: z.string().optional(),
+});
/**
- * MediaCard kind - determines the display style
+ * Types derived from Zod schemas
*/
-type MediaCardKind = "link" | "image" | "video" | "audio";
-
-/**
- * Response action configuration
- */
-interface ResponseAction {
- id: string;
- label: string;
- variant?: "default" | "secondary" | "outline" | "destructive" | "ghost";
- confirmLabel?: string;
-}
+type AspectRatio = z.infer;
+type MediaCardKind = z.infer;
+type ResponseAction = z.infer;
+export type SerializableMediaCard = z.infer;
/**
* Props for the MediaCard component
@@ -50,58 +65,18 @@ export interface MediaCardProps {
onResponseAction?: (id: string) => void;
}
-/**
- * Serializable schema for MediaCard props (for tool results)
- */
-export interface SerializableMediaCard {
- id: string;
- assetId: string;
- kind: MediaCardKind;
- href?: string;
- src?: string;
- title: string;
- description?: string;
- thumb?: string;
- ratio?: AspectRatio;
- domain?: string;
-}
-
/**
* Parse and validate serializable media card from tool result
*/
export function parseSerializableMediaCard(result: unknown): SerializableMediaCard {
- if (typeof result !== "object" || result === null) {
- throw new Error("Invalid media card result: expected object");
+ const parsed = SerializableMediaCardSchema.safeParse(result);
+
+ if (!parsed.success) {
+ console.warn("Invalid media card data:", parsed.error.issues);
+ throw new Error(`Invalid media card: ${parsed.error.issues.map(i => i.message).join(", ")}`);
}
-
- const obj = result as Record;
-
- // Validate required fields
- if (typeof obj.id !== "string") {
- throw new Error("Invalid media card: missing id");
- }
- if (typeof obj.assetId !== "string") {
- throw new Error("Invalid media card: missing assetId");
- }
- if (typeof obj.kind !== "string") {
- throw new Error("Invalid media card: missing kind");
- }
- if (typeof obj.title !== "string") {
- throw new Error("Invalid media card: missing title");
- }
-
- return {
- id: obj.id,
- assetId: obj.assetId,
- kind: obj.kind as MediaCardKind,
- href: typeof obj.href === "string" ? obj.href : undefined,
- src: typeof obj.src === "string" ? obj.src : undefined,
- title: obj.title,
- description: typeof obj.description === "string" ? obj.description : undefined,
- thumb: typeof obj.thumb === "string" ? obj.thumb : undefined,
- ratio: typeof obj.ratio === "string" ? (obj.ratio as AspectRatio) : undefined,
- domain: typeof obj.domain === "string" ? obj.domain : undefined,
- };
+
+ return parsed.data;
}
/**