diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py index 23db4f813..e6b2aca06 100644 --- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py +++ b/surfsense_backend/app/agents/new_chat/chat_deepagent.py @@ -14,6 +14,7 @@ from langgraph.types import Checkpointer from sqlalchemy.ext.asyncio import AsyncSession from app.agents.new_chat.context import SurfSenseContextSchema +from app.agents.new_chat.display_image import create_display_image_tool from app.agents.new_chat.knowledge_base import create_search_knowledge_base_tool from app.agents.new_chat.link_preview import create_link_preview_tool from app.agents.new_chat.podcast import create_generate_podcast_tool @@ -36,6 +37,7 @@ def create_surfsense_deep_agent( enable_citations: bool = True, enable_podcast: bool = True, enable_link_preview: bool = True, + enable_display_image: bool = True, additional_tools: Sequence[BaseTool] | None = None, ): """ @@ -57,6 +59,8 @@ def create_surfsense_deep_agent( When True and user_id is provided, the agent can generate podcasts. enable_link_preview: Whether to include the link preview tool (default: True). When True, the agent can fetch and display rich link previews. + enable_display_image: Whether to include the display image tool (default: True). + When True, the agent can display images with metadata. additional_tools: Optional sequence of additional tools to inject into the agent. The search_knowledge_base tool will always be included. @@ -87,6 +91,11 @@ def create_surfsense_deep_agent( link_preview_tool = create_link_preview_tool() tools.append(link_preview_tool) + # Add display image tool if enabled + if enable_display_image: + display_image_tool = create_display_image_tool() + tools.append(display_image_tool) + if additional_tools: tools.extend(additional_tools) diff --git a/surfsense_backend/app/agents/new_chat/display_image.py b/surfsense_backend/app/agents/new_chat/display_image.py new file mode 100644 index 000000000..03153300a --- /dev/null +++ b/surfsense_backend/app/agents/new_chat/display_image.py @@ -0,0 +1,106 @@ +""" +Display image tool for the new chat agent. + +This module provides a tool for displaying images in the chat UI +with metadata like title, description, and source attribution. +""" + +import hashlib +from typing import Any +from urllib.parse import urlparse + +from langchain_core.tools import tool + + +def extract_domain(url: str) -> str: + """Extract the domain from a URL.""" + try: + parsed = urlparse(url) + domain = parsed.netloc + # Remove 'www.' prefix if present + if domain.startswith("www."): + domain = domain[4:] + return domain + except Exception: + return "" + + +def generate_image_id(src: str) -> str: + """Generate a unique ID for an image.""" + hash_val = hashlib.md5(src.encode()).hexdigest()[:12] + return f"image-{hash_val}" + + +def create_display_image_tool(): + """ + Factory function to create the display_image tool. + + Returns: + A configured tool function for displaying images. + """ + + @tool + async def display_image( + src: str, + alt: str = "Image", + title: str | None = None, + description: str | None = None, + ) -> dict[str, Any]: + """ + Display an image in the chat with metadata. + + Use this tool when you want to show an image to the user. + This displays the image with an optional title, description, + and source attribution. + + Common use cases: + - Showing an image from a URL the user mentioned + - Displaying a diagram or chart you're referencing + - Showing example images when explaining concepts + + Args: + src: The URL of the image to display (must be a valid HTTP/HTTPS URL) + alt: Alternative text describing the image (for accessibility) + title: Optional title to display below the image + description: Optional description providing context about the image + + Returns: + A dictionary containing image metadata for the UI to render: + - id: Unique identifier for this image + - assetId: The image URL (for deduplication) + - src: The image URL + - alt: Alt text for accessibility + - title: Image title (if provided) + - description: Image description (if provided) + - domain: Source domain + """ + image_id = generate_image_id(src) + + # Ensure URL has protocol + if not src.startswith(("http://", "https://")): + src = f"https://{src}" + + domain = extract_domain(src) + + # Determine aspect ratio based on common image sources + ratio = "16:9" # Default + if "unsplash.com" in src or "pexels.com" in src: + ratio = "16:9" + elif "imgur.com" in src: + ratio = "auto" + elif "github.com" in src or "githubusercontent.com" in src: + ratio = "auto" + + return { + "id": image_id, + "assetId": src, + "src": src, + "alt": alt, + "title": title, + "description": description, + "domain": domain, + "ratio": ratio, + } + + return display_image + diff --git a/surfsense_backend/app/agents/new_chat/system_prompt.py b/surfsense_backend/app/agents/new_chat/system_prompt.py index 32e649e5d..74abadd38 100644 --- a/surfsense_backend/app/agents/new_chat/system_prompt.py +++ b/surfsense_backend/app/agents/new_chat/system_prompt.py @@ -158,6 +158,21 @@ You have access to the following tools: - url: The URL to fetch metadata for (must be a valid HTTP/HTTPS URL) - Returns: A rich preview card with title, description, thumbnail, and domain - The preview card will automatically be displayed in the chat. + +4. display_image: Display an image in the chat with metadata. + - Use this tool when you want to show an image to the user. + - This displays the image with an optional title, description, and source attribution. + - Common use cases: + * Showing an image from a URL mentioned in the conversation + * Displaying a diagram, chart, or illustration you're referencing + * Showing visual examples when explaining concepts + - Args: + - src: The URL of the image to display (must be a valid HTTP/HTTPS image URL) + - alt: Alternative text describing the image (for accessibility) + - title: Optional title to display below the image + - description: Optional description providing context about the image + - Returns: An image card with the image, title, and description + - The image will automatically be displayed in the chat. - User: "Fetch all my notes and what's in them?" @@ -184,6 +199,12 @@ You have access to the following tools: - User: "https://github.com/some/repo" - Call: `link_preview(url="https://github.com/some/repo")` + +- User: "Show me this image: https://example.com/image.png" + - Call: `display_image(src="https://example.com/image.png", alt="User shared image")` + +- User: "Can you display a diagram of a neural network?" + - Call: `display_image(src="https://example.com/neural-network.png", alt="Neural network diagram", title="Neural Network Architecture", description="A visual representation of a neural network with input, hidden, and output layers")` {citation_section} """ diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py index 853fb3392..6e2b30c6a 100644 --- a/surfsense_backend/app/tasks/chat/stream_new_chat.py +++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py @@ -298,6 +298,27 @@ async def stream_new_chat( status="in_progress", items=last_active_step_items, ) + elif tool_name == "display_image": + src = ( + tool_input.get("src", "") + if isinstance(tool_input, dict) + else str(tool_input) + ) + title = ( + tool_input.get("title", "") + if isinstance(tool_input, dict) + else "" + ) + last_active_step_title = "Displaying image" + last_active_step_items = [ + f"Image: {title[:50] if title else src[:50]}{'...' if len(title or src) > 50 else ''}" + ] + yield streaming_service.format_thinking_step( + step_id=tool_step_id, + title="Displaying image", + status="in_progress", + items=last_active_step_items, + ) elif tool_name == "generate_podcast": podcast_title = ( tool_input.get("podcast_title", "SurfSense Podcast") @@ -367,6 +388,16 @@ async def stream_new_chat( f"Fetching link preview: {url[:80]}{'...' if len(url) > 80 else ''}", "info", ) + elif tool_name == "display_image": + src = ( + tool_input.get("src", "") + if isinstance(tool_input, dict) + else str(tool_input) + ) + yield streaming_service.format_terminal_info( + f"Displaying image: {src[:60]}{'...' if len(src) > 60 else ''}", + "info", + ) elif tool_name == "generate_podcast": title = ( tool_input.get("podcast_title", "SurfSense Podcast") @@ -453,6 +484,24 @@ async def stream_new_chat( status="completed", items=completed_items, ) + elif tool_name == "display_image": + # Build completion items for image display + if isinstance(tool_output, dict): + title = tool_output.get("title", "") + alt = tool_output.get("alt", "Image") + display_name = title or alt + completed_items = [ + *last_active_step_items, + f"Showing: {display_name[:50]}{'...' if len(display_name) > 50 else ''}", + ] + else: + completed_items = [*last_active_step_items, "Image displayed"] + yield streaming_service.format_thinking_step( + step_id=original_step_id, + title="Displaying image", + status="completed", + items=completed_items, + ) elif tool_name == "generate_podcast": # Build detailed completion items based on podcast status podcast_status = ( @@ -566,6 +615,21 @@ async def stream_new_chat( f"Link preview failed: {error_msg}", "error", ) + elif tool_name == "display_image": + # Stream the full image result so frontend can render the Image component + yield streaming_service.format_tool_output_available( + tool_call_id, + tool_output + if isinstance(tool_output, dict) + else {"result": tool_output}, + ) + # Send terminal message + if isinstance(tool_output, dict): + title = tool_output.get("title") or tool_output.get("alt", "Image") + yield streaming_service.format_terminal_info( + f"Image displayed: {title[:40]}{'...' if len(title) > 40 else ''}", + "success", + ) elif tool_name == "search_knowledge_base": # Don't stream the full output for search (can be very large), just acknowledge yield streaming_service.format_tool_output_available( diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 4d45db118..7c6bd34c1 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -12,6 +12,7 @@ import { toast } from "sonner"; import { Thread } from "@/components/assistant-ui/thread"; import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast"; import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview"; +import { DisplayImageToolUI } from "@/components/tool-ui/display-image"; import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; import { getBearerToken } from "@/lib/auth-utils"; import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter"; @@ -80,7 +81,7 @@ function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike { /** * Tools that should render custom UI in the chat. */ -const TOOLS_WITH_UI = new Set(["generate_podcast", "link_preview"]); +const TOOLS_WITH_UI = new Set(["generate_podcast", "link_preview", "display_image"]); /** * Type for thinking step data from the backend @@ -591,6 +592,7 @@ export default function NewChatPage() { +
diff --git a/surfsense_web/components/tool-ui/display-image.tsx b/surfsense_web/components/tool-ui/display-image.tsx new file mode 100644 index 000000000..37bdad5fa --- /dev/null +++ b/surfsense_web/components/tool-ui/display-image.tsx @@ -0,0 +1,160 @@ +"use client"; + +import { makeAssistantToolUI } from "@assistant-ui/react"; +import { AlertCircleIcon, ImageIcon } from "lucide-react"; +import { + Image, + ImageErrorBoundary, + ImageLoading, + parseSerializableImage, +} from "@/components/tool-ui/image"; + +/** + * Type definitions for the display_image tool + */ +interface DisplayImageArgs { + src: string; + alt?: string; + title?: string; + description?: string; +} + +interface DisplayImageResult { + id: string; + assetId: string; + src: string; + alt: string; + title?: string; + description?: string; + domain?: string; + ratio?: string; + error?: string; +} + +/** + * Error state component shown when image display fails + */ +function ImageErrorState({ src, error }: { src: string; error: string }) { + return ( +
+
+
+ +
+
+

Failed to display image

+

{src}

+

{error}

+
+
+
+ ); +} + +/** + * Cancelled state component + */ +function ImageCancelledState({ src }: { src: string }) { + return ( +
+

+ + Image: {src} +

+
+ ); +} + +/** + * Parsed Image component with error handling + */ +function ParsedImage({ result }: { result: unknown }) { + const image = parseSerializableImage(result); + + return ( + { + if (id === "open" && image.src) { + window.open(image.src, "_blank", "noopener,noreferrer"); + } + }} + /> + ); +} + +/** + * Display Image Tool UI Component + * + * This component is registered with assistant-ui to render an image + * when the display_image tool is called by the agent. + * + * It displays images with: + * - Title and description + * - Source attribution + * - Hover overlay effects + * - Click to open full size + */ +export const DisplayImageToolUI = makeAssistantToolUI< + DisplayImageArgs, + DisplayImageResult +>({ + toolName: "display_image", + render: function DisplayImageUI({ args, result, status }) { + const src = args.src || "Unknown"; + + // Loading state - tool is still running + if (status.type === "running" || status.type === "requires-action") { + return ( +
+ +
+ ); + } + + // Incomplete/cancelled state + if (status.type === "incomplete") { + if (status.reason === "cancelled") { + return ; + } + if (status.reason === "error") { + return ( + + ); + } + } + + // No result yet + if (!result) { + return ( +
+ +
+ ); + } + + // Error result from the tool + if (result.error) { + return ; + } + + // Success - render the image + return ( +
+ + + +
+ ); + }, +}); + +export type { DisplayImageArgs, DisplayImageResult }; + diff --git a/surfsense_web/components/tool-ui/image/index.tsx b/surfsense_web/components/tool-ui/image/index.tsx new file mode 100644 index 000000000..97c983625 --- /dev/null +++ b/surfsense_web/components/tool-ui/image/index.tsx @@ -0,0 +1,332 @@ +"use client"; + +import { ExternalLinkIcon, ImageIcon, Loader2 } from "lucide-react"; +import NextImage from "next/image"; +import { Component, type ReactNode, useState } from "react"; +import { Badge } from "@/components/ui/badge"; +import { Card } from "@/components/ui/card"; +import { cn } from "@/lib/utils"; + +/** + * Aspect ratio options for images + */ +type AspectRatio = "1:1" | "4:3" | "16:9" | "9:16" | "auto"; + +/** + * Image fit options + */ +type ImageFit = "cover" | "contain"; + +/** + * Source attribution + */ +interface ImageSource { + label: string; + iconUrl?: string; + url?: string; +} + +/** + * Props for the Image component + */ +export interface ImageProps { + id: string; + assetId: string; + src: string; + alt: string; + title?: string; + description?: string; + href?: string; + domain?: string; + ratio?: AspectRatio; + fit?: ImageFit; + source?: ImageSource; + maxWidth?: string; + className?: string; +} + +/** + * Serializable schema for Image props (for tool results) + */ +export interface SerializableImage { + id: string; + assetId: string; + src: string; + alt: string; + title?: string; + description?: string; + href?: string; + domain?: string; + ratio?: AspectRatio; + source?: ImageSource; +} + +/** + * Parse and validate serializable image from tool result + */ +export function parseSerializableImage(result: unknown): SerializableImage { + if (typeof result !== "object" || result === null) { + throw new Error("Invalid image result: expected object"); + } + + const obj = result as Record; + + // Validate required fields + if (typeof obj.id !== "string") { + throw new Error("Invalid image: missing id"); + } + if (typeof obj.assetId !== "string") { + throw new Error("Invalid image: missing assetId"); + } + if (typeof obj.src !== "string") { + throw new Error("Invalid image: missing src"); + } + if (typeof obj.alt !== "string") { + throw new Error("Invalid image: missing alt"); + } + + return { + id: obj.id, + assetId: obj.assetId, + src: obj.src, + alt: obj.alt, + title: typeof obj.title === "string" ? obj.title : undefined, + description: typeof obj.description === "string" ? obj.description : undefined, + href: typeof obj.href === "string" ? obj.href : undefined, + domain: typeof obj.domain === "string" ? obj.domain : undefined, + ratio: typeof obj.ratio === "string" ? (obj.ratio as AspectRatio) : undefined, + source: typeof obj.source === "object" && obj.source !== null ? (obj.source as ImageSource) : undefined, + }; +} + +/** + * Get aspect ratio class based on ratio prop + */ +function getAspectRatioClass(ratio?: AspectRatio): string { + switch (ratio) { + case "1:1": + return "aspect-square"; + case "4:3": + return "aspect-[4/3]"; + case "16:9": + return "aspect-video"; + case "9:16": + return "aspect-[9/16]"; + case "auto": + default: + return "aspect-[4/3]"; + } +} + +/** + * Error boundary for Image component + */ +interface ImageErrorBoundaryState { + hasError: boolean; + error?: Error; +} + +export class ImageErrorBoundary extends Component< + { children: ReactNode }, + ImageErrorBoundaryState +> { + constructor(props: { children: ReactNode }) { + super(props); + this.state = { hasError: false }; + } + + static getDerivedStateFromError(error: Error): ImageErrorBoundaryState { + return { hasError: true, error }; + } + + render() { + if (this.state.hasError) { + return ( + +
+
+ +

Failed to load image

+
+
+ + ); + } + + return this.props.children; + } +} + +/** + * Loading skeleton for Image + */ +export function ImageSkeleton({ maxWidth = "420px" }: { maxWidth?: string }) { + return ( + +
+ +
+ + ); +} + +/** + * Image Loading State + */ +export function ImageLoading({ title = "Loading image..." }: { title?: string }) { + return ( + +
+
+ +

{title}

+
+
+ + ); +} + +/** + * Image Component + * + * Display images with metadata and attribution. + * Features hover overlay with title and source attribution. + */ +export function Image({ + id, + src, + alt, + title, + description, + href, + domain, + ratio = "4:3", + fit = "cover", + source, + maxWidth = "420px", + className, +}: ImageProps) { + const [isHovered, setIsHovered] = useState(false); + const [imageError, setImageError] = useState(false); + const aspectRatioClass = getAspectRatioClass(ratio); + const displayDomain = domain || source?.label; + + const handleClick = () => { + const targetUrl = href || source?.url || src; + if (targetUrl) { + window.open(targetUrl, "_blank", "noopener,noreferrer"); + } + }; + + if (imageError) { + return ( + +
+
+ +

Image not available

+
+
+ + ); + } + + return ( + setIsHovered(true)} + onMouseLeave={() => setIsHovered(false)} + onKeyDown={(e) => { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + handleClick(); + } + }} + role="button" + tabIndex={0} + > +
+ {/* Image */} + setImageError(true)} + /> + + {/* Hover overlay - appears on hover */} +
+ {/* Content at bottom */} +
+ {/* Title */} + {title && ( +

+ {title} +

+ )} + + {/* Description */} + {description && ( +

+ {description} +

+ )} + + {/* Source attribution */} + {displayDomain && ( +
+ {source?.iconUrl ? ( + + ) : ( + + )} + {displayDomain} +
+ )} +
+
+ + {/* Always visible domain badge (bottom right, shown when NOT hovered) */} + {displayDomain && !isHovered && ( +
+ + {displayDomain} + +
+ )} +
+ + ); +} diff --git a/surfsense_web/components/tool-ui/index.ts b/surfsense_web/components/tool-ui/index.ts index aebe400d5..b9500e7cd 100644 --- a/surfsense_web/components/tool-ui/index.ts +++ b/surfsense_web/components/tool-ui/index.ts @@ -32,3 +32,17 @@ export { type MediaCardProps, type SerializableMediaCard, } from "./media-card"; +export { + Image, + ImageErrorBoundary, + ImageLoading, + ImageSkeleton, + parseSerializableImage, + type ImageProps, + type SerializableImage, +} from "./image"; +export { + DisplayImageToolUI, + type DisplayImageArgs, + type DisplayImageResult, +} from "./display-image";