feat: introduce display image tool for enhanced image rendering in chat with metadata support

surfsense_backend/app/agents/new_chat/chat_deepagent.py

@@ -14,6 +14,7 @@ from langgraph.types import Checkpointer
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.agents.new_chat.context import SurfSenseContextSchema
+from app.agents.new_chat.display_image import create_display_image_tool
 from app.agents.new_chat.knowledge_base import create_search_knowledge_base_tool
 from app.agents.new_chat.link_preview import create_link_preview_tool
 from app.agents.new_chat.podcast import create_generate_podcast_tool
@@ -36,6 +37,7 @@ def create_surfsense_deep_agent(
     enable_citations: bool = True,
     enable_podcast: bool = True,
     enable_link_preview: bool = True,
+    enable_display_image: bool = True,
     additional_tools: Sequence[BaseTool] | None = None,
 ):
     """
@@ -57,6 +59,8 @@ def create_surfsense_deep_agent(
                        When True and user_id is provided, the agent can generate podcasts.
         enable_link_preview: Whether to include the link preview tool (default: True).
                             When True, the agent can fetch and display rich link previews.
+        enable_display_image: Whether to include the display image tool (default: True).
+                             When True, the agent can display images with metadata.
         additional_tools: Optional sequence of additional tools to inject into the agent.
                          The search_knowledge_base tool will always be included.
 
@@ -87,6 +91,11 @@ def create_surfsense_deep_agent(
         link_preview_tool = create_link_preview_tool()
         tools.append(link_preview_tool)
 
+    # Add display image tool if enabled
+    if enable_display_image:
+        display_image_tool = create_display_image_tool()
+        tools.append(display_image_tool)
+
     if additional_tools:
         tools.extend(additional_tools)
 

surfsense_backend/app/agents/new_chat/display_image.py

@@ -0,0 +1,106 @@
+"""
+Display image tool for the new chat agent.
+
+This module provides a tool for displaying images in the chat UI
+with metadata like title, description, and source attribution.
+"""
+
+import hashlib
+from typing import Any
+from urllib.parse import urlparse
+
+from langchain_core.tools import tool
+
+
+def extract_domain(url: str) -> str:
+    """Extract the domain from a URL."""
+    try:
+        parsed = urlparse(url)
+        domain = parsed.netloc
+        # Remove 'www.' prefix if present
+        if domain.startswith("www."):
+            domain = domain[4:]
+        return domain
+    except Exception:
+        return ""
+
+
+def generate_image_id(src: str) -> str:
+    """Generate a unique ID for an image."""
+    hash_val = hashlib.md5(src.encode()).hexdigest()[:12]
+    return f"image-{hash_val}"
+
+
+def create_display_image_tool():
+    """
+    Factory function to create the display_image tool.
+
+    Returns:
+        A configured tool function for displaying images.
+    """
+
+    @tool
+    async def display_image(
+        src: str,
+        alt: str = "Image",
+        title: str | None = None,
+        description: str | None = None,
+    ) -> dict[str, Any]:
+        """
+        Display an image in the chat with metadata.
+
+        Use this tool when you want to show an image to the user.
+        This displays the image with an optional title, description,
+        and source attribution.
+
+        Common use cases:
+        - Showing an image from a URL the user mentioned
+        - Displaying a diagram or chart you're referencing
+        - Showing example images when explaining concepts
+
+        Args:
+            src: The URL of the image to display (must be a valid HTTP/HTTPS URL)
+            alt: Alternative text describing the image (for accessibility)
+            title: Optional title to display below the image
+            description: Optional description providing context about the image
+
+        Returns:
+            A dictionary containing image metadata for the UI to render:
+            - id: Unique identifier for this image
+            - assetId: The image URL (for deduplication)
+            - src: The image URL
+            - alt: Alt text for accessibility
+            - title: Image title (if provided)
+            - description: Image description (if provided)
+            - domain: Source domain
+        """
+        image_id = generate_image_id(src)
+        
+        # Ensure URL has protocol
+        if not src.startswith(("http://", "https://")):
+            src = f"https://{src}"
+        
+        domain = extract_domain(src)
+        
+        # Determine aspect ratio based on common image sources
+        ratio = "16:9"  # Default
+        if "unsplash.com" in src or "pexels.com" in src:
+            ratio = "16:9"
+        elif "imgur.com" in src:
+            ratio = "auto"
+        elif "github.com" in src or "githubusercontent.com" in src:
+            ratio = "auto"
+        
+        return {
+            "id": image_id,
+            "assetId": src,
+            "src": src,
+            "alt": alt,
+            "title": title,
+            "description": description,
+            "domain": domain,
+            "ratio": ratio,
+        }
+
+    return display_image
+

surfsense_backend/app/agents/new_chat/system_prompt.py

@@ -158,6 +158,21 @@ You have access to the following tools:
     - url: The URL to fetch metadata for (must be a valid HTTP/HTTPS URL)
   - Returns: A rich preview card with title, description, thumbnail, and domain
   - The preview card will automatically be displayed in the chat.
+
+4. display_image: Display an image in the chat with metadata.
+  - Use this tool when you want to show an image to the user.
+  - This displays the image with an optional title, description, and source attribution.
+  - Common use cases:
+    * Showing an image from a URL mentioned in the conversation
+    * Displaying a diagram, chart, or illustration you're referencing
+    * Showing visual examples when explaining concepts
+  - Args:
+    - src: The URL of the image to display (must be a valid HTTP/HTTPS image URL)
+    - alt: Alternative text describing the image (for accessibility)
+    - title: Optional title to display below the image
+    - description: Optional description providing context about the image
+  - Returns: An image card with the image, title, and description
+  - The image will automatically be displayed in the chat.
 </tools>
 <tool_call_examples>
 - User: "Fetch all my notes and what's in them?"
@@ -184,6 +199,12 @@ You have access to the following tools:
 
 - User: "https://github.com/some/repo"
   - Call: `link_preview(url="https://github.com/some/repo")`
+
+- User: "Show me this image: https://example.com/image.png"
+  - Call: `display_image(src="https://example.com/image.png", alt="User shared image")`
+
+- User: "Can you display a diagram of a neural network?"
+  - Call: `display_image(src="https://example.com/neural-network.png", alt="Neural network diagram", title="Neural Network Architecture", description="A visual representation of a neural network with input, hidden, and output layers")`
 </tool_call_examples>{citation_section}
 """