From 0de9589a7dba6ffc39faa2d060e33828023c6f96 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Sat, 7 Feb 2026 12:53:43 +0530 Subject: [PATCH] feat: add interactive file path cards in chat UI Render filepath code blocks as rich, clickable cards with three variants: knowledge files (navigate to editor), audio files (inline play/pause), and system files (open externally). Adds shell:openPath and shell:readFileBase64 IPC channels, FileCardProvider context, and Streamdown pre override. Co-Authored-By: Claude Opus 4.6 --- apps/x/apps/main/src/ipc.ts | 36 ++- apps/x/apps/renderer/src/App.tsx | 11 +- .../components/ai-elements/file-path-card.tsx | 231 ++++++++++++++++++ .../ai-elements/markdown-code-override.tsx | 27 ++ .../src/components/ai-elements/message.tsx | 2 +- .../renderer/src/components/chat-sidebar.tsx | 12 +- .../src/contexts/file-card-context.tsx | 27 ++ .../src/application/assistant/instructions.ts | 21 +- apps/x/packages/shared/src/ipc.ts | 9 + 9 files changed, 369 insertions(+), 7 deletions(-) create mode 100644 apps/x/apps/renderer/src/components/ai-elements/file-path-card.tsx create mode 100644 apps/x/apps/renderer/src/components/ai-elements/markdown-code-override.tsx create mode 100644 apps/x/apps/renderer/src/contexts/file-card-context.tsx diff --git a/apps/x/apps/main/src/ipc.ts b/apps/x/apps/main/src/ipc.ts index 90377e62..7b42a98e 100644 --- a/apps/x/apps/main/src/ipc.ts +++ b/apps/x/apps/main/src/ipc.ts @@ -1,5 +1,7 @@ -import { ipcMain, BrowserWindow } from 'electron'; +import { ipcMain, BrowserWindow, shell } from 'electron'; import { ipc } from '@x/shared'; +import path from 'node:path'; +import os from 'node:os'; import { connectProvider, disconnectProvider, @@ -455,5 +457,37 @@ export function setupIpcHandlers() { await stateRepo.deleteAgentState(args.agentName); return { success: true }; }, + // Shell integration handlers + 'shell:openPath': async (_event, args) => { + let filePath = args.path; + if (filePath.startsWith('~')) { + filePath = path.join(os.homedir(), filePath.slice(1)); + } + const error = await shell.openPath(filePath); + return { error: error || undefined }; + }, + 'shell:readFileBase64': async (_event, args) => { + let filePath = args.path; + if (filePath.startsWith('~')) { + filePath = path.join(os.homedir(), filePath.slice(1)); + } + const stat = await fs.stat(filePath); + if (stat.size > 10 * 1024 * 1024) { + throw new Error('File too large (>10MB)'); + } + const buffer = await fs.readFile(filePath); + const ext = path.extname(filePath).toLowerCase(); + const mimeMap: Record = { + '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', + '.gif': 'image/gif', '.webp': 'image/webp', '.svg': 'image/svg+xml', + '.bmp': 'image/bmp', '.ico': 'image/x-icon', + '.wav': 'audio/wav', '.mp3': 'audio/mpeg', '.m4a': 'audio/mp4', + '.ogg': 'audio/ogg', '.flac': 'audio/flac', '.aac': 'audio/aac', + '.pdf': 'application/pdf', '.json': 'application/json', + '.txt': 'text/plain', '.md': 'text/markdown', + }; + const mimeType = mimeMap[ext] || 'application/octet-stream'; + return { data: buffer.toString('base64'), mimeType, size: stat.size }; + }, }); } diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index 112b0c6f..d64d9947 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -50,6 +50,8 @@ import { Toaster } from "@/components/ui/sonner" import { stripKnowledgePrefix, toKnowledgePath, wikiLabel } from '@/lib/wiki-links' import { OnboardingModal } from '@/components/onboarding-modal' import { BackgroundTaskDetail } from '@/components/background-task-detail' +import { FileCardProvider } from '@/contexts/file-card-context' +import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-override' import { AgentScheduleConfig } from '@x/shared/dist/agent-schedule.js' import { AgentScheduleState } from '@x/shared/dist/agent-schedule-state.js' @@ -108,6 +110,8 @@ const toToolState = (status: ToolCall['status']): ToolState => { } } +const streamdownComponents = { pre: MarkdownPreOverride } + const DEFAULT_SIDEBAR_WIDTH = 256 const wikiLinkRegex = /\[\[([^[\]]+)\]\]/g const graphPalette = [ @@ -1766,7 +1770,7 @@ function App() { return ( - {item.content} + {item.content} ) @@ -1948,6 +1952,7 @@ function App() { /> ) : ( + { setSelectedPath(path); setIsGraphOpen(false) }}>
@@ -2004,7 +2009,7 @@ function App() { {currentAssistantMessage && ( - {currentAssistantMessage} + {currentAssistantMessage} )} @@ -2042,6 +2047,7 @@ function App() {
+ )} @@ -2071,6 +2077,7 @@ function App() { permissionResponses={permissionResponses} onPermissionResponse={handlePermissionResponse} onAskHumanResponse={handleAskHumanResponse} + onOpenKnowledgeFile={(path) => { setSelectedPath(path); setIsGraphOpen(false) }} /> )} {/* Rendered last so its no-drag region paints over the sidebar drag region */} diff --git a/apps/x/apps/renderer/src/components/ai-elements/file-path-card.tsx b/apps/x/apps/renderer/src/components/ai-elements/file-path-card.tsx new file mode 100644 index 00000000..c178c4d0 --- /dev/null +++ b/apps/x/apps/renderer/src/components/ai-elements/file-path-card.tsx @@ -0,0 +1,231 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { BookOpen, FileIcon, FileText, Image, Music, Pause, Play, Video } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { useFileCard } from '@/contexts/file-card-context' +import { useSidebarSection } from '@/contexts/sidebar-context' +import { wikiLabel } from '@/lib/wiki-links' + +const AUDIO_EXTENSIONS = new Set(['.wav', '.mp3', '.m4a', '.ogg', '.flac', '.aac']) +const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.bmp', '.ico']) +const VIDEO_EXTENSIONS = new Set(['.mp4', '.mov', '.avi', '.mkv', '.webm']) +const DOCUMENT_EXTENSIONS = new Set(['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.txt', '.rtf', '.csv']) + +function getExtension(filePath: string): string { + const dot = filePath.lastIndexOf('.') + return dot >= 0 ? filePath.slice(dot).toLowerCase() : '' +} + +function getFileNameWithoutExt(filePath: string): string { + const name = filePath.split('/').pop() || filePath + const dot = name.lastIndexOf('.') + return dot > 0 ? name.slice(0, dot) : name +} + +function getFileCategory(ext: string): { label: string; icon: typeof FileIcon } { + if (AUDIO_EXTENSIONS.has(ext)) return { label: 'Audio', icon: Music } + if (IMAGE_EXTENSIONS.has(ext)) return { label: 'Image', icon: Image } + if (VIDEO_EXTENSIONS.has(ext)) return { label: 'Video', icon: Video } + if (DOCUMENT_EXTENSIONS.has(ext)) return { label: 'Document', icon: FileText } + if (ext === '.md') return { label: 'Markdown', icon: FileText } + return { label: 'File', icon: FileIcon } +} + +function getExtLabel(ext: string): string { + return ext ? ext.slice(1).toUpperCase() : '' +} + +// Shared card shell used by all variants +function CardShell({ + icon, + title, + subtitle, + onClick, + action, +}: { + icon: React.ReactNode + title: string + subtitle: string + onClick?: () => void + action?: React.ReactNode +}) { + return ( +
{ if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); onClick() } } : undefined} + className="flex items-center gap-3 rounded-xl border border-border bg-card p-3 pr-4 text-left transition-colors hover:bg-accent/50 cursor-pointer w-full my-2" + > +
+ {icon} +
+
+
{title}
+
{subtitle}
+
+ {action} +
+ ) +} + +// --- Knowledge File Card --- + +function KnowledgeFileCard({ filePath }: { filePath: string }) { + const { onOpenKnowledgeFile } = useFileCard() + const { setActiveSection } = useSidebarSection() + const label = wikiLabel(filePath) + const ext = getExtension(filePath) + const extLabel = getExtLabel(ext) + + return ( + } + title={label} + subtitle={extLabel ? `Knowledge \u00b7 ${extLabel}` : 'Knowledge'} + onClick={() => { setActiveSection('knowledge'); onOpenKnowledgeFile(filePath) }} + action={ + + } + /> + ) +} + +// --- Audio File Card --- + +function AudioFileCard({ filePath }: { filePath: string }) { + const [isPlaying, setIsPlaying] = useState(false) + const [isLoading, setIsLoading] = useState(false) + const audioRef = useRef(null) + const ext = getExtension(filePath) + const extLabel = getExtLabel(ext) + + const handlePlayPause = useCallback(async (e: React.MouseEvent) => { + e.stopPropagation() + if (isPlaying && audioRef.current) { + audioRef.current.pause() + setIsPlaying(false) + return + } + + if (!audioRef.current) { + setIsLoading(true) + try { + const result = await window.ipc.invoke('shell:readFileBase64', { path: filePath }) + const dataUrl = `data:${result.mimeType};base64,${result.data}` + const audio = new Audio(dataUrl) + audio.addEventListener('ended', () => setIsPlaying(false)) + audioRef.current = audio + } catch (err) { + console.error('Failed to load audio:', err) + setIsLoading(false) + return + } + setIsLoading(false) + } + + audioRef.current.play() + setIsPlaying(true) + }, [filePath, isPlaying]) + + useEffect(() => { + return () => { + if (audioRef.current) { + audioRef.current.pause() + audioRef.current = null + } + } + }, []) + + const handleOpen = async () => { + await window.ipc.invoke('shell:openPath', { path: filePath }) + } + + return ( + + {isPlaying + ? + : + } + + } + title={getFileNameWithoutExt(filePath)} + subtitle={`Audio \u00b7 ${extLabel}`} + onClick={handleOpen} + action={ + + } + /> + ) +} + +// --- System File Card --- + +function SystemFileCard({ filePath }: { filePath: string }) { + const ext = getExtension(filePath) + const isImage = IMAGE_EXTENSIONS.has(ext) + const [thumbnail, setThumbnail] = useState(null) + const { label: categoryLabel, icon: CategoryIcon } = getFileCategory(ext) + const extLabel = getExtLabel(ext) + + useEffect(() => { + if (!isImage) return + let cancelled = false + window.ipc.invoke('shell:readFileBase64', { path: filePath }) + .then((result) => { + if (!cancelled) { + setThumbnail(`data:${result.mimeType};base64,${result.data}`) + } + }) + .catch(() => {/* ignore thumbnail failures */}) + return () => { cancelled = true } + }, [filePath, isImage]) + + const handleOpen = async () => { + await window.ipc.invoke('shell:openPath', { path: filePath }) + } + + return ( + + : + } + title={getFileNameWithoutExt(filePath)} + subtitle={extLabel ? `${categoryLabel} \u00b7 ${extLabel}` : categoryLabel} + onClick={handleOpen} + action={ + + } + /> + ) +} + +// --- Main FilePathCard --- + +export function FilePathCard({ filePath }: { filePath: string }) { + const trimmed = filePath.trim() + + if (trimmed.startsWith('knowledge/')) { + return + } + + const ext = getExtension(trimmed) + if (AUDIO_EXTENSIONS.has(ext)) { + return + } + + return +} diff --git a/apps/x/apps/renderer/src/components/ai-elements/markdown-code-override.tsx b/apps/x/apps/renderer/src/components/ai-elements/markdown-code-override.tsx new file mode 100644 index 00000000..c1470326 --- /dev/null +++ b/apps/x/apps/renderer/src/components/ai-elements/markdown-code-override.tsx @@ -0,0 +1,27 @@ +import { isValidElement, type JSX } from 'react' +import { FilePathCard } from './file-path-card' + +export function MarkdownPreOverride(props: JSX.IntrinsicElements['pre']) { + const { children, ...rest } = props + + // Check if the child is a with className "language-filepath" + if (isValidElement(children)) { + const childProps = children.props as { className?: string; children?: unknown } + if ( + typeof childProps.className === 'string' && + childProps.className.includes('language-filepath') + ) { + // Extract the text content from the code element + const text = typeof childProps.children === 'string' + ? childProps.children.trim() + : '' + if (text) { + return + } + } + } + + // Passthrough for all other code blocks - return children directly + // so Streamdown's own rendering (syntax highlighting, etc.) is preserved + return 
{children}
+} diff --git a/apps/x/apps/renderer/src/components/ai-elements/message.tsx b/apps/x/apps/renderer/src/components/ai-elements/message.tsx index 635d455c..ec3acfc1 100644 --- a/apps/x/apps/renderer/src/components/ai-elements/message.tsx +++ b/apps/x/apps/renderer/src/components/ai-elements/message.tsx @@ -50,7 +50,7 @@ export const MessageContent = ({ className={cn( "is-user:dark flex w-fit max-w-full min-w-0 flex-col gap-2 overflow-hidden text-sm", "group-[.is-user]:ml-auto group-[.is-user]:rounded-lg group-[.is-user]:bg-secondary group-[.is-user]:px-4 group-[.is-user]:py-3 group-[.is-user]:text-foreground", - "group-[.is-assistant]:text-foreground", + "group-[.is-assistant]:w-full group-[.is-assistant]:text-foreground", className )} {...props} diff --git a/apps/x/apps/renderer/src/components/chat-sidebar.tsx b/apps/x/apps/renderer/src/components/chat-sidebar.tsx index 41ed6e71..48b89b4d 100644 --- a/apps/x/apps/renderer/src/components/chat-sidebar.tsx +++ b/apps/x/apps/renderer/src/components/chat-sidebar.tsx @@ -33,6 +33,8 @@ import { getMentionHighlightSegments } from '@/lib/mention-highlights' import { ToolPermissionRequestEvent, AskHumanRequestEvent } from '@x/shared/src/runs.js' import z from 'zod' import React from 'react' +import { FileCardProvider } from '@/contexts/file-card-context' +import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-override' interface ChatMessage { id: string @@ -103,6 +105,8 @@ const normalizeToolOutput = (output: ToolCall['result'] | undefined, status: Too return output } +const streamdownComponents = { pre: MarkdownPreOverride } + const MIN_WIDTH = 300 const MAX_WIDTH = 700 const DEFAULT_WIDTH = 400 @@ -131,6 +135,7 @@ interface ChatSidebarProps { permissionResponses?: Map onPermissionResponse?: (toolCallId: string, subflow: string[], response: 'approve' | 'deny') => void onAskHumanResponse?: (toolCallId: string, subflow: string[], response: string) => void + onOpenKnowledgeFile?: (path: string) => void } export function ChatSidebar({ @@ -156,6 +161,7 @@ export function ChatSidebar({ permissionResponses = new Map(), onPermissionResponse, onAskHumanResponse, + onOpenKnowledgeFile, }: ChatSidebarProps) { const [width, setWidth] = useState(defaultWidth) const [isResizing, setIsResizing] = useState(false) @@ -391,7 +397,7 @@ export function ChatSidebar({ {item.role === 'assistant' ? ( - {item.content} + {item.content} ) : ( item.content )} @@ -480,6 +486,7 @@ export function ChatSidebar({ {/* Conversation area */} + {})}>
@@ -538,7 +545,7 @@ export function ChatSidebar({ {currentAssistantMessage && ( - {currentAssistantMessage} + {currentAssistantMessage} )} @@ -650,6 +657,7 @@ export function ChatSidebar({ )}
+ )} diff --git a/apps/x/apps/renderer/src/contexts/file-card-context.tsx b/apps/x/apps/renderer/src/contexts/file-card-context.tsx new file mode 100644 index 00000000..08910b12 --- /dev/null +++ b/apps/x/apps/renderer/src/contexts/file-card-context.tsx @@ -0,0 +1,27 @@ +import { createContext, useContext, type ReactNode } from 'react' + +interface FileCardContextType { + onOpenKnowledgeFile: (path: string) => void +} + +const FileCardContext = createContext(null) + +export function useFileCard() { + const ctx = useContext(FileCardContext) + if (!ctx) throw new Error('useFileCard must be used within FileCardProvider') + return ctx +} + +export function FileCardProvider({ + onOpenKnowledgeFile, + children, +}: { + onOpenKnowledgeFile: (path: string) => void + children: ReactNode +}) { + return ( + + {children} + + ) +} diff --git a/apps/x/packages/core/src/application/assistant/instructions.ts b/apps/x/packages/core/src/application/assistant/instructions.ts index 57f3a446..1104df90 100644 --- a/apps/x/packages/core/src/application/assistant/instructions.ts +++ b/apps/x/packages/core/src/application/assistant/instructions.ts @@ -179,4 +179,23 @@ When a user asks for ANY task that might require external capabilities (web sear **Only \`executeCommand\` (shell/bash commands) goes through the approval flow.** If you need to delete a file, use the \`workspace-remove\` builtin tool, not \`executeCommand\` with \`rm\`. If you need to create a file, use \`workspace-writeFile\`, not \`executeCommand\` with \`touch\` or \`echo >\`. -Rowboat's internal builtin tools never require approval — only shell commands via \`executeCommand\` do.`; +Rowboat's internal builtin tools never require approval — only shell commands via \`executeCommand\` do. + +## File Path References + +When you reference a file path in your response (whether a knowledge base file or a file on the user's system), ALWAYS wrap it in a filepath code block: + +\`\`\`filepath +knowledge/People/Sarah Chen.md +\`\`\` + +\`\`\`filepath +~/Desktop/report.pdf +\`\`\` + +This renders as an interactive card in the UI. Use this format for: +- Knowledge base file paths (knowledge/...) +- Files on the user's machine (~/Desktop/..., /Users/..., etc.) +- Audio files, images, documents, or any file reference + +Never output raw file paths in plain text when they could be wrapped in a filepath block.`; diff --git a/apps/x/packages/shared/src/ipc.ts b/apps/x/packages/shared/src/ipc.ts index 67b29a28..175c409e 100644 --- a/apps/x/packages/shared/src/ipc.ts +++ b/apps/x/packages/shared/src/ipc.ts @@ -386,6 +386,15 @@ const ipcSchemas = { success: z.literal(true), }), }, + // Shell integration channels + 'shell:openPath': { + req: z.object({ path: z.string() }), + res: z.object({ error: z.string().optional() }), + }, + 'shell:readFileBase64': { + req: z.object({ path: z.string() }), + res: z.object({ data: z.string(), mimeType: z.string(), size: z.number() }), + }, } as const; // ============================================================================