diff --git a/apps/x/apps/main/src/ipc.ts b/apps/x/apps/main/src/ipc.ts index 6efd48df..2795bfcd 100644 --- a/apps/x/apps/main/src/ipc.ts +++ b/apps/x/apps/main/src/ipc.ts @@ -1,5 +1,7 @@ -import { ipcMain, BrowserWindow } from 'electron'; +import { ipcMain, BrowserWindow, shell } from 'electron'; import { ipc } from '@x/shared'; +import path from 'node:path'; +import os from 'node:os'; import { connectProvider, disconnectProvider, @@ -420,5 +422,37 @@ export function setupIpcHandlers() { await stateRepo.deleteAgentState(args.agentName); return { success: true }; }, + // Shell integration handlers + 'shell:openPath': async (_event, args) => { + let filePath = args.path; + if (filePath.startsWith('~')) { + filePath = path.join(os.homedir(), filePath.slice(1)); + } + const error = await shell.openPath(filePath); + return { error: error || undefined }; + }, + 'shell:readFileBase64': async (_event, args) => { + let filePath = args.path; + if (filePath.startsWith('~')) { + filePath = path.join(os.homedir(), filePath.slice(1)); + } + const stat = await fs.stat(filePath); + if (stat.size > 10 * 1024 * 1024) { + throw new Error('File too large (>10MB)'); + } + const buffer = await fs.readFile(filePath); + const ext = path.extname(filePath).toLowerCase(); + const mimeMap: Record = { + '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', + '.gif': 'image/gif', '.webp': 'image/webp', '.svg': 'image/svg+xml', + '.bmp': 'image/bmp', '.ico': 'image/x-icon', + '.wav': 'audio/wav', '.mp3': 'audio/mpeg', '.m4a': 'audio/mp4', + '.ogg': 'audio/ogg', '.flac': 'audio/flac', '.aac': 'audio/aac', + '.pdf': 'application/pdf', '.json': 'application/json', + '.txt': 'text/plain', '.md': 'text/markdown', + }; + const mimeType = mimeMap[ext] || 'application/octet-stream'; + return { data: buffer.toString('base64'), mimeType, size: stat.size }; + }, }); } diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index a0fe4e0e..383a4133 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -52,6 +52,8 @@ import { Toaster } from "@/components/ui/sonner" import { stripKnowledgePrefix, toKnowledgePath, wikiLabel } from '@/lib/wiki-links' import { OnboardingModal } from '@/components/onboarding-modal' import { BackgroundTaskDetail } from '@/components/background-task-detail' +import { FileCardProvider } from '@/contexts/file-card-context' +import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-override' import { AgentScheduleConfig } from '@x/shared/dist/agent-schedule.js' import { AgentScheduleState } from '@x/shared/dist/agent-schedule-state.js' @@ -110,6 +112,8 @@ const toToolState = (status: ToolCall['status']): ToolState => { } } +const streamdownComponents = { pre: MarkdownPreOverride } + const DEFAULT_SIDEBAR_WIDTH = 256 const wikiLinkRegex = /\[\[([^[\]]+)\]\]/g const graphPalette = [ @@ -1728,7 +1732,7 @@ function App() { return ( - {item.content} + {item.content} ) @@ -1939,6 +1943,7 @@ function App() { /> ) : ( + { setSelectedPath(path); setIsGraphOpen(false) }}>
@@ -1995,7 +2000,7 @@ function App() { {currentAssistantMessage && ( - {currentAssistantMessage} + {currentAssistantMessage} )} @@ -2033,6 +2038,7 @@ function App() {
+ )} @@ -2062,6 +2068,7 @@ function App() { permissionResponses={permissionResponses} onPermissionResponse={handlePermissionResponse} onAskHumanResponse={handleAskHumanResponse} + onOpenKnowledgeFile={(path) => { setSelectedPath(path); setIsGraphOpen(false) }} /> )} diff --git a/apps/x/apps/renderer/src/components/ai-elements/file-path-card.tsx b/apps/x/apps/renderer/src/components/ai-elements/file-path-card.tsx new file mode 100644 index 00000000..e989f278 --- /dev/null +++ b/apps/x/apps/renderer/src/components/ai-elements/file-path-card.tsx @@ -0,0 +1,176 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { BookOpen, ExternalLink, FileIcon, Pause, Play } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { useFileCard } from '@/contexts/file-card-context' +import { wikiLabel } from '@/lib/wiki-links' + +const AUDIO_EXTENSIONS = new Set(['.wav', '.mp3', '.m4a', '.ogg', '.flac', '.aac']) +const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.bmp', '.ico']) + +function getExtension(filePath: string): string { + const dot = filePath.lastIndexOf('.') + return dot >= 0 ? filePath.slice(dot).toLowerCase() : '' +} + +function getFileName(filePath: string): string { + return filePath.split('/').pop() || filePath +} + +function truncatePath(filePath: string, maxLen = 40): string { + if (filePath.length <= maxLen) return filePath + const parts = filePath.split('/') + if (parts.length <= 2) return filePath + return `.../${parts.slice(-2).join('/')}` +} + +// --- Knowledge File Card --- + +function KnowledgeFileCard({ filePath }: { filePath: string }) { + const { onOpenKnowledgeFile } = useFileCard() + const label = wikiLabel(filePath) + + return ( + + ) +} + +// --- Audio File Card --- + +function AudioFileCard({ filePath }: { filePath: string }) { + const [isPlaying, setIsPlaying] = useState(false) + const [isLoading, setIsLoading] = useState(false) + const audioRef = useRef(null) + + const handlePlayPause = useCallback(async () => { + if (isPlaying && audioRef.current) { + audioRef.current.pause() + setIsPlaying(false) + return + } + + if (!audioRef.current) { + setIsLoading(true) + try { + const result = await window.ipc.invoke('shell:readFileBase64', { path: filePath }) + const dataUrl = `data:${result.mimeType};base64,${result.data}` + const audio = new Audio(dataUrl) + audio.addEventListener('ended', () => setIsPlaying(false)) + audioRef.current = audio + } catch (err) { + console.error('Failed to load audio:', err) + setIsLoading(false) + return + } + setIsLoading(false) + } + + audioRef.current.play() + setIsPlaying(true) + }, [filePath, isPlaying]) + + useEffect(() => { + return () => { + if (audioRef.current) { + audioRef.current.pause() + audioRef.current = null + } + } + }, []) + + const handleOpen = async () => { + await window.ipc.invoke('shell:openPath', { path: filePath }) + } + + return ( +
+ +
+
{getFileName(filePath)}
+
{truncatePath(filePath)}
+
+ +
+ ) +} + +// --- System File Card --- + +function SystemFileCard({ filePath }: { filePath: string }) { + const ext = getExtension(filePath) + const isImage = IMAGE_EXTENSIONS.has(ext) + const [thumbnail, setThumbnail] = useState(null) + + useEffect(() => { + if (!isImage) return + let cancelled = false + window.ipc.invoke('shell:readFileBase64', { path: filePath }) + .then((result) => { + if (!cancelled) { + setThumbnail(`data:${result.mimeType};base64,${result.data}`) + } + }) + .catch(() => {/* ignore thumbnail failures */}) + return () => { cancelled = true } + }, [filePath, isImage]) + + const handleOpen = async () => { + await window.ipc.invoke('shell:openPath', { path: filePath }) + } + + return ( + + ) +} + +// --- Main FilePathCard --- + +export function FilePathCard({ filePath }: { filePath: string }) { + const trimmed = filePath.trim() + + if (trimmed.startsWith('knowledge/')) { + return + } + + const ext = getExtension(trimmed) + if (AUDIO_EXTENSIONS.has(ext)) { + return + } + + return +} diff --git a/apps/x/apps/renderer/src/components/ai-elements/markdown-code-override.tsx b/apps/x/apps/renderer/src/components/ai-elements/markdown-code-override.tsx new file mode 100644 index 00000000..c1470326 --- /dev/null +++ b/apps/x/apps/renderer/src/components/ai-elements/markdown-code-override.tsx @@ -0,0 +1,27 @@ +import { isValidElement, type JSX } from 'react' +import { FilePathCard } from './file-path-card' + +export function MarkdownPreOverride(props: JSX.IntrinsicElements['pre']) { + const { children, ...rest } = props + + // Check if the child is a with className "language-filepath" + if (isValidElement(children)) { + const childProps = children.props as { className?: string; children?: unknown } + if ( + typeof childProps.className === 'string' && + childProps.className.includes('language-filepath') + ) { + // Extract the text content from the code element + const text = typeof childProps.children === 'string' + ? childProps.children.trim() + : '' + if (text) { + return + } + } + } + + // Passthrough for all other code blocks - return children directly + // so Streamdown's own rendering (syntax highlighting, etc.) is preserved + return 
{children}
+} diff --git a/apps/x/apps/renderer/src/components/chat-sidebar.tsx b/apps/x/apps/renderer/src/components/chat-sidebar.tsx index b5b32e09..79b2ea0b 100644 --- a/apps/x/apps/renderer/src/components/chat-sidebar.tsx +++ b/apps/x/apps/renderer/src/components/chat-sidebar.tsx @@ -33,6 +33,8 @@ import { getMentionHighlightSegments } from '@/lib/mention-highlights' import { ToolPermissionRequestEvent, AskHumanRequestEvent } from '@x/shared/src/runs.js' import z from 'zod' import React from 'react' +import { FileCardProvider } from '@/contexts/file-card-context' +import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-override' interface ChatMessage { id: string @@ -103,6 +105,8 @@ const normalizeToolOutput = (output: ToolCall['result'] | undefined, status: Too return output } +const streamdownComponents = { pre: MarkdownPreOverride } + const MIN_WIDTH = 300 const MAX_WIDTH = 700 const DEFAULT_WIDTH = 400 @@ -131,6 +135,7 @@ interface ChatSidebarProps { permissionResponses?: Map onPermissionResponse?: (toolCallId: string, subflow: string[], response: 'approve' | 'deny') => void onAskHumanResponse?: (toolCallId: string, subflow: string[], response: string) => void + onOpenKnowledgeFile?: (path: string) => void } export function ChatSidebar({ @@ -156,6 +161,7 @@ export function ChatSidebar({ permissionResponses = new Map(), onPermissionResponse, onAskHumanResponse, + onOpenKnowledgeFile, }: ChatSidebarProps) { const [width, setWidth] = useState(defaultWidth) const [isResizing, setIsResizing] = useState(false) @@ -391,7 +397,7 @@ export function ChatSidebar({ {item.role === 'assistant' ? ( - {item.content} + {item.content} ) : ( item.content )} @@ -480,6 +486,7 @@ export function ChatSidebar({ {/* Conversation area */} + {})}>
@@ -538,7 +545,7 @@ export function ChatSidebar({ {currentAssistantMessage && ( - {currentAssistantMessage} + {currentAssistantMessage} )} @@ -650,6 +657,7 @@ export function ChatSidebar({ )}
+ )} diff --git a/apps/x/apps/renderer/src/contexts/file-card-context.tsx b/apps/x/apps/renderer/src/contexts/file-card-context.tsx new file mode 100644 index 00000000..08910b12 --- /dev/null +++ b/apps/x/apps/renderer/src/contexts/file-card-context.tsx @@ -0,0 +1,27 @@ +import { createContext, useContext, type ReactNode } from 'react' + +interface FileCardContextType { + onOpenKnowledgeFile: (path: string) => void +} + +const FileCardContext = createContext(null) + +export function useFileCard() { + const ctx = useContext(FileCardContext) + if (!ctx) throw new Error('useFileCard must be used within FileCardProvider') + return ctx +} + +export function FileCardProvider({ + onOpenKnowledgeFile, + children, +}: { + onOpenKnowledgeFile: (path: string) => void + children: ReactNode +}) { + return ( + + {children} + + ) +} diff --git a/apps/x/packages/core/src/application/assistant/instructions.ts b/apps/x/packages/core/src/application/assistant/instructions.ts index 57f3a446..1104df90 100644 --- a/apps/x/packages/core/src/application/assistant/instructions.ts +++ b/apps/x/packages/core/src/application/assistant/instructions.ts @@ -179,4 +179,23 @@ When a user asks for ANY task that might require external capabilities (web sear **Only \`executeCommand\` (shell/bash commands) goes through the approval flow.** If you need to delete a file, use the \`workspace-remove\` builtin tool, not \`executeCommand\` with \`rm\`. If you need to create a file, use \`workspace-writeFile\`, not \`executeCommand\` with \`touch\` or \`echo >\`. -Rowboat's internal builtin tools never require approval — only shell commands via \`executeCommand\` do.`; +Rowboat's internal builtin tools never require approval — only shell commands via \`executeCommand\` do. + +## File Path References + +When you reference a file path in your response (whether a knowledge base file or a file on the user's system), ALWAYS wrap it in a filepath code block: + +\`\`\`filepath +knowledge/People/Sarah Chen.md +\`\`\` + +\`\`\`filepath +~/Desktop/report.pdf +\`\`\` + +This renders as an interactive card in the UI. Use this format for: +- Knowledge base file paths (knowledge/...) +- Files on the user's machine (~/Desktop/..., /Users/..., etc.) +- Audio files, images, documents, or any file reference + +Never output raw file paths in plain text when they could be wrapped in a filepath block.`; diff --git a/apps/x/packages/shared/src/ipc.ts b/apps/x/packages/shared/src/ipc.ts index 767de9a0..632f34d9 100644 --- a/apps/x/packages/shared/src/ipc.ts +++ b/apps/x/packages/shared/src/ipc.ts @@ -381,6 +381,15 @@ const ipcSchemas = { success: z.literal(true), }), }, + // Shell integration channels + 'shell:openPath': { + req: z.object({ path: z.string() }), + res: z.object({ error: z.string().optional() }), + }, + 'shell:readFileBase64': { + req: z.object({ path: z.string() }), + res: z.object({ data: z.string(), mimeType: z.string(), size: z.number() }), + }, } as const; // ============================================================================