diff --git a/apps/x/VIDEO_MODE.md b/apps/x/VIDEO_MODE.md index 091ef415..63b9621e 100644 --- a/apps/x/VIDEO_MODE.md +++ b/apps/x/VIDEO_MODE.md @@ -132,6 +132,11 @@ Push-to-talk is disabled while a call owns the mic. ## Popout window +- The popout window keeps the Dock icon alive: it uses + `setVisibleOnAllWorkspaces(true)` WITHOUT `visibleOnFullScreen` — that flag + turns the app into a macOS "agent" app and hides its Dock icon while the + window exists (looks like Rowboat vanished). Trade-off: the popout doesn't + hover over other apps' fullscreen Spaces. - Shown iff the derived `callSurface === 'popout'` (effect in `App.tsx`). Renderer asks `video:setPopout {show}`; main creates a frameless, `alwaysOnTop` ('floating'), all-workspaces BrowserWindow at the top-right @@ -163,6 +168,7 @@ Push-to-talk is disabled while a call owns the mic. |--------|-------| | `# Video Mode (Live Camera)` system section — how to use webcam frames, coaching guidance, screen-share rules ("treat the screen as the primary subject", "last screen frame is current"), etiquette (never comment on appearance) | `packages/core/src/agents/runtime.ts:386` (`composeSystemInstructions`, gated on `videoMode`) | | `# Practice Session (Coach Mode)` system section — coaching persona: specific/actionable feedback after each take, one-sentence interjections mid-flow, structured debrief on wrap-up | `composeSystemInstructions`, gated on `coachMode` (directly after the video section) | +| "Driving the app" paragraph in the video-mode section — on calls, prefer app-navigation read-view/open-item (show while telling) over describing or squinting at frames | same `# Video Mode` section; full action docs in the `app-navigation` skill (`application/assistant/skills/app-navigation/skill.ts`) | | Per-message frame context line `[Video mode: N live webcam frames … and M frames of the user's shared screen …]` + group labels | `packages/core/src/agents/runtime.ts` (`convertFromMessages`) | | `videoMode` / `coachMode` composition overrides (session-sticky; flips bust prefix cache) | `packages/core/src/turns/bridges/real-agent-resolver.ts` (`CompositionOverrides`); set from `App.tsx` `sendConfig` | @@ -170,6 +176,28 @@ Voice input/output prompt sections (`# Voice Input`, `# Voice Output`) are reused untouched — calls set `voiceInput` per utterance and force `voiceOutput: 'full'`. +## Driving the app on a call + +The assistant can drive the Rowboat UI itself via the extended +`app-navigation` builtin ("app driver"): `open-view` (any main view), +`read-view` (returns the emails / background agents / chat-history data the +view renders — and the renderer simultaneously navigates there so the user +watches it happen), and `open-item` (a specific email thread, note, +background agent, or past chat, deep-linked on screen). Data comes from the +same core functions the UI's IPC handlers use (`listImportantThreads` / +`searchThreads`, background-task `listTasks`, the sessions container) — no +OCR of screen frames. The renderer applies results via +`applyAppNavigation` in App.tsx, fed from BOTH event paths: the legacy +`runs:events` ref-poll AND a watcher over the session-chat conversation (the +turn runtime does not emit legacy run events — miss this and navigation +silently no-ops while the tool reports success). Session switches seed the +watcher so replaying history never navigates. During a call, visible +navigations also collapse the full-screen call to the pill and focus the app +window (`app:focusMainWindow`) so the user actually sees the screen change. +Card labels live in `lib/chat-conversation.ts`. The call prompt and the +`app-navigation` skill teach the show-while-telling pattern: read-view → +speak the highlights → open-item when the user picks one. + ## Latency Voice-to-voice latency (user stops talking → assistant audio) is engineered diff --git a/apps/x/apps/main/src/ipc.ts b/apps/x/apps/main/src/ipc.ts index dde72aa1..6344b18f 100644 --- a/apps/x/apps/main/src/ipc.ts +++ b/apps/x/apps/main/src/ipc.ts @@ -1835,10 +1835,13 @@ export function setupIpcHandlers() { preload: preloadPath, }, }); - // Float above other apps (and fullscreen spaces on macOS) — the whole - // point is being visible while the user works elsewhere. + // Float above other apps on every workspace. Deliberately NOT + // `visibleOnFullScreen: true`: on macOS that flag hides the app's Dock + // icon for as long as such a window exists (the app becomes an + // "agent" app), which reads as Rowboat having vanished. The trade-off + // is the popout won't hover over other apps' fullscreen Spaces. win.setAlwaysOnTop(true, 'floating'); - win.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }); + win.setVisibleOnAllWorkspaces(true); win.webContents.once('did-finish-load', () => { if (lastVideoPopoutState) { win.webContents.send('video:popout-state', lastVideoPopoutState); @@ -1865,6 +1868,15 @@ export function setupIpcHandlers() { } return {}; }, + 'app:focusMainWindow': async () => { + const main = findMainAppWindow(); + if (main) { + if (main.isMinimized()) main.restore(); + main.show(); + main.focus(); + } + return {}; + }, 'video:getPopoutState': async () => { return { state: lastVideoPopoutState }; }, diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index f0017fb3..cca66c0f 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -97,6 +97,7 @@ import { type ChatViewportAnchorState, type ChatTabViewState, type ConversationItem, + type ToolCall, createEmptyChatTabViewState, getWebSearchCardData, getAppActionCardData, @@ -4569,22 +4570,61 @@ function App() { // External search set by app-navigation tool (passed to BasesView) const [externalBaseSearch, setExternalBaseSearch] = useState(undefined) - // Process pending app-navigation results - useEffect(() => { - const result = pendingAppNavRef.current - if (!result) return - pendingAppNavRef.current = null + // Apply an app-navigation tool result to the UI. Shared by both event + // paths (legacy runs:events and the session-chat turn runtime). + const applyAppNavigation = useCallback((result: Record) => { + // During a call, navigation must be VISIBLE: the full-screen call view + // would cover the very thing being shown — collapse it to the pill — + // and if the user is in another app, bring Rowboat forward. + const visibleActions = ['open-note', 'open-view', 'read-view', 'open-item', 'update-base-view', 'create-base'] + if (inCallRef.current && visibleActions.includes(result.action as string)) { + setCallMinimized(true) + void window.ipc.invoke('app:focusMainWindow', null).catch(() => {}) + } + + // Views the assistant can open (or auto-open while reading them via + // read-view — the user should SEE what's being read). + const navigateToNamedView = (view: string) => { + switch (view) { + case 'graph': void navigateToView({ type: 'graph' }); break + case 'bases': void navigateToView({ type: 'file', path: BASES_DEFAULT_TAB_PATH }); break + case 'home': void navigateToView({ type: 'home' }); break + case 'email': void navigateToView({ type: 'email' }); break + case 'meetings': void navigateToView({ type: 'meetings' }); break + case 'live-notes': void navigateToView({ type: 'live-notes' }); break + case 'bg-tasks': void navigateToView({ type: 'bg-tasks' }); break + case 'chat-history': void navigateToView({ type: 'chat-history' }); break + case 'knowledge': void navigateToView({ type: 'knowledge-view' }); break + case 'workspace': void navigateToView({ type: 'workspace' }); break + case 'code': void navigateToView({ type: 'code' }); break + } + } switch (result.action) { case 'open-note': navigateToFile(result.path as string) break case 'open-view': - if (result.view === 'graph') void navigateToView({ type: 'graph' }) - if (result.view === 'bases') { - void navigateToView({ type: 'file', path: BASES_DEFAULT_TAB_PATH }) + case 'read-view': + navigateToNamedView(result.view as string) + break + case 'open-item': { + switch (result.kind) { + case 'email-thread': + void navigateToView({ type: 'email', threadId: result.threadId as string }) + break + case 'note': + navigateToFile(result.path as string) + break + case 'bg-task': + void navigateToView({ type: 'task', name: result.taskName as string }) + break + case 'session': + void navigateToView({ type: 'chat', runId: result.sessionId as string }) + break } break + } case 'update-base-view': { // Navigate to bases if not already there const targetPath = selectedPath && isBaseFilePath(selectedPath) ? selectedPath : BASES_DEFAULT_TAB_PATH @@ -4664,8 +4704,41 @@ function App() { } break } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [navigateToFile, navigateToView, selectedPath]) + + // Legacy runs:events path: handleRunEvent stashes the result in a ref; + // polled every render (the triggering event always causes one). + useEffect(() => { + const result = pendingAppNavRef.current + if (!result) return + pendingAppNavRef.current = null + applyAppNavigation(result) }) + // Turn-runtime path: the session-chat store surfaces tool results in the + // conversation; apply newly completed app-navigation calls exactly once. + // On session switch/load, everything already in the transcript happened in + // the past — seed as processed without replaying navigations. + const processedAppNavRef = useRef<{ key: string | null; ids: Set }>({ key: null, ids: new Set() }) + useEffect(() => { + const conversation = sessionChat.chatState?.conversation + if (!conversation) return + const completed = conversation.filter( + (item): item is ToolCall => isToolCall(item) && item.name === 'app-navigation' && item.status === 'completed' + ) + if (processedAppNavRef.current.key !== runId) { + processedAppNavRef.current = { key: runId, ids: new Set(completed.map((t) => t.id)) } + return + } + for (const tool of completed) { + if (processedAppNavRef.current.ids.has(tool.id)) continue + processedAppNavRef.current.ids.add(tool.id) + const result = tool.result as Record | undefined + if (result && result.success) applyAppNavigation(result) + } + }, [sessionChat.chatState?.conversation, runId, applyAppNavigation]) + const navigateToFullScreenChat = useCallback(() => { // Only treat this as navigation when coming from another view if (currentViewState.type !== 'chat') { diff --git a/apps/x/apps/renderer/src/lib/chat-conversation.ts b/apps/x/apps/renderer/src/lib/chat-conversation.ts index d39f7863..f9a00c74 100644 --- a/apps/x/apps/renderer/src/lib/chat-conversation.ts +++ b/apps/x/apps/renderer/src/lib/chat-conversation.ts @@ -201,6 +201,22 @@ const summarizeFilterUpdates = (updates: Record): string => { return parts.length > 0 ? parts.join(', ') : 'Updated view' } +const APP_VIEW_LABELS: Record = { + home: 'home', + email: 'email', + meetings: 'meetings', + 'live-notes': 'live notes', + 'bg-tasks': 'background agents', + 'chat-history': 'chat history', + knowledge: 'knowledge', + workspace: 'workspace', + code: 'code', + bases: 'bases', + graph: 'graph', +} + +const appViewLabel = (view: unknown): string => APP_VIEW_LABELS[view as string] ?? String(view ?? 'view') + export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null => { if (tool.name !== 'app-navigation') return null const result = tool.result as Record | undefined @@ -212,7 +228,9 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null = const action = input.action as string switch (action) { case 'open-note': return { action, label: `Opening ${(input.path as string || '').split('/').pop()?.replace(/\.md$/, '') || 'note'}...` } - case 'open-view': return { action, label: `Opening ${input.view} view...` } + case 'open-view': return { action, label: `Opening ${appViewLabel(input.view)}...` } + case 'read-view': return { action, label: `Reading ${appViewLabel(input.view)}...` } + case 'open-item': return { action, label: 'Opening...' } case 'update-base-view': return { action, label: 'Updating view...' } case 'create-base': return { action, label: `Creating "${input.name}"...` } case 'get-base-state': return null // renders as normal tool block @@ -227,7 +245,31 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null = return { action: 'open-note', label: `Opened ${name}` } } case 'open-view': - return { action: 'open-view', label: `Opened ${result.view} view` } + return { action: 'open-view', label: `Opened ${appViewLabel(result.view)}` } + case 'read-view': { + const counted = + (result.threads as unknown[] | undefined)?.length ?? + (result.agents as unknown[] | undefined)?.length ?? + (result.sessions as unknown[] | undefined)?.length + return { + action: 'read-view', + label: counted !== undefined + ? `Read ${appViewLabel(result.view)} (${counted} item${counted === 1 ? '' : 's'})` + : `Read ${appViewLabel(result.view)}`, + } + } + case 'open-item': { + switch (result.kind) { + case 'email-thread': return { action: 'open-item', label: 'Opened email thread' } + case 'note': { + const name = (result.path as string || '').split('/').pop()?.replace(/\.md$/, '') || 'note' + return { action: 'open-item', label: `Opened ${name}` } + } + case 'bg-task': return { action: 'open-item', label: `Opened agent "${result.taskName}"` } + case 'session': return { action: 'open-item', label: 'Opened chat' } + default: return { action: 'open-item', label: 'Opened item' } + } + } case 'update-base-view': return { action: 'update-base-view', diff --git a/apps/x/packages/core/src/agents/runtime.ts b/apps/x/packages/core/src/agents/runtime.ts index d542c09c..5333df6b 100644 --- a/apps/x/packages/core/src/agents/runtime.ts +++ b/apps/x/packages/core/src/agents/runtime.ts @@ -394,6 +394,10 @@ How to use the frames: - When the user practices something performative (a pitch, presentation, interview, talk) or asks for delivery feedback, give specific, actionable coaching grounded in what you actually see — cite concrete observations ("in the last few frames you looked down and away from the camera") rather than generic advice. Cover posture, eye contact, facial expressiveness, gesturing, and visible energy. - If they show something to the camera (an object, document, whiteboard), read or describe it and respond accordingly. +Driving the app: +- You can control the Rowboat app the user is looking at via the app-navigation tool (load the app-navigation skill first): open views, READ a view's contents as data (emails, background agents, chat history), and open specific items on their screen. +- When the user asks about anything that lives inside Rowboat ("what emails do I have?", "what agents are running?", "open the one from Arjun"), prefer driving over describing: read-view shows the view on their screen while returning its data, then answer out loud briefly; open-item when they pick one. Narrate as you act ("pulling up your inbox…"). Reading a view's data beats squinting at screen-share frames — it's exact. + Screen sharing: - The user may also share their screen. Screen-share frames arrive in a separately labeled group after the webcam frames; they show the user's screen, not the user. - When screen frames are present, treat them as the primary subject: the user is usually asking about, or working on, what's visible there. Read the screen carefully — code, documents, error messages, UI state — and help with it concretely. diff --git a/apps/x/packages/core/src/application/assistant/instructions.ts b/apps/x/packages/core/src/application/assistant/instructions.ts index ed3576b0..c5cf9830 100644 --- a/apps/x/packages/core/src/application/assistant/instructions.ts +++ b/apps/x/packages/core/src/application/assistant/instructions.ts @@ -114,7 +114,7 @@ ${codeModeEnabled ? `**Code with Agents:** When users ask you to write code, build a project, create a script, fix a bug, or do any software development task — **including simple things like "create a .c file" or "write a hello-world in Python"** — your FIRST action MUST be \`loadSkill('code-with-agents')\`. Do NOT reach for \`executeCommand\` (PowerShell / bash / shell) or any workspace file tool to do code work yourself before loading this skill. The skill decides whether to delegate to Claude Code / Codex (via acpx) or hand control back to you, and it presents the user a one-click choice when needed. Paths outside the Rowboat workspace root (e.g. \`G:/...\`, \`~/projects/...\`) are NORMAL for coding tasks — do NOT raise "outside workspace" concerns or fall back to your own tools.` : `**Code with Agents (disabled):** Code mode is currently OFF in the user's settings. Do NOT load \`code-with-agents\` and do NOT call acpx. Handle coding requests yourself with your normal tools if you can. After answering, add a final line letting the user know they can delegate coding to Claude Code or Codex by enabling Code Mode in Settings → Code Mode.`} -**App Control:** When users ask you to open notes, show the bases or graph view, filter or search notes, or manage saved views, load the \`app-navigation\` skill first. It provides structured guidance for navigating the app UI and controlling the knowledge base view. +**App Control (drive the app):** You can drive the Rowboat UI the user is looking at — open any view (email, meetings, background agents, chat history, knowledge, workspace, code, bases, graph), READ what a view contains (\`read-view\` returns emails / background agents / past chats as data while showing the view on screen), and open specific items (an email thread, a note, an agent, a past chat). When users ask to open, show, find, or ask about anything that lives inside Rowboat, load the \`app-navigation\` skill first — it documents the show-while-telling pattern. This matters most on calls: navigate so the user sees what you see, then answer briefly. **Background Tasks (Self-Running Work):** Rowboat can run *background tasks* — persistent instructions the agent fires on a schedule and/or in response to incoming emails / calendar events. A bg-task either maintains a snapshot in its \`index.md\` (digest, dashboard, rolling summary) or performs a recurring side-effect (send a Slack message, draft an email, post to a webhook, call an API). This is the flagship surface for *anything recurring*. @@ -293,7 +293,7 @@ ${runtimeContextPrompt} - \`addMcpServer\`, \`listMcpServers\`, \`listMcpTools\`, \`executeMcpTool\` - MCP server management and execution - \`loadSkill\` - Skill loading ${slackToolsLine}- \`web-search\` - Search the web. Returns rich results with full text, highlights, and metadata. The \`category\` parameter defaults to \`general\` (full web search) — only use a specific category like \`news\`, \`company\`, \`research paper\` etc. when the query is clearly about that type. For everyday queries (weather, restaurants, prices, how-to), use \`general\`. -- \`app-navigation\` - Control the app UI: open notes, switch views, filter/search the knowledge base, manage saved views. **Load the \`app-navigation\` skill before using this tool.** +- \`app-navigation\` - Drive the app UI: open any view, read a view's contents (emails / background agents / chat history), open specific items (email thread, note, agent, past chat), filter/search the knowledge base, manage saved views. **Load the \`app-navigation\` skill before using this tool.** - \`browser-control\` - Control the embedded browser pane: open sites, inspect the live page, switch tabs, and interact with indexed page elements. **Load the \`browser-control\` skill before using this tool.** - \`save-to-memory\` - Save observations about the user to the agent memory system. Use this proactively during conversations. ${composioToolsLine} diff --git a/apps/x/packages/core/src/application/assistant/skills/app-navigation/skill.ts b/apps/x/packages/core/src/application/assistant/skills/app-navigation/skill.ts index 98956d30..3f19f0e3 100644 --- a/apps/x/packages/core/src/application/assistant/skills/app-navigation/skill.ts +++ b/apps/x/packages/core/src/application/assistant/skills/app-navigation/skill.ts @@ -1,82 +1,91 @@ export const skill = String.raw` -# App Navigation Skill +# App Driving Skill -You have access to the **app-navigation** tool which lets you control the Rowboat UI directly — opening notes, switching views, filtering the knowledge base, and creating saved views. +You have the **app-navigation** tool: you can DRIVE the Rowboat app the user +is looking at — open any view, read what a view contains, open specific items +(an email thread, a note, a background agent, a past chat), filter the +knowledge base, and manage saved views. Navigation happens on the USER'S +screen: when you open something, they watch it open. + +## The core pattern: show while telling + +When the user asks about something that lives inside Rowboat ("what emails do +I have?", "what background agents are running?", "open the note about Acme"), +don't answer blind. Drive: + +1. **read-view** the relevant view — this returns the actual data AND + navigates the user's screen to that view at the same time. +2. Answer from the returned data, concisely. +3. If they ask about one item ("open the one from Arjun"), **open-item** it — + it appears on their screen — and summarize what's in it if useful. + +This matters most during a call: the user is talking to you hands-free and +watching the screen. Navigate so they see what you see, and keep spoken +answers short. ## Actions +### read-view — read a view's contents (and show it) +Returns the same data the view renders; the app simultaneously navigates to +that view so the user sees it. + +- ` + "`view: \"email\"`" + ` → latest important inbox threads: ` + "`{ threadId, subject, from, date, unread, summary }`" + `. + Pass ` + "`query`" + ` to search instead (sender name, subject words — e.g. ` + "`query: \"from Arjun\"`" + ` or just ` + "`\"Arjun\"`" + `). +- ` + "`view: \"bg-tasks\"`" + ` → background agents: ` + "`{ name, slug, active, triggers, lastRunAt, lastRunSummary, lastRunError }`" + `. +- ` + "`view: \"chat-history\"`" + ` → past chats: ` + "`{ sessionId, title, updatedAt, turnCount }`" + `. +- ` + "`limit`" + ` (optional, default 15). + +For notes, meetings, and live notes use the ` + "`file-*`" + ` tools (they are +markdown files in the workspace) and then open-note / open-item to show them. + +### open-item — open one specific thing on screen +- ` + "`kind: \"email-thread\"`" + ` + ` + "`threadId`" + ` (from read-view email) +- ` + "`kind: \"note\"`" + ` + ` + "`path`" + ` +- ` + "`kind: \"bg-task\"`" + ` + ` + "`taskName`" + ` (from read-view bg-tasks; validated against real tasks) +- ` + "`kind: \"session\"`" + ` + ` + "`sessionId`" + ` (from read-view chat-history) + +### open-view — just switch the screen +` + "`view`" + `: ` + "`home | email | meetings | live-notes | bg-tasks | chat-history | knowledge | workspace | code | bases | graph`" + ` +Use when the user asks to "go to"/"show" a view without a question to answer. + ### open-note -Open a specific knowledge file in the editor pane. +Open a knowledge file in the editor. ` + "`path`" + `: full workspace-relative path +(e.g. ` + "`knowledge/People/John Smith.md`" + `). Use ` + "`file-grep`" + ` first if unsure +of the exact path. -**When to use:** When the user asks to see, open, or view a specific note (e.g., "open John's note", "show me the Acme project page"). +### update-base-view / get-base-state / create-base +Knowledge-base table control (unchanged): +- ` + "`update-base-view`" + `: ` + "`filters`" + ` (` + "`set/add/remove/clear`" + ` of ` + "`{category, value}`" + `), + ` + "`sort`" + ` (` + "`{field, dir}`" + `), ` + "`search`" + `. **Never pass ` + "`columns`" + ` unless the user + explicitly asks to change columns** — it overrides their layout. +- ` + "`get-base-state`" + `: available filter categories/values and note count. +- ` + "`create-base`" + `: save the current view configuration under ` + "`name`" + `. -**Parameters:** -- ` + "`path`" + `: Full workspace-relative path (e.g., ` + "`knowledge/People/John Smith.md`" + `) +## Worked examples -**Tips:** -- Use ` + "`file-grep`" + ` first to find the exact path if you're unsure of the filename. -- Always pass the full ` + "`knowledge/...`" + ` path, not just the filename. +**"What emails do I have?"** (on a call) +1. ` + "`app-navigation({ action: \"read-view\", view: \"email\" })`" + ` — email view opens on their screen. +2. Speak the highlights: "You've got six new ones — the ones that matter are from Arjun about the deck and from Stripe about billing." -### open-view -Switch the UI to the graph or bases view. +**"Open the one from Arjun."** +1. Find Arjun's thread in the data you already have (or ` + "`read-view`" + ` with ` + "`query: \"Arjun\"`" + `). +2. ` + "`app-navigation({ action: \"open-item\", kind: \"email-thread\", threadId: \"...\" })`" + ` +3. "It's open — he's asking whether Thursday works for the pitch review." -**When to use:** When the user asks to see the knowledge graph, view all notes, or open the bases/table view. +**"What background agents do I have?"** +1. ` + "`app-navigation({ action: \"read-view\", view: \"bg-tasks\" })`" + ` +2. "Three: the inbox summarizer ran an hour ago, the meeting-prep agent is active, and the Linear digest failed its last run — want me to open that one?" -**Parameters:** -- ` + "`view`" + `: ` + "`\"graph\"`" + ` or ` + "`\"bases\"`" + ` +**"Show me all active customers"** +1. ` + "`get-base-state`" + ` to see available categories, then +2. ` + "`update-base-view`" + ` with ` + "`filters.set: [{ category: \"relationship\", value: \"customer\" }]`" + ` -### update-base-view -Change filters, columns, sort order, or search in the bases (table) view. - -**When to use:** When the user asks to find, filter, sort, or search notes. Examples: "show me all active customers", "filter by topic=hiring", "sort by name", "search for pricing". - -**Parameters:** -- ` + "`filters`" + `: Object with ` + "`set`" + `, ` + "`add`" + `, ` + "`remove`" + `, or ` + "`clear`" + ` — each takes an array of ` + "`{ category, value }`" + ` pairs. - - ` + "`set`" + `: Replace ALL current filters with these. - - ` + "`add`" + `: Append filters without removing existing ones. - - ` + "`remove`" + `: Remove specific filters. - - ` + "`clear: true`" + `: Remove all filters. -- ` + "`columns`" + `: Object with ` + "`set`" + `, ` + "`add`" + `, or ` + "`remove`" + ` — each takes an array of column names (frontmatter keys). -- ` + "`sort`" + `: ` + "`{ field, dir }`" + ` where dir is ` + "`\"asc\"`" + ` or ` + "`\"desc\"`" + `. -- ` + "`search`" + `: Free-text search string. - -**Tips:** -- If unsure what categories/values are available, call ` + "`get-base-state`" + ` first. -- For "show me X", prefer ` + "`filters.set`" + ` to start fresh rather than ` + "`filters.add`" + `. -- Categories come from frontmatter keys (e.g., relationship, status, topic, type). -- **CRITICAL: Do NOT pass ` + "`columns`" + ` unless the user explicitly asks to show/hide specific columns.** Omit the ` + "`columns`" + ` parameter entirely when only filtering, sorting, or searching. Passing ` + "`columns`" + ` will override the user's current column layout and can make the view appear empty. - -### get-base-state -Retrieve information about what's in the knowledge base — available filter categories, values, and note count. - -**When to use:** When you need to know what properties exist before filtering, or when the user asks "what can I filter by?", "how many notes are there?", etc. - -**Parameters:** -- ` + "`base_name`" + ` (optional): Name of a saved base to inspect. - -### create-base -Save the current view configuration as a named base. - -**When to use:** When the user asks to save a filtered view, create a saved search, or says "save this as [name]". - -**Parameters:** -- ` + "`name`" + `: Human-readable name for the base. - -## Workflow Example - -1. User: "Show me all people who are customers" -2. First, check what properties are available: - ` + "`app-navigation({ action: \"get-base-state\" })`" + ` -3. Apply filters based on the available properties: - ` + "`app-navigation({ action: \"update-base-view\", filters: { set: [{ category: \"relationship\", value: \"customer\" }] } })`" + ` -4. If the user wants to save it: - ` + "`app-navigation({ action: \"create-base\", name: \"Customers\" })`" + ` - -## Important Notes -- The ` + "`update-base-view`" + ` action will automatically navigate to the bases view if the user isn't already there. -- ` + "`open-note`" + ` validates that the file exists before navigating. -- Filter categories and values come from frontmatter in knowledge files. -- **Never send ` + "`columns`" + ` or ` + "`sort`" + ` with ` + "`update-base-view`" + ` unless the user specifically asks to change them.** Only pass the parameters you intend to change — omitted parameters are left untouched. +## Notes +- read-view/open-view/open-item change what the user is looking at — that is + the point, but don't bounce their screen around needlessly; navigate when + it serves the question. +- open-note and open-item validate the target exists before navigating. +- update-base-view auto-navigates to the bases view. `; export default skill; diff --git a/apps/x/packages/core/src/application/lib/builtin-tools.ts b/apps/x/packages/core/src/application/lib/builtin-tools.ts index 3d80f0d2..49145e70 100644 --- a/apps/x/packages/core/src/application/lib/builtin-tools.ts +++ b/apps/x/packages/core/src/application/lib/builtin-tools.ts @@ -23,6 +23,9 @@ import { ICodeModeConfigRepo } from "../../code-mode/repo.js"; import type { ApprovalPolicy } from "@x/shared/dist/code-mode.js"; import type { ICodeProjectsRepo } from "../../code-mode/projects/repo.js"; import * as gitService from "../../code-mode/git/service.js"; +import { listImportantThreads, searchThreads } from "../../knowledge/sync_gmail.js"; +import { listTasks as listBackgroundTasks } from "../../background-tasks/fileops.js"; +import type { ISessions } from "../../sessions/api.js"; // Inputs for the bg-task builtin tools. Reuse the canonical schema field // descriptions; only `triggers` gets a tighter contextual override (the @@ -1065,13 +1068,21 @@ export const BuiltinTools: z.infer = { // ============================================================================ 'app-navigation': { - description: 'Control the app UI - navigate to notes, switch views, filter/search the knowledge base, and manage saved views.', + description: 'Drive the Rowboat app UI: navigate to any view, read what a view contains (emails, background agents, chat history), open specific items (an email thread, a note, an agent, a past chat), filter/search the knowledge base, and manage saved views. Use it to SHOW the user things while telling them — navigation happens on their screen.', inputSchema: z.object({ - action: z.enum(["open-note", "open-view", "update-base-view", "get-base-state", "create-base"]).describe("The navigation action to perform"), + action: z.enum(["open-note", "open-view", "read-view", "open-item", "update-base-view", "get-base-state", "create-base"]).describe("The navigation action to perform"), // open-note path: z.string().optional().describe("Knowledge file path for open-note, e.g. knowledge/People/John.md"), - // open-view - view: z.enum(["bases", "graph"]).optional().describe("Which view to open (for open-view action)"), + // open-view / read-view + view: z.enum(["home", "email", "meetings", "live-notes", "bg-tasks", "chat-history", "knowledge", "workspace", "code", "bases", "graph"]).optional().describe("Which view to open (open-view) or read (read-view; supported for read: email, bg-tasks, chat-history)"), + // read-view (email) + query: z.string().optional().describe("For read-view on email: search query (sender name, subject words, etc.). Omit to list the latest important inbox threads."), + limit: z.number().int().min(1).max(50).optional().describe("For read-view: max items to return (default 15)"), + // open-item + kind: z.enum(["email-thread", "note", "bg-task", "session"]).optional().describe("What to open (for open-item)"), + threadId: z.string().optional().describe("Gmail thread id (open-item kind=email-thread; get it from read-view email)"), + taskName: z.string().optional().describe("Background task/agent name (open-item kind=bg-task; get it from read-view bg-tasks)"), + sessionId: z.string().optional().describe("Chat session id (open-item kind=session; get it from read-view chat-history)"), // update-base-view filters: z.object({ set: z.array(z.object({ category: z.string(), value: z.string() })).optional().describe("Replace all filters with these"), @@ -1117,6 +1128,110 @@ export const BuiltinTools: z.infer = { return { success: true, action: 'open-view', view }; } + case 'read-view': { + // Returns the same data the view renders, so the assistant + // can answer precisely — and the renderer navigates to the + // view at the same time so the user SEES what's being read. + const view = input.view as string; + const limit = (input.limit as number | undefined) ?? 15; + try { + switch (view) { + case 'email': { + const query = (input.query as string | undefined)?.trim(); + const result = query + ? await searchThreads(query, { limit }) + : listImportantThreads({ limit }); + const threads = (result.threads ?? []).slice(0, limit).map((t) => ({ + threadId: t.threadId, + subject: t.subject ?? '(no subject)', + from: t.from ?? '', + date: t.date ?? '', + unread: t.unread ?? false, + summary: t.summary ? t.summary.slice(0, 200) : undefined, + })); + return { success: true, action: 'read-view', view, query, threads }; + } + case 'bg-tasks': { + const { items } = await listBackgroundTasks({ limit }); + const agents = items.map((t) => ({ + name: t.name, + slug: t.slug, + active: t.active, + triggers: t.triggers, + lastRunAt: t.lastRunAt, + lastRunSummary: t.lastRunSummary ? t.lastRunSummary.slice(0, 200) : undefined, + lastRunError: t.lastRunError ? t.lastRunError.slice(0, 200) : undefined, + })); + return { success: true, action: 'read-view', view, agents }; + } + case 'chat-history': { + const sessions = container.resolve('sessions') + .listSessions() + .slice(0, limit) + .map((s) => ({ + sessionId: s.sessionId, + title: s.title ?? '(untitled)', + updatedAt: s.updatedAt, + turnCount: s.turnCount, + })); + return { success: true, action: 'read-view', view, sessions }; + } + default: + return { + success: false, + error: `read-view supports: email, bg-tasks, chat-history. For notes/meetings/live-notes use the file-* tools (they are files under the workspace); for other views use open-view and describe what you need.`, + }; + } + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : `Failed to read ${view}`, + }; + } + } + + case 'open-item': { + const kind = input.kind as string; + switch (kind) { + case 'email-thread': { + const threadId = input.threadId as string | undefined; + if (!threadId) return { success: false, error: 'threadId is required for kind=email-thread' }; + return { success: true, action: 'open-item', kind, threadId }; + } + case 'note': { + const filePath = input.path as string | undefined; + if (!filePath) return { success: false, error: 'path is required for kind=note' }; + const result = await files.exists(filePath); + if (!result.exists) return { success: false, error: `File not found: ${filePath}` }; + return { success: true, action: 'open-item', kind, path: filePath }; + } + case 'bg-task': { + const taskName = input.taskName as string | undefined; + if (!taskName) return { success: false, error: 'taskName is required for kind=bg-task' }; + // Validate (and canonicalize) against the real task list. + const { items: tasks } = await listBackgroundTasks({}); + const match = tasks.find( + (t) => t.name === taskName || t.slug === taskName + || t.name.toLowerCase() === taskName.toLowerCase(), + ); + if (!match) { + return { + success: false, + error: `No background task named "${taskName}". Known tasks: ${tasks.map((t) => t.name).join(', ') || '(none)'}`, + }; + } + return { success: true, action: 'open-item', kind, taskName: match.name }; + } + case 'session': { + const sessionId = input.sessionId as string | undefined; + if (!sessionId) return { success: false, error: 'sessionId is required for kind=session' }; + return { success: true, action: 'open-item', kind, sessionId }; + } + default: + return { success: false, error: `Unknown item kind: ${kind}` }; + } + } + case 'update-base-view': { const updates: Record = {}; if (input.filters) updates.filters = input.filters; diff --git a/apps/x/packages/shared/src/ipc.ts b/apps/x/packages/shared/src/ipc.ts index a6784839..fba8aec7 100644 --- a/apps/x/packages/shared/src/ipc.ts +++ b/apps/x/packages/shared/src/ipc.ts @@ -649,6 +649,12 @@ const ipcSchemas = { }), res: z.null(), }, + // Bring the main app window to the foreground (e.g. the assistant navigated + // the UI during a call while the user was in another app). + 'app:focusMainWindow': { + req: z.null(), + res: z.object({}), + }, 'app:takeMeetingNotes': { req: z.object({ // Pass the raw calendar event JSON through; renderer adapts to its existing flow.