mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-12 21:02:17 +02:00
add better app navigation on call
This commit is contained in:
Arjun
parent
5b2ecc6720
commit
82abfe9a04
9 changed files with 374 additions and 85 deletions
|
|
@ -132,6 +132,11 @@ Push-to-talk is disabled while a call owns the mic.
|
|||
|
||||
## Popout window
|
||||
|
||||
- The popout window keeps the Dock icon alive: it uses
|
||||
`setVisibleOnAllWorkspaces(true)` WITHOUT `visibleOnFullScreen` — that flag
|
||||
turns the app into a macOS "agent" app and hides its Dock icon while the
|
||||
window exists (looks like Rowboat vanished). Trade-off: the popout doesn't
|
||||
hover over other apps' fullscreen Spaces.
|
||||
- Shown iff the derived `callSurface === 'popout'` (effect in `App.tsx`).
|
||||
Renderer asks `video:setPopout {show}`; main creates a frameless,
|
||||
`alwaysOnTop` ('floating'), all-workspaces BrowserWindow at the top-right
|
||||
|
|
@ -163,6 +168,7 @@ Push-to-talk is disabled while a call owns the mic.
|
|||
|--------|-------|
|
||||
| `# Video Mode (Live Camera)` system section — how to use webcam frames, coaching guidance, screen-share rules ("treat the screen as the primary subject", "last screen frame is current"), etiquette (never comment on appearance) | `packages/core/src/agents/runtime.ts:386` (`composeSystemInstructions`, gated on `videoMode`) |
|
||||
| `# Practice Session (Coach Mode)` system section — coaching persona: specific/actionable feedback after each take, one-sentence interjections mid-flow, structured debrief on wrap-up | `composeSystemInstructions`, gated on `coachMode` (directly after the video section) |
|
||||
| "Driving the app" paragraph in the video-mode section — on calls, prefer app-navigation read-view/open-item (show while telling) over describing or squinting at frames | same `# Video Mode` section; full action docs in the `app-navigation` skill (`application/assistant/skills/app-navigation/skill.ts`) |
|
||||
| Per-message frame context line `[Video mode: N live webcam frames … and M frames of the user's shared screen …]` + group labels | `packages/core/src/agents/runtime.ts` (`convertFromMessages`) |
|
||||
| `videoMode` / `coachMode` composition overrides (session-sticky; flips bust prefix cache) | `packages/core/src/turns/bridges/real-agent-resolver.ts` (`CompositionOverrides`); set from `App.tsx` `sendConfig` |
|
||||
|
||||
|
|
@ -170,6 +176,28 @@ Voice input/output prompt sections (`# Voice Input`, `# Voice Output`) are
|
|||
reused untouched — calls set `voiceInput` per utterance and force
|
||||
`voiceOutput: 'full'`.
|
||||
|
||||
## Driving the app on a call
|
||||
|
||||
The assistant can drive the Rowboat UI itself via the extended
|
||||
`app-navigation` builtin ("app driver"): `open-view` (any main view),
|
||||
`read-view` (returns the emails / background agents / chat-history data the
|
||||
view renders — and the renderer simultaneously navigates there so the user
|
||||
watches it happen), and `open-item` (a specific email thread, note,
|
||||
background agent, or past chat, deep-linked on screen). Data comes from the
|
||||
same core functions the UI's IPC handlers use (`listImportantThreads` /
|
||||
`searchThreads`, background-task `listTasks`, the sessions container) — no
|
||||
OCR of screen frames. The renderer applies results via
|
||||
`applyAppNavigation` in App.tsx, fed from BOTH event paths: the legacy
|
||||
`runs:events` ref-poll AND a watcher over the session-chat conversation (the
|
||||
turn runtime does not emit legacy run events — miss this and navigation
|
||||
silently no-ops while the tool reports success). Session switches seed the
|
||||
watcher so replaying history never navigates. During a call, visible
|
||||
navigations also collapse the full-screen call to the pill and focus the app
|
||||
window (`app:focusMainWindow`) so the user actually sees the screen change.
|
||||
Card labels live in `lib/chat-conversation.ts`. The call prompt and the
|
||||
`app-navigation` skill teach the show-while-telling pattern: read-view →
|
||||
speak the highlights → open-item when the user picks one.
|
||||
|
||||
## Latency
|
||||
|
||||
Voice-to-voice latency (user stops talking → assistant audio) is engineered
|
||||
|
|
|
|||
|
|
@ -1835,10 +1835,13 @@ export function setupIpcHandlers() {
|
|||
preload: preloadPath,
|
||||
},
|
||||
});
|
||||
// Float above other apps (and fullscreen spaces on macOS) — the whole
|
||||
// point is being visible while the user works elsewhere.
|
||||
// Float above other apps on every workspace. Deliberately NOT
|
||||
// `visibleOnFullScreen: true`: on macOS that flag hides the app's Dock
|
||||
// icon for as long as such a window exists (the app becomes an
|
||||
// "agent" app), which reads as Rowboat having vanished. The trade-off
|
||||
// is the popout won't hover over other apps' fullscreen Spaces.
|
||||
win.setAlwaysOnTop(true, 'floating');
|
||||
win.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
|
||||
win.setVisibleOnAllWorkspaces(true);
|
||||
win.webContents.once('did-finish-load', () => {
|
||||
if (lastVideoPopoutState) {
|
||||
win.webContents.send('video:popout-state', lastVideoPopoutState);
|
||||
|
|
@ -1865,6 +1868,15 @@ export function setupIpcHandlers() {
|
|||
}
|
||||
return {};
|
||||
},
|
||||
'app:focusMainWindow': async () => {
|
||||
const main = findMainAppWindow();
|
||||
if (main) {
|
||||
if (main.isMinimized()) main.restore();
|
||||
main.show();
|
||||
main.focus();
|
||||
}
|
||||
return {};
|
||||
},
|
||||
'video:getPopoutState': async () => {
|
||||
return { state: lastVideoPopoutState };
|
||||
},
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@ import {
|
|||
type ChatViewportAnchorState,
|
||||
type ChatTabViewState,
|
||||
type ConversationItem,
|
||||
type ToolCall,
|
||||
createEmptyChatTabViewState,
|
||||
getWebSearchCardData,
|
||||
getAppActionCardData,
|
||||
|
|
@ -4569,22 +4570,61 @@ function App() {
|
|||
// External search set by app-navigation tool (passed to BasesView)
|
||||
const [externalBaseSearch, setExternalBaseSearch] = useState<string | undefined>(undefined)
|
||||
|
||||
// Process pending app-navigation results
|
||||
useEffect(() => {
|
||||
const result = pendingAppNavRef.current
|
||||
if (!result) return
|
||||
pendingAppNavRef.current = null
|
||||
// Apply an app-navigation tool result to the UI. Shared by both event
|
||||
// paths (legacy runs:events and the session-chat turn runtime).
|
||||
const applyAppNavigation = useCallback((result: Record<string, unknown>) => {
|
||||
// During a call, navigation must be VISIBLE: the full-screen call view
|
||||
// would cover the very thing being shown — collapse it to the pill —
|
||||
// and if the user is in another app, bring Rowboat forward.
|
||||
const visibleActions = ['open-note', 'open-view', 'read-view', 'open-item', 'update-base-view', 'create-base']
|
||||
if (inCallRef.current && visibleActions.includes(result.action as string)) {
|
||||
setCallMinimized(true)
|
||||
void window.ipc.invoke('app:focusMainWindow', null).catch(() => {})
|
||||
}
|
||||
|
||||
// Views the assistant can open (or auto-open while reading them via
|
||||
// read-view — the user should SEE what's being read).
|
||||
const navigateToNamedView = (view: string) => {
|
||||
switch (view) {
|
||||
case 'graph': void navigateToView({ type: 'graph' }); break
|
||||
case 'bases': void navigateToView({ type: 'file', path: BASES_DEFAULT_TAB_PATH }); break
|
||||
case 'home': void navigateToView({ type: 'home' }); break
|
||||
case 'email': void navigateToView({ type: 'email' }); break
|
||||
case 'meetings': void navigateToView({ type: 'meetings' }); break
|
||||
case 'live-notes': void navigateToView({ type: 'live-notes' }); break
|
||||
case 'bg-tasks': void navigateToView({ type: 'bg-tasks' }); break
|
||||
case 'chat-history': void navigateToView({ type: 'chat-history' }); break
|
||||
case 'knowledge': void navigateToView({ type: 'knowledge-view' }); break
|
||||
case 'workspace': void navigateToView({ type: 'workspace' }); break
|
||||
case 'code': void navigateToView({ type: 'code' }); break
|
||||
}
|
||||
}
|
||||
|
||||
switch (result.action) {
|
||||
case 'open-note':
|
||||
navigateToFile(result.path as string)
|
||||
break
|
||||
case 'open-view':
|
||||
if (result.view === 'graph') void navigateToView({ type: 'graph' })
|
||||
if (result.view === 'bases') {
|
||||
void navigateToView({ type: 'file', path: BASES_DEFAULT_TAB_PATH })
|
||||
case 'read-view':
|
||||
navigateToNamedView(result.view as string)
|
||||
break
|
||||
case 'open-item': {
|
||||
switch (result.kind) {
|
||||
case 'email-thread':
|
||||
void navigateToView({ type: 'email', threadId: result.threadId as string })
|
||||
break
|
||||
case 'note':
|
||||
navigateToFile(result.path as string)
|
||||
break
|
||||
case 'bg-task':
|
||||
void navigateToView({ type: 'task', name: result.taskName as string })
|
||||
break
|
||||
case 'session':
|
||||
void navigateToView({ type: 'chat', runId: result.sessionId as string })
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
case 'update-base-view': {
|
||||
// Navigate to bases if not already there
|
||||
const targetPath = selectedPath && isBaseFilePath(selectedPath) ? selectedPath : BASES_DEFAULT_TAB_PATH
|
||||
|
|
@ -4664,8 +4704,41 @@ function App() {
|
|||
}
|
||||
break
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [navigateToFile, navigateToView, selectedPath])
|
||||
|
||||
// Legacy runs:events path: handleRunEvent stashes the result in a ref;
|
||||
// polled every render (the triggering event always causes one).
|
||||
useEffect(() => {
|
||||
const result = pendingAppNavRef.current
|
||||
if (!result) return
|
||||
pendingAppNavRef.current = null
|
||||
applyAppNavigation(result)
|
||||
})
|
||||
|
||||
// Turn-runtime path: the session-chat store surfaces tool results in the
|
||||
// conversation; apply newly completed app-navigation calls exactly once.
|
||||
// On session switch/load, everything already in the transcript happened in
|
||||
// the past — seed as processed without replaying navigations.
|
||||
const processedAppNavRef = useRef<{ key: string | null; ids: Set<string> }>({ key: null, ids: new Set() })
|
||||
useEffect(() => {
|
||||
const conversation = sessionChat.chatState?.conversation
|
||||
if (!conversation) return
|
||||
const completed = conversation.filter(
|
||||
(item): item is ToolCall => isToolCall(item) && item.name === 'app-navigation' && item.status === 'completed'
|
||||
)
|
||||
if (processedAppNavRef.current.key !== runId) {
|
||||
processedAppNavRef.current = { key: runId, ids: new Set(completed.map((t) => t.id)) }
|
||||
return
|
||||
}
|
||||
for (const tool of completed) {
|
||||
if (processedAppNavRef.current.ids.has(tool.id)) continue
|
||||
processedAppNavRef.current.ids.add(tool.id)
|
||||
const result = tool.result as Record<string, unknown> | undefined
|
||||
if (result && result.success) applyAppNavigation(result)
|
||||
}
|
||||
}, [sessionChat.chatState?.conversation, runId, applyAppNavigation])
|
||||
|
||||
const navigateToFullScreenChat = useCallback(() => {
|
||||
// Only treat this as navigation when coming from another view
|
||||
if (currentViewState.type !== 'chat') {
|
||||
|
|
|
|||
|
|
@ -201,6 +201,22 @@ const summarizeFilterUpdates = (updates: Record<string, unknown>): string => {
|
|||
return parts.length > 0 ? parts.join(', ') : 'Updated view'
|
||||
}
|
||||
|
||||
const APP_VIEW_LABELS: Record<string, string> = {
|
||||
home: 'home',
|
||||
email: 'email',
|
||||
meetings: 'meetings',
|
||||
'live-notes': 'live notes',
|
||||
'bg-tasks': 'background agents',
|
||||
'chat-history': 'chat history',
|
||||
knowledge: 'knowledge',
|
||||
workspace: 'workspace',
|
||||
code: 'code',
|
||||
bases: 'bases',
|
||||
graph: 'graph',
|
||||
}
|
||||
|
||||
const appViewLabel = (view: unknown): string => APP_VIEW_LABELS[view as string] ?? String(view ?? 'view')
|
||||
|
||||
export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null => {
|
||||
if (tool.name !== 'app-navigation') return null
|
||||
const result = tool.result as Record<string, unknown> | undefined
|
||||
|
|
@ -212,7 +228,9 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null =
|
|||
const action = input.action as string
|
||||
switch (action) {
|
||||
case 'open-note': return { action, label: `Opening ${(input.path as string || '').split('/').pop()?.replace(/\.md$/, '') || 'note'}...` }
|
||||
case 'open-view': return { action, label: `Opening ${input.view} view...` }
|
||||
case 'open-view': return { action, label: `Opening ${appViewLabel(input.view)}...` }
|
||||
case 'read-view': return { action, label: `Reading ${appViewLabel(input.view)}...` }
|
||||
case 'open-item': return { action, label: 'Opening...' }
|
||||
case 'update-base-view': return { action, label: 'Updating view...' }
|
||||
case 'create-base': return { action, label: `Creating "${input.name}"...` }
|
||||
case 'get-base-state': return null // renders as normal tool block
|
||||
|
|
@ -227,7 +245,31 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null =
|
|||
return { action: 'open-note', label: `Opened ${name}` }
|
||||
}
|
||||
case 'open-view':
|
||||
return { action: 'open-view', label: `Opened ${result.view} view` }
|
||||
return { action: 'open-view', label: `Opened ${appViewLabel(result.view)}` }
|
||||
case 'read-view': {
|
||||
const counted =
|
||||
(result.threads as unknown[] | undefined)?.length ??
|
||||
(result.agents as unknown[] | undefined)?.length ??
|
||||
(result.sessions as unknown[] | undefined)?.length
|
||||
return {
|
||||
action: 'read-view',
|
||||
label: counted !== undefined
|
||||
? `Read ${appViewLabel(result.view)} (${counted} item${counted === 1 ? '' : 's'})`
|
||||
: `Read ${appViewLabel(result.view)}`,
|
||||
}
|
||||
}
|
||||
case 'open-item': {
|
||||
switch (result.kind) {
|
||||
case 'email-thread': return { action: 'open-item', label: 'Opened email thread' }
|
||||
case 'note': {
|
||||
const name = (result.path as string || '').split('/').pop()?.replace(/\.md$/, '') || 'note'
|
||||
return { action: 'open-item', label: `Opened ${name}` }
|
||||
}
|
||||
case 'bg-task': return { action: 'open-item', label: `Opened agent "${result.taskName}"` }
|
||||
case 'session': return { action: 'open-item', label: 'Opened chat' }
|
||||
default: return { action: 'open-item', label: 'Opened item' }
|
||||
}
|
||||
}
|
||||
case 'update-base-view':
|
||||
return {
|
||||
action: 'update-base-view',
|
||||
|
|
|
|||
|
|
@ -394,6 +394,10 @@ How to use the frames:
|
|||
- When the user practices something performative (a pitch, presentation, interview, talk) or asks for delivery feedback, give specific, actionable coaching grounded in what you actually see — cite concrete observations ("in the last few frames you looked down and away from the camera") rather than generic advice. Cover posture, eye contact, facial expressiveness, gesturing, and visible energy.
|
||||
- If they show something to the camera (an object, document, whiteboard), read or describe it and respond accordingly.
|
||||
|
||||
Driving the app:
|
||||
- You can control the Rowboat app the user is looking at via the app-navigation tool (load the app-navigation skill first): open views, READ a view's contents as data (emails, background agents, chat history), and open specific items on their screen.
|
||||
- When the user asks about anything that lives inside Rowboat ("what emails do I have?", "what agents are running?", "open the one from Arjun"), prefer driving over describing: read-view shows the view on their screen while returning its data, then answer out loud briefly; open-item when they pick one. Narrate as you act ("pulling up your inbox…"). Reading a view's data beats squinting at screen-share frames — it's exact.
|
||||
|
||||
Screen sharing:
|
||||
- The user may also share their screen. Screen-share frames arrive in a separately labeled group after the webcam frames; they show the user's screen, not the user.
|
||||
- When screen frames are present, treat them as the primary subject: the user is usually asking about, or working on, what's visible there. Read the screen carefully — code, documents, error messages, UI state — and help with it concretely.
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ ${codeModeEnabled
|
|||
? `**Code with Agents:** When users ask you to write code, build a project, create a script, fix a bug, or do any software development task — **including simple things like "create a .c file" or "write a hello-world in Python"** — your FIRST action MUST be \`loadSkill('code-with-agents')\`. Do NOT reach for \`executeCommand\` (PowerShell / bash / shell) or any workspace file tool to do code work yourself before loading this skill. The skill decides whether to delegate to Claude Code / Codex (via acpx) or hand control back to you, and it presents the user a one-click choice when needed. Paths outside the Rowboat workspace root (e.g. \`G:/...\`, \`~/projects/...\`) are NORMAL for coding tasks — do NOT raise "outside workspace" concerns or fall back to your own tools.`
|
||||
: `**Code with Agents (disabled):** Code mode is currently OFF in the user's settings. Do NOT load \`code-with-agents\` and do NOT call acpx. Handle coding requests yourself with your normal tools if you can. After answering, add a final line letting the user know they can delegate coding to Claude Code or Codex by enabling Code Mode in Settings → Code Mode.`}
|
||||
|
||||
**App Control:** When users ask you to open notes, show the bases or graph view, filter or search notes, or manage saved views, load the \`app-navigation\` skill first. It provides structured guidance for navigating the app UI and controlling the knowledge base view.
|
||||
**App Control (drive the app):** You can drive the Rowboat UI the user is looking at — open any view (email, meetings, background agents, chat history, knowledge, workspace, code, bases, graph), READ what a view contains (\`read-view\` returns emails / background agents / past chats as data while showing the view on screen), and open specific items (an email thread, a note, an agent, a past chat). When users ask to open, show, find, or ask about anything that lives inside Rowboat, load the \`app-navigation\` skill first — it documents the show-while-telling pattern. This matters most on calls: navigate so the user sees what you see, then answer briefly.
|
||||
|
||||
**Background Tasks (Self-Running Work):** Rowboat can run *background tasks* — persistent instructions the agent fires on a schedule and/or in response to incoming emails / calendar events. A bg-task either maintains a snapshot in its \`index.md\` (digest, dashboard, rolling summary) or performs a recurring side-effect (send a Slack message, draft an email, post to a webhook, call an API). This is the flagship surface for *anything recurring*.
|
||||
|
||||
|
|
@ -293,7 +293,7 @@ ${runtimeContextPrompt}
|
|||
- \`addMcpServer\`, \`listMcpServers\`, \`listMcpTools\`, \`executeMcpTool\` - MCP server management and execution
|
||||
- \`loadSkill\` - Skill loading
|
||||
${slackToolsLine}- \`web-search\` - Search the web. Returns rich results with full text, highlights, and metadata. The \`category\` parameter defaults to \`general\` (full web search) — only use a specific category like \`news\`, \`company\`, \`research paper\` etc. when the query is clearly about that type. For everyday queries (weather, restaurants, prices, how-to), use \`general\`.
|
||||
- \`app-navigation\` - Control the app UI: open notes, switch views, filter/search the knowledge base, manage saved views. **Load the \`app-navigation\` skill before using this tool.**
|
||||
- \`app-navigation\` - Drive the app UI: open any view, read a view's contents (emails / background agents / chat history), open specific items (email thread, note, agent, past chat), filter/search the knowledge base, manage saved views. **Load the \`app-navigation\` skill before using this tool.**
|
||||
- \`browser-control\` - Control the embedded browser pane: open sites, inspect the live page, switch tabs, and interact with indexed page elements. **Load the \`browser-control\` skill before using this tool.**
|
||||
- \`save-to-memory\` - Save observations about the user to the agent memory system. Use this proactively during conversations.
|
||||
${composioToolsLine}
|
||||
|
|
|
|||
|
|
@ -1,82 +1,91 @@
|
|||
export const skill = String.raw`
|
||||
# App Navigation Skill
|
||||
# App Driving Skill
|
||||
|
||||
You have access to the **app-navigation** tool which lets you control the Rowboat UI directly — opening notes, switching views, filtering the knowledge base, and creating saved views.
|
||||
You have the **app-navigation** tool: you can DRIVE the Rowboat app the user
|
||||
is looking at — open any view, read what a view contains, open specific items
|
||||
(an email thread, a note, a background agent, a past chat), filter the
|
||||
knowledge base, and manage saved views. Navigation happens on the USER'S
|
||||
screen: when you open something, they watch it open.
|
||||
|
||||
## The core pattern: show while telling
|
||||
|
||||
When the user asks about something that lives inside Rowboat ("what emails do
|
||||
I have?", "what background agents are running?", "open the note about Acme"),
|
||||
don't answer blind. Drive:
|
||||
|
||||
1. **read-view** the relevant view — this returns the actual data AND
|
||||
navigates the user's screen to that view at the same time.
|
||||
2. Answer from the returned data, concisely.
|
||||
3. If they ask about one item ("open the one from Arjun"), **open-item** it —
|
||||
it appears on their screen — and summarize what's in it if useful.
|
||||
|
||||
This matters most during a call: the user is talking to you hands-free and
|
||||
watching the screen. Navigate so they see what you see, and keep spoken
|
||||
answers short.
|
||||
|
||||
## Actions
|
||||
|
||||
### read-view — read a view's contents (and show it)
|
||||
Returns the same data the view renders; the app simultaneously navigates to
|
||||
that view so the user sees it.
|
||||
|
||||
- ` + "`view: \"email\"`" + ` → latest important inbox threads: ` + "`{ threadId, subject, from, date, unread, summary }`" + `.
|
||||
Pass ` + "`query`" + ` to search instead (sender name, subject words — e.g. ` + "`query: \"from Arjun\"`" + ` or just ` + "`\"Arjun\"`" + `).
|
||||
- ` + "`view: \"bg-tasks\"`" + ` → background agents: ` + "`{ name, slug, active, triggers, lastRunAt, lastRunSummary, lastRunError }`" + `.
|
||||
- ` + "`view: \"chat-history\"`" + ` → past chats: ` + "`{ sessionId, title, updatedAt, turnCount }`" + `.
|
||||
- ` + "`limit`" + ` (optional, default 15).
|
||||
|
||||
For notes, meetings, and live notes use the ` + "`file-*`" + ` tools (they are
|
||||
markdown files in the workspace) and then open-note / open-item to show them.
|
||||
|
||||
### open-item — open one specific thing on screen
|
||||
- ` + "`kind: \"email-thread\"`" + ` + ` + "`threadId`" + ` (from read-view email)
|
||||
- ` + "`kind: \"note\"`" + ` + ` + "`path`" + `
|
||||
- ` + "`kind: \"bg-task\"`" + ` + ` + "`taskName`" + ` (from read-view bg-tasks; validated against real tasks)
|
||||
- ` + "`kind: \"session\"`" + ` + ` + "`sessionId`" + ` (from read-view chat-history)
|
||||
|
||||
### open-view — just switch the screen
|
||||
` + "`view`" + `: ` + "`home | email | meetings | live-notes | bg-tasks | chat-history | knowledge | workspace | code | bases | graph`" + `
|
||||
Use when the user asks to "go to"/"show" a view without a question to answer.
|
||||
|
||||
### open-note
|
||||
Open a specific knowledge file in the editor pane.
|
||||
Open a knowledge file in the editor. ` + "`path`" + `: full workspace-relative path
|
||||
(e.g. ` + "`knowledge/People/John Smith.md`" + `). Use ` + "`file-grep`" + ` first if unsure
|
||||
of the exact path.
|
||||
|
||||
**When to use:** When the user asks to see, open, or view a specific note (e.g., "open John's note", "show me the Acme project page").
|
||||
### update-base-view / get-base-state / create-base
|
||||
Knowledge-base table control (unchanged):
|
||||
- ` + "`update-base-view`" + `: ` + "`filters`" + ` (` + "`set/add/remove/clear`" + ` of ` + "`{category, value}`" + `),
|
||||
` + "`sort`" + ` (` + "`{field, dir}`" + `), ` + "`search`" + `. **Never pass ` + "`columns`" + ` unless the user
|
||||
explicitly asks to change columns** — it overrides their layout.
|
||||
- ` + "`get-base-state`" + `: available filter categories/values and note count.
|
||||
- ` + "`create-base`" + `: save the current view configuration under ` + "`name`" + `.
|
||||
|
||||
**Parameters:**
|
||||
- ` + "`path`" + `: Full workspace-relative path (e.g., ` + "`knowledge/People/John Smith.md`" + `)
|
||||
## Worked examples
|
||||
|
||||
**Tips:**
|
||||
- Use ` + "`file-grep`" + ` first to find the exact path if you're unsure of the filename.
|
||||
- Always pass the full ` + "`knowledge/...`" + ` path, not just the filename.
|
||||
**"What emails do I have?"** (on a call)
|
||||
1. ` + "`app-navigation({ action: \"read-view\", view: \"email\" })`" + ` — email view opens on their screen.
|
||||
2. Speak the highlights: "You've got six new ones — the ones that matter are from Arjun about the deck and from Stripe about billing."
|
||||
|
||||
### open-view
|
||||
Switch the UI to the graph or bases view.
|
||||
**"Open the one from Arjun."**
|
||||
1. Find Arjun's thread in the data you already have (or ` + "`read-view`" + ` with ` + "`query: \"Arjun\"`" + `).
|
||||
2. ` + "`app-navigation({ action: \"open-item\", kind: \"email-thread\", threadId: \"...\" })`" + `
|
||||
3. "It's open — he's asking whether Thursday works for the pitch review."
|
||||
|
||||
**When to use:** When the user asks to see the knowledge graph, view all notes, or open the bases/table view.
|
||||
**"What background agents do I have?"**
|
||||
1. ` + "`app-navigation({ action: \"read-view\", view: \"bg-tasks\" })`" + `
|
||||
2. "Three: the inbox summarizer ran an hour ago, the meeting-prep agent is active, and the Linear digest failed its last run — want me to open that one?"
|
||||
|
||||
**Parameters:**
|
||||
- ` + "`view`" + `: ` + "`\"graph\"`" + ` or ` + "`\"bases\"`" + `
|
||||
**"Show me all active customers"**
|
||||
1. ` + "`get-base-state`" + ` to see available categories, then
|
||||
2. ` + "`update-base-view`" + ` with ` + "`filters.set: [{ category: \"relationship\", value: \"customer\" }]`" + `
|
||||
|
||||
### update-base-view
|
||||
Change filters, columns, sort order, or search in the bases (table) view.
|
||||
|
||||
**When to use:** When the user asks to find, filter, sort, or search notes. Examples: "show me all active customers", "filter by topic=hiring", "sort by name", "search for pricing".
|
||||
|
||||
**Parameters:**
|
||||
- ` + "`filters`" + `: Object with ` + "`set`" + `, ` + "`add`" + `, ` + "`remove`" + `, or ` + "`clear`" + ` — each takes an array of ` + "`{ category, value }`" + ` pairs.
|
||||
- ` + "`set`" + `: Replace ALL current filters with these.
|
||||
- ` + "`add`" + `: Append filters without removing existing ones.
|
||||
- ` + "`remove`" + `: Remove specific filters.
|
||||
- ` + "`clear: true`" + `: Remove all filters.
|
||||
- ` + "`columns`" + `: Object with ` + "`set`" + `, ` + "`add`" + `, or ` + "`remove`" + ` — each takes an array of column names (frontmatter keys).
|
||||
- ` + "`sort`" + `: ` + "`{ field, dir }`" + ` where dir is ` + "`\"asc\"`" + ` or ` + "`\"desc\"`" + `.
|
||||
- ` + "`search`" + `: Free-text search string.
|
||||
|
||||
**Tips:**
|
||||
- If unsure what categories/values are available, call ` + "`get-base-state`" + ` first.
|
||||
- For "show me X", prefer ` + "`filters.set`" + ` to start fresh rather than ` + "`filters.add`" + `.
|
||||
- Categories come from frontmatter keys (e.g., relationship, status, topic, type).
|
||||
- **CRITICAL: Do NOT pass ` + "`columns`" + ` unless the user explicitly asks to show/hide specific columns.** Omit the ` + "`columns`" + ` parameter entirely when only filtering, sorting, or searching. Passing ` + "`columns`" + ` will override the user's current column layout and can make the view appear empty.
|
||||
|
||||
### get-base-state
|
||||
Retrieve information about what's in the knowledge base — available filter categories, values, and note count.
|
||||
|
||||
**When to use:** When you need to know what properties exist before filtering, or when the user asks "what can I filter by?", "how many notes are there?", etc.
|
||||
|
||||
**Parameters:**
|
||||
- ` + "`base_name`" + ` (optional): Name of a saved base to inspect.
|
||||
|
||||
### create-base
|
||||
Save the current view configuration as a named base.
|
||||
|
||||
**When to use:** When the user asks to save a filtered view, create a saved search, or says "save this as [name]".
|
||||
|
||||
**Parameters:**
|
||||
- ` + "`name`" + `: Human-readable name for the base.
|
||||
|
||||
## Workflow Example
|
||||
|
||||
1. User: "Show me all people who are customers"
|
||||
2. First, check what properties are available:
|
||||
` + "`app-navigation({ action: \"get-base-state\" })`" + `
|
||||
3. Apply filters based on the available properties:
|
||||
` + "`app-navigation({ action: \"update-base-view\", filters: { set: [{ category: \"relationship\", value: \"customer\" }] } })`" + `
|
||||
4. If the user wants to save it:
|
||||
` + "`app-navigation({ action: \"create-base\", name: \"Customers\" })`" + `
|
||||
|
||||
## Important Notes
|
||||
- The ` + "`update-base-view`" + ` action will automatically navigate to the bases view if the user isn't already there.
|
||||
- ` + "`open-note`" + ` validates that the file exists before navigating.
|
||||
- Filter categories and values come from frontmatter in knowledge files.
|
||||
- **Never send ` + "`columns`" + ` or ` + "`sort`" + ` with ` + "`update-base-view`" + ` unless the user specifically asks to change them.** Only pass the parameters you intend to change — omitted parameters are left untouched.
|
||||
## Notes
|
||||
- read-view/open-view/open-item change what the user is looking at — that is
|
||||
the point, but don't bounce their screen around needlessly; navigate when
|
||||
it serves the question.
|
||||
- open-note and open-item validate the target exists before navigating.
|
||||
- update-base-view auto-navigates to the bases view.
|
||||
`;
|
||||
|
||||
export default skill;
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ import { ICodeModeConfigRepo } from "../../code-mode/repo.js";
|
|||
import type { ApprovalPolicy } from "@x/shared/dist/code-mode.js";
|
||||
import type { ICodeProjectsRepo } from "../../code-mode/projects/repo.js";
|
||||
import * as gitService from "../../code-mode/git/service.js";
|
||||
import { listImportantThreads, searchThreads } from "../../knowledge/sync_gmail.js";
|
||||
import { listTasks as listBackgroundTasks } from "../../background-tasks/fileops.js";
|
||||
import type { ISessions } from "../../sessions/api.js";
|
||||
|
||||
// Inputs for the bg-task builtin tools. Reuse the canonical schema field
|
||||
// descriptions; only `triggers` gets a tighter contextual override (the
|
||||
|
|
@ -1065,13 +1068,21 @@ export const BuiltinTools: z.infer<typeof BuiltinToolsSchema> = {
|
|||
// ============================================================================
|
||||
|
||||
'app-navigation': {
|
||||
description: 'Control the app UI - navigate to notes, switch views, filter/search the knowledge base, and manage saved views.',
|
||||
description: 'Drive the Rowboat app UI: navigate to any view, read what a view contains (emails, background agents, chat history), open specific items (an email thread, a note, an agent, a past chat), filter/search the knowledge base, and manage saved views. Use it to SHOW the user things while telling them — navigation happens on their screen.',
|
||||
inputSchema: z.object({
|
||||
action: z.enum(["open-note", "open-view", "update-base-view", "get-base-state", "create-base"]).describe("The navigation action to perform"),
|
||||
action: z.enum(["open-note", "open-view", "read-view", "open-item", "update-base-view", "get-base-state", "create-base"]).describe("The navigation action to perform"),
|
||||
// open-note
|
||||
path: z.string().optional().describe("Knowledge file path for open-note, e.g. knowledge/People/John.md"),
|
||||
// open-view
|
||||
view: z.enum(["bases", "graph"]).optional().describe("Which view to open (for open-view action)"),
|
||||
// open-view / read-view
|
||||
view: z.enum(["home", "email", "meetings", "live-notes", "bg-tasks", "chat-history", "knowledge", "workspace", "code", "bases", "graph"]).optional().describe("Which view to open (open-view) or read (read-view; supported for read: email, bg-tasks, chat-history)"),
|
||||
// read-view (email)
|
||||
query: z.string().optional().describe("For read-view on email: search query (sender name, subject words, etc.). Omit to list the latest important inbox threads."),
|
||||
limit: z.number().int().min(1).max(50).optional().describe("For read-view: max items to return (default 15)"),
|
||||
// open-item
|
||||
kind: z.enum(["email-thread", "note", "bg-task", "session"]).optional().describe("What to open (for open-item)"),
|
||||
threadId: z.string().optional().describe("Gmail thread id (open-item kind=email-thread; get it from read-view email)"),
|
||||
taskName: z.string().optional().describe("Background task/agent name (open-item kind=bg-task; get it from read-view bg-tasks)"),
|
||||
sessionId: z.string().optional().describe("Chat session id (open-item kind=session; get it from read-view chat-history)"),
|
||||
// update-base-view
|
||||
filters: z.object({
|
||||
set: z.array(z.object({ category: z.string(), value: z.string() })).optional().describe("Replace all filters with these"),
|
||||
|
|
@ -1117,6 +1128,110 @@ export const BuiltinTools: z.infer<typeof BuiltinToolsSchema> = {
|
|||
return { success: true, action: 'open-view', view };
|
||||
}
|
||||
|
||||
case 'read-view': {
|
||||
// Returns the same data the view renders, so the assistant
|
||||
// can answer precisely — and the renderer navigates to the
|
||||
// view at the same time so the user SEES what's being read.
|
||||
const view = input.view as string;
|
||||
const limit = (input.limit as number | undefined) ?? 15;
|
||||
try {
|
||||
switch (view) {
|
||||
case 'email': {
|
||||
const query = (input.query as string | undefined)?.trim();
|
||||
const result = query
|
||||
? await searchThreads(query, { limit })
|
||||
: listImportantThreads({ limit });
|
||||
const threads = (result.threads ?? []).slice(0, limit).map((t) => ({
|
||||
threadId: t.threadId,
|
||||
subject: t.subject ?? '(no subject)',
|
||||
from: t.from ?? '',
|
||||
date: t.date ?? '',
|
||||
unread: t.unread ?? false,
|
||||
summary: t.summary ? t.summary.slice(0, 200) : undefined,
|
||||
}));
|
||||
return { success: true, action: 'read-view', view, query, threads };
|
||||
}
|
||||
case 'bg-tasks': {
|
||||
const { items } = await listBackgroundTasks({ limit });
|
||||
const agents = items.map((t) => ({
|
||||
name: t.name,
|
||||
slug: t.slug,
|
||||
active: t.active,
|
||||
triggers: t.triggers,
|
||||
lastRunAt: t.lastRunAt,
|
||||
lastRunSummary: t.lastRunSummary ? t.lastRunSummary.slice(0, 200) : undefined,
|
||||
lastRunError: t.lastRunError ? t.lastRunError.slice(0, 200) : undefined,
|
||||
}));
|
||||
return { success: true, action: 'read-view', view, agents };
|
||||
}
|
||||
case 'chat-history': {
|
||||
const sessions = container.resolve<ISessions>('sessions')
|
||||
.listSessions()
|
||||
.slice(0, limit)
|
||||
.map((s) => ({
|
||||
sessionId: s.sessionId,
|
||||
title: s.title ?? '(untitled)',
|
||||
updatedAt: s.updatedAt,
|
||||
turnCount: s.turnCount,
|
||||
}));
|
||||
return { success: true, action: 'read-view', view, sessions };
|
||||
}
|
||||
default:
|
||||
return {
|
||||
success: false,
|
||||
error: `read-view supports: email, bg-tasks, chat-history. For notes/meetings/live-notes use the file-* tools (they are files under the workspace); for other views use open-view and describe what you need.`,
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : `Failed to read ${view}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
case 'open-item': {
|
||||
const kind = input.kind as string;
|
||||
switch (kind) {
|
||||
case 'email-thread': {
|
||||
const threadId = input.threadId as string | undefined;
|
||||
if (!threadId) return { success: false, error: 'threadId is required for kind=email-thread' };
|
||||
return { success: true, action: 'open-item', kind, threadId };
|
||||
}
|
||||
case 'note': {
|
||||
const filePath = input.path as string | undefined;
|
||||
if (!filePath) return { success: false, error: 'path is required for kind=note' };
|
||||
const result = await files.exists(filePath);
|
||||
if (!result.exists) return { success: false, error: `File not found: ${filePath}` };
|
||||
return { success: true, action: 'open-item', kind, path: filePath };
|
||||
}
|
||||
case 'bg-task': {
|
||||
const taskName = input.taskName as string | undefined;
|
||||
if (!taskName) return { success: false, error: 'taskName is required for kind=bg-task' };
|
||||
// Validate (and canonicalize) against the real task list.
|
||||
const { items: tasks } = await listBackgroundTasks({});
|
||||
const match = tasks.find(
|
||||
(t) => t.name === taskName || t.slug === taskName
|
||||
|| t.name.toLowerCase() === taskName.toLowerCase(),
|
||||
);
|
||||
if (!match) {
|
||||
return {
|
||||
success: false,
|
||||
error: `No background task named "${taskName}". Known tasks: ${tasks.map((t) => t.name).join(', ') || '(none)'}`,
|
||||
};
|
||||
}
|
||||
return { success: true, action: 'open-item', kind, taskName: match.name };
|
||||
}
|
||||
case 'session': {
|
||||
const sessionId = input.sessionId as string | undefined;
|
||||
if (!sessionId) return { success: false, error: 'sessionId is required for kind=session' };
|
||||
return { success: true, action: 'open-item', kind, sessionId };
|
||||
}
|
||||
default:
|
||||
return { success: false, error: `Unknown item kind: ${kind}` };
|
||||
}
|
||||
}
|
||||
|
||||
case 'update-base-view': {
|
||||
const updates: Record<string, unknown> = {};
|
||||
if (input.filters) updates.filters = input.filters;
|
||||
|
|
|
|||
|
|
@ -649,6 +649,12 @@ const ipcSchemas = {
|
|||
}),
|
||||
res: z.null(),
|
||||
},
|
||||
// Bring the main app window to the foreground (e.g. the assistant navigated
|
||||
// the UI during a call while the user was in another app).
|
||||
'app:focusMainWindow': {
|
||||
req: z.null(),
|
||||
res: z.object({}),
|
||||
},
|
||||
'app:takeMeetingNotes': {
|
||||
req: z.object({
|
||||
// Pass the raw calendar event JSON through; renderer adapts to its existing flow.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue