apunkt/rowboat
1
0
Fork 0
mirror of https://github.com/rowboatlabs/rowboat.git synced 2026-07-15 21:11:08 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #663 from rowboatlabs/video1

Browse source 
Video mode with screenshare and app navigation
This commit is contained in:
arkml 2026-07-04 14:27:18 +05:30 • committed by GitHub
commit 51c289ba36
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 2736 additions and 329 deletions
Download patch file Download diff file Expand all files Collapse all files

93
apps/x/packages/core/src/agents/runtime.ts
View file

@ -337,6 +337,9 @@ export interface ComposeSystemInstructionsInput {
searchEnabled: boolean;
codeMode: 'claude' | 'codex' | null;
codeCwd: string | null;
// Optional so legacy callers (old streamAgent path) are unaffected.
videoMode?: boolean;
coachMode?: boolean;
}
// System-prompt assembly, extracted verbatim from streamAgent so the new turn
@ -351,6 +354,8 @@ export function composeSystemInstructions({
searchEnabled,
codeMode,
codeCwd,
videoMode,
coachMode,
}: ComposeSystemInstructionsInput): string {
let instructionsWithDateTime = `${instructions}\n\n${USER_CONTEXT_SYSTEM_INSTRUCTIONS}`;
if (agentNotesContext) {
@ -379,6 +384,43 @@ Do not announce the work directory unless it's relevant. Just use it.`;
if (voiceInput) {
instructionsWithDateTime += `\n\n# Voice Input\nThe user's message was transcribed from speech. Be aware that:\n- There may be transcription errors. Silently correct obvious ones (e.g. homophones, misheard words). If an error is genuinely ambiguous, briefly mention your interpretation (e.g. "I'm assuming you meant X").\n- Spoken messages are often long-winded. The user may ramble, repeat themselves, or correct something they said earlier in the same message. Focus on their final intent, not every word verbatim.`;
}
if (videoMode) {
instructionsWithDateTime += `\n\n# Video Mode (Live Camera)
The user has turned on video mode: their webcam is on, and their messages arrive with a series of live webcam frames (ordered oldest to newest) captured while they were speaking or typing. The frames are a live view of the user themselves — not a document or file to analyze.
How to use the frames:
- Use them for what visual awareness genuinely adds: the user's expressions, body language, posture, gestures, eye contact with the camera, visible energy, anything they hold up to the camera, and their surroundings when relevant.
- Compare across frames to notice change over time within a message (e.g. increasingly slouched, started smiling, looked away for most of the message).
- When the user practices something performative (a pitch, presentation, interview, talk) or asks for delivery feedback, give specific, actionable coaching grounded in what you actually see — cite concrete observations ("in the last few frames you looked down and away from the camera") rather than generic advice. Cover posture, eye contact, facial expressiveness, gesturing, and visible energy.
- If they show something to the camera (an object, document, whiteboard), read or describe it and respond accordingly.
Driving the app:
- You can control the Rowboat app the user is looking at via the app-navigation tool (load the app-navigation skill first): open views, READ a view's contents as data (emails, background agents, chat history), and open specific items on their screen.
- When the user asks about anything that lives inside Rowboat ("what emails do I have?", "what agents are running?", "open the one from Arjun"), prefer driving over describing: read-view shows the view on their screen while returning its data, then answer out loud briefly; open-item when they pick one. Narrate as you act ("pulling up your inbox…"). Reading a view's data beats squinting at screen-share frames — it's exact.
Screen sharing:
- The user may also share their screen. Screen-share frames arrive in a separately labeled group after the webcam frames; they show the user's screen, not the user.
- When screen frames are present, treat them as the primary subject: the user is usually asking about, or working on, what's visible there. Read the screen carefully — code, documents, error messages, UI state — and help with it concretely.
- The LAST screen frame is the most current view of their screen; earlier ones show how it changed while they spoke.
- Frames are downscaled captures: small text may be hard to read. If something crucial is illegible, say what you need ("zoom into the error message" / "make that panel bigger") rather than guessing.
Etiquette:
- Do not narrate or list what you see in every response. Bring up visual observations only when relevant to the user's request or clearly worth mentioning.
- Never comment on the user's physical appearance, attractiveness, or personal attributes — visual feedback is strictly about delivery, expression, and body language.
- Frames are periodic snapshots, not continuous video; moments between frames are missing. Don't claim certainty about motion you couldn't have seen.`;
}
if (coachMode) {
instructionsWithDateTime += `\n\n# Practice Session (Coach Mode)
The user started a practice session: they are rehearsing something performative — a pitch, presentation, interview answer, or talk — and want live coaching. You are their coach for this session.
How to coach:
- Watch and listen for delivery: pacing, filler words, rambling, structure, and (from the webcam frames) posture, eye contact with the camera, facial expressiveness, gesturing, and visible energy.
- After each take or answer, give brief, specific, actionable feedback: 2-3 concrete observations, quoting what you actually saw or heard ("you looked down during the ask", "the opening 'um, so, basically' undercuts your hook"). Then let them go again.
- If they are clearly mid-flow, keep any interjection to one short sentence — or stay silent and save it for the break.
- Ask what they're practicing for at the start if it isn't obvious (investor pitch? interview? conference talk?) and tailor feedback to that audience.
- When they say they're done or wrap up, give a structured debrief: what's working, the top 3 things to improve, and one concrete drill or reframe to try next time.
- Be encouraging but honest — vague praise wastes their rehearsal time. Never comment on physical appearance; delivery, expression, and body language only.`;
}
if (voiceOutput === 'summary') {
instructionsWithDateTime += `\n\n# Voice Output (MANDATORY — READ THIS FIRST)\nThe user has voice output enabled. THIS IS YOUR #1 PRIORITY: you MUST start your response with <voice></voice> tags. If your response does not begin with <voice> tags, the user will hear nothing — which is a broken experience. NEVER skip this.\n\nRules:\n1. YOUR VERY FIRST OUTPUT MUST BE A <voice> TAG. No exceptions. Do not start with markdown, headings, or any other text. The literal first characters of your response must be "<voice>".\n2. Place ALL <voice> tags at the BEGINNING of your response, before any detailed content. Do NOT intersperse <voice> tags throughout the response.\n3. Wrap EACH spoken sentence in its own separate <voice> tag so it can be spoken incrementally. Do NOT wrap everything in a single <voice> block.\n4. Use voice as a TL;DR and navigation aid — do NOT read the entire response aloud.\n5. After all <voice> tags, you may include detailed written content (markdown, tables, code, etc.) that will be shown visually but not spoken.\n\n## Examples\n\nExample 1 — User asks: "what happened in my meeting with Alex yesterday?"\n\n<voice>Your meeting with Alex covered three main things: the Q2 roadmap timeline, hiring for the backend role, and the client demo next week.</voice>\n<voice>I've pulled out the key details and action items below — the demo prep notes are at the end.</voice>\n\n## Meeting with Alex — March 11\n### Roadmap\n- Agreed to push Q2 launch to April 15...\n(detailed written content continues)\n\nExample 2 — User asks: "summarize my emails"\n\n<voice>You have five new emails since this morning.</voice>\n<voice>Two are from your team — Jordan sent the RFC you requested and Taylor flagged a contract issue.</voice>\n<voice>There's also a warm intro from a VC partner connecting you with someone at a prospective customer.</voice>\n<voice>I've drafted responses for three of them. The details and drafts are below.</voice>\n\n(email blocks, tables, and detailed content follow)\n\nExample 3 — User asks: "what's on my calendar today?"\n\n<voice>You've got a pretty packed day — seven meetings starting with standup at 9.</voice>\n<voice>The big ones are your investor call at 11, lunch with a partner from your lead VC at 12:30, and a customer call at 4.</voice>\n<voice>Your only free block for deep work is 2:30 to 4.</voice>\n\n(calendar block with full event details follows)\n\nExample 4 — User asks: "draft an email to Sam with our metrics"\n\n<voice>Done — I've drafted the email to Sam with your latest WAU and churn numbers.</voice>\n<voice>Take a look at the draft below and send it when you're ready.</voice>\n\n(email block with draft follows)\n\nREMEMBER: If you do not start with <voice> tags, the user hears silence. Always speak first, then write.`;
} else if (voiceOutput === 'full') {
@ -942,15 +984,27 @@ export function convertFromMessages(messages: z.infer<typeof Message>[]): ModelM
providerOptions,
});
} else {
// New content parts array — collapse to text for LLM
// New content parts array — collapse text/attachments to text
// for the LLM; inline image parts (video-mode webcam and
// screen-share frames) are passed through as real multimodal
// image parts, grouped under labeled text headers so the
// model knows which images show the user vs their screen.
const textSegments: string[] = userMessageContextPrefix ? [userMessageContextPrefix] : [];
const attachmentLines: string[] = [];
type EncodedImagePart = { type: "image"; image: string; mediaType: string };
const cameraParts: EncodedImagePart[] = [];
const screenParts: EncodedImagePart[] = [];
const frameTimes: string[] = [];
for (const part of msg.content) {
if (part.type === "attachment") {
const sizeStr = part.size ? `, ${formatBytes(part.size)}` : '';
const lineStr = part.lineNumber ? ` (line ${part.lineNumber})` : '';
attachmentLines.push(`- ${part.filename} (${part.mimeType}${sizeStr}) at ${part.path}${lineStr}`);
} else if (part.type === "image") {
const target = part.source === "screen" ? screenParts : cameraParts;
target.push({ type: "image", image: part.data, mediaType: part.mediaType });
if (part.capturedAt) frameTimes.push(part.capturedAt);
} else {
textSegments.push(part.text);
}
@ -964,11 +1018,38 @@ export function convertFromMessages(messages: z.infer<typeof Message>[]): ModelM
}
}
result.push({
role: "user",
content: textSegments.join("\n"),
providerOptions,
});
const imageCount = cameraParts.length + screenParts.length;
if (imageCount > 0) {
const span = frameTimes.length >= 2
? ` spanning ${frameTimes[0]} to ${frameTimes[frameTimes.length - 1]}`
: frameTimes.length === 1
? ` captured at ${frameTimes[0]}`
: '';
const kinds: string[] = [];
if (cameraParts.length > 0) kinds.push(`${cameraParts.length} live webcam frame${cameraParts.length === 1 ? '' : 's'} of the user`);
if (screenParts.length > 0) kinds.push(`${screenParts.length} frame${screenParts.length === 1 ? '' : 's'} of the user's shared screen`);
textSegments.push(`[Video mode: ${kinds.join(' and ')} attached below, each group oldest to newest,${span ? span + ',' : ''} recorded while they composed this message.]`);
const content: Array<{ type: "text"; text: string } | EncodedImagePart> = [
{ type: "text", text: textSegments.join("\n") },
];
if (cameraParts.length > 0) {
content.push({ type: "text", text: "Webcam frames (oldest to newest):" }, ...cameraParts);
}
if (screenParts.length > 0) {
content.push({ type: "text", text: "Screen-share frames (oldest to newest):" }, ...screenParts);
}
result.push({
role: "user",
content,
providerOptions,
});
} else {
result.push({
role: "user",
content: textSegments.join("\n"),
providerOptions,
});
}
}
break;
}

4
apps/x/packages/core/src/application/assistant/instructions.ts
View file

@ -114,7 +114,7 @@ ${codeModeEnabled
? `**Code with Agents:** When users ask you to write code, build a project, create a script, fix a bug, or do any software development task — **including simple things like "create a .c file" or "write a hello-world in Python"** — your FIRST action MUST be \`loadSkill('code-with-agents')\`. Do NOT reach for \`executeCommand\` (PowerShell / bash / shell) or any workspace file tool to do code work yourself before loading this skill. The skill decides whether to delegate to Claude Code / Codex (via acpx) or hand control back to you, and it presents the user a one-click choice when needed. Paths outside the Rowboat workspace root (e.g. \`G:/...\`, \`~/projects/...\`) are NORMAL for coding tasks — do NOT raise "outside workspace" concerns or fall back to your own tools.`
: `**Code with Agents (disabled):** Code mode is currently OFF in the user's settings. Do NOT load \`code-with-agents\` and do NOT call acpx. Handle coding requests yourself with your normal tools if you can. After answering, add a final line letting the user know they can delegate coding to Claude Code or Codex by enabling Code Mode in Settings → Code Mode.`}
**App Control:** When users ask you to open notes, show the bases or graph view, filter or search notes, or manage saved views, load the \`app-navigation\` skill first. It provides structured guidance for navigating the app UI and controlling the knowledge base view.
**App Control (drive the app):** You can drive the Rowboat UI the user is looking at — open any view (email, meetings, background agents, chat history, knowledge, workspace, code, bases, graph), READ what a view contains (\`read-view\` returns emails / background agents / past chats as data while showing the view on screen), and open specific items (an email thread, a note, an agent, a past chat). When users ask to open, show, find, or ask about anything that lives inside Rowboat, load the \`app-navigation\` skill first — it documents the show-while-telling pattern. This matters most on calls: navigate so the user sees what you see, then answer briefly.
**Background Tasks (Self-Running Work):** Rowboat can run *background tasks* — persistent instructions the agent fires on a schedule and/or in response to incoming emails / calendar events. A bg-task either maintains a snapshot in its \`index.md\` (digest, dashboard, rolling summary) or performs a recurring side-effect (send a Slack message, draft an email, post to a webhook, call an API). This is the flagship surface for *anything recurring*.
@ -293,7 +293,7 @@ ${runtimeContextPrompt}
- \`addMcpServer\`, \`listMcpServers\`, \`listMcpTools\`, \`executeMcpTool\` - MCP server management and execution
- \`loadSkill\` - Skill loading
${slackToolsLine}- \`web-search\` - Search the web. Returns rich results with full text, highlights, and metadata. The \`category\` parameter defaults to \`general\` (full web search) — only use a specific category like \`news\`, \`company\`, \`research paper\` etc. when the query is clearly about that type. For everyday queries (weather, restaurants, prices, how-to), use \`general\`.
- \`app-navigation\` - Control the app UI: open notes, switch views, filter/search the knowledge base, manage saved views. **Load the \`app-navigation\` skill before using this tool.**
- \`app-navigation\` - Drive the app UI: open any view, read a view's contents (emails / background agents / chat history), open specific items (email thread, note, agent, past chat), filter/search the knowledge base, manage saved views. **Load the \`app-navigation\` skill before using this tool.**
- \`browser-control\` - Control the embedded browser pane: open sites, inspect the live page, switch tabs, and interact with indexed page elements. **Load the \`browser-control\` skill before using this tool.**
- \`save-to-memory\` - Save observations about the user to the agent memory system. Use this proactively during conversations.
${composioToolsLine}

141
apps/x/packages/core/src/application/assistant/skills/app-navigation/skill.ts
View file

@ -1,82 +1,91 @@
export const skill = String.raw`
# App Navigation Skill
# App Driving Skill
You have access to the **app-navigation** tool which lets you control the Rowboat UI directly — opening notes, switching views, filtering the knowledge base, and creating saved views.
You have the **app-navigation** tool: you can DRIVE the Rowboat app the user
is looking at — open any view, read what a view contains, open specific items
(an email thread, a note, a background agent, a past chat), filter the
knowledge base, and manage saved views. Navigation happens on the USER'S
screen: when you open something, they watch it open.
## The core pattern: show while telling
When the user asks about something that lives inside Rowboat ("what emails do
I have?", "what background agents are running?", "open the note about Acme"),
don't answer blind. Drive:
1. **read-view** the relevant view — this returns the actual data AND
navigates the user's screen to that view at the same time.
2. Answer from the returned data, concisely.
3. If they ask about one item ("open the one from Arjun"), **open-item** it —
it appears on their screen — and summarize what's in it if useful.
This matters most during a call: the user is talking to you hands-free and
watching the screen. Navigate so they see what you see, and keep spoken
answers short.
## Actions
### read-view — read a view's contents (and show it)
Returns the same data the view renders; the app simultaneously navigates to
that view so the user sees it.
- ` + "`view: \"email\"`" + ` → latest important inbox threads: ` + "`{ threadId, subject, from, date, unread, summary }`" + `.
Pass ` + "`query`" + ` to search instead (sender name, subject words — e.g. ` + "`query: \"from Arjun\"`" + ` or just ` + "`\"Arjun\"`" + `).
- ` + "`view: \"bg-tasks\"`" + ` → background agents: ` + "`{ name, slug, active, triggers, lastRunAt, lastRunSummary, lastRunError }`" + `.
- ` + "`view: \"chat-history\"`" + ` → past chats: ` + "`{ sessionId, title, updatedAt, turnCount }`" + `.
- ` + "`limit`" + ` (optional, default 15).
For notes, meetings, and live notes use the ` + "`file-*`" + ` tools (they are
markdown files in the workspace) and then open-note / open-item to show them.
### open-item — open one specific thing on screen
- ` + "`kind: \"email-thread\"`" + ` + ` + "`threadId`" + ` (from read-view email)
- ` + "`kind: \"note\"`" + ` + ` + "`path`" + `
- ` + "`kind: \"bg-task\"`" + ` + ` + "`taskName`" + ` (from read-view bg-tasks; validated against real tasks)
- ` + "`kind: \"session\"`" + ` + ` + "`sessionId`" + ` (from read-view chat-history)
### open-view — just switch the screen
` + "`view`" + `: ` + "`home | email | meetings | live-notes | bg-tasks | chat-history | knowledge | workspace | code | bases | graph`" + `
Use when the user asks to "go to"/"show" a view without a question to answer.
### open-note
Open a specific knowledge file in the editor pane.
Open a knowledge file in the editor. ` + "`path`" + `: full workspace-relative path
(e.g. ` + "`knowledge/People/John Smith.md`" + `). Use ` + "`file-grep`" + ` first if unsure
of the exact path.
**When to use:** When the user asks to see, open, or view a specific note (e.g., "open John's note", "show me the Acme project page").
### update-base-view / get-base-state / create-base
Knowledge-base table control (unchanged):
- ` + "`update-base-view`" + `: ` + "`filters`" + ` (` + "`set/add/remove/clear`" + ` of ` + "`{category, value}`" + `),
` + "`sort`" + ` (` + "`{field, dir}`" + `), ` + "`search`" + `. **Never pass ` + "`columns`" + ` unless the user
explicitly asks to change columns** — it overrides their layout.
- ` + "`get-base-state`" + `: available filter categories/values and note count.
- ` + "`create-base`" + `: save the current view configuration under ` + "`name`" + `.
**Parameters:**
- ` + "`path`" + `: Full workspace-relative path (e.g., ` + "`knowledge/People/John Smith.md`" + `)
## Worked examples
**Tips:**
- Use ` + "`file-grep`" + ` first to find the exact path if you're unsure of the filename.
- Always pass the full ` + "`knowledge/...`" + ` path, not just the filename.
**"What emails do I have?"** (on a call)
1. ` + "`app-navigation({ action: \"read-view\", view: \"email\" })`" + ` — email view opens on their screen.
2. Speak the highlights: "You've got six new ones — the ones that matter are from Arjun about the deck and from Stripe about billing."
### open-view
Switch the UI to the graph or bases view.
**"Open the one from Arjun."**
1. Find Arjun's thread in the data you already have (or ` + "`read-view`" + ` with ` + "`query: \"Arjun\"`" + `).
2. ` + "`app-navigation({ action: \"open-item\", kind: \"email-thread\", threadId: \"...\" })`" + `
3. "It's open — he's asking whether Thursday works for the pitch review."
**When to use:** When the user asks to see the knowledge graph, view all notes, or open the bases/table view.
**"What background agents do I have?"**
1. ` + "`app-navigation({ action: \"read-view\", view: \"bg-tasks\" })`" + `
2. "Three: the inbox summarizer ran an hour ago, the meeting-prep agent is active, and the Linear digest failed its last run — want me to open that one?"
**Parameters:**
- ` + "`view`" + `: ` + "`\"graph\"`" + ` or ` + "`\"bases\"`" + `
**"Show me all active customers"**
1. ` + "`get-base-state`" + ` to see available categories, then
2. ` + "`update-base-view`" + ` with ` + "`filters.set: [{ category: \"relationship\", value: \"customer\" }]`" + `
### update-base-view
Change filters, columns, sort order, or search in the bases (table) view.
**When to use:** When the user asks to find, filter, sort, or search notes. Examples: "show me all active customers", "filter by topic=hiring", "sort by name", "search for pricing".
**Parameters:**
- ` + "`filters`" + `: Object with ` + "`set`" + `, ` + "`add`" + `, ` + "`remove`" + `, or ` + "`clear`" + ` — each takes an array of ` + "`{ category, value }`" + ` pairs.
- ` + "`set`" + `: Replace ALL current filters with these.
- ` + "`add`" + `: Append filters without removing existing ones.
- ` + "`remove`" + `: Remove specific filters.
- ` + "`clear: true`" + `: Remove all filters.
- ` + "`columns`" + `: Object with ` + "`set`" + `, ` + "`add`" + `, or ` + "`remove`" + ` — each takes an array of column names (frontmatter keys).
- ` + "`sort`" + `: ` + "`{ field, dir }`" + ` where dir is ` + "`\"asc\"`" + ` or ` + "`\"desc\"`" + `.
- ` + "`search`" + `: Free-text search string.
**Tips:**
- If unsure what categories/values are available, call ` + "`get-base-state`" + ` first.
- For "show me X", prefer ` + "`filters.set`" + ` to start fresh rather than ` + "`filters.add`" + `.
- Categories come from frontmatter keys (e.g., relationship, status, topic, type).
- **CRITICAL: Do NOT pass ` + "`columns`" + ` unless the user explicitly asks to show/hide specific columns.** Omit the ` + "`columns`" + ` parameter entirely when only filtering, sorting, or searching. Passing ` + "`columns`" + ` will override the user's current column layout and can make the view appear empty.
### get-base-state
Retrieve information about what's in the knowledge base — available filter categories, values, and note count.
**When to use:** When you need to know what properties exist before filtering, or when the user asks "what can I filter by?", "how many notes are there?", etc.
**Parameters:**
- ` + "`base_name`" + ` (optional): Name of a saved base to inspect.
### create-base
Save the current view configuration as a named base.
**When to use:** When the user asks to save a filtered view, create a saved search, or says "save this as [name]".
**Parameters:**
- ` + "`name`" + `: Human-readable name for the base.
## Workflow Example
1. User: "Show me all people who are customers"
2. First, check what properties are available:
` + "`app-navigation({ action: \"get-base-state\" })`" + `
3. Apply filters based on the available properties:
` + "`app-navigation({ action: \"update-base-view\", filters: { set: [{ category: \"relationship\", value: \"customer\" }] } })`" + `
4. If the user wants to save it:
` + "`app-navigation({ action: \"create-base\", name: \"Customers\" })`" + `
## Important Notes
- The ` + "`update-base-view`" + ` action will automatically navigate to the bases view if the user isn't already there.
- ` + "`open-note`" + ` validates that the file exists before navigating.
- Filter categories and values come from frontmatter in knowledge files.
- **Never send ` + "`columns`" + ` or ` + "`sort`" + ` with ` + "`update-base-view`" + ` unless the user specifically asks to change them.** Only pass the parameters you intend to change — omitted parameters are left untouched.
## Notes
- read-view/open-view/open-item change what the user is looking at — that is
the point, but don't bounce their screen around needlessly; navigate when
it serves the question.
- open-note and open-item validate the target exists before navigating.
- update-base-view auto-navigates to the bases view.
`;
export default skill;

123
apps/x/packages/core/src/application/lib/builtin-tools.ts
View file

@ -23,6 +23,9 @@ import { ICodeModeConfigRepo } from "../../code-mode/repo.js";
import type { ApprovalPolicy } from "@x/shared/dist/code-mode.js";
import type { ICodeProjectsRepo } from "../../code-mode/projects/repo.js";
import * as gitService from "../../code-mode/git/service.js";
import { listImportantThreads, searchThreads } from "../../knowledge/sync_gmail.js";
import { listTasks as listBackgroundTasks } from "../../background-tasks/fileops.js";
import type { ISessions } from "../../sessions/api.js";
// Inputs for the bg-task builtin tools. Reuse the canonical schema field
// descriptions; only `triggers` gets a tighter contextual override (the
@ -1065,13 +1068,21 @@ export const BuiltinTools: z.infer<typeof BuiltinToolsSchema> = {
// ============================================================================
'app-navigation': {
description: 'Control the app UI - navigate to notes, switch views, filter/search the knowledge base, and manage saved views.',
description: 'Drive the Rowboat app UI: navigate to any view, read what a view contains (emails, background agents, chat history), open specific items (an email thread, a note, an agent, a past chat), filter/search the knowledge base, and manage saved views. Use it to SHOW the user things while telling them — navigation happens on their screen.',
inputSchema: z.object({
action: z.enum(["open-note", "open-view", "update-base-view", "get-base-state", "create-base"]).describe("The navigation action to perform"),
action: z.enum(["open-note", "open-view", "read-view", "open-item", "update-base-view", "get-base-state", "create-base"]).describe("The navigation action to perform"),
// open-note
path: z.string().optional().describe("Knowledge file path for open-note, e.g. knowledge/People/John.md"),
// open-view
view: z.enum(["bases", "graph"]).optional().describe("Which view to open (for open-view action)"),
// open-view / read-view
view: z.enum(["home", "email", "meetings", "live-notes", "bg-tasks", "chat-history", "knowledge", "workspace", "code", "bases", "graph"]).optional().describe("Which view to open (open-view) or read (read-view; supported for read: email, bg-tasks, chat-history)"),
// read-view (email)
query: z.string().optional().describe("For read-view on email: search query (sender name, subject words, etc.). Omit to list the latest important inbox threads."),
limit: z.number().int().min(1).max(50).optional().describe("For read-view: max items to return (default 15)"),
// open-item
kind: z.enum(["email-thread", "note", "bg-task", "session"]).optional().describe("What to open (for open-item)"),
threadId: z.string().optional().describe("Gmail thread id (open-item kind=email-thread; get it from read-view email)"),
taskName: z.string().optional().describe("Background task/agent name (open-item kind=bg-task; get it from read-view bg-tasks)"),
sessionId: z.string().optional().describe("Chat session id (open-item kind=session; get it from read-view chat-history)"),
// update-base-view
filters: z.object({
set: z.array(z.object({ category: z.string(), value: z.string() })).optional().describe("Replace all filters with these"),
@ -1117,6 +1128,110 @@ export const BuiltinTools: z.infer<typeof BuiltinToolsSchema> = {
return { success: true, action: 'open-view', view };
}
case 'read-view': {
// Returns the same data the view renders, so the assistant
// can answer precisely — and the renderer navigates to the
// view at the same time so the user SEES what's being read.
const view = input.view as string;
const limit = (input.limit as number | undefined) ?? 15;
try {
switch (view) {
case 'email': {
const query = (input.query as string | undefined)?.trim();
const result = query
? await searchThreads(query, { limit })
: listImportantThreads({ limit });
const threads = (result.threads ?? []).slice(0, limit).map((t) => ({
threadId: t.threadId,
subject: t.subject ?? '(no subject)',
from: t.from ?? '',
date: t.date ?? '',
unread: t.unread ?? false,
summary: t.summary ? t.summary.slice(0, 200) : undefined,
}));
return { success: true, action: 'read-view', view, query, threads };
}
case 'bg-tasks': {
const { items } = await listBackgroundTasks({ limit });
const agents = items.map((t) => ({
name: t.name,
slug: t.slug,
active: t.active,
triggers: t.triggers,
lastRunAt: t.lastRunAt,
lastRunSummary: t.lastRunSummary ? t.lastRunSummary.slice(0, 200) : undefined,
lastRunError: t.lastRunError ? t.lastRunError.slice(0, 200) : undefined,
}));
return { success: true, action: 'read-view', view, agents };
}
case 'chat-history': {
const sessions = container.resolve<ISessions>('sessions')
.listSessions()
.slice(0, limit)
.map((s) => ({
sessionId: s.sessionId,
title: s.title ?? '(untitled)',
updatedAt: s.updatedAt,
turnCount: s.turnCount,
}));
return { success: true, action: 'read-view', view, sessions };
}
default:
return {
success: false,
error: `read-view supports: email, bg-tasks, chat-history. For notes/meetings/live-notes use the file-* tools (they are files under the workspace); for other views use open-view and describe what you need.`,
};
}
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : `Failed to read ${view}`,
};
}
}
case 'open-item': {
const kind = input.kind as string;
switch (kind) {
case 'email-thread': {
const threadId = input.threadId as string | undefined;
if (!threadId) return { success: false, error: 'threadId is required for kind=email-thread' };
return { success: true, action: 'open-item', kind, threadId };
}
case 'note': {
const filePath = input.path as string | undefined;
if (!filePath) return { success: false, error: 'path is required for kind=note' };
const result = await files.exists(filePath);
if (!result.exists) return { success: false, error: `File not found: ${filePath}` };
return { success: true, action: 'open-item', kind, path: filePath };
}
case 'bg-task': {
const taskName = input.taskName as string | undefined;
if (!taskName) return { success: false, error: 'taskName is required for kind=bg-task' };
// Validate (and canonicalize) against the real task list.
const { items: tasks } = await listBackgroundTasks({});
const match = tasks.find(
(t) => t.name === taskName || t.slug === taskName
|| t.name.toLowerCase() === taskName.toLowerCase(),
);
if (!match) {
return {
success: false,
error: `No background task named "${taskName}". Known tasks: ${tasks.map((t) => t.name).join(', ') || '(none)'}`,
};
}
return { success: true, action: 'open-item', kind, taskName: match.name };
}
case 'session': {
const sessionId = input.sessionId as string | undefined;
if (!sessionId) return { success: false, error: 'sessionId is required for kind=session' };
return { success: true, action: 'open-item', kind, sessionId };
}
default:
return { success: false, error: `Unknown item kind: ${kind}` };
}
}
case 'update-base-view': {
const updates: Record<string, unknown> = {};
if (input.filters) updates.filters = input.filters;

4
apps/x/packages/core/src/turns/bridges/real-agent-resolver.ts
View file

@ -54,6 +54,8 @@ const CompositionOverrides = z.object({
searchEnabled: z.boolean().optional(),
codeMode: z.enum(["claude", "codex"]).nullable().optional(),
codeCwd: z.string().nullable().optional(),
videoMode: z.boolean().optional(),
coachMode: z.boolean().optional(),
});
export interface RealAgentResolverDeps {
@ -121,6 +123,8 @@ export class RealAgentResolver implements IAgentResolver {
searchEnabled: composition.searchEnabled ?? false,
codeMode: composition.codeMode ?? null,
codeCwd: composition.codeCwd ?? null,
videoMode: composition.videoMode ?? false,
coachMode: composition.coachMode ?? false,
});
const tools = await this.resolveTools(agent);

104
apps/x/packages/core/src/voice/voice.ts
View file

@ -32,46 +32,57 @@ export async function getVoiceConfig(): Promise<VoiceConfig> {
};
}
export async function synthesizeSpeech(text: string): Promise<{ audioBase64: string; mimeType: string }> {
async function resolveTtsEndpoint(streaming: boolean): Promise<{ url: string; headers: Record<string, string> }> {
const config = await getVoiceConfig();
const signedIn = await isSignedIn();
let url: string;
let headers: Record<string, string>;
if (signedIn) {
const voiceId = config.elevenlabs?.voiceId || 's3TPKV1kjDlVtZbl4Ksh';
const accessToken = await getAccessToken();
url = `${API_URL}/v1/voice/text-to-speech/${voiceId}`;
headers = {
'Authorization': `Bearer ${accessToken}`,
'Content-Type': 'application/json',
// The proxy has no dedicated /stream route — the same endpoint is
// used and the body is consumed progressively; if the proxy buffers,
// streaming degrades to today's full-body latency, never worse.
return {
url: `${API_URL}/v1/voice/text-to-speech/${voiceId}`,
headers: {
'Authorization': `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
};
console.log('[voice] synthesizing speech via Rowboat proxy, text length:', text.length, 'voiceId:', voiceId);
} else {
if (!config.elevenlabs) {
throw new Error(`ElevenLabs not configured. Create ${path.join(WorkDir, 'config', 'elevenlabs.json')} with { "apiKey": "<your-key>" }`);
}
const voiceId = config.elevenlabs.voiceId || 's3TPKV1kjDlVtZbl4Ksh';
url = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`;
headers = {
}
if (!config.elevenlabs) {
throw new Error(`ElevenLabs not configured. Create ${path.join(WorkDir, 'config', 'elevenlabs.json')} with { "apiKey": "<your-key>" }`);
}
const voiceId = config.elevenlabs.voiceId || 's3TPKV1kjDlVtZbl4Ksh';
return {
url: `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}${streaming ? '/stream' : ''}`,
headers: {
'xi-api-key': config.elevenlabs.apiKey,
'Content-Type': 'application/json',
};
console.log('[voice] synthesizing speech via ElevenLabs, text length:', text.length, 'voiceId:', voiceId);
}
},
};
}
function ttsRequestBody(text: string): string {
return JSON.stringify({
text,
model_id: 'eleven_flash_v2_5',
voice_settings: {
stability: 0.5,
similarity_boost: 0.75,
},
});
}
export async function synthesizeSpeech(text: string): Promise<{ audioBase64: string; mimeType: string }> {
const { url, headers } = await resolveTtsEndpoint(false);
console.log('[voice] synthesizing speech, text length:', text.length);
const response = await fetch(url, {
method: 'POST',
headers,
body: JSON.stringify({
text,
model_id: 'eleven_flash_v2_5',
voice_settings: {
stability: 0.5,
similarity_boost: 0.75,
},
}),
body: ttsRequestBody(text),
});
if (!response.ok) {
@ -85,3 +96,42 @@ export async function synthesizeSpeech(text: string): Promise<{ audioBase64: str
console.log('[voice] synthesized audio, base64 length:', audioBase64.length);
return { audioBase64, mimeType: 'audio/mpeg' };
}
/**
* Streaming synthesis: invokes `onChunk` with MP3 bytes as they arrive so
* playback can start on the first chunk. Resolves when the stream ends;
* rejects on HTTP/stream errors. Abort via the provided signal.
*/
export async function synthesizeSpeechStream(
text: string,
onChunk: (chunk: Buffer) => void,
signal?: AbortSignal,
): Promise<void> {
const { url, headers } = await resolveTtsEndpoint(true);
console.log('[voice] streaming speech synthesis, text length:', text.length);
const response = await fetch(url, {
method: 'POST',
headers,
body: ttsRequestBody(text),
signal: signal ?? null,
});
if (!response.ok) {
const errText = await response.text().catch(() => 'Unknown error');
console.error('[voice] TTS stream API error:', response.status, errText);
throw new Error(`TTS API error ${response.status}: ${errText}`);
}
if (!response.body) {
throw new Error('TTS API returned no body');
}
const reader = response.body.getReader();
for (;;) {
const { done, value } = await reader.read();
if (done) break;
if (value && value.byteLength > 0) {
onChunk(Buffer.from(value));
}
}
}