From b3330c09c450396fbf954671b31c026999ddb10f Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Thu, 9 Jul 2026 22:31:02 +0530 Subject: [PATCH 1/6] =?UTF-8?q?refactor(x):=20agent=20registry=20=E2=80=94?= =?UTF-8?q?=20one=20table=20instead=20of=20the=20loadAgent=20ladder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Built-in agent identity moves to agents/registry.ts: a table of definitions with builders, an alias (rowboatx → copilot) expressed as shared entries, and declared traits. One prompt-file loader replaces the five copy-pasted frontmatter-parsing blocks, and the stringly "copilot" || "rowboatx" comparisons in the resolver and the legacy runtime become hasWorkspaceContext(agentId) trait lookups. loadAgent keeps its signature and its legacy import path via re-export; the user-agents repo fallback resolves the container lazily so the registry adds no static edge into the DI graph. Co-Authored-By: Claude Fable 5 --- .../packages/core/src/agents/registry.test.ts | 80 +++++++++ apps/x/packages/core/src/agents/registry.ts | 96 ++++++++++ apps/x/packages/core/src/agents/runtime.ts | 166 +----------------- .../src/turns/bridges/real-agent-resolver.ts | 9 +- 4 files changed, 186 insertions(+), 165 deletions(-) create mode 100644 apps/x/packages/core/src/agents/registry.test.ts create mode 100644 apps/x/packages/core/src/agents/registry.ts diff --git a/apps/x/packages/core/src/agents/registry.test.ts b/apps/x/packages/core/src/agents/registry.test.ts new file mode 100644 index 00000000..dffe0994 --- /dev/null +++ b/apps/x/packages/core/src/agents/registry.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { + agentFromRaw, + builtinAgentIds, + hasWorkspaceContext, + loadAgent, +} from "./registry.js"; + +describe("agent registry", () => { + it("knows every historical builtin id, including the rowboatx alias", () => { + expect(builtinAgentIds().sort()).toEqual( + [ + "copilot", + "rowboatx", + "live-note-agent", + "background-task-agent", + "note_creation", + "note_curation", + "labeling_agent", + "note_tagging_agent", + "inline_task_agent", + "agent_notes_agent", + ].sort(), + ); + }); + + it("grants workspace context to the copilot ids only", () => { + expect(hasWorkspaceContext("copilot")).toBe(true); + expect(hasWorkspaceContext("rowboatx")).toBe(true); + expect(hasWorkspaceContext("background-task-agent")).toBe(false); + expect(hasWorkspaceContext("note_creation")).toBe(false); + expect(hasWorkspaceContext("some-user-agent")).toBe(false); + expect(hasWorkspaceContext(null)).toBe(false); + expect(hasWorkspaceContext(undefined)).toBe(false); + }); + + it("loads the prompt-file knowledge agents with frontmatter config applied", async () => { + // These ship as raw strings with YAML frontmatter; the shared loader + // replaces five copy-pasted parsing blocks, so pin its behavior on + // the real bundled prompts. + for (const id of [ + "note_creation", + "note_curation", + "labeling_agent", + "note_tagging_agent", + "inline_task_agent", + "agent_notes_agent", + ]) { + const agent = await loadAgent(id); + expect(agent.name).toBe(id); + expect(agent.instructions.length).toBeGreaterThan(0); + // Frontmatter must be consumed, not left in the instructions. + expect(agent.instructions.startsWith("---")).toBe(false); + } + }); + + it("agentFromRaw parses frontmatter into agent config and strips it", () => { + const raw = [ + "---", + "tools:", + " file-readText:", + " type: builtin", + " name: file-readText", + "---", + "Do the thing.", + ].join("\n"); + const agent = agentFromRaw("tester", raw); + expect(agent.name).toBe("tester"); + expect(agent.instructions).toBe("Do the thing."); + expect(agent.tools).toEqual({ + "file-readText": { type: "builtin", name: "file-readText" }, + }); + }); + + it("agentFromRaw passes frontmatter-less prompts through verbatim", () => { + const agent = agentFromRaw("plain", "Just instructions."); + expect(agent.instructions).toBe("Just instructions."); + expect(agent.tools).toBeUndefined(); + }); +}); diff --git a/apps/x/packages/core/src/agents/registry.ts b/apps/x/packages/core/src/agents/registry.ts new file mode 100644 index 00000000..e8ab223e --- /dev/null +++ b/apps/x/packages/core/src/agents/registry.ts @@ -0,0 +1,96 @@ +import type { z } from "zod"; +import { parse } from "yaml"; +import { Agent } from "@x/shared/dist/agent.js"; +import { buildCopilotAgent } from "../application/assistant/agent.js"; +import { buildBackgroundTaskAgent } from "../background-tasks/agent.js"; +import { buildLiveNoteAgent } from "../knowledge/live-note/agent.js"; +import { getRaw as getNoteCreationRaw } from "../knowledge/note_creation.js"; +import { getRaw as getNoteCurationRaw } from "../knowledge/note_curation.js"; +import { getRaw as getLabelingAgentRaw } from "../knowledge/labeling_agent.js"; +import { getRaw as getNoteTaggingAgentRaw } from "../knowledge/note_tagging_agent.js"; +import { getRaw as getInlineTaskAgentRaw } from "../knowledge/inline_task_agent.js"; +import { getRaw as getAgentNotesAgentRaw } from "../knowledge/agent_notes_agent.js"; +import type { IAgentsRepo } from "./repo.js"; + +// The registry of built-in agents: one table instead of the historical +// if (id === ...) ladder. An entry owns its builder and its traits; traits +// replace stringly-typed id comparisons ("copilot" || "rowboatx") scattered +// across the assembly layer. User-defined agents are the fallthrough, +// fetched from the agents repo. + +export interface AgentTraits { + // Receives workspace context — agent notes and the user work directory — + // composed into its system prompt. + workspaceContext?: boolean; +} + +interface BuiltinAgentDefinition { + build: () => Promise> | z.infer; + traits?: AgentTraits; +} + +// Prompt-file agents: instructions ship as a raw string whose optional YAML +// frontmatter carries agent config (tools, model). One loader replaces the +// five copy-pasted parsing blocks the ladder accumulated. +export function agentFromRaw(id: string, raw: string): z.infer { + let agent: z.infer = { name: id, instructions: raw }; + if (raw.startsWith("---")) { + const end = raw.indexOf("\n---", 3); + if (end !== -1) { + const frontmatter = raw.slice(3, end).trim(); + const content = raw.slice(end + 4).trim(); + const yaml: unknown = parse(frontmatter); + const parsed = Agent.omit({ name: true, instructions: true }).parse(yaml); + agent = { ...agent, ...parsed, instructions: content }; + } + } + return agent; +} + +function promptFileAgent(id: string, getRaw: () => string): BuiltinAgentDefinition { + return { build: () => agentFromRaw(id, getRaw()) }; +} + +// "rowboatx" is a legacy alias for the copilot: both ids share one +// definition object. +const COPILOT: BuiltinAgentDefinition = { + build: buildCopilotAgent, + traits: { workspaceContext: true }, +}; + +const builtinAgents: Record = { + copilot: COPILOT, + rowboatx: COPILOT, + "live-note-agent": { build: buildLiveNoteAgent }, + "background-task-agent": { build: buildBackgroundTaskAgent }, + note_creation: promptFileAgent("note_creation", getNoteCreationRaw), + note_curation: promptFileAgent("note_curation", getNoteCurationRaw), + labeling_agent: promptFileAgent("labeling_agent", getLabelingAgentRaw), + note_tagging_agent: promptFileAgent("note_tagging_agent", getNoteTaggingAgentRaw), + inline_task_agent: promptFileAgent("inline_task_agent", getInlineTaskAgentRaw), + agent_notes_agent: promptFileAgent("agent_notes_agent", getAgentNotesAgentRaw), +}; + +export function builtinAgentIds(): string[] { + return Object.keys(builtinAgents); +} + +// Trait lookup for assembly decisions. Unknown/user agents have no traits. +export function hasWorkspaceContext(agentId: string | null | undefined): boolean { + return ( + agentId != null && + builtinAgents[agentId]?.traits?.workspaceContext === true + ); +} + +export async function loadAgent(id: string): Promise> { + const builtin = builtinAgents[id]; + if (builtin) { + return builtin.build(); + } + // User-defined agents. The container is imported lazily so this module + // adds no static edge into the DI graph (mirrors spawn-agent). + const { default: container } = await import("../di/container.js"); + const repo = container.resolve("agentsRepo"); + return repo.fetch(id); +} diff --git a/apps/x/packages/core/src/agents/runtime.ts b/apps/x/packages/core/src/agents/runtime.ts index 95cdd51b..d4a8fa77 100644 --- a/apps/x/packages/core/src/agents/runtime.ts +++ b/apps/x/packages/core/src/agents/runtime.ts @@ -11,19 +11,15 @@ import { execTool } from "../application/lib/exec-tool.js"; import { TOOL_ADDITIONS_KEY } from "../application/lib/tool-additions.js"; import { AskHumanRequestEvent, RunEvent, ToolPermissionMetadata, ToolPermissionRequestEvent } from "@x/shared/dist/runs.js"; import { BuiltinTools } from "../application/lib/builtin-tools.js"; -import { buildCopilotAgent } from "../application/assistant/agent.js"; -import { buildLiveNoteAgent } from "../knowledge/live-note/agent.js"; -import { buildBackgroundTaskAgent } from "../background-tasks/agent.js"; +import { hasWorkspaceContext, loadAgent } from "./registry.js"; import { isBlocked, extractCommandNames } from "../application/lib/command-executor.js"; import { getFileAccessAllowList, type FileAccessGrant, type FileAccessOperation } from "../config/security.js"; import { resolveFilePathForPermission } from "../filesystem/files.js"; -import container from "../di/container.js"; import { notifyIfEnabled } from "../application/notification/notifier.js"; import { IModelConfigRepo } from "../models/repo.js"; import { createLanguageModel } from "../models/models.js"; import { chatActivity } from "../application/lib/chat-activity.js"; import { resolveProviderConfig } from "../models/defaults.js"; -import { IAgentsRepo } from "./repo.js"; import { IMonotonicallyIncreasingIdGenerator } from "../application/lib/id-gen.js"; import { IBus } from "../application/lib/bus.js"; import { IMessageQueue, type MiddlePaneContext } from "../application/lib/message-queue.js"; @@ -31,15 +27,8 @@ import { IRunsRepo } from "../runs/repo.js"; import { IRunsLock } from "../runs/lock.js"; import { IAbortRegistry } from "../runs/abort-registry.js"; import { PrefixLogger } from "@x/shared"; -import { parse } from "yaml"; import { captureLlmUsage } from "../analytics/usage.js"; import { enterUseCase, withUseCase, type UseCase } from "../analytics/use_case.js"; -import { getRaw as getNoteCreationRaw } from "../knowledge/note_creation.js"; -import { getRaw as getNoteCurationRaw } from "../knowledge/note_curation.js"; -import { getRaw as getLabelingAgentRaw } from "../knowledge/labeling_agent.js"; -import { getRaw as getNoteTaggingAgentRaw } from "../knowledge/note_tagging_agent.js"; -import { getRaw as getInlineTaskAgentRaw } from "../knowledge/inline_task_agent.js"; -import { getRaw as getAgentNotesAgentRaw } from "../knowledge/agent_notes_agent.js"; import { classifyToolPermissions, type AutoPermissionCandidate } from "../security/auto-permission-classifier.js"; const AGENT_NOTES_DIR = path.join(WorkDir, 'knowledge', 'Agent Notes'); @@ -240,10 +229,6 @@ export function loadAgentNotesContext(): string | null { return `# Agent Memory\n\n${sections.join('\n\n')}`; } -function isCopilotLikeAgent(agentName: string | null | undefined): boolean { - return agentName === 'copilot' || agentName === 'rowboatx'; -} - function formatCurrentDateTime(now: Date): string { return now.toLocaleString('en-US', { weekday: 'long', @@ -283,7 +268,7 @@ function buildUserMessageContext({ }): z.infer { return { currentDateTime: formatCurrentDateTime(new Date()), - ...(isCopilotLikeAgent(agentName) + ...(hasWorkspaceContext(agentName) ? { middlePane: toUserMessageContextMiddlePane(middlePaneContext) } : {}), }; @@ -790,148 +775,9 @@ function formatLlmStreamError(rawError: unknown): string { return lines.length ? lines.join("\n") : "Model stream error"; } -export async function loadAgent(id: string): Promise> { - if (id === "copilot" || id === "rowboatx") { - return buildCopilotAgent(); - } - - if (id === "live-note-agent") { - return buildLiveNoteAgent(); - } - - if (id === "background-task-agent") { - return buildBackgroundTaskAgent(); - } - - if (id === 'note_creation' || id === 'note_curation') { - const raw = id === 'note_curation' ? getNoteCurationRaw() : getNoteCreationRaw(); - let agent: z.infer = { - name: id, - instructions: raw, - }; - - // Parse frontmatter if present - if (raw.startsWith("---")) { - const end = raw.indexOf("\n---", 3); - if (end !== -1) { - const fm = raw.slice(3, end).trim(); - const content = raw.slice(end + 4).trim(); - const yaml = parse(fm); - const parsed = Agent.omit({ name: true, instructions: true }).parse(yaml); - agent = { - ...agent, - ...parsed, - instructions: content, - }; - } - } - - return agent; - } - - if (id === 'labeling_agent') { - const labelingAgentRaw = getLabelingAgentRaw(); - let agent: z.infer = { - name: id, - instructions: labelingAgentRaw, - }; - - if (labelingAgentRaw.startsWith("---")) { - const end = labelingAgentRaw.indexOf("\n---", 3); - if (end !== -1) { - const fm = labelingAgentRaw.slice(3, end).trim(); - const content = labelingAgentRaw.slice(end + 4).trim(); - const yaml = parse(fm); - const parsed = Agent.omit({ name: true, instructions: true }).parse(yaml); - agent = { - ...agent, - ...parsed, - instructions: content, - }; - } - } - - return agent; - } - - if (id === 'note_tagging_agent') { - const noteTaggingAgentRaw = getNoteTaggingAgentRaw(); - let agent: z.infer = { - name: id, - instructions: noteTaggingAgentRaw, - }; - - if (noteTaggingAgentRaw.startsWith("---")) { - const end = noteTaggingAgentRaw.indexOf("\n---", 3); - if (end !== -1) { - const fm = noteTaggingAgentRaw.slice(3, end).trim(); - const content = noteTaggingAgentRaw.slice(end + 4).trim(); - const yaml = parse(fm); - const parsed = Agent.omit({ name: true, instructions: true }).parse(yaml); - agent = { - ...agent, - ...parsed, - instructions: content, - }; - } - } - - return agent; - } - - if (id === 'inline_task_agent') { - const inlineTaskAgentRaw = getInlineTaskAgentRaw(); - let agent: z.infer = { - name: id, - instructions: inlineTaskAgentRaw, - }; - - if (inlineTaskAgentRaw.startsWith("---")) { - const end = inlineTaskAgentRaw.indexOf("\n---", 3); - if (end !== -1) { - const fm = inlineTaskAgentRaw.slice(3, end).trim(); - const content = inlineTaskAgentRaw.slice(end + 4).trim(); - const yaml = parse(fm); - const parsed = Agent.omit({ name: true, instructions: true }).parse(yaml); - agent = { - ...agent, - ...parsed, - instructions: content, - }; - } - } - - return agent; - } - - if (id === 'agent_notes_agent') { - const agentNotesAgentRaw = getAgentNotesAgentRaw(); - let agent: z.infer = { - name: id, - instructions: agentNotesAgentRaw, - }; - - if (agentNotesAgentRaw.startsWith("---")) { - const end = agentNotesAgentRaw.indexOf("\n---", 3); - if (end !== -1) { - const fm = agentNotesAgentRaw.slice(3, end).trim(); - const content = agentNotesAgentRaw.slice(end + 4).trim(); - const yaml = parse(fm); - const parsed = Agent.omit({ name: true, instructions: true }).parse(yaml); - agent = { - ...agent, - ...parsed, - instructions: content, - }; - } - } - - return agent; - } - - const repo = container.resolve('agentsRepo'); - return await repo.fetch(id); -} +// Agent identity now lives in the registry (registry.ts); re-exported here +// so legacy callers keep their import path until the runs engine dies. +export { loadAgent }; function formatBytes(bytes: number): string { if (bytes < 1024) return `${bytes} B`; @@ -1604,7 +1450,7 @@ export async function* streamAgent({ loopLogger.log('running llm turn'); // stream agent response and build message const messageBuilder = new StreamStepMessageBuilder(); - const composeCopilotContext = state.agentName === 'copilot' || state.agentName === 'rowboatx'; + const composeCopilotContext = hasWorkspaceContext(state.agentName); const instructionsWithDateTime = composeSystemInstructions({ instructions: agent.instructions, agentNotesContext: composeCopilotContext ? loadAgentNotesContext() : null, diff --git a/apps/x/packages/core/src/turns/bridges/real-agent-resolver.ts b/apps/x/packages/core/src/turns/bridges/real-agent-resolver.ts index a2cdaf6f..7450d51a 100644 --- a/apps/x/packages/core/src/turns/bridges/real-agent-resolver.ts +++ b/apps/x/packages/core/src/turns/bridges/real-agent-resolver.ts @@ -8,10 +8,10 @@ import { } from "@x/shared/dist/turns.js"; import { composeSystemInstructions, - loadAgent, loadAgentNotesContext, loadUserWorkDir, } from "../../agents/runtime.js"; +import { hasWorkspaceContext, loadAgent } from "../../agents/registry.js"; import { BuiltinTools } from "../../application/lib/builtin-tools.js"; import { skillToolNames } from "../../application/assistant/skills/index.js"; import { getDefaultModelAndProvider } from "../../models/defaults.js"; @@ -124,10 +124,9 @@ export class RealAgentResolver { requested.overrides?.composition ?? {}, ); const composition = parsed.success ? parsed.data : {}; - // Agent notes and work-dir context are copilot-scoped, matching the - // old runtime's behavior. - const copilotContext = - requested.agentId === "copilot" || requested.agentId === "rowboatx"; + // Agent notes and work-dir context are scoped to agents with the + // workspaceContext trait (the copilot), per the agent registry. + const copilotContext = hasWorkspaceContext(requested.agentId); const systemPrompt = composeSystemInstructions({ instructions: agent.instructions, agentNotesContext: copilotContext ? this.loadNotes() : null, From bd34531305a590be8746d65f071b7389a9c4d900 Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Thu, 9 Jul 2026 22:32:47 +0530 Subject: [PATCH 2/6] refactor(x): extract the system-prompt composer from the legacy runtime file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit composeSystemInstructions (+ the hidden-user-context block) moves verbatim to agents/compose-instructions.ts, owned by the assembly layer; the legacy engine and old import paths keep working via re-export. New golden-bytes snapshot tests pin the composed output for a 12-case composition matrix plus the block ordering — the safety net for folding the mode blocks into capability records: prefix caching and snapshot inheritance require byte-identical prompts, so any restructuring must keep these snapshots green. Co-Authored-By: Claude Fable 5 --- .../compose-instructions.test.ts.snap | 526 ++++++++++++++++++ .../src/agents/compose-instructions.test.ts | 101 ++++ .../core/src/agents/compose-instructions.ts | 142 +++++ apps/x/packages/core/src/agents/runtime.ts | 142 +---- .../src/turns/bridges/real-agent-resolver.ts | 2 +- 5 files changed, 777 insertions(+), 136 deletions(-) create mode 100644 apps/x/packages/core/src/agents/__snapshots__/compose-instructions.test.ts.snap create mode 100644 apps/x/packages/core/src/agents/compose-instructions.test.ts create mode 100644 apps/x/packages/core/src/agents/compose-instructions.ts diff --git a/apps/x/packages/core/src/agents/__snapshots__/compose-instructions.test.ts.snap b/apps/x/packages/core/src/agents/__snapshots__/compose-instructions.test.ts.snap new file mode 100644 index 00000000..4a3dadc5 --- /dev/null +++ b/apps/x/packages/core/src/agents/__snapshots__/compose-instructions.test.ts.snap @@ -0,0 +1,526 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`composeSystemInstructions golden bytes > agent notes 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer. + +# Agent Memory + +Remember X." +`; + +exports[`composeSystemInstructions golden bytes > base: no modes active 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer." +`; + +exports[`composeSystemInstructions golden bytes > coach mode 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer. + +# Practice Session (Coach Mode) +The user started a practice session: they are rehearsing something performative — a pitch, presentation, interview answer, or talk — and want live coaching. You are their coach for this session. + +How to coach: +- Watch and listen for delivery: pacing, filler words, rambling, structure, and (from the webcam frames) posture, eye contact with the camera, facial expressiveness, gesturing, and visible energy. +- After each take or answer, give brief, specific, actionable feedback: 2-3 concrete observations, quoting what you actually saw or heard ("you looked down during the ask", "the opening 'um, so, basically' undercuts your hook"). Then let them go again. +- If they are clearly mid-flow, keep any interjection to one short sentence — or stay silent and save it for the break. +- Ask what they're practicing for at the start if it isn't obvious (investor pitch? interview? conference talk?) and tailor feedback to that audience. +- When they say they're done or wrap up, give a structured debrief: what's working, the top 3 things to improve, and one concrete drill or reframe to try next time. +- Be encouraging but honest — vague praise wastes their rehearsal time. Never comment on physical appearance; delivery, expression, and body language only." +`; + +exports[`composeSystemInstructions golden bytes > code mode claude with cwd 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer. + +# Code Mode (Active) — Agent: Claude Code +The user has turned on **code mode** and the composer chip is set to **Claude Code** (\`claude\`). For EVERY coding task this turn, use **Claude Code**, and narrate that agent ("Using Claude Code to …"). + +The chip is the single source of truth for which agent runs: +- Do NOT carry over a different agent from earlier in this thread — even if a previous run used the other agent, use **Claude Code** now. +- Do NOT switch agents based on an in-chat text request ("use codex", "switch to claude"). The agent only changes when the user toggles the chip; if they ask in chat, tell them to toggle the chip. + +**How to run coding work — call the \`code_agent_run\` tool** with: +- \`agent\`: \`claude\` (always — match the chip). +- \`cwd\`: \`/tmp/project\` (always — this coding session is pinned to that directory; never use another path). +- \`prompt\`: a clear, self-contained coding instruction. + +The tool runs the agent on-device and streams its tool calls, file diffs, and plan into the chat; any action needing approval surfaces as an inline permission card, so you do NOT pre-confirm with an in-chat "reply yes". This chat keeps ONE persistent agent session, so follow-up coding requests automatically resume with full context — just call \`code_agent_run\` again. Do NOT shell out to \`acpx\` or \`executeCommand\` for coding, and do NOT fall back to your own file tools. + +If the user's message is clearly NOT a coding request (small talk, an unrelated question), answer directly without invoking the coding agent. Code mode signals readiness, not that every message must route through the agent." +`; + +exports[`composeSystemInstructions golden bytes > code mode codex without cwd 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer. + +# Code Mode (Active) — Agent: Codex +The user has turned on **code mode** and the composer chip is set to **Codex** (\`codex\`). For EVERY coding task this turn, use **Codex**, and narrate that agent ("Using Codex to …"). + +The chip is the single source of truth for which agent runs: +- Do NOT carry over a different agent from earlier in this thread — even if a previous run used the other agent, use **Codex** now. +- Do NOT switch agents based on an in-chat text request ("use codex", "switch to claude"). The agent only changes when the user toggles the chip; if they ask in chat, tell them to toggle the chip. + +**How to run coding work — call the \`code_agent_run\` tool** with: +- \`agent\`: \`codex\` (always — match the chip). +- \`cwd\`: the absolute project/working directory (resolve it per the code-with-agents skill — a path the user named, the "# User Work Directory" block, or ask once). +- \`prompt\`: a clear, self-contained coding instruction. + +The tool runs the agent on-device and streams its tool calls, file diffs, and plan into the chat; any action needing approval surfaces as an inline permission card, so you do NOT pre-confirm with an in-chat "reply yes". This chat keeps ONE persistent agent session, so follow-up coding requests automatically resume with full context — just call \`code_agent_run\` again. Do NOT shell out to \`acpx\` or \`executeCommand\` for coding, and do NOT fall back to your own file tools. + +If the user's message is clearly NOT a coding request (small talk, an unrelated question), answer directly without invoking the coding agent. Code mode signals readiness, not that every message must route through the agent." +`; + +exports[`composeSystemInstructions golden bytes > everything on 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer. + +# Agent Memory + +Remember X. + +# User Work Directory +The user has chosen the following directory as their current **work directory**: + +\`/Users/test/Documents/Work\` + +Treat this as the **default location** for file operations whenever the user refers to files generically: +- "list the files", "show me what's in here", "what's the latest report" — list or look in the work directory. +- "save this", "export it", "write that to a file" — write the output into the work directory unless the user names another location. +- "open the file I was just working on", "the doc from earlier" — assume the work directory first. + +Use absolute paths rooted at this directory with the \`file-*\` tools. For example, list with \`file-list({ path: "/Users/test/Documents/Work" })\`, read text with \`file-readText\`, and write text with \`file-writeText\`. For PDFs, Office docs, images, scanned docs, and other non-text files, use \`parseFile\` or \`LLMParse\` with the absolute path; you do NOT need to copy the file into the workspace first. + +**Exceptions — these ALWAYS take precedence over the work directory default:** +1. **Knowledge base questions.** If the user asks about anything in the knowledge graph (notes, people, organizations, projects, topics) or paths starting with \`knowledge/\`, use file tools against \`knowledge/\` as documented above. Do NOT redirect those into the work directory. +2. **Explicit paths.** If the user names a different directory or gives an absolute/relative path (e.g. "in ~/Downloads", "from /tmp/foo", "the Desktop"), honor that path exactly and ignore the work-directory default for that request. +3. **Workspace-specific operations.** Anything that obviously belongs in the Rowboat workspace (config files, MCP servers, agent schedules, etc.) stays in the workspace, not the work directory. + +Do not announce the work directory unless it's relevant. Just use it. + +# Voice Input +The user's message was transcribed from speech. Be aware that: +- There may be transcription errors. Silently correct obvious ones (e.g. homophones, misheard words). If an error is genuinely ambiguous, briefly mention your interpretation (e.g. "I'm assuming you meant X"). +- Spoken messages are often long-winded. The user may ramble, repeat themselves, or correct something they said earlier in the same message. Focus on their final intent, not every word verbatim. + +# Video Mode (Live Camera) +The user has turned on video mode: their webcam is on, and their messages arrive with a series of live webcam frames (ordered oldest to newest) captured while they were speaking or typing. The frames are a live view of the user themselves — not a document or file to analyze. + +How to use the frames: +- Use them for what visual awareness genuinely adds: the user's expressions, body language, posture, gestures, eye contact with the camera, visible energy, anything they hold up to the camera, and their surroundings when relevant. +- Compare across frames to notice change over time within a message (e.g. increasingly slouched, started smiling, looked away for most of the message). +- When the user practices something performative (a pitch, presentation, interview, talk) or asks for delivery feedback, give specific, actionable coaching grounded in what you actually see — cite concrete observations ("in the last few frames you looked down and away from the camera") rather than generic advice. Cover posture, eye contact, facial expressiveness, gesturing, and visible energy. +- If they show something to the camera (an object, document, whiteboard), read or describe it and respond accordingly. + +Driving the app: +- You can control the Rowboat app the user is looking at via the app-navigation tool (load the app-navigation skill first): open views, READ a view's contents as data (emails, background agents, chat history), and open specific items on their screen. +- When the user asks about anything that lives inside Rowboat ("what emails do I have?", "what agents are running?", "open the one from Arjun"), prefer driving over describing: read-view shows the view on their screen while returning its data, then answer out loud briefly; open-item when they pick one. Narrate as you act ("pulling up your inbox…"). Reading a view's data beats squinting at screen-share frames — it's exact. + +Screen sharing: +- The user may also share their screen. Screen-share frames arrive in a separately labeled group after the webcam frames; they show the user's screen, not the user. +- When screen frames are present, treat them as the primary subject: the user is usually asking about, or working on, what's visible there. Read the screen carefully — code, documents, error messages, UI state — and help with it concretely. +- The LAST screen frame is the most current view of their screen; earlier ones show how it changed while they spoke. +- Frames are downscaled captures: small text may be hard to read. If something crucial is illegible, say what you need ("zoom into the error message" / "make that panel bigger") rather than guessing. + +Etiquette: +- Do not narrate or list what you see in every response. Bring up visual observations only when relevant to the user's request or clearly worth mentioning. +- Never comment on the user's physical appearance, attractiveness, or personal attributes — visual feedback is strictly about delivery, expression, and body language. +- Frames are periodic snapshots, not continuous video; moments between frames are missing. Don't claim certainty about motion you couldn't have seen. + +# Practice Session (Coach Mode) +The user started a practice session: they are rehearsing something performative — a pitch, presentation, interview answer, or talk — and want live coaching. You are their coach for this session. + +How to coach: +- Watch and listen for delivery: pacing, filler words, rambling, structure, and (from the webcam frames) posture, eye contact with the camera, facial expressiveness, gesturing, and visible energy. +- After each take or answer, give brief, specific, actionable feedback: 2-3 concrete observations, quoting what you actually saw or heard ("you looked down during the ask", "the opening 'um, so, basically' undercuts your hook"). Then let them go again. +- If they are clearly mid-flow, keep any interjection to one short sentence — or stay silent and save it for the break. +- Ask what they're practicing for at the start if it isn't obvious (investor pitch? interview? conference talk?) and tailor feedback to that audience. +- When they say they're done or wrap up, give a structured debrief: what's working, the top 3 things to improve, and one concrete drill or reframe to try next time. +- Be encouraging but honest — vague praise wastes their rehearsal time. Never comment on physical appearance; delivery, expression, and body language only. + +# Voice Output (MANDATORY — READ THIS FIRST) +The user has voice output enabled. THIS IS YOUR #1 PRIORITY: you MUST start your response with tags. If your response does not begin with tags, the user will hear nothing — which is a broken experience. NEVER skip this. + +Rules: +1. YOUR VERY FIRST OUTPUT MUST BE A TAG. No exceptions. Do not start with markdown, headings, or any other text. The literal first characters of your response must be "". +2. Place ALL tags at the BEGINNING of your response, before any detailed content. Do NOT intersperse tags throughout the response. +3. Wrap EACH spoken sentence in its own separate tag so it can be spoken incrementally. Do NOT wrap everything in a single block. +4. Use voice as a TL;DR and navigation aid — do NOT read the entire response aloud. +5. After all tags, you may include detailed written content (markdown, tables, code, etc.) that will be shown visually but not spoken. + +## Examples + +Example 1 — User asks: "what happened in my meeting with Alex yesterday?" + +Your meeting with Alex covered three main things: the Q2 roadmap timeline, hiring for the backend role, and the client demo next week. +I've pulled out the key details and action items below — the demo prep notes are at the end. + +## Meeting with Alex — March 11 +### Roadmap +- Agreed to push Q2 launch to April 15... +(detailed written content continues) + +Example 2 — User asks: "summarize my emails" + +You have five new emails since this morning. +Two are from your team — Jordan sent the RFC you requested and Taylor flagged a contract issue. +There's also a warm intro from a VC partner connecting you with someone at a prospective customer. +I've drafted responses for three of them. The details and drafts are below. + +(email blocks, tables, and detailed content follow) + +Example 3 — User asks: "what's on my calendar today?" + +You've got a pretty packed day — seven meetings starting with standup at 9. +The big ones are your investor call at 11, lunch with a partner from your lead VC at 12:30, and a customer call at 4. +Your only free block for deep work is 2:30 to 4. + +(calendar block with full event details follows) + +Example 4 — User asks: "draft an email to Sam with our metrics" + +Done — I've drafted the email to Sam with your latest WAU and churn numbers. +Take a look at the draft below and send it when you're ready. + +(email block with draft follows) + +REMEMBER: If you do not start with tags, the user hears silence. Always speak first, then write. + +# Search +The user has requested a search. Use the web-search tool to answer their query. + +# Code Mode (Active) — Agent: Claude Code +The user has turned on **code mode** and the composer chip is set to **Claude Code** (\`claude\`). For EVERY coding task this turn, use **Claude Code**, and narrate that agent ("Using Claude Code to …"). + +The chip is the single source of truth for which agent runs: +- Do NOT carry over a different agent from earlier in this thread — even if a previous run used the other agent, use **Claude Code** now. +- Do NOT switch agents based on an in-chat text request ("use codex", "switch to claude"). The agent only changes when the user toggles the chip; if they ask in chat, tell them to toggle the chip. + +**How to run coding work — call the \`code_agent_run\` tool** with: +- \`agent\`: \`claude\` (always — match the chip). +- \`cwd\`: \`/tmp/project\` (always — this coding session is pinned to that directory; never use another path). +- \`prompt\`: a clear, self-contained coding instruction. + +The tool runs the agent on-device and streams its tool calls, file diffs, and plan into the chat; any action needing approval surfaces as an inline permission card, so you do NOT pre-confirm with an in-chat "reply yes". This chat keeps ONE persistent agent session, so follow-up coding requests automatically resume with full context — just call \`code_agent_run\` again. Do NOT shell out to \`acpx\` or \`executeCommand\` for coding, and do NOT fall back to your own file tools. + +If the user's message is clearly NOT a coding request (small talk, an unrelated question), answer directly without invoking the coding agent. Code mode signals readiness, not that every message must route through the agent." +`; + +exports[`composeSystemInstructions golden bytes > search enabled 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer. + +# Search +The user has requested a search. Use the web-search tool to answer their query." +`; + +exports[`composeSystemInstructions golden bytes > video mode 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer. + +# Video Mode (Live Camera) +The user has turned on video mode: their webcam is on, and their messages arrive with a series of live webcam frames (ordered oldest to newest) captured while they were speaking or typing. The frames are a live view of the user themselves — not a document or file to analyze. + +How to use the frames: +- Use them for what visual awareness genuinely adds: the user's expressions, body language, posture, gestures, eye contact with the camera, visible energy, anything they hold up to the camera, and their surroundings when relevant. +- Compare across frames to notice change over time within a message (e.g. increasingly slouched, started smiling, looked away for most of the message). +- When the user practices something performative (a pitch, presentation, interview, talk) or asks for delivery feedback, give specific, actionable coaching grounded in what you actually see — cite concrete observations ("in the last few frames you looked down and away from the camera") rather than generic advice. Cover posture, eye contact, facial expressiveness, gesturing, and visible energy. +- If they show something to the camera (an object, document, whiteboard), read or describe it and respond accordingly. + +Driving the app: +- You can control the Rowboat app the user is looking at via the app-navigation tool (load the app-navigation skill first): open views, READ a view's contents as data (emails, background agents, chat history), and open specific items on their screen. +- When the user asks about anything that lives inside Rowboat ("what emails do I have?", "what agents are running?", "open the one from Arjun"), prefer driving over describing: read-view shows the view on their screen while returning its data, then answer out loud briefly; open-item when they pick one. Narrate as you act ("pulling up your inbox…"). Reading a view's data beats squinting at screen-share frames — it's exact. + +Screen sharing: +- The user may also share their screen. Screen-share frames arrive in a separately labeled group after the webcam frames; they show the user's screen, not the user. +- When screen frames are present, treat them as the primary subject: the user is usually asking about, or working on, what's visible there. Read the screen carefully — code, documents, error messages, UI state — and help with it concretely. +- The LAST screen frame is the most current view of their screen; earlier ones show how it changed while they spoke. +- Frames are downscaled captures: small text may be hard to read. If something crucial is illegible, say what you need ("zoom into the error message" / "make that panel bigger") rather than guessing. + +Etiquette: +- Do not narrate or list what you see in every response. Bring up visual observations only when relevant to the user's request or clearly worth mentioning. +- Never comment on the user's physical appearance, attractiveness, or personal attributes — visual feedback is strictly about delivery, expression, and body language. +- Frames are periodic snapshots, not continuous video; moments between frames are missing. Don't claim certainty about motion you couldn't have seen." +`; + +exports[`composeSystemInstructions golden bytes > voice input 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer. + +# Voice Input +The user's message was transcribed from speech. Be aware that: +- There may be transcription errors. Silently correct obvious ones (e.g. homophones, misheard words). If an error is genuinely ambiguous, briefly mention your interpretation (e.g. "I'm assuming you meant X"). +- Spoken messages are often long-winded. The user may ramble, repeat themselves, or correct something they said earlier in the same message. Focus on their final intent, not every word verbatim." +`; + +exports[`composeSystemInstructions golden bytes > voice output full 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer. + +# Voice Output — Full Read-Aloud (MANDATORY — READ THIS FIRST) +The user wants your ENTIRE response spoken aloud. THIS IS YOUR #1 PRIORITY: every single sentence must be wrapped in tags. If you write anything outside tags, the user will not hear it — which is a broken experience. NEVER skip this. + +Rules: +1. YOUR VERY FIRST OUTPUT MUST BE A TAG. No exceptions. The literal first characters of your response must be "". +2. Wrap EACH sentence in its own separate tag so it can be spoken incrementally. +3. Write your response in a natural, conversational style suitable for listening — no markdown headings, bullet points, or formatting symbols. Use plain spoken language. +4. Structure the content as if you are speaking to the user directly. Use transitions like "first", "also", "one more thing" instead of visual formatting. +5. EVERY sentence MUST be inside a tag. Do not leave ANY content outside tags. If it's not in a tag, the user cannot hear it. + +## Examples + +Example 1 — User asks: "what happened in my meeting with Alex yesterday?" + +Your meeting with Alex covered three main things. +First, you discussed the Q2 roadmap timeline and agreed to push the launch to April. +Second, you talked about hiring for the backend role — Alex will send over two candidates by Friday. +And lastly, the client demo is next week on Thursday at 2pm, and you're handling the intro slides. + +Example 2 — User asks: "summarize my emails" + +You've got five new emails since this morning. +Two are from your team — Jordan sent the RFC you asked for, and Taylor flagged a contract issue that needs your sign-off. +There's a warm intro from a VC partner connecting you with an engineering lead at a potential customer. +And someone from a prospective client wants to confirm your API tier before your call this afternoon. +I've drafted replies for three of them — the metrics update, the intro, and the API question. +The only one I left for you is Taylor's contract redline, since that needs your judgment on the liability cap. + +Example 3 — User asks: "what's on my calendar today?" + +You've got a packed day — seven meetings starting with standup at 9. +The highlights are your investor call at 11, lunch with a VC partner at 12:30, and a customer call at 4. +Your only open block for deep work is 2:30 to 4, so plan accordingly. +Oh, and your 1-on-1 with your co-founder is at 5:30 — that's a walking meeting. + +Example 4 — User asks: "how are our metrics looking?" + +Metrics are looking strong this week. +You hit 2,573 weekly active users, which is up 12% week over week. +That means you've crossed the 2,500 milestone — worth calling out in your next investor update. +Churn is down to 4.1%, improving month over month. +The trailing 8-week compound growth rate is about 10%. + +REMEMBER: Start with immediately. No preamble, no markdown before it. Speak first." +`; + +exports[`composeSystemInstructions golden bytes > voice output summary 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer. + +# Voice Output (MANDATORY — READ THIS FIRST) +The user has voice output enabled. THIS IS YOUR #1 PRIORITY: you MUST start your response with tags. If your response does not begin with tags, the user will hear nothing — which is a broken experience. NEVER skip this. + +Rules: +1. YOUR VERY FIRST OUTPUT MUST BE A TAG. No exceptions. Do not start with markdown, headings, or any other text. The literal first characters of your response must be "". +2. Place ALL tags at the BEGINNING of your response, before any detailed content. Do NOT intersperse tags throughout the response. +3. Wrap EACH spoken sentence in its own separate tag so it can be spoken incrementally. Do NOT wrap everything in a single block. +4. Use voice as a TL;DR and navigation aid — do NOT read the entire response aloud. +5. After all tags, you may include detailed written content (markdown, tables, code, etc.) that will be shown visually but not spoken. + +## Examples + +Example 1 — User asks: "what happened in my meeting with Alex yesterday?" + +Your meeting with Alex covered three main things: the Q2 roadmap timeline, hiring for the backend role, and the client demo next week. +I've pulled out the key details and action items below — the demo prep notes are at the end. + +## Meeting with Alex — March 11 +### Roadmap +- Agreed to push Q2 launch to April 15... +(detailed written content continues) + +Example 2 — User asks: "summarize my emails" + +You have five new emails since this morning. +Two are from your team — Jordan sent the RFC you requested and Taylor flagged a contract issue. +There's also a warm intro from a VC partner connecting you with someone at a prospective customer. +I've drafted responses for three of them. The details and drafts are below. + +(email blocks, tables, and detailed content follow) + +Example 3 — User asks: "what's on my calendar today?" + +You've got a pretty packed day — seven meetings starting with standup at 9. +The big ones are your investor call at 11, lunch with a partner from your lead VC at 12:30, and a customer call at 4. +Your only free block for deep work is 2:30 to 4. + +(calendar block with full event details follows) + +Example 4 — User asks: "draft an email to Sam with our metrics" + +Done — I've drafted the email to Sam with your latest WAU and churn numbers. +Take a look at the draft below and send it when you're ready. + +(email block with draft follows) + +REMEMBER: If you do not start with tags, the user hears silence. Always speak first, then write." +`; + +exports[`composeSystemInstructions golden bytes > work directory 1`] = ` +"You are the test agent. + +# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer. + +# User Work Directory +The user has chosen the following directory as their current **work directory**: + +\`/Users/test/Documents/Work\` + +Treat this as the **default location** for file operations whenever the user refers to files generically: +- "list the files", "show me what's in here", "what's the latest report" — list or look in the work directory. +- "save this", "export it", "write that to a file" — write the output into the work directory unless the user names another location. +- "open the file I was just working on", "the doc from earlier" — assume the work directory first. + +Use absolute paths rooted at this directory with the \`file-*\` tools. For example, list with \`file-list({ path: "/Users/test/Documents/Work" })\`, read text with \`file-readText\`, and write text with \`file-writeText\`. For PDFs, Office docs, images, scanned docs, and other non-text files, use \`parseFile\` or \`LLMParse\` with the absolute path; you do NOT need to copy the file into the workspace first. + +**Exceptions — these ALWAYS take precedence over the work directory default:** +1. **Knowledge base questions.** If the user asks about anything in the knowledge graph (notes, people, organizations, projects, topics) or paths starting with \`knowledge/\`, use file tools against \`knowledge/\` as documented above. Do NOT redirect those into the work directory. +2. **Explicit paths.** If the user names a different directory or gives an absolute/relative path (e.g. "in ~/Downloads", "from /tmp/foo", "the Desktop"), honor that path exactly and ignore the work-directory default for that request. +3. **Workspace-specific operations.** Anything that obviously belongs in the Rowboat workspace (config files, MCP servers, agent schedules, etc.) stays in the workspace, not the work directory. + +Do not announce the work directory unless it's relevant. Just use it." +`; diff --git a/apps/x/packages/core/src/agents/compose-instructions.test.ts b/apps/x/packages/core/src/agents/compose-instructions.test.ts new file mode 100644 index 00000000..30d2e4e0 --- /dev/null +++ b/apps/x/packages/core/src/agents/compose-instructions.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; +import { + composeSystemInstructions, + type ComposeSystemInstructionsInput, +} from "./compose-instructions.js"; + +// Golden-bytes characterization of system-prompt composition. These +// snapshots pin the EXACT output for a representative composition matrix: +// provider prefix caching and agent-snapshot inheritance both depend on +// byte-identical prompts for identical inputs, so any restructuring of the +// composer (capability folding) must keep these snapshots green. An +// intentional prompt-text change is allowed — but it must be intentional, +// reviewed as a snapshot diff. + +function input( + overrides: Partial = {}, +): ComposeSystemInstructionsInput { + return { + instructions: "You are the test agent.", + agentNotesContext: null, + userWorkDir: null, + voiceInput: false, + voiceOutput: null, + searchEnabled: false, + codeMode: null, + codeCwd: null, + videoMode: false, + coachMode: false, + ...overrides, + }; +} + +const MATRIX: Array<[name: string, overrides: Partial]> = [ + ["base: no modes active", {}], + ["agent notes", { agentNotesContext: "# Agent Memory\n\nRemember X." }], + ["work directory", { userWorkDir: "/Users/test/Documents/Work" }], + ["voice input", { voiceInput: true }], + ["voice output summary", { voiceOutput: "summary" }], + ["voice output full", { voiceOutput: "full" }], + ["search enabled", { searchEnabled: true }], + ["video mode", { videoMode: true }], + ["coach mode", { coachMode: true }], + ["code mode claude with cwd", { codeMode: "claude", codeCwd: "/tmp/project" }], + ["code mode codex without cwd", { codeMode: "codex" }], + [ + "everything on", + { + agentNotesContext: "# Agent Memory\n\nRemember X.", + userWorkDir: "/Users/test/Documents/Work", + voiceInput: true, + voiceOutput: "summary", + searchEnabled: true, + codeMode: "claude", + codeCwd: "/tmp/project", + videoMode: true, + coachMode: true, + }, + ], +]; + +describe("composeSystemInstructions golden bytes", () => { + for (const [name, overrides] of MATRIX) { + it(name, () => { + expect(composeSystemInstructions(input(overrides))).toMatchSnapshot(); + }); + } + + it("mode blocks append in a fixed order after the base instructions", () => { + const composed = composeSystemInstructions( + input({ + agentNotesContext: "NOTES", + userWorkDir: "/w", + voiceInput: true, + voiceOutput: "summary", + searchEnabled: true, + codeMode: "claude", + codeCwd: "/c", + videoMode: true, + coachMode: true, + }), + ); + const markers = [ + "You are the test agent.", + "# Hidden User Context", + "NOTES", + "# User Work Directory", + "# Voice Input", + "# Video Mode (Live Camera)", + "# Practice Session (Coach Mode)", + "# Voice Output (MANDATORY — READ THIS FIRST)", + "# Search", + "# Code Mode (Active)", + ]; + let last = -1; + for (const marker of markers) { + const at = composed.indexOf(marker); + expect(at, `marker missing or out of order: ${marker}`).toBeGreaterThan(last); + last = at; + } + }); +}); diff --git a/apps/x/packages/core/src/agents/compose-instructions.ts b/apps/x/packages/core/src/agents/compose-instructions.ts new file mode 100644 index 00000000..691443ee --- /dev/null +++ b/apps/x/packages/core/src/agents/compose-instructions.ts @@ -0,0 +1,142 @@ +// System-prompt composition for agent assembly: the base instructions plus +// the mode blocks (voice, video, coach, search, code) appended per turn +// composition. Extracted verbatim from the legacy streamAgent path so both +// engines compose byte-identical prompts; compose-instructions.test.ts pins +// the output bytes (golden snapshots) that step-by-step restructuring must +// preserve. Pure: callers load agent notes / work dir themselves. + +const USER_CONTEXT_SYSTEM_INSTRUCTIONS = `# Hidden User Context +User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". + +Use "Current date and time" for temporal reasoning. + +If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. + +If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. + +If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. + +If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer.`; + + +export interface ComposeSystemInstructionsInput { + instructions: string; + agentNotesContext: string | null; + userWorkDir: string | null; + voiceInput: boolean; + voiceOutput: 'summary' | 'full' | null; + searchEnabled: boolean; + codeMode: 'claude' | 'codex' | null; + codeCwd: string | null; + // Optional so legacy callers (old streamAgent path) are unaffected. + videoMode?: boolean; + coachMode?: boolean; +} + +// System-prompt assembly, extracted verbatim from streamAgent so the new turn +// runtime's agent resolver composes byte-identical prompts. Pure: callers +// load agent notes / work dir themselves. +export function composeSystemInstructions({ + instructions, + agentNotesContext, + userWorkDir, + voiceInput, + voiceOutput, + searchEnabled, + codeMode, + codeCwd, + videoMode, + coachMode, +}: ComposeSystemInstructionsInput): string { + let instructionsWithDateTime = `${instructions}\n\n${USER_CONTEXT_SYSTEM_INSTRUCTIONS}`; + if (agentNotesContext) { + instructionsWithDateTime += `\n\n${agentNotesContext}`; + } + if (userWorkDir) { + instructionsWithDateTime += `\n\n# User Work Directory +The user has chosen the following directory as their current **work directory**: + +\`${userWorkDir}\` + +Treat this as the **default location** for file operations whenever the user refers to files generically: +- "list the files", "show me what's in here", "what's the latest report" — list or look in the work directory. +- "save this", "export it", "write that to a file" — write the output into the work directory unless the user names another location. +- "open the file I was just working on", "the doc from earlier" — assume the work directory first. + +Use absolute paths rooted at this directory with the \`file-*\` tools. For example, list with \`file-list({ path: "${userWorkDir}" })\`, read text with \`file-readText\`, and write text with \`file-writeText\`. For PDFs, Office docs, images, scanned docs, and other non-text files, use \`parseFile\` or \`LLMParse\` with the absolute path; you do NOT need to copy the file into the workspace first. + +**Exceptions — these ALWAYS take precedence over the work directory default:** +1. **Knowledge base questions.** If the user asks about anything in the knowledge graph (notes, people, organizations, projects, topics) or paths starting with \`knowledge/\`, use file tools against \`knowledge/\` as documented above. Do NOT redirect those into the work directory. +2. **Explicit paths.** If the user names a different directory or gives an absolute/relative path (e.g. "in ~/Downloads", "from /tmp/foo", "the Desktop"), honor that path exactly and ignore the work-directory default for that request. +3. **Workspace-specific operations.** Anything that obviously belongs in the Rowboat workspace (config files, MCP servers, agent schedules, etc.) stays in the workspace, not the work directory. + +Do not announce the work directory unless it's relevant. Just use it.`; + } + if (voiceInput) { + instructionsWithDateTime += `\n\n# Voice Input\nThe user's message was transcribed from speech. Be aware that:\n- There may be transcription errors. Silently correct obvious ones (e.g. homophones, misheard words). If an error is genuinely ambiguous, briefly mention your interpretation (e.g. "I'm assuming you meant X").\n- Spoken messages are often long-winded. The user may ramble, repeat themselves, or correct something they said earlier in the same message. Focus on their final intent, not every word verbatim.`; + } + if (videoMode) { + instructionsWithDateTime += `\n\n# Video Mode (Live Camera) +The user has turned on video mode: their webcam is on, and their messages arrive with a series of live webcam frames (ordered oldest to newest) captured while they were speaking or typing. The frames are a live view of the user themselves — not a document or file to analyze. + +How to use the frames: +- Use them for what visual awareness genuinely adds: the user's expressions, body language, posture, gestures, eye contact with the camera, visible energy, anything they hold up to the camera, and their surroundings when relevant. +- Compare across frames to notice change over time within a message (e.g. increasingly slouched, started smiling, looked away for most of the message). +- When the user practices something performative (a pitch, presentation, interview, talk) or asks for delivery feedback, give specific, actionable coaching grounded in what you actually see — cite concrete observations ("in the last few frames you looked down and away from the camera") rather than generic advice. Cover posture, eye contact, facial expressiveness, gesturing, and visible energy. +- If they show something to the camera (an object, document, whiteboard), read or describe it and respond accordingly. + +Driving the app: +- You can control the Rowboat app the user is looking at via the app-navigation tool (load the app-navigation skill first): open views, READ a view's contents as data (emails, background agents, chat history), and open specific items on their screen. +- When the user asks about anything that lives inside Rowboat ("what emails do I have?", "what agents are running?", "open the one from Arjun"), prefer driving over describing: read-view shows the view on their screen while returning its data, then answer out loud briefly; open-item when they pick one. Narrate as you act ("pulling up your inbox…"). Reading a view's data beats squinting at screen-share frames — it's exact. + +Screen sharing: +- The user may also share their screen. Screen-share frames arrive in a separately labeled group after the webcam frames; they show the user's screen, not the user. +- When screen frames are present, treat them as the primary subject: the user is usually asking about, or working on, what's visible there. Read the screen carefully — code, documents, error messages, UI state — and help with it concretely. +- The LAST screen frame is the most current view of their screen; earlier ones show how it changed while they spoke. +- Frames are downscaled captures: small text may be hard to read. If something crucial is illegible, say what you need ("zoom into the error message" / "make that panel bigger") rather than guessing. + +Etiquette: +- Do not narrate or list what you see in every response. Bring up visual observations only when relevant to the user's request or clearly worth mentioning. +- Never comment on the user's physical appearance, attractiveness, or personal attributes — visual feedback is strictly about delivery, expression, and body language. +- Frames are periodic snapshots, not continuous video; moments between frames are missing. Don't claim certainty about motion you couldn't have seen.`; + } + if (coachMode) { + instructionsWithDateTime += `\n\n# Practice Session (Coach Mode) +The user started a practice session: they are rehearsing something performative — a pitch, presentation, interview answer, or talk — and want live coaching. You are their coach for this session. + +How to coach: +- Watch and listen for delivery: pacing, filler words, rambling, structure, and (from the webcam frames) posture, eye contact with the camera, facial expressiveness, gesturing, and visible energy. +- After each take or answer, give brief, specific, actionable feedback: 2-3 concrete observations, quoting what you actually saw or heard ("you looked down during the ask", "the opening 'um, so, basically' undercuts your hook"). Then let them go again. +- If they are clearly mid-flow, keep any interjection to one short sentence — or stay silent and save it for the break. +- Ask what they're practicing for at the start if it isn't obvious (investor pitch? interview? conference talk?) and tailor feedback to that audience. +- When they say they're done or wrap up, give a structured debrief: what's working, the top 3 things to improve, and one concrete drill or reframe to try next time. +- Be encouraging but honest — vague praise wastes their rehearsal time. Never comment on physical appearance; delivery, expression, and body language only.`; + } + if (voiceOutput === 'summary') { + instructionsWithDateTime += `\n\n# Voice Output (MANDATORY — READ THIS FIRST)\nThe user has voice output enabled. THIS IS YOUR #1 PRIORITY: you MUST start your response with tags. If your response does not begin with tags, the user will hear nothing — which is a broken experience. NEVER skip this.\n\nRules:\n1. YOUR VERY FIRST OUTPUT MUST BE A TAG. No exceptions. Do not start with markdown, headings, or any other text. The literal first characters of your response must be "".\n2. Place ALL tags at the BEGINNING of your response, before any detailed content. Do NOT intersperse tags throughout the response.\n3. Wrap EACH spoken sentence in its own separate tag so it can be spoken incrementally. Do NOT wrap everything in a single block.\n4. Use voice as a TL;DR and navigation aid — do NOT read the entire response aloud.\n5. After all tags, you may include detailed written content (markdown, tables, code, etc.) that will be shown visually but not spoken.\n\n## Examples\n\nExample 1 — User asks: "what happened in my meeting with Alex yesterday?"\n\nYour meeting with Alex covered three main things: the Q2 roadmap timeline, hiring for the backend role, and the client demo next week.\nI've pulled out the key details and action items below — the demo prep notes are at the end.\n\n## Meeting with Alex — March 11\n### Roadmap\n- Agreed to push Q2 launch to April 15...\n(detailed written content continues)\n\nExample 2 — User asks: "summarize my emails"\n\nYou have five new emails since this morning.\nTwo are from your team — Jordan sent the RFC you requested and Taylor flagged a contract issue.\nThere's also a warm intro from a VC partner connecting you with someone at a prospective customer.\nI've drafted responses for three of them. The details and drafts are below.\n\n(email blocks, tables, and detailed content follow)\n\nExample 3 — User asks: "what's on my calendar today?"\n\nYou've got a pretty packed day — seven meetings starting with standup at 9.\nThe big ones are your investor call at 11, lunch with a partner from your lead VC at 12:30, and a customer call at 4.\nYour only free block for deep work is 2:30 to 4.\n\n(calendar block with full event details follows)\n\nExample 4 — User asks: "draft an email to Sam with our metrics"\n\nDone — I've drafted the email to Sam with your latest WAU and churn numbers.\nTake a look at the draft below and send it when you're ready.\n\n(email block with draft follows)\n\nREMEMBER: If you do not start with tags, the user hears silence. Always speak first, then write.`; + } else if (voiceOutput === 'full') { + instructionsWithDateTime += `\n\n# Voice Output — Full Read-Aloud (MANDATORY — READ THIS FIRST)\nThe user wants your ENTIRE response spoken aloud. THIS IS YOUR #1 PRIORITY: every single sentence must be wrapped in tags. If you write anything outside tags, the user will not hear it — which is a broken experience. NEVER skip this.\n\nRules:\n1. YOUR VERY FIRST OUTPUT MUST BE A TAG. No exceptions. The literal first characters of your response must be "".\n2. Wrap EACH sentence in its own separate tag so it can be spoken incrementally.\n3. Write your response in a natural, conversational style suitable for listening — no markdown headings, bullet points, or formatting symbols. Use plain spoken language.\n4. Structure the content as if you are speaking to the user directly. Use transitions like "first", "also", "one more thing" instead of visual formatting.\n5. EVERY sentence MUST be inside a tag. Do not leave ANY content outside tags. If it's not in a tag, the user cannot hear it.\n\n## Examples\n\nExample 1 — User asks: "what happened in my meeting with Alex yesterday?"\n\nYour meeting with Alex covered three main things.\nFirst, you discussed the Q2 roadmap timeline and agreed to push the launch to April.\nSecond, you talked about hiring for the backend role — Alex will send over two candidates by Friday.\nAnd lastly, the client demo is next week on Thursday at 2pm, and you're handling the intro slides.\n\nExample 2 — User asks: "summarize my emails"\n\nYou've got five new emails since this morning.\nTwo are from your team — Jordan sent the RFC you asked for, and Taylor flagged a contract issue that needs your sign-off.\nThere's a warm intro from a VC partner connecting you with an engineering lead at a potential customer.\nAnd someone from a prospective client wants to confirm your API tier before your call this afternoon.\nI've drafted replies for three of them — the metrics update, the intro, and the API question.\nThe only one I left for you is Taylor's contract redline, since that needs your judgment on the liability cap.\n\nExample 3 — User asks: "what's on my calendar today?"\n\nYou've got a packed day — seven meetings starting with standup at 9.\nThe highlights are your investor call at 11, lunch with a VC partner at 12:30, and a customer call at 4.\nYour only open block for deep work is 2:30 to 4, so plan accordingly.\nOh, and your 1-on-1 with your co-founder is at 5:30 — that's a walking meeting.\n\nExample 4 — User asks: "how are our metrics looking?"\n\nMetrics are looking strong this week.\nYou hit 2,573 weekly active users, which is up 12% week over week.\nThat means you've crossed the 2,500 milestone — worth calling out in your next investor update.\nChurn is down to 4.1%, improving month over month.\nThe trailing 8-week compound growth rate is about 10%.\n\nREMEMBER: Start with immediately. No preamble, no markdown before it. Speak first.`; + } + if (searchEnabled) { + instructionsWithDateTime += `\n\n# Search\nThe user has requested a search. Use the web-search tool to answer their query.`; + } + if (codeMode) { + const agentDisplay = codeMode === 'claude' ? 'Claude Code' : 'Codex'; + instructionsWithDateTime += `\n\n# Code Mode (Active) — Agent: ${agentDisplay} +The user has turned on **code mode** and the composer chip is set to **${agentDisplay}** (\`${codeMode}\`). For EVERY coding task this turn, use **${agentDisplay}**, and narrate that agent ("Using ${agentDisplay} to …"). + +The chip is the single source of truth for which agent runs: +- Do NOT carry over a different agent from earlier in this thread — even if a previous run used the other agent, use **${agentDisplay}** now. +- Do NOT switch agents based on an in-chat text request ("use codex", "switch to claude"). The agent only changes when the user toggles the chip; if they ask in chat, tell them to toggle the chip. + +**How to run coding work — call the \`code_agent_run\` tool** with: +- \`agent\`: \`${codeMode}\` (always — match the chip). +- \`cwd\`: ${codeCwd ? `\`${codeCwd}\` (always — this coding session is pinned to that directory; never use another path)` : `the absolute project/working directory (resolve it per the code-with-agents skill — a path the user named, the "# User Work Directory" block, or ask once)`}. +- \`prompt\`: a clear, self-contained coding instruction. + +The tool runs the agent on-device and streams its tool calls, file diffs, and plan into the chat; any action needing approval surfaces as an inline permission card, so you do NOT pre-confirm with an in-chat "reply yes". This chat keeps ONE persistent agent session, so follow-up coding requests automatically resume with full context — just call \`code_agent_run\` again. Do NOT shell out to \`acpx\` or \`executeCommand\` for coding, and do NOT fall back to your own file tools. + +If the user's message is clearly NOT a coding request (small talk, an unrelated question), answer directly without invoking the coding agent. Code mode signals readiness, not that every message must route through the agent.`; + } + return instructionsWithDateTime; +} diff --git a/apps/x/packages/core/src/agents/runtime.ts b/apps/x/packages/core/src/agents/runtime.ts index d4a8fa77..58dedcb5 100644 --- a/apps/x/packages/core/src/agents/runtime.ts +++ b/apps/x/packages/core/src/agents/runtime.ts @@ -302,141 +302,13 @@ ${sections.join('\n\n')} `; } -const USER_CONTEXT_SYSTEM_INSTRUCTIONS = `# Hidden User Context -User messages may include a hidden "# User Context" section before "# User Message". Treat it as runtime metadata captured when that specific user message was sent. The actual user-authored text starts under "# User Message". - -Use "Current date and time" for temporal reasoning. - -If Middle pane context is present, it reflects what the user had open at the time of that specific message and overrides earlier middle-pane references. If the conversation history references a different note or browser page, the user had since closed or navigated away from it. Do not treat earlier context as current. - -If Middle pane state is empty, the user was not looking at any relevant note or web page at that point. Answer the user's message on its own merits. - -If Middle pane state is note, the supplied path and content are available so you can reference the note when relevant. The user may or may not be talking about this note. Do NOT assume every message is about it. Only reference or act on this note when the user's message clearly relates to it, such as "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly the note's content. For unrelated questions, ignore this note entirely and answer normally. Do not mention that you can see this note unless it is relevant to the answer. - -If Middle pane state is browser, only the URL and page title are supplied; the page content itself is NOT included. If you need the page content to answer, use the browser tools available to you to read the page. The user may or may not be talking about this page. Only reference or act on this page when the user's message clearly relates to it, such as "this page", "this article", "what I'm looking at", "this site", or "summarize this". For unrelated questions, ignore this page entirely and answer normally. Do not mention that you can see the browser unless it is relevant to the answer.`; - - -export interface ComposeSystemInstructionsInput { - instructions: string; - agentNotesContext: string | null; - userWorkDir: string | null; - voiceInput: boolean; - voiceOutput: 'summary' | 'full' | null; - searchEnabled: boolean; - codeMode: 'claude' | 'codex' | null; - codeCwd: string | null; - // Optional so legacy callers (old streamAgent path) are unaffected. - videoMode?: boolean; - coachMode?: boolean; -} - -// System-prompt assembly, extracted verbatim from streamAgent so the new turn -// runtime's agent resolver composes byte-identical prompts. Pure: callers -// load agent notes / work dir themselves. -export function composeSystemInstructions({ - instructions, - agentNotesContext, - userWorkDir, - voiceInput, - voiceOutput, - searchEnabled, - codeMode, - codeCwd, - videoMode, - coachMode, -}: ComposeSystemInstructionsInput): string { - let instructionsWithDateTime = `${instructions}\n\n${USER_CONTEXT_SYSTEM_INSTRUCTIONS}`; - if (agentNotesContext) { - instructionsWithDateTime += `\n\n${agentNotesContext}`; - } - if (userWorkDir) { - instructionsWithDateTime += `\n\n# User Work Directory -The user has chosen the following directory as their current **work directory**: - -\`${userWorkDir}\` - -Treat this as the **default location** for file operations whenever the user refers to files generically: -- "list the files", "show me what's in here", "what's the latest report" — list or look in the work directory. -- "save this", "export it", "write that to a file" — write the output into the work directory unless the user names another location. -- "open the file I was just working on", "the doc from earlier" — assume the work directory first. - -Use absolute paths rooted at this directory with the \`file-*\` tools. For example, list with \`file-list({ path: "${userWorkDir}" })\`, read text with \`file-readText\`, and write text with \`file-writeText\`. For PDFs, Office docs, images, scanned docs, and other non-text files, use \`parseFile\` or \`LLMParse\` with the absolute path; you do NOT need to copy the file into the workspace first. - -**Exceptions — these ALWAYS take precedence over the work directory default:** -1. **Knowledge base questions.** If the user asks about anything in the knowledge graph (notes, people, organizations, projects, topics) or paths starting with \`knowledge/\`, use file tools against \`knowledge/\` as documented above. Do NOT redirect those into the work directory. -2. **Explicit paths.** If the user names a different directory or gives an absolute/relative path (e.g. "in ~/Downloads", "from /tmp/foo", "the Desktop"), honor that path exactly and ignore the work-directory default for that request. -3. **Workspace-specific operations.** Anything that obviously belongs in the Rowboat workspace (config files, MCP servers, agent schedules, etc.) stays in the workspace, not the work directory. - -Do not announce the work directory unless it's relevant. Just use it.`; - } - if (voiceInput) { - instructionsWithDateTime += `\n\n# Voice Input\nThe user's message was transcribed from speech. Be aware that:\n- There may be transcription errors. Silently correct obvious ones (e.g. homophones, misheard words). If an error is genuinely ambiguous, briefly mention your interpretation (e.g. "I'm assuming you meant X").\n- Spoken messages are often long-winded. The user may ramble, repeat themselves, or correct something they said earlier in the same message. Focus on their final intent, not every word verbatim.`; - } - if (videoMode) { - instructionsWithDateTime += `\n\n# Video Mode (Live Camera) -The user has turned on video mode: their webcam is on, and their messages arrive with a series of live webcam frames (ordered oldest to newest) captured while they were speaking or typing. The frames are a live view of the user themselves — not a document or file to analyze. - -How to use the frames: -- Use them for what visual awareness genuinely adds: the user's expressions, body language, posture, gestures, eye contact with the camera, visible energy, anything they hold up to the camera, and their surroundings when relevant. -- Compare across frames to notice change over time within a message (e.g. increasingly slouched, started smiling, looked away for most of the message). -- When the user practices something performative (a pitch, presentation, interview, talk) or asks for delivery feedback, give specific, actionable coaching grounded in what you actually see — cite concrete observations ("in the last few frames you looked down and away from the camera") rather than generic advice. Cover posture, eye contact, facial expressiveness, gesturing, and visible energy. -- If they show something to the camera (an object, document, whiteboard), read or describe it and respond accordingly. - -Driving the app: -- You can control the Rowboat app the user is looking at via the app-navigation tool (load the app-navigation skill first): open views, READ a view's contents as data (emails, background agents, chat history), and open specific items on their screen. -- When the user asks about anything that lives inside Rowboat ("what emails do I have?", "what agents are running?", "open the one from Arjun"), prefer driving over describing: read-view shows the view on their screen while returning its data, then answer out loud briefly; open-item when they pick one. Narrate as you act ("pulling up your inbox…"). Reading a view's data beats squinting at screen-share frames — it's exact. - -Screen sharing: -- The user may also share their screen. Screen-share frames arrive in a separately labeled group after the webcam frames; they show the user's screen, not the user. -- When screen frames are present, treat them as the primary subject: the user is usually asking about, or working on, what's visible there. Read the screen carefully — code, documents, error messages, UI state — and help with it concretely. -- The LAST screen frame is the most current view of their screen; earlier ones show how it changed while they spoke. -- Frames are downscaled captures: small text may be hard to read. If something crucial is illegible, say what you need ("zoom into the error message" / "make that panel bigger") rather than guessing. - -Etiquette: -- Do not narrate or list what you see in every response. Bring up visual observations only when relevant to the user's request or clearly worth mentioning. -- Never comment on the user's physical appearance, attractiveness, or personal attributes — visual feedback is strictly about delivery, expression, and body language. -- Frames are periodic snapshots, not continuous video; moments between frames are missing. Don't claim certainty about motion you couldn't have seen.`; - } - if (coachMode) { - instructionsWithDateTime += `\n\n# Practice Session (Coach Mode) -The user started a practice session: they are rehearsing something performative — a pitch, presentation, interview answer, or talk — and want live coaching. You are their coach for this session. - -How to coach: -- Watch and listen for delivery: pacing, filler words, rambling, structure, and (from the webcam frames) posture, eye contact with the camera, facial expressiveness, gesturing, and visible energy. -- After each take or answer, give brief, specific, actionable feedback: 2-3 concrete observations, quoting what you actually saw or heard ("you looked down during the ask", "the opening 'um, so, basically' undercuts your hook"). Then let them go again. -- If they are clearly mid-flow, keep any interjection to one short sentence — or stay silent and save it for the break. -- Ask what they're practicing for at the start if it isn't obvious (investor pitch? interview? conference talk?) and tailor feedback to that audience. -- When they say they're done or wrap up, give a structured debrief: what's working, the top 3 things to improve, and one concrete drill or reframe to try next time. -- Be encouraging but honest — vague praise wastes their rehearsal time. Never comment on physical appearance; delivery, expression, and body language only.`; - } - if (voiceOutput === 'summary') { - instructionsWithDateTime += `\n\n# Voice Output (MANDATORY — READ THIS FIRST)\nThe user has voice output enabled. THIS IS YOUR #1 PRIORITY: you MUST start your response with tags. If your response does not begin with tags, the user will hear nothing — which is a broken experience. NEVER skip this.\n\nRules:\n1. YOUR VERY FIRST OUTPUT MUST BE A TAG. No exceptions. Do not start with markdown, headings, or any other text. The literal first characters of your response must be "".\n2. Place ALL tags at the BEGINNING of your response, before any detailed content. Do NOT intersperse tags throughout the response.\n3. Wrap EACH spoken sentence in its own separate tag so it can be spoken incrementally. Do NOT wrap everything in a single block.\n4. Use voice as a TL;DR and navigation aid — do NOT read the entire response aloud.\n5. After all tags, you may include detailed written content (markdown, tables, code, etc.) that will be shown visually but not spoken.\n\n## Examples\n\nExample 1 — User asks: "what happened in my meeting with Alex yesterday?"\n\nYour meeting with Alex covered three main things: the Q2 roadmap timeline, hiring for the backend role, and the client demo next week.\nI've pulled out the key details and action items below — the demo prep notes are at the end.\n\n## Meeting with Alex — March 11\n### Roadmap\n- Agreed to push Q2 launch to April 15...\n(detailed written content continues)\n\nExample 2 — User asks: "summarize my emails"\n\nYou have five new emails since this morning.\nTwo are from your team — Jordan sent the RFC you requested and Taylor flagged a contract issue.\nThere's also a warm intro from a VC partner connecting you with someone at a prospective customer.\nI've drafted responses for three of them. The details and drafts are below.\n\n(email blocks, tables, and detailed content follow)\n\nExample 3 — User asks: "what's on my calendar today?"\n\nYou've got a pretty packed day — seven meetings starting with standup at 9.\nThe big ones are your investor call at 11, lunch with a partner from your lead VC at 12:30, and a customer call at 4.\nYour only free block for deep work is 2:30 to 4.\n\n(calendar block with full event details follows)\n\nExample 4 — User asks: "draft an email to Sam with our metrics"\n\nDone — I've drafted the email to Sam with your latest WAU and churn numbers.\nTake a look at the draft below and send it when you're ready.\n\n(email block with draft follows)\n\nREMEMBER: If you do not start with tags, the user hears silence. Always speak first, then write.`; - } else if (voiceOutput === 'full') { - instructionsWithDateTime += `\n\n# Voice Output — Full Read-Aloud (MANDATORY — READ THIS FIRST)\nThe user wants your ENTIRE response spoken aloud. THIS IS YOUR #1 PRIORITY: every single sentence must be wrapped in tags. If you write anything outside tags, the user will not hear it — which is a broken experience. NEVER skip this.\n\nRules:\n1. YOUR VERY FIRST OUTPUT MUST BE A TAG. No exceptions. The literal first characters of your response must be "".\n2. Wrap EACH sentence in its own separate tag so it can be spoken incrementally.\n3. Write your response in a natural, conversational style suitable for listening — no markdown headings, bullet points, or formatting symbols. Use plain spoken language.\n4. Structure the content as if you are speaking to the user directly. Use transitions like "first", "also", "one more thing" instead of visual formatting.\n5. EVERY sentence MUST be inside a tag. Do not leave ANY content outside tags. If it's not in a tag, the user cannot hear it.\n\n## Examples\n\nExample 1 — User asks: "what happened in my meeting with Alex yesterday?"\n\nYour meeting with Alex covered three main things.\nFirst, you discussed the Q2 roadmap timeline and agreed to push the launch to April.\nSecond, you talked about hiring for the backend role — Alex will send over two candidates by Friday.\nAnd lastly, the client demo is next week on Thursday at 2pm, and you're handling the intro slides.\n\nExample 2 — User asks: "summarize my emails"\n\nYou've got five new emails since this morning.\nTwo are from your team — Jordan sent the RFC you asked for, and Taylor flagged a contract issue that needs your sign-off.\nThere's a warm intro from a VC partner connecting you with an engineering lead at a potential customer.\nAnd someone from a prospective client wants to confirm your API tier before your call this afternoon.\nI've drafted replies for three of them — the metrics update, the intro, and the API question.\nThe only one I left for you is Taylor's contract redline, since that needs your judgment on the liability cap.\n\nExample 3 — User asks: "what's on my calendar today?"\n\nYou've got a packed day — seven meetings starting with standup at 9.\nThe highlights are your investor call at 11, lunch with a VC partner at 12:30, and a customer call at 4.\nYour only open block for deep work is 2:30 to 4, so plan accordingly.\nOh, and your 1-on-1 with your co-founder is at 5:30 — that's a walking meeting.\n\nExample 4 — User asks: "how are our metrics looking?"\n\nMetrics are looking strong this week.\nYou hit 2,573 weekly active users, which is up 12% week over week.\nThat means you've crossed the 2,500 milestone — worth calling out in your next investor update.\nChurn is down to 4.1%, improving month over month.\nThe trailing 8-week compound growth rate is about 10%.\n\nREMEMBER: Start with immediately. No preamble, no markdown before it. Speak first.`; - } - if (searchEnabled) { - instructionsWithDateTime += `\n\n# Search\nThe user has requested a search. Use the web-search tool to answer their query.`; - } - if (codeMode) { - const agentDisplay = codeMode === 'claude' ? 'Claude Code' : 'Codex'; - instructionsWithDateTime += `\n\n# Code Mode (Active) — Agent: ${agentDisplay} -The user has turned on **code mode** and the composer chip is set to **${agentDisplay}** (\`${codeMode}\`). For EVERY coding task this turn, use **${agentDisplay}**, and narrate that agent ("Using ${agentDisplay} to …"). - -The chip is the single source of truth for which agent runs: -- Do NOT carry over a different agent from earlier in this thread — even if a previous run used the other agent, use **${agentDisplay}** now. -- Do NOT switch agents based on an in-chat text request ("use codex", "switch to claude"). The agent only changes when the user toggles the chip; if they ask in chat, tell them to toggle the chip. - -**How to run coding work — call the \`code_agent_run\` tool** with: -- \`agent\`: \`${codeMode}\` (always — match the chip). -- \`cwd\`: ${codeCwd ? `\`${codeCwd}\` (always — this coding session is pinned to that directory; never use another path)` : `the absolute project/working directory (resolve it per the code-with-agents skill — a path the user named, the "# User Work Directory" block, or ask once)`}. -- \`prompt\`: a clear, self-contained coding instruction. - -The tool runs the agent on-device and streams its tool calls, file diffs, and plan into the chat; any action needing approval surfaces as an inline permission card, so you do NOT pre-confirm with an in-chat "reply yes". This chat keeps ONE persistent agent session, so follow-up coding requests automatically resume with full context — just call \`code_agent_run\` again. Do NOT shell out to \`acpx\` or \`executeCommand\` for coding, and do NOT fall back to your own file tools. - -If the user's message is clearly NOT a coding request (small talk, an unrelated question), answer directly without invoking the coding agent. Code mode signals readiness, not that every message must route through the agent.`; - } - return instructionsWithDateTime; -} +// System-prompt assembly lives in compose-instructions.ts (owned by the +// assembly layer); re-exported so legacy callers keep their import path. +export { + composeSystemInstructions, + type ComposeSystemInstructionsInput, +} from "./compose-instructions.js"; +import { composeSystemInstructions } from "./compose-instructions.js"; export interface IAgentRuntime { trigger(runId: string): Promise; diff --git a/apps/x/packages/core/src/turns/bridges/real-agent-resolver.ts b/apps/x/packages/core/src/turns/bridges/real-agent-resolver.ts index 7450d51a..24dd787e 100644 --- a/apps/x/packages/core/src/turns/bridges/real-agent-resolver.ts +++ b/apps/x/packages/core/src/turns/bridges/real-agent-resolver.ts @@ -6,8 +6,8 @@ import { SPAWN_AGENT_TOOL_NAME, type ToolDescriptor, } from "@x/shared/dist/turns.js"; +import { composeSystemInstructions } from "../../agents/compose-instructions.js"; import { - composeSystemInstructions, loadAgentNotesContext, loadUserWorkDir, } from "../../agents/runtime.js"; From 484a9f0495cd95925ee547cdd680b00f725432cb Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Thu, 9 Jul 2026 22:35:41 +0530 Subject: [PATCH 3/6] =?UTF-8?q?refactor(x):=20capability=20record=20?= =?UTF-8?q?=E2=80=94=20skills=20become=20the=20model-activated=20subset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit capabilities/types.ts introduces the assembly unit: id/title/summary + lazy guidance + owned tools (what skills already were), plus an activation axis ('model' | 'app' | 'always') and an eager promptFragment(ctx) for app/always entries — pure over a persisted composition context so composed prompts stay byte-identical. SkillDefinition is now CapabilityDefinition & {content}: bundled skills are unchanged model-activated entries, the loadSkill catalog filters to that subset, and disk skills remain structurally limited to it (eager fragments and app activation are bundled-only powers — a disk file must never gain a standing system-prompt injection channel). Co-Authored-By: Claude Fable 5 --- .../assistant/capabilities/types.ts | 56 +++++++++++++++++++ .../assistant/skills/index.test.ts | 22 ++++++++ .../src/application/assistant/skills/index.ts | 26 ++++----- 3 files changed, 91 insertions(+), 13 deletions(-) create mode 100644 apps/x/packages/core/src/application/assistant/capabilities/types.ts diff --git a/apps/x/packages/core/src/application/assistant/capabilities/types.ts b/apps/x/packages/core/src/application/assistant/capabilities/types.ts new file mode 100644 index 00000000..f0fee3cd --- /dev/null +++ b/apps/x/packages/core/src/application/assistant/capabilities/types.ts @@ -0,0 +1,56 @@ +// The capability record: the unit of agent assembly. A capability owns what +// it contributes to an agent — lazy guidance the model can pull in, tools, +// and (for app/always activation) an eager system-prompt fragment. +// +// Activation is the axis that distinguishes the historical concepts: +// - 'model' — today's skills: a catalog line in the prompt; the model calls +// loadSkill when it recognizes the task; guidance arrives as a +// tool result and declared tools attach mid-turn. +// - 'app' — today's modes (voice, video, coach, code): the app toggles a +// fact about the world; the fragment is composed into the +// system prompt from token zero and tools attach at assembly. +// - 'always' — unconditional contributions (workspace context traits). +// +// Trust boundary: disk-loaded skills (~/.rowboat/skills, ~/.agents/skills) +// are structurally limited to the 'model' subset — eager prompt fragments +// and app activation are bundled-only powers. A model-activated skill's +// prose lands in conversation because the model chose to read it; a disk +// file injecting into every turn's system prompt would be a standing +// prompt-injection channel. + +export type CapabilityActivation = "model" | "app" | "always"; + +// Inputs an eager fragment may read. All fields mirror persisted +// RequestedAgent.overrides.composition values, resolved before composition — +// a fragment must be a pure function of this context so composed prompts +// stay byte-identical for identical inputs (provider prefix caching and +// agent-snapshot inheritance both depend on it). +export interface CapabilityContext { + voiceInput: boolean; + voiceOutput: "summary" | "full" | null; + searchEnabled: boolean; + codeMode: "claude" | "codex" | null; + codeCwd: string | null; + videoMode: boolean; + coachMode: boolean; +} + +export interface CapabilityDefinition { + id: string; + title: string; + summary: string; + // Defaults to 'model' (the historical skill behavior). + activation?: CapabilityActivation; + // Lazy guidance returned by loadSkill (model activation). + content?: string; + // BuiltinTools keys this capability owns. + tools?: string[]; + // Eager system-prompt fragment (app/always activation). Returns null + // when the capability contributes nothing for this context. MUST be + // pure: same context, same bytes. + promptFragment?: (ctx: CapabilityContext) => string | null; +} + +export function isModelActivated(def: Pick): boolean { + return def.activation === undefined || def.activation === "model"; +} diff --git a/apps/x/packages/core/src/application/assistant/skills/index.test.ts b/apps/x/packages/core/src/application/assistant/skills/index.test.ts index e8b4e6ed..02f38e54 100644 --- a/apps/x/packages/core/src/application/assistant/skills/index.test.ts +++ b/apps/x/packages/core/src/application/assistant/skills/index.test.ts @@ -146,3 +146,25 @@ describe("skills index (disk skill merge layer)", () => { expect(skills.resolveSkill(skillFile)).toBeNull(); }); }); + +describe("capability record (types)", () => { + it("defaults to model activation — every bundled skill is model-activated", async () => { + const { isModelActivated } = await import("../capabilities/types.js"); + // Bundled definitions carry no activation field today; they are the + // model-activated subset by default and must stay in the catalog. + expect(isModelActivated({})).toBe(true); + expect(isModelActivated({ activation: "model" })).toBe(true); + expect(isModelActivated({ activation: "app" })).toBe(false); + expect(isModelActivated({ activation: "always" })).toBe(false); + }); + + it("keeps app/always capabilities out of the loadSkill catalog", async () => { + const { buildSkillCatalog, availableSkills } = await import("./index.js"); + const catalog = buildSkillCatalog(); + // Every advertised id resolves via loadSkill (the catalog and the + // resolver operate on the same model-activated subset). + for (const id of availableSkills) { + expect(catalog).toContain(id); + } + }); +}); diff --git a/apps/x/packages/core/src/application/assistant/skills/index.ts b/apps/x/packages/core/src/application/assistant/skills/index.ts index 2d4e80e1..85b82e56 100644 --- a/apps/x/packages/core/src/application/assistant/skills/index.ts +++ b/apps/x/packages/core/src/application/assistant/skills/index.ts @@ -1,5 +1,6 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; +import { isModelActivated, type CapabilityDefinition } from "../capabilities/types.js"; import { loadDiskSkills } from "./disk-loader.js"; import builtinToolsSkill from "./builtin-tools/skill.js"; import deletionGuardrailsSkill from "./deletion-guardrails/skill.js"; @@ -25,17 +26,12 @@ const CATALOG_PREFIX = "src/application/assistant/skills"; // console.log(liveNoteSkill); -type SkillDefinition = { - id: string; // Also used as folder name - title: string; - summary: string; - content: string; - // BuiltinTools keys this skill owns. Loading the skill attaches these as - // native tool definitions (they are NOT in the copilot's base toolset). - // Names are validated against the live catalog where descriptors are - // built (loadSkill / the agent resolver); unknown names are dropped there. - tools?: string[]; -}; +// A skill is the model-activated subset of the capability record +// (capabilities/types.ts): lazy guidance the model pulls in via loadSkill, +// plus the BuiltinTools it owns. `id` doubles as the folder name. Tool names +// are validated against the live catalog where descriptors are built +// (loadSkill / the agent resolver); unknown names are dropped there. +type SkillDefinition = CapabilityDefinition & { content: string }; type ResolvedSkill = { id: string; @@ -242,9 +238,13 @@ function getSkillEntries(): SkillEntry[] { * Reads the live skill set, so it reflects disk skills added/removed at runtime. */ export function buildSkillCatalog(options?: { excludeIds?: string[] }): string { + // The catalog advertises the model-activated subset only: app/always + // capabilities are composed into the system prompt by the assembly layer, + // not offered for loadSkill. + const modelEntries = getSkillEntries().filter(isModelActivated); const entries = options?.excludeIds - ? getSkillEntries().filter(e => !options.excludeIds!.includes(e.id)) - : getSkillEntries(); + ? modelEntries.filter(e => !options.excludeIds!.includes(e.id)) + : modelEntries; const sections = entries.map((entry) => [ `## ${entry.title}`, `- **Skill file:** \`${entry.catalogPath}\``, From e194e32a23df10c3654ee2f877dbedf55a9214e8 Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Thu, 9 Jul 2026 22:38:06 +0530 Subject: [PATCH 4/6] refactor(x): modes become app-activated capabilities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The six mode blocks (voice input, video, coach, voice output summary/full, search, code mode) move out of the composer's if-chain into capability records (capabilities/modes.ts), each owning its fragment text as a pure function of the composition context — the code-mode fragment keeps its chip/cwd parameterization. The composer now iterates MODE_CAPABILITIES in declared order, which is the fixed total order that keeps composed prompts byte-stable; the 13 golden snapshot tests pass unchanged, proving byte-identical output. Fragment text was extracted from the if-chain programmatically (not retyped) so the bytes could not drift. Adding a mode is now one record in one file instead of a flag threaded through the resolver plus a concat site in the composer. Co-Authored-By: Claude Fable 5 --- .../core/src/agents/compose-instructions.ts | 85 +++-------- .../assistant/capabilities/modes.ts | 134 ++++++++++++++++++ 2 files changed, 154 insertions(+), 65 deletions(-) create mode 100644 apps/x/packages/core/src/application/assistant/capabilities/modes.ts diff --git a/apps/x/packages/core/src/agents/compose-instructions.ts b/apps/x/packages/core/src/agents/compose-instructions.ts index 691443ee..2cda8819 100644 --- a/apps/x/packages/core/src/agents/compose-instructions.ts +++ b/apps/x/packages/core/src/agents/compose-instructions.ts @@ -1,3 +1,6 @@ +import { MODE_CAPABILITIES } from "../application/assistant/capabilities/modes.js"; +import type { CapabilityContext } from "../application/assistant/capabilities/types.js"; + // System-prompt composition for agent assembly: the base instructions plus // the mode blocks (voice, video, coach, search, code) appended per turn // composition. Extracted verbatim from the legacy streamAgent path so both @@ -72,71 +75,23 @@ Use absolute paths rooted at this directory with the \`file-*\` tools. For examp Do not announce the work directory unless it's relevant. Just use it.`; } - if (voiceInput) { - instructionsWithDateTime += `\n\n# Voice Input\nThe user's message was transcribed from speech. Be aware that:\n- There may be transcription errors. Silently correct obvious ones (e.g. homophones, misheard words). If an error is genuinely ambiguous, briefly mention your interpretation (e.g. "I'm assuming you meant X").\n- Spoken messages are often long-winded. The user may ramble, repeat themselves, or correct something they said earlier in the same message. Focus on their final intent, not every word verbatim.`; - } - if (videoMode) { - instructionsWithDateTime += `\n\n# Video Mode (Live Camera) -The user has turned on video mode: their webcam is on, and their messages arrive with a series of live webcam frames (ordered oldest to newest) captured while they were speaking or typing. The frames are a live view of the user themselves — not a document or file to analyze. - -How to use the frames: -- Use them for what visual awareness genuinely adds: the user's expressions, body language, posture, gestures, eye contact with the camera, visible energy, anything they hold up to the camera, and their surroundings when relevant. -- Compare across frames to notice change over time within a message (e.g. increasingly slouched, started smiling, looked away for most of the message). -- When the user practices something performative (a pitch, presentation, interview, talk) or asks for delivery feedback, give specific, actionable coaching grounded in what you actually see — cite concrete observations ("in the last few frames you looked down and away from the camera") rather than generic advice. Cover posture, eye contact, facial expressiveness, gesturing, and visible energy. -- If they show something to the camera (an object, document, whiteboard), read or describe it and respond accordingly. - -Driving the app: -- You can control the Rowboat app the user is looking at via the app-navigation tool (load the app-navigation skill first): open views, READ a view's contents as data (emails, background agents, chat history), and open specific items on their screen. -- When the user asks about anything that lives inside Rowboat ("what emails do I have?", "what agents are running?", "open the one from Arjun"), prefer driving over describing: read-view shows the view on their screen while returning its data, then answer out loud briefly; open-item when they pick one. Narrate as you act ("pulling up your inbox…"). Reading a view's data beats squinting at screen-share frames — it's exact. - -Screen sharing: -- The user may also share their screen. Screen-share frames arrive in a separately labeled group after the webcam frames; they show the user's screen, not the user. -- When screen frames are present, treat them as the primary subject: the user is usually asking about, or working on, what's visible there. Read the screen carefully — code, documents, error messages, UI state — and help with it concretely. -- The LAST screen frame is the most current view of their screen; earlier ones show how it changed while they spoke. -- Frames are downscaled captures: small text may be hard to read. If something crucial is illegible, say what you need ("zoom into the error message" / "make that panel bigger") rather than guessing. - -Etiquette: -- Do not narrate or list what you see in every response. Bring up visual observations only when relevant to the user's request or clearly worth mentioning. -- Never comment on the user's physical appearance, attractiveness, or personal attributes — visual feedback is strictly about delivery, expression, and body language. -- Frames are periodic snapshots, not continuous video; moments between frames are missing. Don't claim certainty about motion you couldn't have seen.`; - } - if (coachMode) { - instructionsWithDateTime += `\n\n# Practice Session (Coach Mode) -The user started a practice session: they are rehearsing something performative — a pitch, presentation, interview answer, or talk — and want live coaching. You are their coach for this session. - -How to coach: -- Watch and listen for delivery: pacing, filler words, rambling, structure, and (from the webcam frames) posture, eye contact with the camera, facial expressiveness, gesturing, and visible energy. -- After each take or answer, give brief, specific, actionable feedback: 2-3 concrete observations, quoting what you actually saw or heard ("you looked down during the ask", "the opening 'um, so, basically' undercuts your hook"). Then let them go again. -- If they are clearly mid-flow, keep any interjection to one short sentence — or stay silent and save it for the break. -- Ask what they're practicing for at the start if it isn't obvious (investor pitch? interview? conference talk?) and tailor feedback to that audience. -- When they say they're done or wrap up, give a structured debrief: what's working, the top 3 things to improve, and one concrete drill or reframe to try next time. -- Be encouraging but honest — vague praise wastes their rehearsal time. Never comment on physical appearance; delivery, expression, and body language only.`; - } - if (voiceOutput === 'summary') { - instructionsWithDateTime += `\n\n# Voice Output (MANDATORY — READ THIS FIRST)\nThe user has voice output enabled. THIS IS YOUR #1 PRIORITY: you MUST start your response with tags. If your response does not begin with tags, the user will hear nothing — which is a broken experience. NEVER skip this.\n\nRules:\n1. YOUR VERY FIRST OUTPUT MUST BE A TAG. No exceptions. Do not start with markdown, headings, or any other text. The literal first characters of your response must be "".\n2. Place ALL tags at the BEGINNING of your response, before any detailed content. Do NOT intersperse tags throughout the response.\n3. Wrap EACH spoken sentence in its own separate tag so it can be spoken incrementally. Do NOT wrap everything in a single block.\n4. Use voice as a TL;DR and navigation aid — do NOT read the entire response aloud.\n5. After all tags, you may include detailed written content (markdown, tables, code, etc.) that will be shown visually but not spoken.\n\n## Examples\n\nExample 1 — User asks: "what happened in my meeting with Alex yesterday?"\n\nYour meeting with Alex covered three main things: the Q2 roadmap timeline, hiring for the backend role, and the client demo next week.\nI've pulled out the key details and action items below — the demo prep notes are at the end.\n\n## Meeting with Alex — March 11\n### Roadmap\n- Agreed to push Q2 launch to April 15...\n(detailed written content continues)\n\nExample 2 — User asks: "summarize my emails"\n\nYou have five new emails since this morning.\nTwo are from your team — Jordan sent the RFC you requested and Taylor flagged a contract issue.\nThere's also a warm intro from a VC partner connecting you with someone at a prospective customer.\nI've drafted responses for three of them. The details and drafts are below.\n\n(email blocks, tables, and detailed content follow)\n\nExample 3 — User asks: "what's on my calendar today?"\n\nYou've got a pretty packed day — seven meetings starting with standup at 9.\nThe big ones are your investor call at 11, lunch with a partner from your lead VC at 12:30, and a customer call at 4.\nYour only free block for deep work is 2:30 to 4.\n\n(calendar block with full event details follows)\n\nExample 4 — User asks: "draft an email to Sam with our metrics"\n\nDone — I've drafted the email to Sam with your latest WAU and churn numbers.\nTake a look at the draft below and send it when you're ready.\n\n(email block with draft follows)\n\nREMEMBER: If you do not start with tags, the user hears silence. Always speak first, then write.`; - } else if (voiceOutput === 'full') { - instructionsWithDateTime += `\n\n# Voice Output — Full Read-Aloud (MANDATORY — READ THIS FIRST)\nThe user wants your ENTIRE response spoken aloud. THIS IS YOUR #1 PRIORITY: every single sentence must be wrapped in tags. If you write anything outside tags, the user will not hear it — which is a broken experience. NEVER skip this.\n\nRules:\n1. YOUR VERY FIRST OUTPUT MUST BE A TAG. No exceptions. The literal first characters of your response must be "".\n2. Wrap EACH sentence in its own separate tag so it can be spoken incrementally.\n3. Write your response in a natural, conversational style suitable for listening — no markdown headings, bullet points, or formatting symbols. Use plain spoken language.\n4. Structure the content as if you are speaking to the user directly. Use transitions like "first", "also", "one more thing" instead of visual formatting.\n5. EVERY sentence MUST be inside a tag. Do not leave ANY content outside tags. If it's not in a tag, the user cannot hear it.\n\n## Examples\n\nExample 1 — User asks: "what happened in my meeting with Alex yesterday?"\n\nYour meeting with Alex covered three main things.\nFirst, you discussed the Q2 roadmap timeline and agreed to push the launch to April.\nSecond, you talked about hiring for the backend role — Alex will send over two candidates by Friday.\nAnd lastly, the client demo is next week on Thursday at 2pm, and you're handling the intro slides.\n\nExample 2 — User asks: "summarize my emails"\n\nYou've got five new emails since this morning.\nTwo are from your team — Jordan sent the RFC you asked for, and Taylor flagged a contract issue that needs your sign-off.\nThere's a warm intro from a VC partner connecting you with an engineering lead at a potential customer.\nAnd someone from a prospective client wants to confirm your API tier before your call this afternoon.\nI've drafted replies for three of them — the metrics update, the intro, and the API question.\nThe only one I left for you is Taylor's contract redline, since that needs your judgment on the liability cap.\n\nExample 3 — User asks: "what's on my calendar today?"\n\nYou've got a packed day — seven meetings starting with standup at 9.\nThe highlights are your investor call at 11, lunch with a VC partner at 12:30, and a customer call at 4.\nYour only open block for deep work is 2:30 to 4, so plan accordingly.\nOh, and your 1-on-1 with your co-founder is at 5:30 — that's a walking meeting.\n\nExample 4 — User asks: "how are our metrics looking?"\n\nMetrics are looking strong this week.\nYou hit 2,573 weekly active users, which is up 12% week over week.\nThat means you've crossed the 2,500 milestone — worth calling out in your next investor update.\nChurn is down to 4.1%, improving month over month.\nThe trailing 8-week compound growth rate is about 10%.\n\nREMEMBER: Start with immediately. No preamble, no markdown before it. Speak first.`; - } - if (searchEnabled) { - instructionsWithDateTime += `\n\n# Search\nThe user has requested a search. Use the web-search tool to answer their query.`; - } - if (codeMode) { - const agentDisplay = codeMode === 'claude' ? 'Claude Code' : 'Codex'; - instructionsWithDateTime += `\n\n# Code Mode (Active) — Agent: ${agentDisplay} -The user has turned on **code mode** and the composer chip is set to **${agentDisplay}** (\`${codeMode}\`). For EVERY coding task this turn, use **${agentDisplay}**, and narrate that agent ("Using ${agentDisplay} to …"). - -The chip is the single source of truth for which agent runs: -- Do NOT carry over a different agent from earlier in this thread — even if a previous run used the other agent, use **${agentDisplay}** now. -- Do NOT switch agents based on an in-chat text request ("use codex", "switch to claude"). The agent only changes when the user toggles the chip; if they ask in chat, tell them to toggle the chip. - -**How to run coding work — call the \`code_agent_run\` tool** with: -- \`agent\`: \`${codeMode}\` (always — match the chip). -- \`cwd\`: ${codeCwd ? `\`${codeCwd}\` (always — this coding session is pinned to that directory; never use another path)` : `the absolute project/working directory (resolve it per the code-with-agents skill — a path the user named, the "# User Work Directory" block, or ask once)`}. -- \`prompt\`: a clear, self-contained coding instruction. - -The tool runs the agent on-device and streams its tool calls, file diffs, and plan into the chat; any action needing approval surfaces as an inline permission card, so you do NOT pre-confirm with an in-chat "reply yes". This chat keeps ONE persistent agent session, so follow-up coding requests automatically resume with full context — just call \`code_agent_run\` again. Do NOT shell out to \`acpx\` or \`executeCommand\` for coding, and do NOT fall back to your own file tools. - -If the user's message is clearly NOT a coding request (small talk, an unrelated question), answer directly without invoking the coding agent. Code mode signals readiness, not that every message must route through the agent.`; + // App-activated mode capabilities compose here, in MODE_CAPABILITIES + // order — a fixed total order, so identical inputs yield identical + // bytes. The fragment text lives with the capability records. + const ctx: CapabilityContext = { + voiceInput, + voiceOutput, + searchEnabled, + codeMode, + codeCwd, + videoMode: videoMode ?? false, + coachMode: coachMode ?? false, + }; + for (const capability of MODE_CAPABILITIES) { + const fragment = capability.promptFragment?.(ctx); + if (fragment) { + instructionsWithDateTime += `\n\n${fragment}`; + } } return instructionsWithDateTime; } diff --git a/apps/x/packages/core/src/application/assistant/capabilities/modes.ts b/apps/x/packages/core/src/application/assistant/capabilities/modes.ts new file mode 100644 index 00000000..b3b88fd4 --- /dev/null +++ b/apps/x/packages/core/src/application/assistant/capabilities/modes.ts @@ -0,0 +1,134 @@ +import type { CapabilityContext, CapabilityDefinition } from "./types.js"; + +// The app-activated capabilities: the modes the app (not the model) toggles — +// facts about the world like "the camera is on" whose guidance must be in the +// system prompt from token zero. Fragment text is extracted VERBATIM from the +// historical composeSystemInstructions if-chain; the golden snapshot tests in +// agents/compose-instructions.test.ts pin the composed bytes. +// +// Array order IS composition order (a fixed total order keeps composed +// prompts byte-stable). Tool ownership note: code-mode conceptually owns +// code_agent_run/launch-code-task, but they stay in COPILOT_BASE_TOOLS until +// the legacy runs engine (which cannot attach tools mid-run) is retired. + +export const MODE_CAPABILITIES: readonly CapabilityDefinition[] = [ + { + id: "voice-input", + title: "Voice Input", + summary: "The user's message was transcribed from speech.", + activation: "app", + promptFragment: (ctx: CapabilityContext) => + ctx.voiceInput ? VOICE_INPUT : null, + }, + { + id: "video-mode", + title: "Video Mode (Live Camera)", + summary: "Webcam (and optionally screen-share) frames arrive with the user's messages.", + activation: "app", + promptFragment: (ctx: CapabilityContext) => + ctx.videoMode ? VIDEO_MODE : null, + }, + { + id: "coach-mode", + title: "Practice Session (Coach Mode)", + summary: "The user is rehearsing something performative and wants live coaching.", + activation: "app", + promptFragment: (ctx: CapabilityContext) => + ctx.coachMode ? COACH_MODE : null, + }, + { + id: "voice-output", + title: "Voice Output", + summary: "Responses must lead with tags for TTS.", + activation: "app", + promptFragment: (ctx: CapabilityContext) => + ctx.voiceOutput === "summary" + ? VOICE_OUTPUT_SUMMARY + : ctx.voiceOutput === "full" + ? VOICE_OUTPUT_FULL + : null, + }, + { + id: "search", + title: "Search", + summary: "The user has requested a web search.", + activation: "app", + promptFragment: (ctx: CapabilityContext) => + ctx.searchEnabled ? SEARCH : null, + }, + { + id: "code-mode", + title: "Code Mode", + summary: "Coding work routes to the on-device coding agent selected by the composer chip.", + activation: "app", + promptFragment: (ctx: CapabilityContext) => { + const { codeMode, codeCwd } = ctx; + if (!codeMode) return null; + const agentDisplay = codeMode === "claude" ? "Claude Code" : "Codex"; + return CODE_MODE_TEMPLATE(agentDisplay, codeMode, codeCwd); + }, + }, +]; + +const VOICE_INPUT = `# Voice Input\nThe user's message was transcribed from speech. Be aware that:\n- There may be transcription errors. Silently correct obvious ones (e.g. homophones, misheard words). If an error is genuinely ambiguous, briefly mention your interpretation (e.g. "I'm assuming you meant X").\n- Spoken messages are often long-winded. The user may ramble, repeat themselves, or correct something they said earlier in the same message. Focus on their final intent, not every word verbatim.`; + +const VIDEO_MODE = `# Video Mode (Live Camera) +The user has turned on video mode: their webcam is on, and their messages arrive with a series of live webcam frames (ordered oldest to newest) captured while they were speaking or typing. The frames are a live view of the user themselves — not a document or file to analyze. + +How to use the frames: +- Use them for what visual awareness genuinely adds: the user's expressions, body language, posture, gestures, eye contact with the camera, visible energy, anything they hold up to the camera, and their surroundings when relevant. +- Compare across frames to notice change over time within a message (e.g. increasingly slouched, started smiling, looked away for most of the message). +- When the user practices something performative (a pitch, presentation, interview, talk) or asks for delivery feedback, give specific, actionable coaching grounded in what you actually see — cite concrete observations ("in the last few frames you looked down and away from the camera") rather than generic advice. Cover posture, eye contact, facial expressiveness, gesturing, and visible energy. +- If they show something to the camera (an object, document, whiteboard), read or describe it and respond accordingly. + +Driving the app: +- You can control the Rowboat app the user is looking at via the app-navigation tool (load the app-navigation skill first): open views, READ a view's contents as data (emails, background agents, chat history), and open specific items on their screen. +- When the user asks about anything that lives inside Rowboat ("what emails do I have?", "what agents are running?", "open the one from Arjun"), prefer driving over describing: read-view shows the view on their screen while returning its data, then answer out loud briefly; open-item when they pick one. Narrate as you act ("pulling up your inbox…"). Reading a view's data beats squinting at screen-share frames — it's exact. + +Screen sharing: +- The user may also share their screen. Screen-share frames arrive in a separately labeled group after the webcam frames; they show the user's screen, not the user. +- When screen frames are present, treat them as the primary subject: the user is usually asking about, or working on, what's visible there. Read the screen carefully — code, documents, error messages, UI state — and help with it concretely. +- The LAST screen frame is the most current view of their screen; earlier ones show how it changed while they spoke. +- Frames are downscaled captures: small text may be hard to read. If something crucial is illegible, say what you need ("zoom into the error message" / "make that panel bigger") rather than guessing. + +Etiquette: +- Do not narrate or list what you see in every response. Bring up visual observations only when relevant to the user's request or clearly worth mentioning. +- Never comment on the user's physical appearance, attractiveness, or personal attributes — visual feedback is strictly about delivery, expression, and body language. +- Frames are periodic snapshots, not continuous video; moments between frames are missing. Don't claim certainty about motion you couldn't have seen.`; + +const COACH_MODE = `# Practice Session (Coach Mode) +The user started a practice session: they are rehearsing something performative — a pitch, presentation, interview answer, or talk — and want live coaching. You are their coach for this session. + +How to coach: +- Watch and listen for delivery: pacing, filler words, rambling, structure, and (from the webcam frames) posture, eye contact with the camera, facial expressiveness, gesturing, and visible energy. +- After each take or answer, give brief, specific, actionable feedback: 2-3 concrete observations, quoting what you actually saw or heard ("you looked down during the ask", "the opening 'um, so, basically' undercuts your hook"). Then let them go again. +- If they are clearly mid-flow, keep any interjection to one short sentence — or stay silent and save it for the break. +- Ask what they're practicing for at the start if it isn't obvious (investor pitch? interview? conference talk?) and tailor feedback to that audience. +- When they say they're done or wrap up, give a structured debrief: what's working, the top 3 things to improve, and one concrete drill or reframe to try next time. +- Be encouraging but honest — vague praise wastes their rehearsal time. Never comment on physical appearance; delivery, expression, and body language only.`; + +const VOICE_OUTPUT_SUMMARY = `# Voice Output (MANDATORY — READ THIS FIRST)\nThe user has voice output enabled. THIS IS YOUR #1 PRIORITY: you MUST start your response with tags. If your response does not begin with tags, the user will hear nothing — which is a broken experience. NEVER skip this.\n\nRules:\n1. YOUR VERY FIRST OUTPUT MUST BE A TAG. No exceptions. Do not start with markdown, headings, or any other text. The literal first characters of your response must be "".\n2. Place ALL tags at the BEGINNING of your response, before any detailed content. Do NOT intersperse tags throughout the response.\n3. Wrap EACH spoken sentence in its own separate tag so it can be spoken incrementally. Do NOT wrap everything in a single block.\n4. Use voice as a TL;DR and navigation aid — do NOT read the entire response aloud.\n5. After all tags, you may include detailed written content (markdown, tables, code, etc.) that will be shown visually but not spoken.\n\n## Examples\n\nExample 1 — User asks: "what happened in my meeting with Alex yesterday?"\n\nYour meeting with Alex covered three main things: the Q2 roadmap timeline, hiring for the backend role, and the client demo next week.\nI've pulled out the key details and action items below — the demo prep notes are at the end.\n\n## Meeting with Alex — March 11\n### Roadmap\n- Agreed to push Q2 launch to April 15...\n(detailed written content continues)\n\nExample 2 — User asks: "summarize my emails"\n\nYou have five new emails since this morning.\nTwo are from your team — Jordan sent the RFC you requested and Taylor flagged a contract issue.\nThere's also a warm intro from a VC partner connecting you with someone at a prospective customer.\nI've drafted responses for three of them. The details and drafts are below.\n\n(email blocks, tables, and detailed content follow)\n\nExample 3 — User asks: "what's on my calendar today?"\n\nYou've got a pretty packed day — seven meetings starting with standup at 9.\nThe big ones are your investor call at 11, lunch with a partner from your lead VC at 12:30, and a customer call at 4.\nYour only free block for deep work is 2:30 to 4.\n\n(calendar block with full event details follows)\n\nExample 4 — User asks: "draft an email to Sam with our metrics"\n\nDone — I've drafted the email to Sam with your latest WAU and churn numbers.\nTake a look at the draft below and send it when you're ready.\n\n(email block with draft follows)\n\nREMEMBER: If you do not start with tags, the user hears silence. Always speak first, then write.`; + +const VOICE_OUTPUT_FULL = `# Voice Output — Full Read-Aloud (MANDATORY — READ THIS FIRST)\nThe user wants your ENTIRE response spoken aloud. THIS IS YOUR #1 PRIORITY: every single sentence must be wrapped in tags. If you write anything outside tags, the user will not hear it — which is a broken experience. NEVER skip this.\n\nRules:\n1. YOUR VERY FIRST OUTPUT MUST BE A TAG. No exceptions. The literal first characters of your response must be "".\n2. Wrap EACH sentence in its own separate tag so it can be spoken incrementally.\n3. Write your response in a natural, conversational style suitable for listening — no markdown headings, bullet points, or formatting symbols. Use plain spoken language.\n4. Structure the content as if you are speaking to the user directly. Use transitions like "first", "also", "one more thing" instead of visual formatting.\n5. EVERY sentence MUST be inside a tag. Do not leave ANY content outside tags. If it's not in a tag, the user cannot hear it.\n\n## Examples\n\nExample 1 — User asks: "what happened in my meeting with Alex yesterday?"\n\nYour meeting with Alex covered three main things.\nFirst, you discussed the Q2 roadmap timeline and agreed to push the launch to April.\nSecond, you talked about hiring for the backend role — Alex will send over two candidates by Friday.\nAnd lastly, the client demo is next week on Thursday at 2pm, and you're handling the intro slides.\n\nExample 2 — User asks: "summarize my emails"\n\nYou've got five new emails since this morning.\nTwo are from your team — Jordan sent the RFC you asked for, and Taylor flagged a contract issue that needs your sign-off.\nThere's a warm intro from a VC partner connecting you with an engineering lead at a potential customer.\nAnd someone from a prospective client wants to confirm your API tier before your call this afternoon.\nI've drafted replies for three of them — the metrics update, the intro, and the API question.\nThe only one I left for you is Taylor's contract redline, since that needs your judgment on the liability cap.\n\nExample 3 — User asks: "what's on my calendar today?"\n\nYou've got a packed day — seven meetings starting with standup at 9.\nThe highlights are your investor call at 11, lunch with a VC partner at 12:30, and a customer call at 4.\nYour only open block for deep work is 2:30 to 4, so plan accordingly.\nOh, and your 1-on-1 with your co-founder is at 5:30 — that's a walking meeting.\n\nExample 4 — User asks: "how are our metrics looking?"\n\nMetrics are looking strong this week.\nYou hit 2,573 weekly active users, which is up 12% week over week.\nThat means you've crossed the 2,500 milestone — worth calling out in your next investor update.\nChurn is down to 4.1%, improving month over month.\nThe trailing 8-week compound growth rate is about 10%.\n\nREMEMBER: Start with immediately. No preamble, no markdown before it. Speak first.`; + +const SEARCH = `# Search\nThe user has requested a search. Use the web-search tool to answer their query.`; + +const CODE_MODE_TEMPLATE = ( + agentDisplay: string, + codeMode: "claude" | "codex", + codeCwd: string | null, +): string => `# Code Mode (Active) — Agent: ${agentDisplay} +The user has turned on **code mode** and the composer chip is set to **${agentDisplay}** (\`${codeMode}\`). For EVERY coding task this turn, use **${agentDisplay}**, and narrate that agent ("Using ${agentDisplay} to …"). + +The chip is the single source of truth for which agent runs: +- Do NOT carry over a different agent from earlier in this thread — even if a previous run used the other agent, use **${agentDisplay}** now. +- Do NOT switch agents based on an in-chat text request ("use codex", "switch to claude"). The agent only changes when the user toggles the chip; if they ask in chat, tell them to toggle the chip. + +**How to run coding work — call the \`code_agent_run\` tool** with: +- \`agent\`: \`${codeMode}\` (always — match the chip). +- \`cwd\`: ${codeCwd ? `\`${codeCwd}\` (always — this coding session is pinned to that directory; never use another path)` : `the absolute project/working directory (resolve it per the code-with-agents skill — a path the user named, the "# User Work Directory" block, or ask once)`}. +- \`prompt\`: a clear, self-contained coding instruction. + +The tool runs the agent on-device and streams its tool calls, file diffs, and plan into the chat; any action needing approval surfaces as an inline permission card, so you do NOT pre-confirm with an in-chat "reply yes". This chat keeps ONE persistent agent session, so follow-up coding requests automatically resume with full context — just call \`code_agent_run\` again. Do NOT shell out to \`acpx\` or \`executeCommand\` for coding, and do NOT fall back to your own file tools. + +If the user's message is clearly NOT a coding request (small talk, an unrelated question), answer directly without invoking the coding agent. Code mode signals readiness, not that every message must route through the agent.`; From f2a874a21447b8fab1c80b0757129477406c02c8 Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Thu, 9 Jul 2026 22:40:27 +0530 Subject: [PATCH 5/6] refactor(x): workspace context becomes an always-activated capability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent-notes and work-directory prompt blocks move out of the composer into WORKSPACE_CONTEXT_CAPABILITY (activation: 'always'), whose fragment is pure over the resolved inputs — the resolver still loads notes/work-dir only for agents with the workspaceContext trait and passes null otherwise, so trait gating is unchanged. The composer now iterates one capability list (workspace + modes) in fixed order; golden snapshots pass unchanged, byte-identical output. The composer's own body is down to: base instructions + hidden-user-context + the capability loop. Co-Authored-By: Claude Fable 5 --- .../core/src/agents/compose-instructions.ts | 28 ++---------- .../assistant/capabilities/types.ts | 4 ++ .../assistant/capabilities/workspace.ts | 43 +++++++++++++++++++ 3 files changed, 51 insertions(+), 24 deletions(-) create mode 100644 apps/x/packages/core/src/application/assistant/capabilities/workspace.ts diff --git a/apps/x/packages/core/src/agents/compose-instructions.ts b/apps/x/packages/core/src/agents/compose-instructions.ts index 2cda8819..025f9196 100644 --- a/apps/x/packages/core/src/agents/compose-instructions.ts +++ b/apps/x/packages/core/src/agents/compose-instructions.ts @@ -1,4 +1,5 @@ import { MODE_CAPABILITIES } from "../application/assistant/capabilities/modes.js"; +import { WORKSPACE_CONTEXT_CAPABILITY } from "../application/assistant/capabilities/workspace.js"; import type { CapabilityContext } from "../application/assistant/capabilities/types.js"; // System-prompt composition for agent assembly: the base instructions plus @@ -52,33 +53,12 @@ export function composeSystemInstructions({ coachMode, }: ComposeSystemInstructionsInput): string { let instructionsWithDateTime = `${instructions}\n\n${USER_CONTEXT_SYSTEM_INSTRUCTIONS}`; - if (agentNotesContext) { - instructionsWithDateTime += `\n\n${agentNotesContext}`; - } - if (userWorkDir) { - instructionsWithDateTime += `\n\n# User Work Directory -The user has chosen the following directory as their current **work directory**: - -\`${userWorkDir}\` - -Treat this as the **default location** for file operations whenever the user refers to files generically: -- "list the files", "show me what's in here", "what's the latest report" — list or look in the work directory. -- "save this", "export it", "write that to a file" — write the output into the work directory unless the user names another location. -- "open the file I was just working on", "the doc from earlier" — assume the work directory first. - -Use absolute paths rooted at this directory with the \`file-*\` tools. For example, list with \`file-list({ path: "${userWorkDir}" })\`, read text with \`file-readText\`, and write text with \`file-writeText\`. For PDFs, Office docs, images, scanned docs, and other non-text files, use \`parseFile\` or \`LLMParse\` with the absolute path; you do NOT need to copy the file into the workspace first. - -**Exceptions — these ALWAYS take precedence over the work directory default:** -1. **Knowledge base questions.** If the user asks about anything in the knowledge graph (notes, people, organizations, projects, topics) or paths starting with \`knowledge/\`, use file tools against \`knowledge/\` as documented above. Do NOT redirect those into the work directory. -2. **Explicit paths.** If the user names a different directory or gives an absolute/relative path (e.g. "in ~/Downloads", "from /tmp/foo", "the Desktop"), honor that path exactly and ignore the work-directory default for that request. -3. **Workspace-specific operations.** Anything that obviously belongs in the Rowboat workspace (config files, MCP servers, agent schedules, etc.) stays in the workspace, not the work directory. - -Do not announce the work directory unless it's relevant. Just use it.`; - } // App-activated mode capabilities compose here, in MODE_CAPABILITIES // order — a fixed total order, so identical inputs yield identical // bytes. The fragment text lives with the capability records. const ctx: CapabilityContext = { + agentNotesContext, + userWorkDir, voiceInput, voiceOutput, searchEnabled, @@ -87,7 +67,7 @@ Do not announce the work directory unless it's relevant. Just use it.`; videoMode: videoMode ?? false, coachMode: coachMode ?? false, }; - for (const capability of MODE_CAPABILITIES) { + for (const capability of [WORKSPACE_CONTEXT_CAPABILITY, ...MODE_CAPABILITIES]) { const fragment = capability.promptFragment?.(ctx); if (fragment) { instructionsWithDateTime += `\n\n${fragment}`; diff --git a/apps/x/packages/core/src/application/assistant/capabilities/types.ts b/apps/x/packages/core/src/application/assistant/capabilities/types.ts index f0fee3cd..6a5c7129 100644 --- a/apps/x/packages/core/src/application/assistant/capabilities/types.ts +++ b/apps/x/packages/core/src/application/assistant/capabilities/types.ts @@ -26,6 +26,10 @@ export type CapabilityActivation = "model" | "app" | "always"; // stay byte-identical for identical inputs (provider prefix caching and // agent-snapshot inheritance both depend on it). export interface CapabilityContext { + // Workspace context (workspaceContext trait agents only; the resolver + // loads these and leaves them null for everyone else). + agentNotesContext: string | null; + userWorkDir: string | null; voiceInput: boolean; voiceOutput: "summary" | "full" | null; searchEnabled: boolean; diff --git a/apps/x/packages/core/src/application/assistant/capabilities/workspace.ts b/apps/x/packages/core/src/application/assistant/capabilities/workspace.ts new file mode 100644 index 00000000..77fb7511 --- /dev/null +++ b/apps/x/packages/core/src/application/assistant/capabilities/workspace.ts @@ -0,0 +1,43 @@ +import type { CapabilityContext, CapabilityDefinition } from "./types.js"; + +// The always-activated workspace-context capability: agent notes and the +// user work directory, composed for agents with the workspaceContext trait +// (the resolver loads both inputs and leaves them null for everyone else). +// Fragment text extracted verbatim from the historical composer; the golden +// snapshots in agents/compose-instructions.test.ts pin the bytes. + +export const WORKSPACE_CONTEXT_CAPABILITY: CapabilityDefinition = { + id: "workspace-context", + title: "Workspace Context", + summary: "Agent notes and the user's chosen work directory.", + activation: "always", + promptFragment: (ctx: CapabilityContext) => { + const parts: string[] = []; + if (ctx.agentNotesContext) { + parts.push(ctx.agentNotesContext); + } + if (ctx.userWorkDir) { + parts.push(WORK_DIR_TEMPLATE(ctx.userWorkDir)); + } + return parts.length > 0 ? parts.join("\n\n") : null; + }, +}; + +const WORK_DIR_TEMPLATE = (userWorkDir: string): string => `# User Work Directory +The user has chosen the following directory as their current **work directory**: + +\`${userWorkDir}\` + +Treat this as the **default location** for file operations whenever the user refers to files generically: +- "list the files", "show me what's in here", "what's the latest report" — list or look in the work directory. +- "save this", "export it", "write that to a file" — write the output into the work directory unless the user names another location. +- "open the file I was just working on", "the doc from earlier" — assume the work directory first. + +Use absolute paths rooted at this directory with the \`file-*\` tools. For example, list with \`file-list({ path: "${userWorkDir}" })\`, read text with \`file-readText\`, and write text with \`file-writeText\`. For PDFs, Office docs, images, scanned docs, and other non-text files, use \`parseFile\` or \`LLMParse\` with the absolute path; you do NOT need to copy the file into the workspace first. + +**Exceptions — these ALWAYS take precedence over the work directory default:** +1. **Knowledge base questions.** If the user asks about anything in the knowledge graph (notes, people, organizations, projects, topics) or paths starting with \`knowledge/\`, use file tools against \`knowledge/\` as documented above. Do NOT redirect those into the work directory. +2. **Explicit paths.** If the user names a different directory or gives an absolute/relative path (e.g. "in ~/Downloads", "from /tmp/foo", "the Desktop"), honor that path exactly and ignore the work-directory default for that request. +3. **Workspace-specific operations.** Anything that obviously belongs in the Rowboat workspace (config files, MCP servers, agent schedules, etc.) stays in the workspace, not the work directory. + +Do not announce the work directory unless it's relevant. Just use it.`; From a9a342653f06d69cdf78f9616f817804d36d7405 Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Thu, 9 Jul 2026 23:10:58 +0530 Subject: [PATCH 6/6] =?UTF-8?q?fix(x):=20review=20fixes=20=E2=80=94=20prot?= =?UTF-8?q?otype-safe=20registry=20lookup,=20shared=20parsing,=20slimmer?= =?UTF-8?q?=20records?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Findings from an independent review of the branch: - loadAgent's plain-object lookup traversed Object.prototype, so a user-defined agent named "constructor"/"toString" threw a TypeError instead of falling through to the agents repo as it did before the registry (Object.hasOwn now guards both lookups; regression test pins the fallthrough without depending on the environment). - agentFromRaw now uses the same parseFrontmatter helper FSAgentsRepo uses, so builtin and user agents can never drift on one file format. - The fused workspace-context capability splits into agent-notes and work-directory records — the composer's own separator joining makes the output byte-identical, so the fusion bought nothing. - Mode records drop their write-only title/summary (optional on the base record, still required on skills where the catalog renders them), the composer body is reindented and iterates a hoisted PROMPT_CAPABILITIES list, and the two single-consumer re-export shims in the legacy runtime file are deleted (consumers retargeted). Golden snapshots unchanged throughout: composed prompts remain byte-identical. Deferred to the follow-up PR with availability wiring: discriminated-union typing on activation + fencing resolveSkill, and unifying the three mirrored composition shapes. Co-Authored-By: Claude Fable 5 --- .../core/src/agents/compose-instructions.ts | 54 +++++++++++-------- .../packages/core/src/agents/registry.test.ts | 18 +++++++ apps/x/packages/core/src/agents/registry.ts | 33 ++++++------ apps/x/packages/core/src/agents/runtime.ts | 10 ---- .../assistant/capabilities/modes.ts | 12 ----- .../assistant/capabilities/types.ts | 8 ++- .../assistant/capabilities/workspace.ts | 32 ++++++----- .../src/application/assistant/skills/index.ts | 9 +++- apps/x/packages/core/src/runs/runs.ts | 2 +- .../turns/bridges/real-agent-resolver.test.ts | 2 +- 10 files changed, 97 insertions(+), 83 deletions(-) diff --git a/apps/x/packages/core/src/agents/compose-instructions.ts b/apps/x/packages/core/src/agents/compose-instructions.ts index 025f9196..2b30402e 100644 --- a/apps/x/packages/core/src/agents/compose-instructions.ts +++ b/apps/x/packages/core/src/agents/compose-instructions.ts @@ -1,7 +1,17 @@ import { MODE_CAPABILITIES } from "../application/assistant/capabilities/modes.js"; -import { WORKSPACE_CONTEXT_CAPABILITY } from "../application/assistant/capabilities/workspace.js"; +import { + AGENT_NOTES_CAPABILITY, + WORK_DIRECTORY_CAPABILITY, +} from "../application/assistant/capabilities/workspace.js"; import type { CapabilityContext } from "../application/assistant/capabilities/types.js"; +// Everything that composes into the system prompt, in composition order. +const PROMPT_CAPABILITIES = [ + AGENT_NOTES_CAPABILITY, + WORK_DIRECTORY_CAPABILITY, + ...MODE_CAPABILITIES, +] as const; + // System-prompt composition for agent assembly: the base instructions plus // the mode blocks (voice, video, coach, search, code) appended per turn // composition. Extracted verbatim from the legacy streamAgent path so both @@ -52,26 +62,26 @@ export function composeSystemInstructions({ videoMode, coachMode, }: ComposeSystemInstructionsInput): string { - let instructionsWithDateTime = `${instructions}\n\n${USER_CONTEXT_SYSTEM_INSTRUCTIONS}`; - // App-activated mode capabilities compose here, in MODE_CAPABILITIES - // order — a fixed total order, so identical inputs yield identical - // bytes. The fragment text lives with the capability records. - const ctx: CapabilityContext = { - agentNotesContext, - userWorkDir, - voiceInput, - voiceOutput, - searchEnabled, - codeMode, - codeCwd, - videoMode: videoMode ?? false, - coachMode: coachMode ?? false, - }; - for (const capability of [WORKSPACE_CONTEXT_CAPABILITY, ...MODE_CAPABILITIES]) { - const fragment = capability.promptFragment?.(ctx); - if (fragment) { - instructionsWithDateTime += `\n\n${fragment}`; - } + let composed = `${instructions}\n\n${USER_CONTEXT_SYSTEM_INSTRUCTIONS}`; + // Capabilities compose in PROMPT_CAPABILITIES order — a fixed total + // order, so identical inputs yield identical bytes. The fragment text + // lives with the capability records. + const ctx: CapabilityContext = { + agentNotesContext, + userWorkDir, + voiceInput, + voiceOutput, + searchEnabled, + codeMode, + codeCwd, + videoMode: videoMode ?? false, + coachMode: coachMode ?? false, + }; + for (const capability of PROMPT_CAPABILITIES) { + const fragment = capability.promptFragment?.(ctx); + if (fragment) { + composed += `\n\n${fragment}`; } - return instructionsWithDateTime; + } + return composed; } diff --git a/apps/x/packages/core/src/agents/registry.test.ts b/apps/x/packages/core/src/agents/registry.test.ts index dffe0994..d89a66ba 100644 --- a/apps/x/packages/core/src/agents/registry.test.ts +++ b/apps/x/packages/core/src/agents/registry.test.ts @@ -77,4 +77,22 @@ describe("agent registry", () => { expect(agent.instructions).toBe("Just instructions."); expect(agent.tools).toBeUndefined(); }); + + it("ids colliding with Object.prototype fall through to the repo, not the table", async () => { + // A plain-object lookup would resolve builtinAgents['constructor'] to + // the inherited Object constructor and crash on .build() before ever + // reaching the user-agents repo. Whether the repo then resolves or + // rejects depends on the environment; the contract under test is only + // that the table path is not taken. + for (const id of ["constructor", "toString", "valueOf", "hasOwnProperty"]) { + expect(hasWorkspaceContext(id)).toBe(false); + const outcome = await loadAgent(id).then( + () => null, + (error: unknown) => String(error), + ); + if (outcome !== null) { + expect(outcome).not.toMatch(/is not a function/); + } + } + }); }); diff --git a/apps/x/packages/core/src/agents/registry.ts b/apps/x/packages/core/src/agents/registry.ts index e8ab223e..9508ea7f 100644 --- a/apps/x/packages/core/src/agents/registry.ts +++ b/apps/x/packages/core/src/agents/registry.ts @@ -1,6 +1,6 @@ import type { z } from "zod"; -import { parse } from "yaml"; import { Agent } from "@x/shared/dist/agent.js"; +import { parseFrontmatter } from "../application/lib/parse-frontmatter.js"; import { buildCopilotAgent } from "../application/assistant/agent.js"; import { buildBackgroundTaskAgent } from "../background-tasks/agent.js"; import { buildLiveNoteAgent } from "../knowledge/live-note/agent.js"; @@ -30,21 +30,16 @@ interface BuiltinAgentDefinition { } // Prompt-file agents: instructions ship as a raw string whose optional YAML -// frontmatter carries agent config (tools, model). One loader replaces the -// five copy-pasted parsing blocks the ladder accumulated. +// frontmatter carries agent config (tools, model). Parsing goes through the +// same parseFrontmatter helper FSAgentsRepo uses for user agents, so builtin +// and user-defined agents can never drift on the same file format. export function agentFromRaw(id: string, raw: string): z.infer { - let agent: z.infer = { name: id, instructions: raw }; - if (raw.startsWith("---")) { - const end = raw.indexOf("\n---", 3); - if (end !== -1) { - const frontmatter = raw.slice(3, end).trim(); - const content = raw.slice(end + 4).trim(); - const yaml: unknown = parse(frontmatter); - const parsed = Agent.omit({ name: true, instructions: true }).parse(yaml); - agent = { ...agent, ...parsed, instructions: content }; - } + const { frontmatter, content } = parseFrontmatter(raw); + if (frontmatter === null) { + return { name: id, instructions: raw }; } - return agent; + const parsed = Agent.omit({ name: true, instructions: true }).parse(frontmatter); + return { name: id, ...parsed, instructions: content }; } function promptFileAgent(id: string, getRaw: () => string): BuiltinAgentDefinition { @@ -79,12 +74,18 @@ export function builtinAgentIds(): string[] { export function hasWorkspaceContext(agentId: string | null | undefined): boolean { return ( agentId != null && - builtinAgents[agentId]?.traits?.workspaceContext === true + Object.hasOwn(builtinAgents, agentId) && + builtinAgents[agentId].traits?.workspaceContext === true ); } export async function loadAgent(id: string): Promise> { - const builtin = builtinAgents[id]; + // Object.hasOwn: a plain lookup would traverse Object.prototype, so a + // user agent named "constructor"/"toString" would hit an inherited + // function instead of falling through to the repo. + const builtin = Object.hasOwn(builtinAgents, id) + ? builtinAgents[id] + : undefined; if (builtin) { return builtin.build(); } diff --git a/apps/x/packages/core/src/agents/runtime.ts b/apps/x/packages/core/src/agents/runtime.ts index 58dedcb5..ac68c3ee 100644 --- a/apps/x/packages/core/src/agents/runtime.ts +++ b/apps/x/packages/core/src/agents/runtime.ts @@ -302,12 +302,6 @@ ${sections.join('\n\n')} `; } -// System-prompt assembly lives in compose-instructions.ts (owned by the -// assembly layer); re-exported so legacy callers keep their import path. -export { - composeSystemInstructions, - type ComposeSystemInstructionsInput, -} from "./compose-instructions.js"; import { composeSystemInstructions } from "./compose-instructions.js"; export interface IAgentRuntime { @@ -647,10 +641,6 @@ function formatLlmStreamError(rawError: unknown): string { return lines.length ? lines.join("\n") : "Model stream error"; } -// Agent identity now lives in the registry (registry.ts); re-exported here -// so legacy callers keep their import path until the runs engine dies. -export { loadAgent }; - function formatBytes(bytes: number): string { if (bytes < 1024) return `${bytes} B`; if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; diff --git a/apps/x/packages/core/src/application/assistant/capabilities/modes.ts b/apps/x/packages/core/src/application/assistant/capabilities/modes.ts index b3b88fd4..e8641747 100644 --- a/apps/x/packages/core/src/application/assistant/capabilities/modes.ts +++ b/apps/x/packages/core/src/application/assistant/capabilities/modes.ts @@ -14,32 +14,24 @@ import type { CapabilityContext, CapabilityDefinition } from "./types.js"; export const MODE_CAPABILITIES: readonly CapabilityDefinition[] = [ { id: "voice-input", - title: "Voice Input", - summary: "The user's message was transcribed from speech.", activation: "app", promptFragment: (ctx: CapabilityContext) => ctx.voiceInput ? VOICE_INPUT : null, }, { id: "video-mode", - title: "Video Mode (Live Camera)", - summary: "Webcam (and optionally screen-share) frames arrive with the user's messages.", activation: "app", promptFragment: (ctx: CapabilityContext) => ctx.videoMode ? VIDEO_MODE : null, }, { id: "coach-mode", - title: "Practice Session (Coach Mode)", - summary: "The user is rehearsing something performative and wants live coaching.", activation: "app", promptFragment: (ctx: CapabilityContext) => ctx.coachMode ? COACH_MODE : null, }, { id: "voice-output", - title: "Voice Output", - summary: "Responses must lead with tags for TTS.", activation: "app", promptFragment: (ctx: CapabilityContext) => ctx.voiceOutput === "summary" @@ -50,16 +42,12 @@ export const MODE_CAPABILITIES: readonly CapabilityDefinition[] = [ }, { id: "search", - title: "Search", - summary: "The user has requested a web search.", activation: "app", promptFragment: (ctx: CapabilityContext) => ctx.searchEnabled ? SEARCH : null, }, { id: "code-mode", - title: "Code Mode", - summary: "Coding work routes to the on-device coding agent selected by the composer chip.", activation: "app", promptFragment: (ctx: CapabilityContext) => { const { codeMode, codeCwd } = ctx; diff --git a/apps/x/packages/core/src/application/assistant/capabilities/types.ts b/apps/x/packages/core/src/application/assistant/capabilities/types.ts index 6a5c7129..a3ebb7d5 100644 --- a/apps/x/packages/core/src/application/assistant/capabilities/types.ts +++ b/apps/x/packages/core/src/application/assistant/capabilities/types.ts @@ -41,8 +41,12 @@ export interface CapabilityContext { export interface CapabilityDefinition { id: string; - title: string; - summary: string; + // Catalog metadata: required for model-activated entries (rendered in + // the loadSkill catalog); app/always entries omit them — nothing + // consumes them there, and stale copies of fragment headings would + // only drift into misinformation. + title?: string; + summary?: string; // Defaults to 'model' (the historical skill behavior). activation?: CapabilityActivation; // Lazy guidance returned by loadSkill (model activation). diff --git a/apps/x/packages/core/src/application/assistant/capabilities/workspace.ts b/apps/x/packages/core/src/application/assistant/capabilities/workspace.ts index 77fb7511..1864ff5c 100644 --- a/apps/x/packages/core/src/application/assistant/capabilities/workspace.ts +++ b/apps/x/packages/core/src/application/assistant/capabilities/workspace.ts @@ -1,26 +1,24 @@ import type { CapabilityContext, CapabilityDefinition } from "./types.js"; -// The always-activated workspace-context capability: agent notes and the +// The always-activated workspace-context capabilities: agent notes and the // user work directory, composed for agents with the workspaceContext trait // (the resolver loads both inputs and leaves them null for everyone else). -// Fragment text extracted verbatim from the historical composer; the golden -// snapshots in agents/compose-instructions.test.ts pin the bytes. +// Two independent records — the composer's own '\n\n' joining makes their +// output byte-identical to the historical fused block. Fragment text +// extracted verbatim from the historical composer; the golden snapshots in +// agents/compose-instructions.test.ts pin the bytes. -export const WORKSPACE_CONTEXT_CAPABILITY: CapabilityDefinition = { - id: "workspace-context", - title: "Workspace Context", - summary: "Agent notes and the user's chosen work directory.", +export const AGENT_NOTES_CAPABILITY: CapabilityDefinition = { + id: "agent-notes", activation: "always", - promptFragment: (ctx: CapabilityContext) => { - const parts: string[] = []; - if (ctx.agentNotesContext) { - parts.push(ctx.agentNotesContext); - } - if (ctx.userWorkDir) { - parts.push(WORK_DIR_TEMPLATE(ctx.userWorkDir)); - } - return parts.length > 0 ? parts.join("\n\n") : null; - }, + promptFragment: (ctx: CapabilityContext) => ctx.agentNotesContext, +}; + +export const WORK_DIRECTORY_CAPABILITY: CapabilityDefinition = { + id: "work-directory", + activation: "always", + promptFragment: (ctx: CapabilityContext) => + ctx.userWorkDir ? WORK_DIR_TEMPLATE(ctx.userWorkDir) : null, }; const WORK_DIR_TEMPLATE = (userWorkDir: string): string => `# User Work Directory diff --git a/apps/x/packages/core/src/application/assistant/skills/index.ts b/apps/x/packages/core/src/application/assistant/skills/index.ts index 85b82e56..9848c53b 100644 --- a/apps/x/packages/core/src/application/assistant/skills/index.ts +++ b/apps/x/packages/core/src/application/assistant/skills/index.ts @@ -28,10 +28,15 @@ const CATALOG_PREFIX = "src/application/assistant/skills"; // A skill is the model-activated subset of the capability record // (capabilities/types.ts): lazy guidance the model pulls in via loadSkill, -// plus the BuiltinTools it owns. `id` doubles as the folder name. Tool names +// plus the BuiltinTools it owns. `id` doubles as the folder name; title and +// summary are required here because the catalog renders them. Tool names // are validated against the live catalog where descriptors are built // (loadSkill / the agent resolver); unknown names are dropped there. -type SkillDefinition = CapabilityDefinition & { content: string }; +type SkillDefinition = CapabilityDefinition & { + title: string; + summary: string; + content: string; +}; type ResolvedSkill = { id: string; diff --git a/apps/x/packages/core/src/runs/runs.ts b/apps/x/packages/core/src/runs/runs.ts index e61b2309..c79e8077 100644 --- a/apps/x/packages/core/src/runs/runs.ts +++ b/apps/x/packages/core/src/runs/runs.ts @@ -11,7 +11,7 @@ import { IRunsLock } from "./lock.js"; import { forceCloseAllMcpClients } from "../mcp/mcp.js"; import { extractCommandNames } from "../application/lib/command-executor.js"; import { addFileAccessGrant, addToSecurityConfig } from "../config/security.js"; -import { loadAgent } from "../agents/runtime.js"; +import { loadAgent } from "../agents/registry.js"; import { getDefaultModelAndProvider } from "../models/defaults.js"; export async function createRun(opts: z.infer): Promise> { diff --git a/apps/x/packages/core/src/turns/bridges/real-agent-resolver.test.ts b/apps/x/packages/core/src/turns/bridges/real-agent-resolver.test.ts index e33bb3aa..1f7f46ec 100644 --- a/apps/x/packages/core/src/turns/bridges/real-agent-resolver.test.ts +++ b/apps/x/packages/core/src/turns/bridges/real-agent-resolver.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { z } from "zod"; import type { Agent } from "@x/shared/dist/agent.js"; -import { composeSystemInstructions } from "../../agents/runtime.js"; +import { composeSystemInstructions } from "../../agents/compose-instructions.js"; import type { BuiltinTools } from "../../application/lib/builtin-tools.js"; import { RealAgentResolver } from "./real-agent-resolver.js";