From 99f196e16ce7cb926da713bcf0fbd0b641418251 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:09:32 +0530 Subject: [PATCH 01/10] feat(core): elide oversized historic tool results from model context Tool results from prior turns (skill loads, file reads, HTTP fetches) dominate resent context and are rarely needed verbatim. A decorator over the context resolver now replaces prior-turn tool results above a size threshold with a short placeholder telling the model to re-run the tool if it needs the output. The current turn's in-flight results are always sent verbatim, the durable log is untouched, and elision is a pure per-message function so resolved prefixes stay byte-stable for provider prefix caching. Policy lives in config/context.json (elideHistoricToolResults, default on; elideHistoricToolResultsThresholdChars, default 10000). The inspect CLI composes through the same decorated resolver so debug output still matches transmitted bytes. Co-Authored-By: Claude Fable 5 --- .../packages/core/docs/turn-runtime-design.md | 12 + apps/x/packages/core/src/di/container.ts | 7 +- .../core/src/turns/context-elision.test.ts | 245 ++++++++++++++++++ .../core/src/turns/context-elision.ts | 129 +++++++++ apps/x/packages/core/src/turns/inspect-cli.ts | 4 +- 5 files changed, 393 insertions(+), 4 deletions(-) create mode 100644 apps/x/packages/core/src/turns/context-elision.test.ts create mode 100644 apps/x/packages/core/src/turns/context-elision.ts diff --git a/apps/x/packages/core/docs/turn-runtime-design.md b/apps/x/packages/core/docs/turn-runtime-design.md index d199fabe..0dcc8026 100644 --- a/apps/x/packages/core/docs/turn-runtime-design.md +++ b/apps/x/packages/core/docs/turn-runtime-design.md @@ -390,6 +390,18 @@ Rules: It rejects the execution and does not append `turn_failed`. - The reducer treats `context` as opaque data and never resolves references. +The app wires the resolver through a decorator (`context-elision.ts`) that +applies transmit-time elision to the materialized prefix: tool results from +prior turns above a size threshold are replaced with a short placeholder +telling the model to re-run the tool if it needs the output. The durable log +is untouched; only the transmitted bytes change. Elision is a pure function +of each message, so resolved prefixes stay byte-stable across calls (provider +prefix caches keep working), and the current turn's own in-flight tool +results never pass through the resolver, so they are always sent verbatim. +Policy lives in `config/context.json` (`elideHistoricToolResults`, default +true; `elideHistoricToolResultsThresholdChars`, default 10000). The inspect +CLI composes through the same decorated resolver. + ### 6.7 Agent snapshot inheritance Session turns whose resolved system prompt and tool set are byte-identical diff --git a/apps/x/packages/core/src/di/container.ts b/apps/x/packages/core/src/di/container.ts index 77bda6bb..306ebc33 100644 --- a/apps/x/packages/core/src/di/container.ts +++ b/apps/x/packages/core/src/di/container.ts @@ -31,7 +31,8 @@ import type { INotificationService } from "../application/notification/service.j import { SystemClock, type IClock } from "../turns/clock.js"; import { FSTurnRepo } from "../turns/fs-repo.js"; import type { ITurnRepo } from "../turns/repo.js"; -import { TurnRepoContextResolver, type IContextResolver } from "../turns/context-resolver.js"; +import type { IContextResolver } from "../turns/context-resolver.js"; +import { createContextResolver } from "../turns/context-elision.js"; import { EmitterTurnLifecycleBus, type ITurnLifecycleBus } from "../turns/bus.js"; import { RealUsageReporter } from "../turns/bridges/real-usage-reporter.js"; import type { IUsageReporter } from "../turns/usage-reporter.js"; @@ -117,7 +118,9 @@ container.register({ turnsRootDir: asValue(path.join(WorkDir, "storage", "turns")), sessionsRootDir: asValue(path.join(WorkDir, "storage", "sessions")), turnRepo: asClass(FSTurnRepo).singleton(), - contextResolver: asClass(TurnRepoContextResolver).singleton(), + contextResolver: asFunction(({ turnRepo }) => + createContextResolver({ turnRepo }), + ).singleton(), lifecycleBus: asClass(EmitterTurnLifecycleBus).singleton(), usageReporter: asClass(RealUsageReporter).singleton(), agentResolver: asFunction(() => new RealAgentResolver()).singleton(), diff --git a/apps/x/packages/core/src/turns/context-elision.test.ts b/apps/x/packages/core/src/turns/context-elision.test.ts new file mode 100644 index 00000000..9543a46f --- /dev/null +++ b/apps/x/packages/core/src/turns/context-elision.test.ts @@ -0,0 +1,245 @@ +import { describe, expect, it } from "vitest"; +import type { z } from "zod"; +import type { TurnContext, TurnEvent } from "@x/shared/dist/turns.js"; +import { + ElidingContextResolver, + elideHistoricToolResults, +} from "./context-elision.js"; +import { TurnRepoContextResolver } from "./context-resolver.js"; +import { InMemoryTurnRepo } from "./in-memory-turn-repo.js"; + +type TEvent = z.infer; + +function user(text: string) { + return { role: "user" as const, content: text }; +} + +function assistant(text: string) { + return { role: "assistant" as const, content: text }; +} + +function toolMsg(content: string, toolName = "file-readText") { + return { + role: "tool" as const, + content, + toolCallId: "tc1", + toolName, + }; +} + +// A completed turn containing one tool round trip whose result has the given +// output, followed by a final text response. +function toolTurnLog( + turnId: string, + context: z.infer, + toolOutput: string, +): TEvent[] { + const ts = "2026-07-02T10:00:00Z"; + const echo = { + toolId: "tool.echo", + name: "echo", + description: "Echo", + inputSchema: {}, + execution: "sync" as const, + requiresHuman: false, + }; + const call = { + role: "assistant" as const, + content: [ + { + type: "tool-call" as const, + toolCallId: "tc1", + toolName: "echo", + arguments: {}, + }, + ], + }; + return [ + { + type: "turn_created", + schemaVersion: 1, + turnId, + ts, + sessionId: "sess-1", + agent: { + requested: { agentId: "copilot" }, + resolved: { + agentId: "copilot", + systemPrompt: "SYS", + model: { provider: "fake", model: "m" }, + tools: [echo], + }, + }, + context, + input: user("do it"), + config: { + autoPermission: false, + humanAvailable: true, + maxModelCalls: 20, + }, + }, + { + type: "model_call_requested", + turnId, + ts, + modelCallIndex: 0, + request: { + ...(Array.isArray(context) ? {} : { contextRef: context }), + messages: + Array.isArray(context) && context.length > 0 + ? ["context", "input"] + : ["input"], + parameters: {}, + }, + }, + { + type: "model_call_completed", + turnId, + ts, + modelCallIndex: 0, + message: call, + finishReason: "tool-calls", + usage: {}, + }, + { + type: "tool_invocation_requested", + turnId, + ts, + toolCallId: "tc1", + toolId: "tool.echo", + toolName: "echo", + execution: "sync", + input: {}, + }, + { + type: "tool_result", + turnId, + ts, + toolCallId: "tc1", + toolName: "echo", + source: "sync", + result: { output: toolOutput, isError: false }, + }, + { + type: "model_call_requested", + turnId, + ts, + modelCallIndex: 1, + request: { + messages: ["assistant:0", "toolResult:tc1"], + parameters: {}, + }, + }, + { + type: "model_call_completed", + turnId, + ts, + modelCallIndex: 1, + message: assistant("done"), + finishReason: "stop", + usage: {}, + }, + { + type: "turn_completed", + turnId, + ts, + output: assistant("done"), + finishReason: "stop", + usage: {}, + }, + ]; +} + +const T1 = "2026-07-02T10-00-00Z-0000001-000"; + +describe("elideHistoricToolResults", () => { + it("replaces tool results above the threshold with a placeholder", () => { + const big = "x".repeat(101); + const elided = elideHistoricToolResults([toolMsg(big)], 100); + expect(elided).toHaveLength(1); + expect(elided[0]).toMatchObject({ + role: "tool", + toolCallId: "tc1", + toolName: "file-readText", + }); + expect(elided[0].content).toContain("elided"); + expect(elided[0].content).toContain("file-readText"); + expect(elided[0].content).toContain("101"); + }); + + it("keeps tool results at or below the threshold verbatim", () => { + const exact = toolMsg("x".repeat(100)); + expect(elideHistoricToolResults([exact], 100)).toEqual([exact]); + }); + + it("leaves non-tool messages untouched", () => { + const messages = [user("q".repeat(500)), assistant("a".repeat(500))]; + expect(elideHistoricToolResults(messages, 100)).toEqual(messages); + }); + + it("is idempotent at realistic thresholds (placeholder is well under them)", () => { + const once = elideHistoricToolResults([toolMsg("x".repeat(5000))], 1000); + expect(once[0].content.length).toBeLessThan(1000); + expect(elideHistoricToolResults(once, 1000)).toEqual(once); + }); + + it("is deterministic for the same input", () => { + const messages = [toolMsg("x".repeat(5000)), user("q"), assistant("a")]; + expect(elideHistoricToolResults(messages, 100)).toEqual( + elideHistoricToolResults(messages, 100), + ); + }); +}); + +describe("ElidingContextResolver", () => { + function resolver( + repo: InMemoryTurnRepo, + policy: { enabled: boolean; thresholdChars: number }, + ) { + return new ElidingContextResolver({ + inner: new TurnRepoContextResolver({ turnRepo: repo }), + loadPolicy: () => policy, + }); + } + + it("elides oversized tool results in the resolved prefix", async () => { + const repo = new InMemoryTurnRepo(); + repo.seed(toolTurnLog(T1, [], "x".repeat(200))); + const resolved = await resolver(repo, { + enabled: true, + thresholdChars: 100, + }).resolve({ previousTurnId: T1 }); + const tool = resolved.find((m) => m.role === "tool"); + expect(tool?.content).toContain("elided"); + // The rest of the transcript is intact. + expect(resolved.map((m) => m.role)).toEqual([ + "user", + "assistant", + "tool", + "assistant", + ]); + }); + + it("returns the prefix unchanged when the policy is disabled", async () => { + const repo = new InMemoryTurnRepo(); + const output = "x".repeat(200); + repo.seed(toolTurnLog(T1, [], output)); + const resolved = await resolver(repo, { + enabled: false, + thresholdChars: 100, + }).resolve({ previousTurnId: T1 }); + const tool = resolved.find((m) => m.role === "tool"); + expect(tool?.content).toBe(output); + }); + + it("keeps small tool results verbatim when enabled", async () => { + const repo = new InMemoryTurnRepo(); + repo.seed(toolTurnLog(T1, [], "small")); + const resolved = await resolver(repo, { + enabled: true, + thresholdChars: 100, + }).resolve({ previousTurnId: T1 }); + const tool = resolved.find((m) => m.role === "tool"); + expect(tool?.content).toBe("small"); + }); +}); diff --git a/apps/x/packages/core/src/turns/context-elision.ts b/apps/x/packages/core/src/turns/context-elision.ts new file mode 100644 index 00000000..6c88445c --- /dev/null +++ b/apps/x/packages/core/src/turns/context-elision.ts @@ -0,0 +1,129 @@ +import fs from "fs"; +import path from "path"; +import { z } from "zod"; +import type { + ConversationMessage, + ResolvedAgent, + ResolvedAgentSnapshot, + TurnContext, +} from "@x/shared/dist/turns.js"; +import { WorkDir } from "../config/config.js"; +import type { IContextResolver } from "./context-resolver.js"; +import { TurnRepoContextResolver } from "./context-resolver.js"; +import type { ITurnRepo } from "./repo.js"; + +// Transmit-time elision of historic tool results (the cross-turn prefix +// only — the current turn's own messages never pass through the resolver, so +// in-flight tool results are always sent verbatim). Large tool outputs from +// earlier turns (skill loads, file reads, HTTP fetches) dominate resent +// context; the model rarely needs them verbatim and can re-run the tool when +// it does. Elision is a pure function of each message's content, so resolved +// prefixes stay byte-stable across calls and turns (provider prefix caches +// keep working), and the durable JSONL log is untouched — only the +// transmitted bytes change. + +export interface ToolResultElisionPolicy { + enabled: boolean; + thresholdChars: number; +} + +export const DEFAULT_ELISION_POLICY: ToolResultElisionPolicy = { + enabled: true, + thresholdChars: 10_000, +}; + +const ContextConfig = z.object({ + elideHistoricToolResults: z.boolean().optional(), + elideHistoricToolResultsThresholdChars: z.number().int().min(0).optional(), +}); + +const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json"); + +// Read the elision policy from config/context.json, falling back to defaults +// for missing keys or an unreadable file. Read per resolve so a config edit +// applies to the next turn without a restart. +export function loadElisionPolicy(): ToolResultElisionPolicy { + try { + if (!fs.existsSync(CONTEXT_CONFIG_PATH)) { + return DEFAULT_ELISION_POLICY; + } + const raw = fs.readFileSync(CONTEXT_CONFIG_PATH, "utf-8"); + const parsed = ContextConfig.parse(JSON.parse(raw)); + return { + enabled: + parsed.elideHistoricToolResults ?? + DEFAULT_ELISION_POLICY.enabled, + thresholdChars: + parsed.elideHistoricToolResultsThresholdChars ?? + DEFAULT_ELISION_POLICY.thresholdChars, + }; + } catch { + return DEFAULT_ELISION_POLICY; + } +} + +export function elideHistoricToolResults( + messages: Array>, + thresholdChars: number, +): Array> { + return messages.map((message) => { + if ( + message.role !== "tool" || + message.content.length <= thresholdChars + ) { + return message; + } + return { + ...message, + content: `[Tool result elided to save context: "${message.toolName}" returned ${message.content.length} characters in an earlier turn. Call the tool again if you need this output now.]`, + }; + }); +} + +// IContextResolver decorator: applies the elision policy to the materialized +// cross-turn prefix. Agent snapshot resolution is delegated untouched. +export class ElidingContextResolver implements IContextResolver { + private readonly inner: IContextResolver; + private readonly loadPolicy: () => ToolResultElisionPolicy; + + constructor({ + inner, + loadPolicy, + }: { + inner: IContextResolver; + loadPolicy?: () => ToolResultElisionPolicy; + }) { + this.inner = inner; + this.loadPolicy = loadPolicy ?? loadElisionPolicy; + } + + async resolve( + context: z.infer, + ): Promise>> { + const prefix = await this.inner.resolve(context); + const policy = this.loadPolicy(); + if (!policy.enabled) { + return prefix; + } + return elideHistoricToolResults(prefix, policy.thresholdChars); + } + + resolveAgent( + resolved: z.infer, + ): Promise> { + return this.inner.resolveAgent(resolved); + } +} + +// The one context resolver the app should construct (DI container and the +// inspect CLI both use this), so the debug view reproduces the same bytes +// the loop transmits. +export function createContextResolver({ + turnRepo, +}: { + turnRepo: ITurnRepo; +}): IContextResolver { + return new ElidingContextResolver({ + inner: new TurnRepoContextResolver({ turnRepo }), + }); +} diff --git a/apps/x/packages/core/src/turns/inspect-cli.ts b/apps/x/packages/core/src/turns/inspect-cli.ts index b662e407..3140148f 100644 --- a/apps/x/packages/core/src/turns/inspect-cli.ts +++ b/apps/x/packages/core/src/turns/inspect-cli.ts @@ -27,7 +27,7 @@ import { convertFromMessages } from "../agents/runtime.js"; import { WorkDir } from "../config/config.js"; import { FSSessionRepo } from "../sessions/fs-repo.js"; import { composeModelRequest } from "./compose-model-request.js"; -import { TurnRepoContextResolver } from "./context-resolver.js"; +import { createContextResolver } from "./context-elision.js"; import { FSTurnRepo } from "./fs-repo.js"; const turnRepo = new FSTurnRepo({ @@ -36,7 +36,7 @@ const turnRepo = new FSTurnRepo({ const sessionRepo = new FSSessionRepo({ sessionsRootDir: path.join(WorkDir, "storage", "sessions"), }); -const resolver = new TurnRepoContextResolver({ turnRepo }); +const resolver = createContextResolver({ turnRepo }); const encode = (messages: Parameters[0]) => convertFromMessages(messages) as unknown as JsonValue[]; From 05d7fa41cf696a9d09bb7ef45ef15f1c07a047eb Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:24:44 +0530 Subject: [PATCH 02/10] feat(core): elide historic video-mode frames from model context Video-mode webcam and screen-share frames matter for the response they were captured for; afterwards the assistant's own text carries the takeaway and fresh frames arrive with every new call message, yet each message's frames (~10k tokens) were resent on every subsequent model call. The elision decorator now strips image parts from prior-turn user messages, leaving a text part recording how many frames of each kind were dropped. The current turn's just-captured frames are always sent verbatim, and the policy generalizes the existing config (elideHistoricImages, default on). Co-Authored-By: Claude Fable 5 --- .../packages/core/docs/turn-runtime-design.md | 20 ++-- .../core/src/turns/context-elision.test.ts | 103 ++++++++++++++++-- .../core/src/turns/context-elision.ts | 102 ++++++++++++----- 3 files changed, 182 insertions(+), 43 deletions(-) diff --git a/apps/x/packages/core/docs/turn-runtime-design.md b/apps/x/packages/core/docs/turn-runtime-design.md index 0dcc8026..cfff8cf2 100644 --- a/apps/x/packages/core/docs/turn-runtime-design.md +++ b/apps/x/packages/core/docs/turn-runtime-design.md @@ -393,14 +393,18 @@ Rules: The app wires the resolver through a decorator (`context-elision.ts`) that applies transmit-time elision to the materialized prefix: tool results from prior turns above a size threshold are replaced with a short placeholder -telling the model to re-run the tool if it needs the output. The durable log -is untouched; only the transmitted bytes change. Elision is a pure function -of each message, so resolved prefixes stay byte-stable across calls (provider -prefix caches keep working), and the current turn's own in-flight tool -results never pass through the resolver, so they are always sent verbatim. -Policy lives in `config/context.json` (`elideHistoricToolResults`, default -true; `elideHistoricToolResultsThresholdChars`, default 10000). The inspect -CLI composes through the same decorated resolver. +telling the model to re-run the tool if it needs the output, and inline +image parts from prior turns (video-mode webcam and screen-share frames) are +replaced with a text part recording how many frames of each kind were +dropped. The durable log is untouched; only the transmitted bytes change. +Elision is a pure function of each message, so resolved prefixes stay +byte-stable across calls (provider prefix caches keep working), and the +current turn's own messages never pass through the resolver, so in-flight +tool results and just-captured frames are always sent verbatim. Policy lives +in `config/context.json` (`elideHistoricToolResults`, default true; +`elideHistoricToolResultsThresholdChars`, default 10000; +`elideHistoricImages`, default true). The inspect CLI composes through the +same decorated resolver. ### 6.7 Agent snapshot inheritance diff --git a/apps/x/packages/core/src/turns/context-elision.test.ts b/apps/x/packages/core/src/turns/context-elision.test.ts index 9543a46f..e08ebfa1 100644 --- a/apps/x/packages/core/src/turns/context-elision.test.ts +++ b/apps/x/packages/core/src/turns/context-elision.test.ts @@ -3,6 +3,7 @@ import type { z } from "zod"; import type { TurnContext, TurnEvent } from "@x/shared/dist/turns.js"; import { ElidingContextResolver, + elideHistoricImages, elideHistoricToolResults, } from "./context-elision.js"; import { TurnRepoContextResolver } from "./context-resolver.js"; @@ -150,6 +151,16 @@ function toolTurnLog( ]; } +function frame(source: "camera" | "screen") { + return { + type: "image" as const, + data: "aGVsbG8=".repeat(50), + mediaType: "image/jpeg", + source, + capturedAt: "2026-07-02T10:00:00Z", + }; +} + const T1 = "2026-07-02T10-00-00Z-0000001-000"; describe("elideHistoricToolResults", () => { @@ -191,10 +202,56 @@ describe("elideHistoricToolResults", () => { }); }); +describe("elideHistoricImages", () => { + it("replaces image parts with a labeled placeholder, keeping other parts", () => { + const message = { + role: "user" as const, + content: [ + { type: "text" as const, text: "how do I look?" }, + frame("camera"), + frame("camera"), + frame("screen"), + ], + }; + const [elided] = elideHistoricImages([message]); + if (typeof elided.content === "string" || elided.role !== "user") { + throw new Error("expected user message with parts"); + } + expect(elided.content.filter((p) => p.type === "image")).toHaveLength(0); + expect(elided.content[0]).toEqual({ type: "text", text: "how do I look?" }); + const placeholder = elided.content[elided.content.length - 1]; + if (placeholder.type !== "text") throw new Error("expected text placeholder"); + expect(placeholder.text).toContain("2 webcam frames"); + expect(placeholder.text).toContain("1 screen-share frame"); + }); + + it("leaves string-content and image-free user messages untouched", () => { + const messages = [ + user("plain"), + { + role: "user" as const, + content: [{ type: "text" as const, text: "no images" }], + }, + assistant("a"), + ]; + expect(elideHistoricImages(messages)).toEqual(messages); + }); +}); + +const POLICY_OFF = { + toolResults: false, + toolResultThresholdChars: 100, + images: false, +}; + describe("ElidingContextResolver", () => { function resolver( repo: InMemoryTurnRepo, - policy: { enabled: boolean; thresholdChars: number }, + policy: { + toolResults: boolean; + toolResultThresholdChars: number; + images: boolean; + }, ) { return new ElidingContextResolver({ inner: new TurnRepoContextResolver({ turnRepo: repo }), @@ -206,8 +263,8 @@ describe("ElidingContextResolver", () => { const repo = new InMemoryTurnRepo(); repo.seed(toolTurnLog(T1, [], "x".repeat(200))); const resolved = await resolver(repo, { - enabled: true, - thresholdChars: 100, + ...POLICY_OFF, + toolResults: true, }).resolve({ previousTurnId: T1 }); const tool = resolved.find((m) => m.role === "tool"); expect(tool?.content).toContain("elided"); @@ -224,10 +281,9 @@ describe("ElidingContextResolver", () => { const repo = new InMemoryTurnRepo(); const output = "x".repeat(200); repo.seed(toolTurnLog(T1, [], output)); - const resolved = await resolver(repo, { - enabled: false, - thresholdChars: 100, - }).resolve({ previousTurnId: T1 }); + const resolved = await resolver(repo, POLICY_OFF).resolve({ + previousTurnId: T1, + }); const tool = resolved.find((m) => m.role === "tool"); expect(tool?.content).toBe(output); }); @@ -236,10 +292,39 @@ describe("ElidingContextResolver", () => { const repo = new InMemoryTurnRepo(); repo.seed(toolTurnLog(T1, [], "small")); const resolved = await resolver(repo, { - enabled: true, - thresholdChars: 100, + ...POLICY_OFF, + toolResults: true, }).resolve({ previousTurnId: T1 }); const tool = resolved.find((m) => m.role === "tool"); expect(tool?.content).toBe("small"); }); + + it("elides images in the resolved prefix when enabled", async () => { + const repo = new InMemoryTurnRepo(); + const log = toolTurnLog(T1, [], "small").map((event) => + event.type === "turn_created" + ? { + ...event, + input: { + role: "user" as const, + content: [ + { type: "text" as const, text: "watch me" }, + frame("camera"), + ], + }, + } + : event, + ); + repo.seed(log); + const resolved = await resolver(repo, { + ...POLICY_OFF, + images: true, + }).resolve({ previousTurnId: T1 }); + const input = resolved[0]; + if (input.role !== "user" || typeof input.content === "string") { + throw new Error("expected user message with parts"); + } + expect(input.content.some((p) => p.type === "image")).toBe(false); + expect(JSON.stringify(input.content)).toContain("1 webcam frame"); + }); }); diff --git a/apps/x/packages/core/src/turns/context-elision.ts b/apps/x/packages/core/src/turns/context-elision.ts index 6c88445c..87a51129 100644 --- a/apps/x/packages/core/src/turns/context-elision.ts +++ b/apps/x/packages/core/src/turns/context-elision.ts @@ -12,29 +12,41 @@ import type { IContextResolver } from "./context-resolver.js"; import { TurnRepoContextResolver } from "./context-resolver.js"; import type { ITurnRepo } from "./repo.js"; -// Transmit-time elision of historic tool results (the cross-turn prefix -// only — the current turn's own messages never pass through the resolver, so -// in-flight tool results are always sent verbatim). Large tool outputs from -// earlier turns (skill loads, file reads, HTTP fetches) dominate resent -// context; the model rarely needs them verbatim and can re-run the tool when -// it does. Elision is a pure function of each message's content, so resolved -// prefixes stay byte-stable across calls and turns (provider prefix caches -// keep working), and the durable JSONL log is untouched — only the -// transmitted bytes change. +// Transmit-time elision of historic context (the cross-turn prefix only — +// the current turn's own messages never pass through the resolver, so +// in-flight tool results and just-captured frames are always sent verbatim). +// +// Two policies, both on by default: +// - Tool results: large tool outputs from earlier turns (skill loads, file +// reads, HTTP fetches) dominate resent context; the model rarely needs +// them verbatim and can re-run the tool when it does. +// - Inline images: video-mode webcam and screen-share frames matter for +// the response they were captured for; afterwards the assistant's own +// text carries the takeaway, and fresh frames arrive with every new call +// message. Unlike tool results they cannot be re-fetched, so the +// placeholder only records what was there. +// +// Elision is a pure function of each message's content, so resolved prefixes +// stay byte-stable across calls and turns (provider prefix caches keep +// working), and the durable JSONL log is untouched — only the transmitted +// bytes change. -export interface ToolResultElisionPolicy { - enabled: boolean; - thresholdChars: number; +export interface ElisionPolicy { + toolResults: boolean; + toolResultThresholdChars: number; + images: boolean; } -export const DEFAULT_ELISION_POLICY: ToolResultElisionPolicy = { - enabled: true, - thresholdChars: 10_000, +export const DEFAULT_ELISION_POLICY: ElisionPolicy = { + toolResults: true, + toolResultThresholdChars: 10_000, + images: true, }; const ContextConfig = z.object({ elideHistoricToolResults: z.boolean().optional(), elideHistoricToolResultsThresholdChars: z.number().int().min(0).optional(), + elideHistoricImages: z.boolean().optional(), }); const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json"); @@ -42,7 +54,7 @@ const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json"); // Read the elision policy from config/context.json, falling back to defaults // for missing keys or an unreadable file. Read per resolve so a config edit // applies to the next turn without a restart. -export function loadElisionPolicy(): ToolResultElisionPolicy { +export function loadElisionPolicy(): ElisionPolicy { try { if (!fs.existsSync(CONTEXT_CONFIG_PATH)) { return DEFAULT_ELISION_POLICY; @@ -50,12 +62,14 @@ export function loadElisionPolicy(): ToolResultElisionPolicy { const raw = fs.readFileSync(CONTEXT_CONFIG_PATH, "utf-8"); const parsed = ContextConfig.parse(JSON.parse(raw)); return { - enabled: + toolResults: parsed.elideHistoricToolResults ?? - DEFAULT_ELISION_POLICY.enabled, - thresholdChars: + DEFAULT_ELISION_POLICY.toolResults, + toolResultThresholdChars: parsed.elideHistoricToolResultsThresholdChars ?? - DEFAULT_ELISION_POLICY.thresholdChars, + DEFAULT_ELISION_POLICY.toolResultThresholdChars, + images: + parsed.elideHistoricImages ?? DEFAULT_ELISION_POLICY.images, }; } catch { return DEFAULT_ELISION_POLICY; @@ -80,18 +94,48 @@ export function elideHistoricToolResults( }); } +export function elideHistoricImages( + messages: Array>, +): Array> { + return messages.map((message) => { + if (message.role !== "user" || typeof message.content === "string") { + return message; + } + const images = message.content.filter((part) => part.type === "image"); + if (images.length === 0) { + return message; + } + const camera = images.filter((part) => part.source !== "screen").length; + const screen = images.length - camera; + const kinds = [ + ...(camera > 0 ? [`${camera} webcam frame${camera === 1 ? "" : "s"}`] : []), + ...(screen > 0 ? [`${screen} screen-share frame${screen === 1 ? "" : "s"}`] : []), + ].join(" and "); + return { + ...message, + content: [ + ...message.content.filter((part) => part.type !== "image"), + { + type: "text" as const, + text: `[Omitted from history to save context: ${kinds} captured while this message was composed. The assistant saw them when responding to this message.]`, + }, + ], + }; + }); +} + // IContextResolver decorator: applies the elision policy to the materialized // cross-turn prefix. Agent snapshot resolution is delegated untouched. export class ElidingContextResolver implements IContextResolver { private readonly inner: IContextResolver; - private readonly loadPolicy: () => ToolResultElisionPolicy; + private readonly loadPolicy: () => ElisionPolicy; constructor({ inner, loadPolicy, }: { inner: IContextResolver; - loadPolicy?: () => ToolResultElisionPolicy; + loadPolicy?: () => ElisionPolicy; }) { this.inner = inner; this.loadPolicy = loadPolicy ?? loadElisionPolicy; @@ -100,12 +144,18 @@ export class ElidingContextResolver implements IContextResolver { async resolve( context: z.infer, ): Promise>> { - const prefix = await this.inner.resolve(context); + let prefix = await this.inner.resolve(context); const policy = this.loadPolicy(); - if (!policy.enabled) { - return prefix; + if (policy.toolResults) { + prefix = elideHistoricToolResults( + prefix, + policy.toolResultThresholdChars, + ); } - return elideHistoricToolResults(prefix, policy.thresholdChars); + if (policy.images) { + prefix = elideHistoricImages(prefix); + } + return prefix; } resolveAgent( From 4fbca2dbadbcc1afd7319021450a7a125940349e Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:33:22 +0530 Subject: [PATCH 03/10] feat(core): elide historic middle-pane note snapshots from model context Every user message sent while a note is open carries a full snapshot of that note in userMessageContext, so a long chat over one open note resends N full copies on every model call. The elision decorator now rewrites prior-turn user messages to keep the pane kind and path but replace note content above a small floor with a placeholder pointing at the still-readable file. The current message's snapshot is untouched, so "summarize this" keeps working; the system prompt already tells the model later middle-pane context overrides earlier. Config: elideHistoricMiddlePaneContent, default on. Co-Authored-By: Claude Fable 5 --- .../packages/core/docs/turn-runtime-design.md | 13 ++-- .../core/src/turns/context-elision.test.ts | 70 +++++++++++++++++++ .../core/src/turns/context-elision.ts | 46 ++++++++++++ 3 files changed, 124 insertions(+), 5 deletions(-) diff --git a/apps/x/packages/core/docs/turn-runtime-design.md b/apps/x/packages/core/docs/turn-runtime-design.md index cfff8cf2..b6bb53b0 100644 --- a/apps/x/packages/core/docs/turn-runtime-design.md +++ b/apps/x/packages/core/docs/turn-runtime-design.md @@ -393,18 +393,21 @@ Rules: The app wires the resolver through a decorator (`context-elision.ts`) that applies transmit-time elision to the materialized prefix: tool results from prior turns above a size threshold are replaced with a short placeholder -telling the model to re-run the tool if it needs the output, and inline -image parts from prior turns (video-mode webcam and screen-share frames) are +telling the model to re-run the tool if it needs the output; inline image +parts from prior turns (video-mode webcam and screen-share frames) are replaced with a text part recording how many frames of each kind were -dropped. The durable log is untouched; only the transmitted bytes change. +dropped; and middle-pane note snapshots on prior-turn user messages keep +their kind and path but have their content replaced with a placeholder +pointing at the still-readable file. The durable log is untouched; only the +transmitted bytes change. Elision is a pure function of each message, so resolved prefixes stay byte-stable across calls (provider prefix caches keep working), and the current turn's own messages never pass through the resolver, so in-flight tool results and just-captured frames are always sent verbatim. Policy lives in `config/context.json` (`elideHistoricToolResults`, default true; `elideHistoricToolResultsThresholdChars`, default 10000; -`elideHistoricImages`, default true). The inspect CLI composes through the -same decorated resolver. +`elideHistoricImages`, default true; `elideHistoricMiddlePaneContent`, +default true). The inspect CLI composes through the same decorated resolver. ### 6.7 Agent snapshot inheritance diff --git a/apps/x/packages/core/src/turns/context-elision.test.ts b/apps/x/packages/core/src/turns/context-elision.test.ts index e08ebfa1..0864d206 100644 --- a/apps/x/packages/core/src/turns/context-elision.test.ts +++ b/apps/x/packages/core/src/turns/context-elision.test.ts @@ -4,6 +4,7 @@ import type { TurnContext, TurnEvent } from "@x/shared/dist/turns.js"; import { ElidingContextResolver, elideHistoricImages, + elideHistoricMiddlePaneContent, elideHistoricToolResults, } from "./context-elision.js"; import { TurnRepoContextResolver } from "./context-resolver.js"; @@ -238,10 +239,60 @@ describe("elideHistoricImages", () => { }); }); +function noteMessage(content: string) { + return { + role: "user" as const, + content: "make this punchier", + userMessageContext: { + currentDateTime: "Thursday, July 2, 2026 at 10:00 AM GMT", + middlePane: { + kind: "note" as const, + path: "knowledge/Notes/Draft.md", + content, + }, + }, + }; +} + +describe("elideHistoricMiddlePaneContent", () => { + it("replaces large note snapshots, keeping kind and path", () => { + const [elided] = elideHistoricMiddlePaneContent([ + noteMessage("n".repeat(600)), + ]); + if (elided.role !== "user") throw new Error("expected user message"); + const middlePane = elided.userMessageContext?.middlePane; + if (middlePane?.kind !== "note") throw new Error("expected note pane"); + expect(middlePane.path).toBe("knowledge/Notes/Draft.md"); + expect(middlePane.content).toContain("omitted from history"); + expect(middlePane.content).toContain("600"); + expect(elided.userMessageContext?.currentDateTime).toBeDefined(); + expect(elided.content).toBe("make this punchier"); + }); + + it("keeps small notes and non-note panes untouched", () => { + const small = noteMessage("short todo list"); + const browser = { + role: "user" as const, + content: "what is this page?", + userMessageContext: { + middlePane: { + kind: "browser" as const, + url: "https://example.com", + title: "Example", + }, + }, + }; + const plain = user("no context at all"); + const messages = [small, browser, plain]; + expect(elideHistoricMiddlePaneContent(messages)).toEqual(messages); + }); +}); + const POLICY_OFF = { toolResults: false, toolResultThresholdChars: 100, images: false, + middlePaneContent: false, }; describe("ElidingContextResolver", () => { @@ -327,4 +378,23 @@ describe("ElidingContextResolver", () => { expect(input.content.some((p) => p.type === "image")).toBe(false); expect(JSON.stringify(input.content)).toContain("1 webcam frame"); }); + + it("elides middle-pane note snapshots in the resolved prefix when enabled", async () => { + const repo = new InMemoryTurnRepo(); + const log = toolTurnLog(T1, [], "small").map((event) => + event.type === "turn_created" + ? { ...event, input: noteMessage("n".repeat(600)) } + : event, + ); + repo.seed(log); + const resolved = await resolver(repo, { + ...POLICY_OFF, + middlePaneContent: true, + }).resolve({ previousTurnId: T1 }); + const input = resolved[0]; + if (input.role !== "user") throw new Error("expected user message"); + const middlePane = input.userMessageContext?.middlePane; + if (middlePane?.kind !== "note") throw new Error("expected note pane"); + expect(middlePane.content).toContain("omitted from history"); + }); }); diff --git a/apps/x/packages/core/src/turns/context-elision.ts b/apps/x/packages/core/src/turns/context-elision.ts index 87a51129..05fbfb43 100644 --- a/apps/x/packages/core/src/turns/context-elision.ts +++ b/apps/x/packages/core/src/turns/context-elision.ts @@ -25,6 +25,11 @@ import type { ITurnRepo } from "./repo.js"; // text carries the takeaway, and fresh frames arrive with every new call // message. Unlike tool results they cannot be re-fetched, so the // placeholder only records what was there. +// - Middle-pane note content: every user message sent while a note is +// open carries a full snapshot of that note in userMessageContext. Only +// the newest snapshot matters (the system prompt already tells the model +// later middle-pane context overrides earlier), and the current file is +// always re-readable at the recorded path. // // Elision is a pure function of each message's content, so resolved prefixes // stay byte-stable across calls and turns (provider prefix caches keep @@ -35,18 +40,25 @@ export interface ElisionPolicy { toolResults: boolean; toolResultThresholdChars: number; images: boolean; + middlePaneContent: boolean; } export const DEFAULT_ELISION_POLICY: ElisionPolicy = { toolResults: true, toolResultThresholdChars: 10_000, images: true, + middlePaneContent: true, }; +// Notes smaller than this stay verbatim in history: below the floor the +// placeholder saves nothing and a short note may carry useful context. +const MIDDLE_PANE_CONTENT_FLOOR_CHARS = 500; + const ContextConfig = z.object({ elideHistoricToolResults: z.boolean().optional(), elideHistoricToolResultsThresholdChars: z.number().int().min(0).optional(), elideHistoricImages: z.boolean().optional(), + elideHistoricMiddlePaneContent: z.boolean().optional(), }); const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json"); @@ -70,6 +82,9 @@ export function loadElisionPolicy(): ElisionPolicy { DEFAULT_ELISION_POLICY.toolResultThresholdChars, images: parsed.elideHistoricImages ?? DEFAULT_ELISION_POLICY.images, + middlePaneContent: + parsed.elideHistoricMiddlePaneContent ?? + DEFAULT_ELISION_POLICY.middlePaneContent, }; } catch { return DEFAULT_ELISION_POLICY; @@ -124,6 +139,34 @@ export function elideHistoricImages( }); } +export function elideHistoricMiddlePaneContent( + messages: Array>, +): Array> { + return messages.map((message) => { + if (message.role !== "user") { + return message; + } + const middlePane = message.userMessageContext?.middlePane; + if ( + !middlePane || + middlePane.kind !== "note" || + middlePane.content.length <= MIDDLE_PANE_CONTENT_FLOOR_CHARS + ) { + return message; + } + return { + ...message, + userMessageContext: { + ...message.userMessageContext, + middlePane: { + ...middlePane, + content: `[Note content (${middlePane.content.length} characters) omitted from history to save context. This snapshot is stale; read the file at the path above if its content is needed now.]`, + }, + }, + }; + }); +} + // IContextResolver decorator: applies the elision policy to the materialized // cross-turn prefix. Agent snapshot resolution is delegated untouched. export class ElidingContextResolver implements IContextResolver { @@ -155,6 +198,9 @@ export class ElidingContextResolver implements IContextResolver { if (policy.images) { prefix = elideHistoricImages(prefix); } + if (policy.middlePaneContent) { + prefix = elideHistoricMiddlePaneContent(prefix); + } return prefix; } From 04348dcc41f2ae85e7b6bfa7c9824f37b239a1a6 Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:22:44 +0530 Subject: [PATCH 04/10] feat(core): keep a head preview in elided tool results A bare placeholder tells the model only the tool name and size; the first 400 characters tell it what the output actually was (a skill guide reads very differently from a fetched page), which is what it needs to judge whether re-running the tool is worth it. The preview is capped at the threshold so small thresholds still shrink content, and the transform stays a pure per-message function (byte-stable prefixes). Co-Authored-By: Claude Fable 5 --- .../packages/core/src/turns/context-elision.test.ts | 12 ++++++++++++ apps/x/packages/core/src/turns/context-elision.ts | 12 +++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/apps/x/packages/core/src/turns/context-elision.test.ts b/apps/x/packages/core/src/turns/context-elision.test.ts index 0864d206..19f9c5ef 100644 --- a/apps/x/packages/core/src/turns/context-elision.test.ts +++ b/apps/x/packages/core/src/turns/context-elision.test.ts @@ -179,6 +179,18 @@ describe("elideHistoricToolResults", () => { expect(elided[0].content).toContain("101"); }); + it("keeps a head preview ahead of the placeholder", () => { + const content = `SKILL GUIDE${"y".repeat(5000)}`; + const [elided] = elideHistoricToolResults([toolMsg(content)], 2500); + // Starts with the first 400 chars verbatim, then the marker. + expect(elided.content.startsWith(content.slice(0, 400))).toBe(true); + expect(elided.content).toContain("Rest of tool result elided"); + expect(elided.content.length).toBeLessThan(700); + // Preview never exceeds a tiny threshold. + const [tiny] = elideHistoricToolResults([toolMsg(content)], 50); + expect(tiny.content.startsWith(`${content.slice(0, 50)}\n[Rest`)).toBe(true); + }); + it("keeps tool results at or below the threshold verbatim", () => { const exact = toolMsg("x".repeat(100)); expect(elideHistoricToolResults([exact], 100)).toEqual([exact]); diff --git a/apps/x/packages/core/src/turns/context-elision.ts b/apps/x/packages/core/src/turns/context-elision.ts index 05fbfb43..22ff2d6f 100644 --- a/apps/x/packages/core/src/turns/context-elision.ts +++ b/apps/x/packages/core/src/turns/context-elision.ts @@ -54,6 +54,9 @@ export const DEFAULT_ELISION_POLICY: ElisionPolicy = { // placeholder saves nothing and a short note may carry useful context. const MIDDLE_PANE_CONTENT_FLOOR_CHARS = 500; +// Head of an elided tool result kept verbatim ahead of the placeholder. +const TOOL_RESULT_PREVIEW_CHARS = 400; + const ContextConfig = z.object({ elideHistoricToolResults: z.boolean().optional(), elideHistoricToolResultsThresholdChars: z.number().int().min(0).optional(), @@ -102,9 +105,16 @@ export function elideHistoricToolResults( ) { return message; } + // Keep a head preview so the model knows what it is declining to + // re-fetch (a skill body reads very differently from a web page). + // Capped at the threshold so tiny thresholds still shrink content. + const preview = message.content.slice( + 0, + Math.min(TOOL_RESULT_PREVIEW_CHARS, thresholdChars), + ); return { ...message, - content: `[Tool result elided to save context: "${message.toolName}" returned ${message.content.length} characters in an earlier turn. Call the tool again if you need this output now.]`, + content: `${preview}\n[Rest of tool result elided to save context: "${message.toolName}" returned ${message.content.length} characters in an earlier turn. Call the tool again if you need the full output now.]`, }; }); } From 331a5eda4e32dc81dc4018093940dcf161d3d802 Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:23:18 +0530 Subject: [PATCH 05/10] feat(core): lower default tool-result elision threshold to 2500 chars Real session data shows ~5k-char skill bodies are the most common oversized historic result; the 10k default replayed them on every model call for the life of the session. With the head preview in place the model retains enough scent to re-load on demand, so the aggressive default is the right trade. Still tunable via config/context.json. Co-Authored-By: Claude Fable 5 --- apps/x/packages/core/docs/turn-runtime-design.md | 2 +- apps/x/packages/core/src/turns/context-elision.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/x/packages/core/docs/turn-runtime-design.md b/apps/x/packages/core/docs/turn-runtime-design.md index b6bb53b0..620a0ad8 100644 --- a/apps/x/packages/core/docs/turn-runtime-design.md +++ b/apps/x/packages/core/docs/turn-runtime-design.md @@ -405,7 +405,7 @@ byte-stable across calls (provider prefix caches keep working), and the current turn's own messages never pass through the resolver, so in-flight tool results and just-captured frames are always sent verbatim. Policy lives in `config/context.json` (`elideHistoricToolResults`, default true; -`elideHistoricToolResultsThresholdChars`, default 10000; +`elideHistoricToolResultsThresholdChars`, default 2500; `elideHistoricImages`, default true; `elideHistoricMiddlePaneContent`, default true). The inspect CLI composes through the same decorated resolver. diff --git a/apps/x/packages/core/src/turns/context-elision.ts b/apps/x/packages/core/src/turns/context-elision.ts index 22ff2d6f..4db63a57 100644 --- a/apps/x/packages/core/src/turns/context-elision.ts +++ b/apps/x/packages/core/src/turns/context-elision.ts @@ -43,9 +43,13 @@ export interface ElisionPolicy { middlePaneContent: boolean; } +// Threshold rationale: ~2,500 chars ≈ 600 tokens. Observed sessions show +// skill bodies at ~5k chars are the most common oversized result, so 10k +// would replay them forever; the head preview plus re-run hint makes the +// lower cut safe. Tunable via config/context.json. export const DEFAULT_ELISION_POLICY: ElisionPolicy = { toolResults: true, - toolResultThresholdChars: 10_000, + toolResultThresholdChars: 2_500, images: true, middlePaneContent: true, }; From 8a952a4b7e0eb94e8cb80b59b88560ac285f4a3e Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:24:46 +0530 Subject: [PATCH 06/10] fix(core): guard against constructing an undecorated context resolver TurnRepoContextResolver was still freely constructible, so a future call site could silently lose elision. The factory now accepts an injectable policy loader (also keeps tests off the machine's real context.json), the raw class carries a do-not-construct note, and factory-level tests pin both the decorator wiring and the default policy behavior. Co-Authored-By: Claude Fable 5 --- .../core/src/turns/context-elision.test.ts | 28 +++++++++++++++++++ .../core/src/turns/context-elision.ts | 4 +++ .../core/src/turns/context-resolver.ts | 4 +++ 3 files changed, 36 insertions(+) diff --git a/apps/x/packages/core/src/turns/context-elision.test.ts b/apps/x/packages/core/src/turns/context-elision.test.ts index 19f9c5ef..88f3ff28 100644 --- a/apps/x/packages/core/src/turns/context-elision.test.ts +++ b/apps/x/packages/core/src/turns/context-elision.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from "vitest"; import type { z } from "zod"; import type { TurnContext, TurnEvent } from "@x/shared/dist/turns.js"; import { + DEFAULT_ELISION_POLICY, ElidingContextResolver, + createContextResolver, elideHistoricImages, elideHistoricMiddlePaneContent, elideHistoricToolResults, @@ -307,6 +309,32 @@ const POLICY_OFF = { middlePaneContent: false, }; +describe("createContextResolver", () => { + // The factory is the app's one sanctioned constructor (DI container and + // inspect CLI). If this stops returning the eliding decorator, every + // construction site silently regresses to full historic replay. + it("wraps the repo resolver in the eliding decorator", () => { + const resolver = createContextResolver({ + turnRepo: new InMemoryTurnRepo(), + }); + expect(resolver).toBeInstanceOf(ElidingContextResolver); + }); + + it("applies the default policy through the decorated resolver", async () => { + const repo = new InMemoryTurnRepo(); + repo.seed(toolTurnLog(T1, [], "x".repeat(5000))); + // Pinned to the shipped defaults (not the machine's context.json) so + // the test is hermetic while still exercising the full wiring. + const resolved = await createContextResolver({ + turnRepo: repo, + loadPolicy: () => DEFAULT_ELISION_POLICY, + }).resolve({ previousTurnId: T1 }); + const tool = resolved.find((m) => m.role === "tool"); + expect(tool?.content).toContain("elided"); + expect(tool?.content.length).toBeLessThan(1000); + }); +}); + describe("ElidingContextResolver", () => { function resolver( repo: InMemoryTurnRepo, diff --git a/apps/x/packages/core/src/turns/context-elision.ts b/apps/x/packages/core/src/turns/context-elision.ts index 4db63a57..22d7d175 100644 --- a/apps/x/packages/core/src/turns/context-elision.ts +++ b/apps/x/packages/core/src/turns/context-elision.ts @@ -230,10 +230,14 @@ export class ElidingContextResolver implements IContextResolver { // the loop transmits. export function createContextResolver({ turnRepo, + loadPolicy, }: { turnRepo: ITurnRepo; + // Injectable for tests; the app default reads config/context.json. + loadPolicy?: () => ElisionPolicy; }): IContextResolver { return new ElidingContextResolver({ inner: new TurnRepoContextResolver({ turnRepo }), + ...(loadPolicy ? { loadPolicy } : {}), }); } diff --git a/apps/x/packages/core/src/turns/context-resolver.ts b/apps/x/packages/core/src/turns/context-resolver.ts index 3785cc26..623bdb4e 100644 --- a/apps/x/packages/core/src/turns/context-resolver.ts +++ b/apps/x/packages/core/src/turns/context-resolver.ts @@ -28,6 +28,10 @@ export interface IContextResolver { ): Promise>; } +// NOTE: app code must not construct this directly — use createContextResolver +// (context-elision.ts), which wraps it in the eliding decorator. A raw +// instance silently transmits full historic tool results, frames, and note +// snapshots. Direct construction is for tests of the raw resolution only. export class TurnRepoContextResolver implements IContextResolver { private readonly turnRepo: ITurnRepo; From 4b4e6af2eaef4108a9694181c311fac2acd0ed40 Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:26:22 +0530 Subject: [PATCH 07/10] docs(core): surface the config-relative recomposition caveat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Elision reads config/context.json at compose time, so the composed payload is no longer a pure function of the durable log — inspecting an old turn after a config edit can show different prefix bytes than were transmitted. Document the exception against §8.3 and make the inspect CLI print the policy in effect so divergence is visible instead of silent. The gold fix (recording the applied policy on the turn) is noted for when exact-bytes replay becomes a hard requirement. Co-Authored-By: Claude Fable 5 --- .../x/packages/core/docs/turn-runtime-design.md | 9 +++++++++ apps/x/packages/core/src/turns/inspect-cli.ts | 17 ++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/apps/x/packages/core/docs/turn-runtime-design.md b/apps/x/packages/core/docs/turn-runtime-design.md index 620a0ad8..5f462bd1 100644 --- a/apps/x/packages/core/docs/turn-runtime-design.md +++ b/apps/x/packages/core/docs/turn-runtime-design.md @@ -409,6 +409,15 @@ in `config/context.json` (`elideHistoricToolResults`, default true; `elideHistoricImages`, default true; `elideHistoricMiddlePaneContent`, default true). The inspect CLI composes through the same decorated resolver. +Caveat: elision makes the composed payload a function of the durable log +PLUS the config in effect at compose time — the one deliberate exception to +"recomposition from durable state alone" (§8.3). Within a turn this is +harmless (policy is loaded once per execution, at resolve time), but +inspecting an old turn after a config change may show different prefix +bytes than were transmitted; the inspect CLI prints the active policy so +the divergence is visible. If exact-bytes replay ever becomes a hard +requirement, record the applied policy on the turn and compose from that. + ### 6.7 Agent snapshot inheritance Session turns whose resolved system prompt and tool set are byte-identical diff --git a/apps/x/packages/core/src/turns/inspect-cli.ts b/apps/x/packages/core/src/turns/inspect-cli.ts index 3140148f..9326a73c 100644 --- a/apps/x/packages/core/src/turns/inspect-cli.ts +++ b/apps/x/packages/core/src/turns/inspect-cli.ts @@ -8,6 +8,10 @@ // messages (user-message context, attachments, tool-result envelopes) are // visible; the file itself stores only structural facts and references. // +// Caveat: historic-context elision reads config/context.json at compose +// time, so recomposition is exact only while that config matches what was +// live when the call ran. The header prints the policy in effect NOW. +// // Session mode prints the session overview (title, turns, statuses, sizes); // pass --turns to cascade full turn inspection for every turn. // @@ -27,7 +31,7 @@ import { convertFromMessages } from "../agents/runtime.js"; import { WorkDir } from "../config/config.js"; import { FSSessionRepo } from "../sessions/fs-repo.js"; import { composeModelRequest } from "./compose-model-request.js"; -import { createContextResolver } from "./context-elision.js"; +import { createContextResolver, loadElisionPolicy } from "./context-elision.js"; import { FSTurnRepo } from "./fs-repo.js"; const turnRepo = new FSTurnRepo({ @@ -81,6 +85,17 @@ async function inspectTurn( console.log( `agent ${agent.agentId} model ${agent.model.provider}/${agent.model.model} calls ${state.modelCalls.length}${inherited}`, ); + const policy = loadElisionPolicy(); + console.log( + `context elision (per config/context.json NOW; transmitted bytes reflect the config live at call time): ` + + [ + policy.toolResults + ? `toolResults>${policy.toolResultThresholdChars}` + : "toolResults off", + policy.images ? "images" : "images off", + policy.middlePaneContent ? "notes" : "notes off", + ].join(", "), + ); for (const call of state.modelCalls) { if (onlyIndex !== undefined && call.index !== onlyIndex) continue; From 30570a932553cd6ddbd45c172929b04adb8444fd Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:27:46 +0530 Subject: [PATCH 08/10] test(core): fill elision coverage gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original suite covered the tool-result path well but left holes: loadElisionPolicy had no tests at all (now: missing file, full config, partial merge, unparseable JSON, and the all-or-nothing malformed-key behavior — via an injectable config path so tests stay off the real WorkDir), images/note policies had no idempotency or determinism tests (the properties prefix caching depends on), the note floor boundary was unpinned, resolveAgent delegation was unverified, and nothing exercised a multi-turn reference chain or the per-resolve hot config reload. Co-Authored-By: Claude Fable 5 --- .../core/src/turns/context-elision.test.ts | 156 +++++++++++++++++- .../core/src/turns/context-elision.ts | 13 +- 2 files changed, 163 insertions(+), 6 deletions(-) diff --git a/apps/x/packages/core/src/turns/context-elision.test.ts b/apps/x/packages/core/src/turns/context-elision.test.ts index 88f3ff28..bb844aa6 100644 --- a/apps/x/packages/core/src/turns/context-elision.test.ts +++ b/apps/x/packages/core/src/turns/context-elision.test.ts @@ -1,4 +1,7 @@ -import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; import type { z } from "zod"; import type { TurnContext, TurnEvent } from "@x/shared/dist/turns.js"; import { @@ -8,6 +11,7 @@ import { elideHistoricImages, elideHistoricMiddlePaneContent, elideHistoricToolResults, + loadElisionPolicy, } from "./context-elision.js"; import { TurnRepoContextResolver } from "./context-resolver.js"; import { InMemoryTurnRepo } from "./in-memory-turn-repo.js"; @@ -437,4 +441,154 @@ describe("ElidingContextResolver", () => { if (middlePane?.kind !== "note") throw new Error("expected note pane"); expect(middlePane.content).toContain("omitted from history"); }); + + it("delegates resolveAgent to the inner resolver untouched", async () => { + const repo = new InMemoryTurnRepo(); + const snapshot = { + agentId: "copilot", + systemPrompt: "SYS", + model: { provider: "fake", model: "m" }, + tools: [], + }; + const agent = await resolver(repo, POLICY_OFF).resolveAgent(snapshot); + expect(agent).toEqual(snapshot); + }); + + it("elides across every turn of a multi-turn reference chain", async () => { + const repo = new InMemoryTurnRepo(); + const T2 = "2026-07-02T11-00-00Z-0000002-000"; + repo.seed(toolTurnLog(T1, [], "a".repeat(200))); + repo.seed(toolTurnLog(T2, { previousTurnId: T1 }, "b".repeat(200))); + const resolved = await resolver(repo, { + ...POLICY_OFF, + toolResults: true, + }).resolve({ previousTurnId: T2 }); + const tools = resolved.filter((m) => m.role === "tool"); + expect(tools).toHaveLength(2); + for (const tool of tools) { + expect(tool.content).toContain("elided"); + } + }); + + it("reloads the policy on every resolve (hot config)", async () => { + const repo = new InMemoryTurnRepo(); + const output = "x".repeat(200); + repo.seed(toolTurnLog(T1, [], output)); + let loads = 0; + const eliding = new ElidingContextResolver({ + inner: new TurnRepoContextResolver({ turnRepo: repo }), + loadPolicy: () => { + loads += 1; + return { ...POLICY_OFF, toolResults: loads > 1 }; + }, + }); + const first = await eliding.resolve({ previousTurnId: T1 }); + const second = await eliding.resolve({ previousTurnId: T1 }); + expect(loads).toBe(2); + expect(first.find((m) => m.role === "tool")?.content).toBe(output); + expect(second.find((m) => m.role === "tool")?.content).toContain("elided"); + }); +}); + +describe("elision policy idempotency and determinism", () => { + // Placeholders must never themselves be elided on a second pass, and the + // transforms must be pure — both are what keeps replayed prefixes + // byte-stable for provider prefix caching. + it("elideHistoricImages is idempotent and deterministic", () => { + const messages = [ + { + role: "user" as const, + content: [ + { type: "text" as const, text: "watch" }, + frame("camera"), + frame("screen"), + ], + }, + ]; + const once = elideHistoricImages(messages); + expect(elideHistoricImages(once)).toEqual(once); + expect(elideHistoricImages(messages)).toEqual(once); + }); + + it("elideHistoricMiddlePaneContent is idempotent and deterministic", () => { + const messages = [noteMessage("n".repeat(600))]; + const once = elideHistoricMiddlePaneContent(messages); + expect(elideHistoricMiddlePaneContent(once)).toEqual(once); + expect(elideHistoricMiddlePaneContent(messages)).toEqual(once); + }); + + it("keeps notes at the floor verbatim and elides just above it", () => { + const atFloor = noteMessage("n".repeat(500)); + expect(elideHistoricMiddlePaneContent([atFloor])).toEqual([atFloor]); + const [above] = elideHistoricMiddlePaneContent([ + noteMessage("n".repeat(501)), + ]); + if (above.role !== "user") throw new Error("expected user message"); + expect(above.userMessageContext?.middlePane?.kind === "note" + ? above.userMessageContext.middlePane.content + : "").toContain("omitted from history"); + }); +}); + +describe("loadElisionPolicy", () => { + let dir: string; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), "elision-config-")); + }); + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + function write(content: string): string { + const p = path.join(dir, "context.json"); + fs.writeFileSync(p, content); + return p; + } + + it("returns defaults when the file is missing", () => { + expect(loadElisionPolicy(path.join(dir, "nope.json"))).toEqual( + DEFAULT_ELISION_POLICY, + ); + }); + + it("applies a full config", () => { + const p = write( + JSON.stringify({ + elideHistoricToolResults: false, + elideHistoricToolResultsThresholdChars: 42, + elideHistoricImages: false, + elideHistoricMiddlePaneContent: false, + }), + ); + expect(loadElisionPolicy(p)).toEqual({ + toolResults: false, + toolResultThresholdChars: 42, + images: false, + middlePaneContent: false, + }); + }); + + it("merges a partial config with defaults", () => { + const p = write(JSON.stringify({ elideHistoricToolResultsThresholdChars: 42 })); + expect(loadElisionPolicy(p)).toEqual({ + ...DEFAULT_ELISION_POLICY, + toolResultThresholdChars: 42, + }); + }); + + it("falls back to defaults on unparseable JSON", () => { + expect(loadElisionPolicy(write("{nope"))).toEqual(DEFAULT_ELISION_POLICY); + }); + + it("discards the whole file when any key is malformed (all-or-nothing)", () => { + const p = write( + JSON.stringify({ + elideHistoricToolResults: false, + elideHistoricToolResultsThresholdChars: "not-a-number", + }), + ); + expect(loadElisionPolicy(p)).toEqual(DEFAULT_ELISION_POLICY); + }); }); diff --git a/apps/x/packages/core/src/turns/context-elision.ts b/apps/x/packages/core/src/turns/context-elision.ts index 22d7d175..04796037 100644 --- a/apps/x/packages/core/src/turns/context-elision.ts +++ b/apps/x/packages/core/src/turns/context-elision.ts @@ -71,14 +71,17 @@ const ContextConfig = z.object({ const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json"); // Read the elision policy from config/context.json, falling back to defaults -// for missing keys or an unreadable file. Read per resolve so a config edit -// applies to the next turn without a restart. -export function loadElisionPolicy(): ElisionPolicy { +// for missing keys or an unreadable file (a single malformed key discards the +// whole file — all-or-nothing by design, so a typo can't half-apply). Read +// per resolve so a config edit applies to the next turn without a restart. +export function loadElisionPolicy( + configPath: string = CONTEXT_CONFIG_PATH, +): ElisionPolicy { try { - if (!fs.existsSync(CONTEXT_CONFIG_PATH)) { + if (!fs.existsSync(configPath)) { return DEFAULT_ELISION_POLICY; } - const raw = fs.readFileSync(CONTEXT_CONFIG_PATH, "utf-8"); + const raw = fs.readFileSync(configPath, "utf-8"); const parsed = ContextConfig.parse(JSON.parse(raw)); return { toolResults: From 7000f34c484c3d9b98f098bfdce5acc8744ff8b9 Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:02:03 +0530 Subject: [PATCH 09/10] test(core): fix type errors in the elision test suite The resolver helper's inline policy type predated middlePaneContent and no longer satisfied ElisionPolicy; use the exported type. Coerce message content to string where the preview assertions call string methods. Co-Authored-By: Claude Fable 5 --- .../core/src/turns/context-elision.test.ts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/apps/x/packages/core/src/turns/context-elision.test.ts b/apps/x/packages/core/src/turns/context-elision.test.ts index bb844aa6..2e3d51f3 100644 --- a/apps/x/packages/core/src/turns/context-elision.test.ts +++ b/apps/x/packages/core/src/turns/context-elision.test.ts @@ -12,6 +12,7 @@ import { elideHistoricMiddlePaneContent, elideHistoricToolResults, loadElisionPolicy, + type ElisionPolicy, } from "./context-elision.js"; import { TurnRepoContextResolver } from "./context-resolver.js"; import { InMemoryTurnRepo } from "./in-memory-turn-repo.js"; @@ -189,12 +190,14 @@ describe("elideHistoricToolResults", () => { const content = `SKILL GUIDE${"y".repeat(5000)}`; const [elided] = elideHistoricToolResults([toolMsg(content)], 2500); // Starts with the first 400 chars verbatim, then the marker. - expect(elided.content.startsWith(content.slice(0, 400))).toBe(true); + expect(String(elided.content).startsWith(content.slice(0, 400))).toBe(true); expect(elided.content).toContain("Rest of tool result elided"); - expect(elided.content.length).toBeLessThan(700); + expect(String(elided.content).length).toBeLessThan(700); // Preview never exceeds a tiny threshold. const [tiny] = elideHistoricToolResults([toolMsg(content)], 50); - expect(tiny.content.startsWith(`${content.slice(0, 50)}\n[Rest`)).toBe(true); + expect(String(tiny.content).startsWith(`${content.slice(0, 50)}\n[Rest`)).toBe( + true, + ); }); it("keeps tool results at or below the threshold verbatim", () => { @@ -340,14 +343,7 @@ describe("createContextResolver", () => { }); describe("ElidingContextResolver", () => { - function resolver( - repo: InMemoryTurnRepo, - policy: { - toolResults: boolean; - toolResultThresholdChars: number; - images: boolean; - }, - ) { + function resolver(repo: InMemoryTurnRepo, policy: ElisionPolicy) { return new ElidingContextResolver({ inner: new TurnRepoContextResolver({ turnRepo: repo }), loadPolicy: () => policy, From cda925ad9d2904bae16d7b8d967da4785321763e Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:03:35 +0530 Subject: [PATCH 10/10] feat(core): Anthropic prompt-cache breakpoints in the model registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Anthropic caching is opt-in, and nothing sent breakpoints: observed sessions show 0% cache hits on Claude models (~1.27M of 1.36M sampled input tokens billed at full rate) vs ~82% implicit hits on Gemini. Two ephemeral breakpoints fix that: the system prompt (whose cache prefix also covers the tool schemas — both immutable per turn by construction) and the last message (Anthropic's incremental-conversation pattern). Conservative simulation on the sampled traffic floors the saving at 44% with no cross-turn reuse; realistic reuse lands 70-85%. Applied in the model registry bridge just before streamText, gated by provider flavor or model id (covers direct Anthropic, OpenRouter, and the gateways — the installed OpenRouter provider reads the same providerOptions.anthropic key). Transport-only: nothing is persisted, message content is untouched, and non-Anthropic requests pass through byte-identical. Verify with cachedInputTokens on model_call_completed. Co-Authored-By: Claude Fable 5 --- .../packages/core/docs/turn-runtime-design.md | 6 ++ .../core/src/models/prompt-caching.test.ts | 100 ++++++++++++++++++ .../core/src/models/prompt-caching.ts | 92 ++++++++++++++++ .../turns/bridges/real-model-registry.test.ts | 58 ++++++++++ .../src/turns/bridges/real-model-registry.ts | 21 +++- 5 files changed, 273 insertions(+), 4 deletions(-) create mode 100644 apps/x/packages/core/src/models/prompt-caching.test.ts create mode 100644 apps/x/packages/core/src/models/prompt-caching.ts diff --git a/apps/x/packages/core/docs/turn-runtime-design.md b/apps/x/packages/core/docs/turn-runtime-design.md index 5f462bd1..2b0e54fa 100644 --- a/apps/x/packages/core/docs/turn-runtime-design.md +++ b/apps/x/packages/core/docs/turn-runtime-design.md @@ -418,6 +418,12 @@ bytes than were transmitted; the inspect CLI prints the active policy so the divergence is visible. If exact-bytes replay ever becomes a hard requirement, record the applied policy on the turn and compose from that. +Relatedly, Anthropic-family requests get cache_control breakpoints stamped +at the transport layer (`models/prompt-caching.ts`, applied inside the +model registry bridge just before streamText). These are provider metadata +only — message content is untouched, nothing is persisted, and inspect +does not render them; non-Anthropic requests pass through byte-identical. + ### 6.7 Agent snapshot inheritance Session turns whose resolved system prompt and tool set are byte-identical diff --git a/apps/x/packages/core/src/models/prompt-caching.test.ts b/apps/x/packages/core/src/models/prompt-caching.test.ts new file mode 100644 index 00000000..3d3fd8f7 --- /dev/null +++ b/apps/x/packages/core/src/models/prompt-caching.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; +import { applyPromptCaching, isAnthropicModel } from "./prompt-caching.js"; + +const BREAKPOINT = { anthropic: { cacheControl: { type: "ephemeral" } } }; + +function prompt(messageCount = 3) { + return { + system: "SYS", + messages: Array.from({ length: messageCount }, (_, i) => ({ + role: i % 2 === 0 ? "user" : "assistant", + content: `m${i}`, + })), + }; +} + +describe("isAnthropicModel", () => { + it("matches the direct provider regardless of model id", () => { + expect(isAnthropicModel("anthropic", "whatever")).toBe(true); + }); + + it("matches aggregator ids by prefix or claude name", () => { + expect(isAnthropicModel("rowboat", "anthropic/claude-opus-4.8")).toBe(true); + expect(isAnthropicModel("openrouter", "anthropic/claude-3.5-sonnet")).toBe(true); + expect(isAnthropicModel("aigateway", "claude-sonnet-5")).toBe(true); + }); + + it("rejects everything else", () => { + expect(isAnthropicModel("openai", "gpt-test")).toBe(false); + expect(isAnthropicModel("rowboat", "google/gemini-3.5-flash")).toBe(false); + expect(isAnthropicModel("ollama", "llama3")).toBe(false); + expect(isAnthropicModel("openai-compatible", "qwen3")).toBe(false); + }); +}); + +describe("applyPromptCaching", () => { + it("returns non-Anthropic prompts unchanged (same references)", () => { + const input = prompt(); + const output = applyPromptCaching("openai", "gpt-test", input); + expect(output).toBe(input); + }); + + it("moves the system prompt into a breakpointed system message", () => { + const { system, messages } = applyPromptCaching("anthropic", "m", prompt()); + expect(system).toBeUndefined(); + expect(messages[0]).toEqual({ + role: "system", + content: "SYS", + providerOptions: BREAKPOINT, + }); + }); + + it("marks only the last message; middle messages stay untouched", () => { + const input = prompt(3); + const { messages } = applyPromptCaching("anthropic", "m", input); + // [system, m0, m1, m2] + expect(messages).toHaveLength(4); + expect(messages[1]).toBe(input.messages[0]); + expect(messages[2]).toBe(input.messages[1]); + expect(messages[3]).toEqual({ + role: "user", + content: "m2", + providerOptions: BREAKPOINT, + }); + }); + + it("does not mutate the input prompt", () => { + const input = prompt(2); + const snapshot = JSON.parse(JSON.stringify(input)); + applyPromptCaching("anthropic", "m", input); + expect(input).toEqual(snapshot); + }); + + it("merges with pre-existing providerOptions on the last message", () => { + const input = { + system: "SYS", + messages: [ + { + role: "user", + content: "hi", + providerOptions: { anthropic: { foo: 1 }, openai: { bar: 2 } }, + }, + ], + }; + const { messages } = applyPromptCaching("anthropic", "m", input); + expect(messages[1]).toEqual({ + role: "user", + content: "hi", + providerOptions: { + openai: { bar: 2 }, + anthropic: { foo: 1, cacheControl: { type: "ephemeral" } }, + }, + }); + }); + + it("is deterministic (same input, same bytes)", () => { + const a = applyPromptCaching("anthropic", "m", prompt()); + const b = applyPromptCaching("anthropic", "m", prompt()); + expect(a).toEqual(b); + }); +}); diff --git a/apps/x/packages/core/src/models/prompt-caching.ts b/apps/x/packages/core/src/models/prompt-caching.ts new file mode 100644 index 00000000..089cc3ff --- /dev/null +++ b/apps/x/packages/core/src/models/prompt-caching.ts @@ -0,0 +1,92 @@ +import type { JsonValue } from "@x/shared/dist/turns.js"; + +// Anthropic prompt caching is opt-in: without explicit cache_control +// breakpoints every call re-bills the full prefix at the base input rate +// (observed sessions: 0% cache hits on Claude vs ~82% implicit hits on +// Gemini, whose caching is automatic). Two breakpoints of the 4 allowed: +// +// 1. the system prompt — Anthropic's cache prefix is tools -> system -> +// messages, so this single breakpoint also covers the tool schemas; +// both are immutable for the life of a turn (agent snapshot), +// 2. the last message — the incremental-conversation pattern: each call +// writes a cache entry at the conversation tip, and the next call's +// lookup reuses the longest previously written prefix. +// +// Writes bill 1.25x on the newly cached segment and reads bill 0.1x, so a +// segment pays for itself on its first re-read; only tokens written and +// never read again (a session's final tail) cost extra, capped at 25%. +// +// This is a transport-layer decoration ONLY: nothing is persisted, so the +// durable log stays provider-agnostic and recomposition is unaffected. +// Non-Anthropic requests pass through byte-identical. Failure degrades +// soft: the AI SDK Anthropic provider validates placement and ignores +// (with a warning) misplaced or excess breakpoints, and the OpenRouter +// provider reads the same providerOptions.anthropic key. +const CACHE_CONTROL: Record = { + cacheControl: { type: "ephemeral" }, +}; + +// Anthropic models arrive three ways: the direct provider (flavor +// "anthropic", any model id), and aggregators (openrouter, aigateway, the +// rowboat gateway) that address them as "anthropic/" or "claude-*". +export function isAnthropicModel(flavor: string, modelId: string): boolean { + if (flavor === "anthropic") { + return true; + } + const id = modelId.toLowerCase(); + return id.startsWith("anthropic/") || id.includes("claude"); +} + +export interface PromptShape { + system?: string; + messages: JsonValue[]; +} + +export function applyPromptCaching( + flavor: string, + modelId: string, + prompt: { system: string; messages: JsonValue[] }, +): PromptShape { + if (!isAnthropicModel(flavor, modelId)) { + return prompt; + } + // The system string moves into the message array so it can carry + // per-message providerOptions (the AI SDK's documented breakpoint + // mechanism); the provider reassembles it into the system field. + const systemMessage: JsonValue = { + role: "system", + content: prompt.system, + providerOptions: { anthropic: CACHE_CONTROL }, + }; + const messages = prompt.messages.map((message, index) => { + if (index !== prompt.messages.length - 1) { + return message; + } + const existing = + typeof message === "object" && message !== null && !Array.isArray(message) + ? message + : {}; + const existingOptions = + "providerOptions" in existing && + typeof existing.providerOptions === "object" && + existing.providerOptions !== null && + !Array.isArray(existing.providerOptions) + ? existing.providerOptions + : {}; + return { + ...existing, + providerOptions: { + ...existingOptions, + anthropic: { + ...(typeof existingOptions.anthropic === "object" && + existingOptions.anthropic !== null && + !Array.isArray(existingOptions.anthropic) + ? existingOptions.anthropic + : {}), + ...CACHE_CONTROL, + }, + }, + }; + }); + return { messages: [systemMessage, ...messages] }; +} diff --git a/apps/x/packages/core/src/turns/bridges/real-model-registry.test.ts b/apps/x/packages/core/src/turns/bridges/real-model-registry.test.ts index ea0fe990..8fbf1a3e 100644 --- a/apps/x/packages/core/src/turns/bridges/real-model-registry.test.ts +++ b/apps/x/packages/core/src/turns/bridges/real-model-registry.test.ts @@ -188,6 +188,64 @@ describe("RealModelRegistry", () => { }); }); + it("applies cache breakpoints for Anthropic-family models", async () => { + const capture: InvokerOptions[] = []; + const fakeModel = { modelId: "claude-test" } as unknown as LanguageModel; + const registry = new RealModelRegistry({ + resolveProvider: async () => ({ flavor: "anthropic" }), + createProviderImpl: (() => ({ + languageModel: () => fakeModel, + })) as never, + invoke: (options) => { + capture.push(options); + return { + fullStream: (async function* () { + yield { type: "finish-step", finishReason: "stop", usage: {} }; + })(), + }; + }, + }); + const model = await registry.resolve({ + provider: "anthropic", + model: "claude-test", + }); + const drained: unknown[] = []; + for await (const event of model.stream(request())) { + drained.push(event); + } + expect(drained.length).toBeGreaterThan(0); + + const [options] = capture; + // System prompt rides the message array with a breakpoint; no + // separate system string is sent. + expect(options.system).toBeUndefined(); + expect(options.messages[0]).toMatchObject({ + role: "system", + content: "SYS", + providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }, + }); + expect(options.messages[options.messages.length - 1]).toMatchObject({ + role: "user", + providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }, + }); + }); + + it("leaves non-Anthropic requests byte-identical", async () => { + const capture: InvokerOptions[] = []; + const registry = makeRegistry( + [{ type: "finish-step", finishReason: "stop", usage: {} }], + capture, + ); + const req = request(); + const originalMessages = JSON.parse(JSON.stringify(req.messages)); + await collect(registry, req); + + const [options] = capture; + expect(options.system).toBe("SYS"); + expect(options.messages).toEqual(originalMessages); + expect(JSON.stringify(options.messages)).not.toContain("cacheControl"); + }); + it("stops promptly when the signal aborts mid-stream", async () => { const controller = new AbortController(); const registry = makeRegistry( diff --git a/apps/x/packages/core/src/turns/bridges/real-model-registry.ts b/apps/x/packages/core/src/turns/bridges/real-model-registry.ts index 36935d8a..40665a44 100644 --- a/apps/x/packages/core/src/turns/bridges/real-model-registry.ts +++ b/apps/x/packages/core/src/turns/bridges/real-model-registry.ts @@ -14,6 +14,7 @@ import type { JsonValue, ModelDescriptor, TurnUsage } from "@x/shared/dist/turns import { convertFromMessages } from "../../agents/runtime.js"; import { resolveProviderConfig } from "../../models/defaults.js"; import { createProvider } from "../../models/models.js"; +import { applyPromptCaching } from "../../models/prompt-caching.js"; import { applyLocalModelSettings } from "../../models/local.js"; import type { IModelRegistry, @@ -26,7 +27,9 @@ import type { // provider. The bridge always requests exactly one step. export type StreamTextInvoker = (options: { model: LanguageModel; - system: string; + // Absent when prompt caching moved the system prompt into messages + // (Anthropic-family models; see models/prompt-caching.ts). + system?: string; messages: ModelMessage[]; tools: ToolSet; abortSignal: AbortSignal; @@ -79,13 +82,16 @@ export class RealModelRegistry implements IModelRegistry { // per-message, so composed requests are byte-stable. encodeMessages: (messages) => convertFromMessages(messages) as unknown as JsonValue[], - stream: (request) => this.run(model, request), + stream: (request) => + this.run(model, request, providerConfig.flavor, descriptor.model), }; } private async *run( model: LanguageModel, request: ModelStreamRequest, + flavor: string, + modelId: string, ): AsyncGenerator { const tools: ToolSet = {}; for (const descriptor of request.tools) { @@ -121,10 +127,17 @@ export class RealModelRegistry implements IModelRegistry { let usage: z.infer = {}; let providerMetadata: JsonValue | undefined; + // Anthropic-family models get cache_control breakpoints; everything + // else passes through byte-identical (transport-only, not persisted). + const prompt = applyPromptCaching(flavor, modelId, { + system: request.systemPrompt, + messages: request.messages, + }); + const result = this.invoke({ model, - system: request.systemPrompt, - messages: request.messages as ModelMessage[], + ...(prompt.system === undefined ? {} : { system: prompt.system }), + messages: prompt.messages as ModelMessage[], tools, abortSignal: request.signal, ...generationParams,