diff --git a/apps/x/packages/core/docs/turn-runtime-design.md b/apps/x/packages/core/docs/turn-runtime-design.md index d199fabe..2b0e54fa 100644 --- a/apps/x/packages/core/docs/turn-runtime-design.md +++ b/apps/x/packages/core/docs/turn-runtime-design.md @@ -390,6 +390,40 @@ Rules: It rejects the execution and does not append `turn_failed`. - The reducer treats `context` as opaque data and never resolves references. +The app wires the resolver through a decorator (`context-elision.ts`) that +applies transmit-time elision to the materialized prefix: tool results from +prior turns above a size threshold are replaced with a short placeholder +telling the model to re-run the tool if it needs the output; inline image +parts from prior turns (video-mode webcam and screen-share frames) are +replaced with a text part recording how many frames of each kind were +dropped; and middle-pane note snapshots on prior-turn user messages keep +their kind and path but have their content replaced with a placeholder +pointing at the still-readable file. The durable log is untouched; only the +transmitted bytes change. +Elision is a pure function of each message, so resolved prefixes stay +byte-stable across calls (provider prefix caches keep working), and the +current turn's own messages never pass through the resolver, so in-flight +tool results and just-captured frames are always sent verbatim. Policy lives +in `config/context.json` (`elideHistoricToolResults`, default true; +`elideHistoricToolResultsThresholdChars`, default 2500; +`elideHistoricImages`, default true; `elideHistoricMiddlePaneContent`, +default true). The inspect CLI composes through the same decorated resolver. + +Caveat: elision makes the composed payload a function of the durable log +PLUS the config in effect at compose time — the one deliberate exception to +"recomposition from durable state alone" (§8.3). Within a turn this is +harmless (policy is loaded once per execution, at resolve time), but +inspecting an old turn after a config change may show different prefix +bytes than were transmitted; the inspect CLI prints the active policy so +the divergence is visible. If exact-bytes replay ever becomes a hard +requirement, record the applied policy on the turn and compose from that. + +Relatedly, Anthropic-family requests get cache_control breakpoints stamped +at the transport layer (`models/prompt-caching.ts`, applied inside the +model registry bridge just before streamText). These are provider metadata +only — message content is untouched, nothing is persisted, and inspect +does not render them; non-Anthropic requests pass through byte-identical. + ### 6.7 Agent snapshot inheritance Session turns whose resolved system prompt and tool set are byte-identical diff --git a/apps/x/packages/core/src/di/container.ts b/apps/x/packages/core/src/di/container.ts index 77bda6bb..306ebc33 100644 --- a/apps/x/packages/core/src/di/container.ts +++ b/apps/x/packages/core/src/di/container.ts @@ -31,7 +31,8 @@ import type { INotificationService } from "../application/notification/service.j import { SystemClock, type IClock } from "../turns/clock.js"; import { FSTurnRepo } from "../turns/fs-repo.js"; import type { ITurnRepo } from "../turns/repo.js"; -import { TurnRepoContextResolver, type IContextResolver } from "../turns/context-resolver.js"; +import type { IContextResolver } from "../turns/context-resolver.js"; +import { createContextResolver } from "../turns/context-elision.js"; import { EmitterTurnLifecycleBus, type ITurnLifecycleBus } from "../turns/bus.js"; import { RealUsageReporter } from "../turns/bridges/real-usage-reporter.js"; import type { IUsageReporter } from "../turns/usage-reporter.js"; @@ -117,7 +118,9 @@ container.register({ turnsRootDir: asValue(path.join(WorkDir, "storage", "turns")), sessionsRootDir: asValue(path.join(WorkDir, "storage", "sessions")), turnRepo: asClass(FSTurnRepo).singleton(), - contextResolver: asClass(TurnRepoContextResolver).singleton(), + contextResolver: asFunction(({ turnRepo }) => + createContextResolver({ turnRepo }), + ).singleton(), lifecycleBus: asClass(EmitterTurnLifecycleBus).singleton(), usageReporter: asClass(RealUsageReporter).singleton(), agentResolver: asFunction(() => new RealAgentResolver()).singleton(), diff --git a/apps/x/packages/core/src/models/prompt-caching.test.ts b/apps/x/packages/core/src/models/prompt-caching.test.ts new file mode 100644 index 00000000..3d3fd8f7 --- /dev/null +++ b/apps/x/packages/core/src/models/prompt-caching.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; +import { applyPromptCaching, isAnthropicModel } from "./prompt-caching.js"; + +const BREAKPOINT = { anthropic: { cacheControl: { type: "ephemeral" } } }; + +function prompt(messageCount = 3) { + return { + system: "SYS", + messages: Array.from({ length: messageCount }, (_, i) => ({ + role: i % 2 === 0 ? "user" : "assistant", + content: `m${i}`, + })), + }; +} + +describe("isAnthropicModel", () => { + it("matches the direct provider regardless of model id", () => { + expect(isAnthropicModel("anthropic", "whatever")).toBe(true); + }); + + it("matches aggregator ids by prefix or claude name", () => { + expect(isAnthropicModel("rowboat", "anthropic/claude-opus-4.8")).toBe(true); + expect(isAnthropicModel("openrouter", "anthropic/claude-3.5-sonnet")).toBe(true); + expect(isAnthropicModel("aigateway", "claude-sonnet-5")).toBe(true); + }); + + it("rejects everything else", () => { + expect(isAnthropicModel("openai", "gpt-test")).toBe(false); + expect(isAnthropicModel("rowboat", "google/gemini-3.5-flash")).toBe(false); + expect(isAnthropicModel("ollama", "llama3")).toBe(false); + expect(isAnthropicModel("openai-compatible", "qwen3")).toBe(false); + }); +}); + +describe("applyPromptCaching", () => { + it("returns non-Anthropic prompts unchanged (same references)", () => { + const input = prompt(); + const output = applyPromptCaching("openai", "gpt-test", input); + expect(output).toBe(input); + }); + + it("moves the system prompt into a breakpointed system message", () => { + const { system, messages } = applyPromptCaching("anthropic", "m", prompt()); + expect(system).toBeUndefined(); + expect(messages[0]).toEqual({ + role: "system", + content: "SYS", + providerOptions: BREAKPOINT, + }); + }); + + it("marks only the last message; middle messages stay untouched", () => { + const input = prompt(3); + const { messages } = applyPromptCaching("anthropic", "m", input); + // [system, m0, m1, m2] + expect(messages).toHaveLength(4); + expect(messages[1]).toBe(input.messages[0]); + expect(messages[2]).toBe(input.messages[1]); + expect(messages[3]).toEqual({ + role: "user", + content: "m2", + providerOptions: BREAKPOINT, + }); + }); + + it("does not mutate the input prompt", () => { + const input = prompt(2); + const snapshot = JSON.parse(JSON.stringify(input)); + applyPromptCaching("anthropic", "m", input); + expect(input).toEqual(snapshot); + }); + + it("merges with pre-existing providerOptions on the last message", () => { + const input = { + system: "SYS", + messages: [ + { + role: "user", + content: "hi", + providerOptions: { anthropic: { foo: 1 }, openai: { bar: 2 } }, + }, + ], + }; + const { messages } = applyPromptCaching("anthropic", "m", input); + expect(messages[1]).toEqual({ + role: "user", + content: "hi", + providerOptions: { + openai: { bar: 2 }, + anthropic: { foo: 1, cacheControl: { type: "ephemeral" } }, + }, + }); + }); + + it("is deterministic (same input, same bytes)", () => { + const a = applyPromptCaching("anthropic", "m", prompt()); + const b = applyPromptCaching("anthropic", "m", prompt()); + expect(a).toEqual(b); + }); +}); diff --git a/apps/x/packages/core/src/models/prompt-caching.ts b/apps/x/packages/core/src/models/prompt-caching.ts new file mode 100644 index 00000000..089cc3ff --- /dev/null +++ b/apps/x/packages/core/src/models/prompt-caching.ts @@ -0,0 +1,92 @@ +import type { JsonValue } from "@x/shared/dist/turns.js"; + +// Anthropic prompt caching is opt-in: without explicit cache_control +// breakpoints every call re-bills the full prefix at the base input rate +// (observed sessions: 0% cache hits on Claude vs ~82% implicit hits on +// Gemini, whose caching is automatic). Two breakpoints of the 4 allowed: +// +// 1. the system prompt — Anthropic's cache prefix is tools -> system -> +// messages, so this single breakpoint also covers the tool schemas; +// both are immutable for the life of a turn (agent snapshot), +// 2. the last message — the incremental-conversation pattern: each call +// writes a cache entry at the conversation tip, and the next call's +// lookup reuses the longest previously written prefix. +// +// Writes bill 1.25x on the newly cached segment and reads bill 0.1x, so a +// segment pays for itself on its first re-read; only tokens written and +// never read again (a session's final tail) cost extra, capped at 25%. +// +// This is a transport-layer decoration ONLY: nothing is persisted, so the +// durable log stays provider-agnostic and recomposition is unaffected. +// Non-Anthropic requests pass through byte-identical. Failure degrades +// soft: the AI SDK Anthropic provider validates placement and ignores +// (with a warning) misplaced or excess breakpoints, and the OpenRouter +// provider reads the same providerOptions.anthropic key. +const CACHE_CONTROL: Record = { + cacheControl: { type: "ephemeral" }, +}; + +// Anthropic models arrive three ways: the direct provider (flavor +// "anthropic", any model id), and aggregators (openrouter, aigateway, the +// rowboat gateway) that address them as "anthropic/" or "claude-*". +export function isAnthropicModel(flavor: string, modelId: string): boolean { + if (flavor === "anthropic") { + return true; + } + const id = modelId.toLowerCase(); + return id.startsWith("anthropic/") || id.includes("claude"); +} + +export interface PromptShape { + system?: string; + messages: JsonValue[]; +} + +export function applyPromptCaching( + flavor: string, + modelId: string, + prompt: { system: string; messages: JsonValue[] }, +): PromptShape { + if (!isAnthropicModel(flavor, modelId)) { + return prompt; + } + // The system string moves into the message array so it can carry + // per-message providerOptions (the AI SDK's documented breakpoint + // mechanism); the provider reassembles it into the system field. + const systemMessage: JsonValue = { + role: "system", + content: prompt.system, + providerOptions: { anthropic: CACHE_CONTROL }, + }; + const messages = prompt.messages.map((message, index) => { + if (index !== prompt.messages.length - 1) { + return message; + } + const existing = + typeof message === "object" && message !== null && !Array.isArray(message) + ? message + : {}; + const existingOptions = + "providerOptions" in existing && + typeof existing.providerOptions === "object" && + existing.providerOptions !== null && + !Array.isArray(existing.providerOptions) + ? existing.providerOptions + : {}; + return { + ...existing, + providerOptions: { + ...existingOptions, + anthropic: { + ...(typeof existingOptions.anthropic === "object" && + existingOptions.anthropic !== null && + !Array.isArray(existingOptions.anthropic) + ? existingOptions.anthropic + : {}), + ...CACHE_CONTROL, + }, + }, + }; + }); + return { messages: [systemMessage, ...messages] }; +} diff --git a/apps/x/packages/core/src/turns/bridges/real-model-registry.test.ts b/apps/x/packages/core/src/turns/bridges/real-model-registry.test.ts index ea0fe990..8fbf1a3e 100644 --- a/apps/x/packages/core/src/turns/bridges/real-model-registry.test.ts +++ b/apps/x/packages/core/src/turns/bridges/real-model-registry.test.ts @@ -188,6 +188,64 @@ describe("RealModelRegistry", () => { }); }); + it("applies cache breakpoints for Anthropic-family models", async () => { + const capture: InvokerOptions[] = []; + const fakeModel = { modelId: "claude-test" } as unknown as LanguageModel; + const registry = new RealModelRegistry({ + resolveProvider: async () => ({ flavor: "anthropic" }), + createProviderImpl: (() => ({ + languageModel: () => fakeModel, + })) as never, + invoke: (options) => { + capture.push(options); + return { + fullStream: (async function* () { + yield { type: "finish-step", finishReason: "stop", usage: {} }; + })(), + }; + }, + }); + const model = await registry.resolve({ + provider: "anthropic", + model: "claude-test", + }); + const drained: unknown[] = []; + for await (const event of model.stream(request())) { + drained.push(event); + } + expect(drained.length).toBeGreaterThan(0); + + const [options] = capture; + // System prompt rides the message array with a breakpoint; no + // separate system string is sent. + expect(options.system).toBeUndefined(); + expect(options.messages[0]).toMatchObject({ + role: "system", + content: "SYS", + providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }, + }); + expect(options.messages[options.messages.length - 1]).toMatchObject({ + role: "user", + providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }, + }); + }); + + it("leaves non-Anthropic requests byte-identical", async () => { + const capture: InvokerOptions[] = []; + const registry = makeRegistry( + [{ type: "finish-step", finishReason: "stop", usage: {} }], + capture, + ); + const req = request(); + const originalMessages = JSON.parse(JSON.stringify(req.messages)); + await collect(registry, req); + + const [options] = capture; + expect(options.system).toBe("SYS"); + expect(options.messages).toEqual(originalMessages); + expect(JSON.stringify(options.messages)).not.toContain("cacheControl"); + }); + it("stops promptly when the signal aborts mid-stream", async () => { const controller = new AbortController(); const registry = makeRegistry( diff --git a/apps/x/packages/core/src/turns/bridges/real-model-registry.ts b/apps/x/packages/core/src/turns/bridges/real-model-registry.ts index 36935d8a..40665a44 100644 --- a/apps/x/packages/core/src/turns/bridges/real-model-registry.ts +++ b/apps/x/packages/core/src/turns/bridges/real-model-registry.ts @@ -14,6 +14,7 @@ import type { JsonValue, ModelDescriptor, TurnUsage } from "@x/shared/dist/turns import { convertFromMessages } from "../../agents/runtime.js"; import { resolveProviderConfig } from "../../models/defaults.js"; import { createProvider } from "../../models/models.js"; +import { applyPromptCaching } from "../../models/prompt-caching.js"; import { applyLocalModelSettings } from "../../models/local.js"; import type { IModelRegistry, @@ -26,7 +27,9 @@ import type { // provider. The bridge always requests exactly one step. export type StreamTextInvoker = (options: { model: LanguageModel; - system: string; + // Absent when prompt caching moved the system prompt into messages + // (Anthropic-family models; see models/prompt-caching.ts). + system?: string; messages: ModelMessage[]; tools: ToolSet; abortSignal: AbortSignal; @@ -79,13 +82,16 @@ export class RealModelRegistry implements IModelRegistry { // per-message, so composed requests are byte-stable. encodeMessages: (messages) => convertFromMessages(messages) as unknown as JsonValue[], - stream: (request) => this.run(model, request), + stream: (request) => + this.run(model, request, providerConfig.flavor, descriptor.model), }; } private async *run( model: LanguageModel, request: ModelStreamRequest, + flavor: string, + modelId: string, ): AsyncGenerator { const tools: ToolSet = {}; for (const descriptor of request.tools) { @@ -121,10 +127,17 @@ export class RealModelRegistry implements IModelRegistry { let usage: z.infer = {}; let providerMetadata: JsonValue | undefined; + // Anthropic-family models get cache_control breakpoints; everything + // else passes through byte-identical (transport-only, not persisted). + const prompt = applyPromptCaching(flavor, modelId, { + system: request.systemPrompt, + messages: request.messages, + }); + const result = this.invoke({ model, - system: request.systemPrompt, - messages: request.messages as ModelMessage[], + ...(prompt.system === undefined ? {} : { system: prompt.system }), + messages: prompt.messages as ModelMessage[], tools, abortSignal: request.signal, ...generationParams, diff --git a/apps/x/packages/core/src/turns/context-elision.test.ts b/apps/x/packages/core/src/turns/context-elision.test.ts new file mode 100644 index 00000000..2e3d51f3 --- /dev/null +++ b/apps/x/packages/core/src/turns/context-elision.test.ts @@ -0,0 +1,590 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import type { z } from "zod"; +import type { TurnContext, TurnEvent } from "@x/shared/dist/turns.js"; +import { + DEFAULT_ELISION_POLICY, + ElidingContextResolver, + createContextResolver, + elideHistoricImages, + elideHistoricMiddlePaneContent, + elideHistoricToolResults, + loadElisionPolicy, + type ElisionPolicy, +} from "./context-elision.js"; +import { TurnRepoContextResolver } from "./context-resolver.js"; +import { InMemoryTurnRepo } from "./in-memory-turn-repo.js"; + +type TEvent = z.infer; + +function user(text: string) { + return { role: "user" as const, content: text }; +} + +function assistant(text: string) { + return { role: "assistant" as const, content: text }; +} + +function toolMsg(content: string, toolName = "file-readText") { + return { + role: "tool" as const, + content, + toolCallId: "tc1", + toolName, + }; +} + +// A completed turn containing one tool round trip whose result has the given +// output, followed by a final text response. +function toolTurnLog( + turnId: string, + context: z.infer, + toolOutput: string, +): TEvent[] { + const ts = "2026-07-02T10:00:00Z"; + const echo = { + toolId: "tool.echo", + name: "echo", + description: "Echo", + inputSchema: {}, + execution: "sync" as const, + requiresHuman: false, + }; + const call = { + role: "assistant" as const, + content: [ + { + type: "tool-call" as const, + toolCallId: "tc1", + toolName: "echo", + arguments: {}, + }, + ], + }; + return [ + { + type: "turn_created", + schemaVersion: 1, + turnId, + ts, + sessionId: "sess-1", + agent: { + requested: { agentId: "copilot" }, + resolved: { + agentId: "copilot", + systemPrompt: "SYS", + model: { provider: "fake", model: "m" }, + tools: [echo], + }, + }, + context, + input: user("do it"), + config: { + autoPermission: false, + humanAvailable: true, + maxModelCalls: 20, + }, + }, + { + type: "model_call_requested", + turnId, + ts, + modelCallIndex: 0, + request: { + ...(Array.isArray(context) ? {} : { contextRef: context }), + messages: + Array.isArray(context) && context.length > 0 + ? ["context", "input"] + : ["input"], + parameters: {}, + }, + }, + { + type: "model_call_completed", + turnId, + ts, + modelCallIndex: 0, + message: call, + finishReason: "tool-calls", + usage: {}, + }, + { + type: "tool_invocation_requested", + turnId, + ts, + toolCallId: "tc1", + toolId: "tool.echo", + toolName: "echo", + execution: "sync", + input: {}, + }, + { + type: "tool_result", + turnId, + ts, + toolCallId: "tc1", + toolName: "echo", + source: "sync", + result: { output: toolOutput, isError: false }, + }, + { + type: "model_call_requested", + turnId, + ts, + modelCallIndex: 1, + request: { + messages: ["assistant:0", "toolResult:tc1"], + parameters: {}, + }, + }, + { + type: "model_call_completed", + turnId, + ts, + modelCallIndex: 1, + message: assistant("done"), + finishReason: "stop", + usage: {}, + }, + { + type: "turn_completed", + turnId, + ts, + output: assistant("done"), + finishReason: "stop", + usage: {}, + }, + ]; +} + +function frame(source: "camera" | "screen") { + return { + type: "image" as const, + data: "aGVsbG8=".repeat(50), + mediaType: "image/jpeg", + source, + capturedAt: "2026-07-02T10:00:00Z", + }; +} + +const T1 = "2026-07-02T10-00-00Z-0000001-000"; + +describe("elideHistoricToolResults", () => { + it("replaces tool results above the threshold with a placeholder", () => { + const big = "x".repeat(101); + const elided = elideHistoricToolResults([toolMsg(big)], 100); + expect(elided).toHaveLength(1); + expect(elided[0]).toMatchObject({ + role: "tool", + toolCallId: "tc1", + toolName: "file-readText", + }); + expect(elided[0].content).toContain("elided"); + expect(elided[0].content).toContain("file-readText"); + expect(elided[0].content).toContain("101"); + }); + + it("keeps a head preview ahead of the placeholder", () => { + const content = `SKILL GUIDE${"y".repeat(5000)}`; + const [elided] = elideHistoricToolResults([toolMsg(content)], 2500); + // Starts with the first 400 chars verbatim, then the marker. + expect(String(elided.content).startsWith(content.slice(0, 400))).toBe(true); + expect(elided.content).toContain("Rest of tool result elided"); + expect(String(elided.content).length).toBeLessThan(700); + // Preview never exceeds a tiny threshold. + const [tiny] = elideHistoricToolResults([toolMsg(content)], 50); + expect(String(tiny.content).startsWith(`${content.slice(0, 50)}\n[Rest`)).toBe( + true, + ); + }); + + it("keeps tool results at or below the threshold verbatim", () => { + const exact = toolMsg("x".repeat(100)); + expect(elideHistoricToolResults([exact], 100)).toEqual([exact]); + }); + + it("leaves non-tool messages untouched", () => { + const messages = [user("q".repeat(500)), assistant("a".repeat(500))]; + expect(elideHistoricToolResults(messages, 100)).toEqual(messages); + }); + + it("is idempotent at realistic thresholds (placeholder is well under them)", () => { + const once = elideHistoricToolResults([toolMsg("x".repeat(5000))], 1000); + expect(once[0].content.length).toBeLessThan(1000); + expect(elideHistoricToolResults(once, 1000)).toEqual(once); + }); + + it("is deterministic for the same input", () => { + const messages = [toolMsg("x".repeat(5000)), user("q"), assistant("a")]; + expect(elideHistoricToolResults(messages, 100)).toEqual( + elideHistoricToolResults(messages, 100), + ); + }); +}); + +describe("elideHistoricImages", () => { + it("replaces image parts with a labeled placeholder, keeping other parts", () => { + const message = { + role: "user" as const, + content: [ + { type: "text" as const, text: "how do I look?" }, + frame("camera"), + frame("camera"), + frame("screen"), + ], + }; + const [elided] = elideHistoricImages([message]); + if (typeof elided.content === "string" || elided.role !== "user") { + throw new Error("expected user message with parts"); + } + expect(elided.content.filter((p) => p.type === "image")).toHaveLength(0); + expect(elided.content[0]).toEqual({ type: "text", text: "how do I look?" }); + const placeholder = elided.content[elided.content.length - 1]; + if (placeholder.type !== "text") throw new Error("expected text placeholder"); + expect(placeholder.text).toContain("2 webcam frames"); + expect(placeholder.text).toContain("1 screen-share frame"); + }); + + it("leaves string-content and image-free user messages untouched", () => { + const messages = [ + user("plain"), + { + role: "user" as const, + content: [{ type: "text" as const, text: "no images" }], + }, + assistant("a"), + ]; + expect(elideHistoricImages(messages)).toEqual(messages); + }); +}); + +function noteMessage(content: string) { + return { + role: "user" as const, + content: "make this punchier", + userMessageContext: { + currentDateTime: "Thursday, July 2, 2026 at 10:00 AM GMT", + middlePane: { + kind: "note" as const, + path: "knowledge/Notes/Draft.md", + content, + }, + }, + }; +} + +describe("elideHistoricMiddlePaneContent", () => { + it("replaces large note snapshots, keeping kind and path", () => { + const [elided] = elideHistoricMiddlePaneContent([ + noteMessage("n".repeat(600)), + ]); + if (elided.role !== "user") throw new Error("expected user message"); + const middlePane = elided.userMessageContext?.middlePane; + if (middlePane?.kind !== "note") throw new Error("expected note pane"); + expect(middlePane.path).toBe("knowledge/Notes/Draft.md"); + expect(middlePane.content).toContain("omitted from history"); + expect(middlePane.content).toContain("600"); + expect(elided.userMessageContext?.currentDateTime).toBeDefined(); + expect(elided.content).toBe("make this punchier"); + }); + + it("keeps small notes and non-note panes untouched", () => { + const small = noteMessage("short todo list"); + const browser = { + role: "user" as const, + content: "what is this page?", + userMessageContext: { + middlePane: { + kind: "browser" as const, + url: "https://example.com", + title: "Example", + }, + }, + }; + const plain = user("no context at all"); + const messages = [small, browser, plain]; + expect(elideHistoricMiddlePaneContent(messages)).toEqual(messages); + }); +}); + +const POLICY_OFF = { + toolResults: false, + toolResultThresholdChars: 100, + images: false, + middlePaneContent: false, +}; + +describe("createContextResolver", () => { + // The factory is the app's one sanctioned constructor (DI container and + // inspect CLI). If this stops returning the eliding decorator, every + // construction site silently regresses to full historic replay. + it("wraps the repo resolver in the eliding decorator", () => { + const resolver = createContextResolver({ + turnRepo: new InMemoryTurnRepo(), + }); + expect(resolver).toBeInstanceOf(ElidingContextResolver); + }); + + it("applies the default policy through the decorated resolver", async () => { + const repo = new InMemoryTurnRepo(); + repo.seed(toolTurnLog(T1, [], "x".repeat(5000))); + // Pinned to the shipped defaults (not the machine's context.json) so + // the test is hermetic while still exercising the full wiring. + const resolved = await createContextResolver({ + turnRepo: repo, + loadPolicy: () => DEFAULT_ELISION_POLICY, + }).resolve({ previousTurnId: T1 }); + const tool = resolved.find((m) => m.role === "tool"); + expect(tool?.content).toContain("elided"); + expect(tool?.content.length).toBeLessThan(1000); + }); +}); + +describe("ElidingContextResolver", () => { + function resolver(repo: InMemoryTurnRepo, policy: ElisionPolicy) { + return new ElidingContextResolver({ + inner: new TurnRepoContextResolver({ turnRepo: repo }), + loadPolicy: () => policy, + }); + } + + it("elides oversized tool results in the resolved prefix", async () => { + const repo = new InMemoryTurnRepo(); + repo.seed(toolTurnLog(T1, [], "x".repeat(200))); + const resolved = await resolver(repo, { + ...POLICY_OFF, + toolResults: true, + }).resolve({ previousTurnId: T1 }); + const tool = resolved.find((m) => m.role === "tool"); + expect(tool?.content).toContain("elided"); + // The rest of the transcript is intact. + expect(resolved.map((m) => m.role)).toEqual([ + "user", + "assistant", + "tool", + "assistant", + ]); + }); + + it("returns the prefix unchanged when the policy is disabled", async () => { + const repo = new InMemoryTurnRepo(); + const output = "x".repeat(200); + repo.seed(toolTurnLog(T1, [], output)); + const resolved = await resolver(repo, POLICY_OFF).resolve({ + previousTurnId: T1, + }); + const tool = resolved.find((m) => m.role === "tool"); + expect(tool?.content).toBe(output); + }); + + it("keeps small tool results verbatim when enabled", async () => { + const repo = new InMemoryTurnRepo(); + repo.seed(toolTurnLog(T1, [], "small")); + const resolved = await resolver(repo, { + ...POLICY_OFF, + toolResults: true, + }).resolve({ previousTurnId: T1 }); + const tool = resolved.find((m) => m.role === "tool"); + expect(tool?.content).toBe("small"); + }); + + it("elides images in the resolved prefix when enabled", async () => { + const repo = new InMemoryTurnRepo(); + const log = toolTurnLog(T1, [], "small").map((event) => + event.type === "turn_created" + ? { + ...event, + input: { + role: "user" as const, + content: [ + { type: "text" as const, text: "watch me" }, + frame("camera"), + ], + }, + } + : event, + ); + repo.seed(log); + const resolved = await resolver(repo, { + ...POLICY_OFF, + images: true, + }).resolve({ previousTurnId: T1 }); + const input = resolved[0]; + if (input.role !== "user" || typeof input.content === "string") { + throw new Error("expected user message with parts"); + } + expect(input.content.some((p) => p.type === "image")).toBe(false); + expect(JSON.stringify(input.content)).toContain("1 webcam frame"); + }); + + it("elides middle-pane note snapshots in the resolved prefix when enabled", async () => { + const repo = new InMemoryTurnRepo(); + const log = toolTurnLog(T1, [], "small").map((event) => + event.type === "turn_created" + ? { ...event, input: noteMessage("n".repeat(600)) } + : event, + ); + repo.seed(log); + const resolved = await resolver(repo, { + ...POLICY_OFF, + middlePaneContent: true, + }).resolve({ previousTurnId: T1 }); + const input = resolved[0]; + if (input.role !== "user") throw new Error("expected user message"); + const middlePane = input.userMessageContext?.middlePane; + if (middlePane?.kind !== "note") throw new Error("expected note pane"); + expect(middlePane.content).toContain("omitted from history"); + }); + + it("delegates resolveAgent to the inner resolver untouched", async () => { + const repo = new InMemoryTurnRepo(); + const snapshot = { + agentId: "copilot", + systemPrompt: "SYS", + model: { provider: "fake", model: "m" }, + tools: [], + }; + const agent = await resolver(repo, POLICY_OFF).resolveAgent(snapshot); + expect(agent).toEqual(snapshot); + }); + + it("elides across every turn of a multi-turn reference chain", async () => { + const repo = new InMemoryTurnRepo(); + const T2 = "2026-07-02T11-00-00Z-0000002-000"; + repo.seed(toolTurnLog(T1, [], "a".repeat(200))); + repo.seed(toolTurnLog(T2, { previousTurnId: T1 }, "b".repeat(200))); + const resolved = await resolver(repo, { + ...POLICY_OFF, + toolResults: true, + }).resolve({ previousTurnId: T2 }); + const tools = resolved.filter((m) => m.role === "tool"); + expect(tools).toHaveLength(2); + for (const tool of tools) { + expect(tool.content).toContain("elided"); + } + }); + + it("reloads the policy on every resolve (hot config)", async () => { + const repo = new InMemoryTurnRepo(); + const output = "x".repeat(200); + repo.seed(toolTurnLog(T1, [], output)); + let loads = 0; + const eliding = new ElidingContextResolver({ + inner: new TurnRepoContextResolver({ turnRepo: repo }), + loadPolicy: () => { + loads += 1; + return { ...POLICY_OFF, toolResults: loads > 1 }; + }, + }); + const first = await eliding.resolve({ previousTurnId: T1 }); + const second = await eliding.resolve({ previousTurnId: T1 }); + expect(loads).toBe(2); + expect(first.find((m) => m.role === "tool")?.content).toBe(output); + expect(second.find((m) => m.role === "tool")?.content).toContain("elided"); + }); +}); + +describe("elision policy idempotency and determinism", () => { + // Placeholders must never themselves be elided on a second pass, and the + // transforms must be pure — both are what keeps replayed prefixes + // byte-stable for provider prefix caching. + it("elideHistoricImages is idempotent and deterministic", () => { + const messages = [ + { + role: "user" as const, + content: [ + { type: "text" as const, text: "watch" }, + frame("camera"), + frame("screen"), + ], + }, + ]; + const once = elideHistoricImages(messages); + expect(elideHistoricImages(once)).toEqual(once); + expect(elideHistoricImages(messages)).toEqual(once); + }); + + it("elideHistoricMiddlePaneContent is idempotent and deterministic", () => { + const messages = [noteMessage("n".repeat(600))]; + const once = elideHistoricMiddlePaneContent(messages); + expect(elideHistoricMiddlePaneContent(once)).toEqual(once); + expect(elideHistoricMiddlePaneContent(messages)).toEqual(once); + }); + + it("keeps notes at the floor verbatim and elides just above it", () => { + const atFloor = noteMessage("n".repeat(500)); + expect(elideHistoricMiddlePaneContent([atFloor])).toEqual([atFloor]); + const [above] = elideHistoricMiddlePaneContent([ + noteMessage("n".repeat(501)), + ]); + if (above.role !== "user") throw new Error("expected user message"); + expect(above.userMessageContext?.middlePane?.kind === "note" + ? above.userMessageContext.middlePane.content + : "").toContain("omitted from history"); + }); +}); + +describe("loadElisionPolicy", () => { + let dir: string; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), "elision-config-")); + }); + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + function write(content: string): string { + const p = path.join(dir, "context.json"); + fs.writeFileSync(p, content); + return p; + } + + it("returns defaults when the file is missing", () => { + expect(loadElisionPolicy(path.join(dir, "nope.json"))).toEqual( + DEFAULT_ELISION_POLICY, + ); + }); + + it("applies a full config", () => { + const p = write( + JSON.stringify({ + elideHistoricToolResults: false, + elideHistoricToolResultsThresholdChars: 42, + elideHistoricImages: false, + elideHistoricMiddlePaneContent: false, + }), + ); + expect(loadElisionPolicy(p)).toEqual({ + toolResults: false, + toolResultThresholdChars: 42, + images: false, + middlePaneContent: false, + }); + }); + + it("merges a partial config with defaults", () => { + const p = write(JSON.stringify({ elideHistoricToolResultsThresholdChars: 42 })); + expect(loadElisionPolicy(p)).toEqual({ + ...DEFAULT_ELISION_POLICY, + toolResultThresholdChars: 42, + }); + }); + + it("falls back to defaults on unparseable JSON", () => { + expect(loadElisionPolicy(write("{nope"))).toEqual(DEFAULT_ELISION_POLICY); + }); + + it("discards the whole file when any key is malformed (all-or-nothing)", () => { + const p = write( + JSON.stringify({ + elideHistoricToolResults: false, + elideHistoricToolResultsThresholdChars: "not-a-number", + }), + ); + expect(loadElisionPolicy(p)).toEqual(DEFAULT_ELISION_POLICY); + }); +}); diff --git a/apps/x/packages/core/src/turns/context-elision.ts b/apps/x/packages/core/src/turns/context-elision.ts new file mode 100644 index 00000000..04796037 --- /dev/null +++ b/apps/x/packages/core/src/turns/context-elision.ts @@ -0,0 +1,246 @@ +import fs from "fs"; +import path from "path"; +import { z } from "zod"; +import type { + ConversationMessage, + ResolvedAgent, + ResolvedAgentSnapshot, + TurnContext, +} from "@x/shared/dist/turns.js"; +import { WorkDir } from "../config/config.js"; +import type { IContextResolver } from "./context-resolver.js"; +import { TurnRepoContextResolver } from "./context-resolver.js"; +import type { ITurnRepo } from "./repo.js"; + +// Transmit-time elision of historic context (the cross-turn prefix only — +// the current turn's own messages never pass through the resolver, so +// in-flight tool results and just-captured frames are always sent verbatim). +// +// Two policies, both on by default: +// - Tool results: large tool outputs from earlier turns (skill loads, file +// reads, HTTP fetches) dominate resent context; the model rarely needs +// them verbatim and can re-run the tool when it does. +// - Inline images: video-mode webcam and screen-share frames matter for +// the response they were captured for; afterwards the assistant's own +// text carries the takeaway, and fresh frames arrive with every new call +// message. Unlike tool results they cannot be re-fetched, so the +// placeholder only records what was there. +// - Middle-pane note content: every user message sent while a note is +// open carries a full snapshot of that note in userMessageContext. Only +// the newest snapshot matters (the system prompt already tells the model +// later middle-pane context overrides earlier), and the current file is +// always re-readable at the recorded path. +// +// Elision is a pure function of each message's content, so resolved prefixes +// stay byte-stable across calls and turns (provider prefix caches keep +// working), and the durable JSONL log is untouched — only the transmitted +// bytes change. + +export interface ElisionPolicy { + toolResults: boolean; + toolResultThresholdChars: number; + images: boolean; + middlePaneContent: boolean; +} + +// Threshold rationale: ~2,500 chars ≈ 600 tokens. Observed sessions show +// skill bodies at ~5k chars are the most common oversized result, so 10k +// would replay them forever; the head preview plus re-run hint makes the +// lower cut safe. Tunable via config/context.json. +export const DEFAULT_ELISION_POLICY: ElisionPolicy = { + toolResults: true, + toolResultThresholdChars: 2_500, + images: true, + middlePaneContent: true, +}; + +// Notes smaller than this stay verbatim in history: below the floor the +// placeholder saves nothing and a short note may carry useful context. +const MIDDLE_PANE_CONTENT_FLOOR_CHARS = 500; + +// Head of an elided tool result kept verbatim ahead of the placeholder. +const TOOL_RESULT_PREVIEW_CHARS = 400; + +const ContextConfig = z.object({ + elideHistoricToolResults: z.boolean().optional(), + elideHistoricToolResultsThresholdChars: z.number().int().min(0).optional(), + elideHistoricImages: z.boolean().optional(), + elideHistoricMiddlePaneContent: z.boolean().optional(), +}); + +const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json"); + +// Read the elision policy from config/context.json, falling back to defaults +// for missing keys or an unreadable file (a single malformed key discards the +// whole file — all-or-nothing by design, so a typo can't half-apply). Read +// per resolve so a config edit applies to the next turn without a restart. +export function loadElisionPolicy( + configPath: string = CONTEXT_CONFIG_PATH, +): ElisionPolicy { + try { + if (!fs.existsSync(configPath)) { + return DEFAULT_ELISION_POLICY; + } + const raw = fs.readFileSync(configPath, "utf-8"); + const parsed = ContextConfig.parse(JSON.parse(raw)); + return { + toolResults: + parsed.elideHistoricToolResults ?? + DEFAULT_ELISION_POLICY.toolResults, + toolResultThresholdChars: + parsed.elideHistoricToolResultsThresholdChars ?? + DEFAULT_ELISION_POLICY.toolResultThresholdChars, + images: + parsed.elideHistoricImages ?? DEFAULT_ELISION_POLICY.images, + middlePaneContent: + parsed.elideHistoricMiddlePaneContent ?? + DEFAULT_ELISION_POLICY.middlePaneContent, + }; + } catch { + return DEFAULT_ELISION_POLICY; + } +} + +export function elideHistoricToolResults( + messages: Array>, + thresholdChars: number, +): Array> { + return messages.map((message) => { + if ( + message.role !== "tool" || + message.content.length <= thresholdChars + ) { + return message; + } + // Keep a head preview so the model knows what it is declining to + // re-fetch (a skill body reads very differently from a web page). + // Capped at the threshold so tiny thresholds still shrink content. + const preview = message.content.slice( + 0, + Math.min(TOOL_RESULT_PREVIEW_CHARS, thresholdChars), + ); + return { + ...message, + content: `${preview}\n[Rest of tool result elided to save context: "${message.toolName}" returned ${message.content.length} characters in an earlier turn. Call the tool again if you need the full output now.]`, + }; + }); +} + +export function elideHistoricImages( + messages: Array>, +): Array> { + return messages.map((message) => { + if (message.role !== "user" || typeof message.content === "string") { + return message; + } + const images = message.content.filter((part) => part.type === "image"); + if (images.length === 0) { + return message; + } + const camera = images.filter((part) => part.source !== "screen").length; + const screen = images.length - camera; + const kinds = [ + ...(camera > 0 ? [`${camera} webcam frame${camera === 1 ? "" : "s"}`] : []), + ...(screen > 0 ? [`${screen} screen-share frame${screen === 1 ? "" : "s"}`] : []), + ].join(" and "); + return { + ...message, + content: [ + ...message.content.filter((part) => part.type !== "image"), + { + type: "text" as const, + text: `[Omitted from history to save context: ${kinds} captured while this message was composed. The assistant saw them when responding to this message.]`, + }, + ], + }; + }); +} + +export function elideHistoricMiddlePaneContent( + messages: Array>, +): Array> { + return messages.map((message) => { + if (message.role !== "user") { + return message; + } + const middlePane = message.userMessageContext?.middlePane; + if ( + !middlePane || + middlePane.kind !== "note" || + middlePane.content.length <= MIDDLE_PANE_CONTENT_FLOOR_CHARS + ) { + return message; + } + return { + ...message, + userMessageContext: { + ...message.userMessageContext, + middlePane: { + ...middlePane, + content: `[Note content (${middlePane.content.length} characters) omitted from history to save context. This snapshot is stale; read the file at the path above if its content is needed now.]`, + }, + }, + }; + }); +} + +// IContextResolver decorator: applies the elision policy to the materialized +// cross-turn prefix. Agent snapshot resolution is delegated untouched. +export class ElidingContextResolver implements IContextResolver { + private readonly inner: IContextResolver; + private readonly loadPolicy: () => ElisionPolicy; + + constructor({ + inner, + loadPolicy, + }: { + inner: IContextResolver; + loadPolicy?: () => ElisionPolicy; + }) { + this.inner = inner; + this.loadPolicy = loadPolicy ?? loadElisionPolicy; + } + + async resolve( + context: z.infer, + ): Promise>> { + let prefix = await this.inner.resolve(context); + const policy = this.loadPolicy(); + if (policy.toolResults) { + prefix = elideHistoricToolResults( + prefix, + policy.toolResultThresholdChars, + ); + } + if (policy.images) { + prefix = elideHistoricImages(prefix); + } + if (policy.middlePaneContent) { + prefix = elideHistoricMiddlePaneContent(prefix); + } + return prefix; + } + + resolveAgent( + resolved: z.infer, + ): Promise> { + return this.inner.resolveAgent(resolved); + } +} + +// The one context resolver the app should construct (DI container and the +// inspect CLI both use this), so the debug view reproduces the same bytes +// the loop transmits. +export function createContextResolver({ + turnRepo, + loadPolicy, +}: { + turnRepo: ITurnRepo; + // Injectable for tests; the app default reads config/context.json. + loadPolicy?: () => ElisionPolicy; +}): IContextResolver { + return new ElidingContextResolver({ + inner: new TurnRepoContextResolver({ turnRepo }), + ...(loadPolicy ? { loadPolicy } : {}), + }); +} diff --git a/apps/x/packages/core/src/turns/context-resolver.ts b/apps/x/packages/core/src/turns/context-resolver.ts index 3785cc26..623bdb4e 100644 --- a/apps/x/packages/core/src/turns/context-resolver.ts +++ b/apps/x/packages/core/src/turns/context-resolver.ts @@ -28,6 +28,10 @@ export interface IContextResolver { ): Promise>; } +// NOTE: app code must not construct this directly — use createContextResolver +// (context-elision.ts), which wraps it in the eliding decorator. A raw +// instance silently transmits full historic tool results, frames, and note +// snapshots. Direct construction is for tests of the raw resolution only. export class TurnRepoContextResolver implements IContextResolver { private readonly turnRepo: ITurnRepo; diff --git a/apps/x/packages/core/src/turns/inspect-cli.ts b/apps/x/packages/core/src/turns/inspect-cli.ts index b662e407..9326a73c 100644 --- a/apps/x/packages/core/src/turns/inspect-cli.ts +++ b/apps/x/packages/core/src/turns/inspect-cli.ts @@ -8,6 +8,10 @@ // messages (user-message context, attachments, tool-result envelopes) are // visible; the file itself stores only structural facts and references. // +// Caveat: historic-context elision reads config/context.json at compose +// time, so recomposition is exact only while that config matches what was +// live when the call ran. The header prints the policy in effect NOW. +// // Session mode prints the session overview (title, turns, statuses, sizes); // pass --turns to cascade full turn inspection for every turn. // @@ -27,7 +31,7 @@ import { convertFromMessages } from "../agents/runtime.js"; import { WorkDir } from "../config/config.js"; import { FSSessionRepo } from "../sessions/fs-repo.js"; import { composeModelRequest } from "./compose-model-request.js"; -import { TurnRepoContextResolver } from "./context-resolver.js"; +import { createContextResolver, loadElisionPolicy } from "./context-elision.js"; import { FSTurnRepo } from "./fs-repo.js"; const turnRepo = new FSTurnRepo({ @@ -36,7 +40,7 @@ const turnRepo = new FSTurnRepo({ const sessionRepo = new FSSessionRepo({ sessionsRootDir: path.join(WorkDir, "storage", "sessions"), }); -const resolver = new TurnRepoContextResolver({ turnRepo }); +const resolver = createContextResolver({ turnRepo }); const encode = (messages: Parameters[0]) => convertFromMessages(messages) as unknown as JsonValue[]; @@ -81,6 +85,17 @@ async function inspectTurn( console.log( `agent ${agent.agentId} model ${agent.model.provider}/${agent.model.model} calls ${state.modelCalls.length}${inherited}`, ); + const policy = loadElisionPolicy(); + console.log( + `context elision (per config/context.json NOW; transmitted bytes reflect the config live at call time): ` + + [ + policy.toolResults + ? `toolResults>${policy.toolResultThresholdChars}` + : "toolResults off", + policy.images ? "images" : "images off", + policy.middlePaneContent ? "notes" : "notes off", + ].join(", "), + ); for (const call of state.modelCalls) { if (onlyIndex !== undefined && call.index !== onlyIndex) continue;