diff --git a/apps/x/packages/core/docs/turn-runtime-design.md b/apps/x/packages/core/docs/turn-runtime-design.md index 0dcc8026..cfff8cf2 100644 --- a/apps/x/packages/core/docs/turn-runtime-design.md +++ b/apps/x/packages/core/docs/turn-runtime-design.md @@ -393,14 +393,18 @@ Rules: The app wires the resolver through a decorator (`context-elision.ts`) that applies transmit-time elision to the materialized prefix: tool results from prior turns above a size threshold are replaced with a short placeholder -telling the model to re-run the tool if it needs the output. The durable log -is untouched; only the transmitted bytes change. Elision is a pure function -of each message, so resolved prefixes stay byte-stable across calls (provider -prefix caches keep working), and the current turn's own in-flight tool -results never pass through the resolver, so they are always sent verbatim. -Policy lives in `config/context.json` (`elideHistoricToolResults`, default -true; `elideHistoricToolResultsThresholdChars`, default 10000). The inspect -CLI composes through the same decorated resolver. +telling the model to re-run the tool if it needs the output, and inline +image parts from prior turns (video-mode webcam and screen-share frames) are +replaced with a text part recording how many frames of each kind were +dropped. The durable log is untouched; only the transmitted bytes change. +Elision is a pure function of each message, so resolved prefixes stay +byte-stable across calls (provider prefix caches keep working), and the +current turn's own messages never pass through the resolver, so in-flight +tool results and just-captured frames are always sent verbatim. Policy lives +in `config/context.json` (`elideHistoricToolResults`, default true; +`elideHistoricToolResultsThresholdChars`, default 10000; +`elideHistoricImages`, default true). The inspect CLI composes through the +same decorated resolver. ### 6.7 Agent snapshot inheritance diff --git a/apps/x/packages/core/src/turns/context-elision.test.ts b/apps/x/packages/core/src/turns/context-elision.test.ts index 9543a46f..e08ebfa1 100644 --- a/apps/x/packages/core/src/turns/context-elision.test.ts +++ b/apps/x/packages/core/src/turns/context-elision.test.ts @@ -3,6 +3,7 @@ import type { z } from "zod"; import type { TurnContext, TurnEvent } from "@x/shared/dist/turns.js"; import { ElidingContextResolver, + elideHistoricImages, elideHistoricToolResults, } from "./context-elision.js"; import { TurnRepoContextResolver } from "./context-resolver.js"; @@ -150,6 +151,16 @@ function toolTurnLog( ]; } +function frame(source: "camera" | "screen") { + return { + type: "image" as const, + data: "aGVsbG8=".repeat(50), + mediaType: "image/jpeg", + source, + capturedAt: "2026-07-02T10:00:00Z", + }; +} + const T1 = "2026-07-02T10-00-00Z-0000001-000"; describe("elideHistoricToolResults", () => { @@ -191,10 +202,56 @@ describe("elideHistoricToolResults", () => { }); }); +describe("elideHistoricImages", () => { + it("replaces image parts with a labeled placeholder, keeping other parts", () => { + const message = { + role: "user" as const, + content: [ + { type: "text" as const, text: "how do I look?" }, + frame("camera"), + frame("camera"), + frame("screen"), + ], + }; + const [elided] = elideHistoricImages([message]); + if (typeof elided.content === "string" || elided.role !== "user") { + throw new Error("expected user message with parts"); + } + expect(elided.content.filter((p) => p.type === "image")).toHaveLength(0); + expect(elided.content[0]).toEqual({ type: "text", text: "how do I look?" }); + const placeholder = elided.content[elided.content.length - 1]; + if (placeholder.type !== "text") throw new Error("expected text placeholder"); + expect(placeholder.text).toContain("2 webcam frames"); + expect(placeholder.text).toContain("1 screen-share frame"); + }); + + it("leaves string-content and image-free user messages untouched", () => { + const messages = [ + user("plain"), + { + role: "user" as const, + content: [{ type: "text" as const, text: "no images" }], + }, + assistant("a"), + ]; + expect(elideHistoricImages(messages)).toEqual(messages); + }); +}); + +const POLICY_OFF = { + toolResults: false, + toolResultThresholdChars: 100, + images: false, +}; + describe("ElidingContextResolver", () => { function resolver( repo: InMemoryTurnRepo, - policy: { enabled: boolean; thresholdChars: number }, + policy: { + toolResults: boolean; + toolResultThresholdChars: number; + images: boolean; + }, ) { return new ElidingContextResolver({ inner: new TurnRepoContextResolver({ turnRepo: repo }), @@ -206,8 +263,8 @@ describe("ElidingContextResolver", () => { const repo = new InMemoryTurnRepo(); repo.seed(toolTurnLog(T1, [], "x".repeat(200))); const resolved = await resolver(repo, { - enabled: true, - thresholdChars: 100, + ...POLICY_OFF, + toolResults: true, }).resolve({ previousTurnId: T1 }); const tool = resolved.find((m) => m.role === "tool"); expect(tool?.content).toContain("elided"); @@ -224,10 +281,9 @@ describe("ElidingContextResolver", () => { const repo = new InMemoryTurnRepo(); const output = "x".repeat(200); repo.seed(toolTurnLog(T1, [], output)); - const resolved = await resolver(repo, { - enabled: false, - thresholdChars: 100, - }).resolve({ previousTurnId: T1 }); + const resolved = await resolver(repo, POLICY_OFF).resolve({ + previousTurnId: T1, + }); const tool = resolved.find((m) => m.role === "tool"); expect(tool?.content).toBe(output); }); @@ -236,10 +292,39 @@ describe("ElidingContextResolver", () => { const repo = new InMemoryTurnRepo(); repo.seed(toolTurnLog(T1, [], "small")); const resolved = await resolver(repo, { - enabled: true, - thresholdChars: 100, + ...POLICY_OFF, + toolResults: true, }).resolve({ previousTurnId: T1 }); const tool = resolved.find((m) => m.role === "tool"); expect(tool?.content).toBe("small"); }); + + it("elides images in the resolved prefix when enabled", async () => { + const repo = new InMemoryTurnRepo(); + const log = toolTurnLog(T1, [], "small").map((event) => + event.type === "turn_created" + ? { + ...event, + input: { + role: "user" as const, + content: [ + { type: "text" as const, text: "watch me" }, + frame("camera"), + ], + }, + } + : event, + ); + repo.seed(log); + const resolved = await resolver(repo, { + ...POLICY_OFF, + images: true, + }).resolve({ previousTurnId: T1 }); + const input = resolved[0]; + if (input.role !== "user" || typeof input.content === "string") { + throw new Error("expected user message with parts"); + } + expect(input.content.some((p) => p.type === "image")).toBe(false); + expect(JSON.stringify(input.content)).toContain("1 webcam frame"); + }); }); diff --git a/apps/x/packages/core/src/turns/context-elision.ts b/apps/x/packages/core/src/turns/context-elision.ts index 6c88445c..87a51129 100644 --- a/apps/x/packages/core/src/turns/context-elision.ts +++ b/apps/x/packages/core/src/turns/context-elision.ts @@ -12,29 +12,41 @@ import type { IContextResolver } from "./context-resolver.js"; import { TurnRepoContextResolver } from "./context-resolver.js"; import type { ITurnRepo } from "./repo.js"; -// Transmit-time elision of historic tool results (the cross-turn prefix -// only — the current turn's own messages never pass through the resolver, so -// in-flight tool results are always sent verbatim). Large tool outputs from -// earlier turns (skill loads, file reads, HTTP fetches) dominate resent -// context; the model rarely needs them verbatim and can re-run the tool when -// it does. Elision is a pure function of each message's content, so resolved -// prefixes stay byte-stable across calls and turns (provider prefix caches -// keep working), and the durable JSONL log is untouched — only the -// transmitted bytes change. +// Transmit-time elision of historic context (the cross-turn prefix only — +// the current turn's own messages never pass through the resolver, so +// in-flight tool results and just-captured frames are always sent verbatim). +// +// Two policies, both on by default: +// - Tool results: large tool outputs from earlier turns (skill loads, file +// reads, HTTP fetches) dominate resent context; the model rarely needs +// them verbatim and can re-run the tool when it does. +// - Inline images: video-mode webcam and screen-share frames matter for +// the response they were captured for; afterwards the assistant's own +// text carries the takeaway, and fresh frames arrive with every new call +// message. Unlike tool results they cannot be re-fetched, so the +// placeholder only records what was there. +// +// Elision is a pure function of each message's content, so resolved prefixes +// stay byte-stable across calls and turns (provider prefix caches keep +// working), and the durable JSONL log is untouched — only the transmitted +// bytes change. -export interface ToolResultElisionPolicy { - enabled: boolean; - thresholdChars: number; +export interface ElisionPolicy { + toolResults: boolean; + toolResultThresholdChars: number; + images: boolean; } -export const DEFAULT_ELISION_POLICY: ToolResultElisionPolicy = { - enabled: true, - thresholdChars: 10_000, +export const DEFAULT_ELISION_POLICY: ElisionPolicy = { + toolResults: true, + toolResultThresholdChars: 10_000, + images: true, }; const ContextConfig = z.object({ elideHistoricToolResults: z.boolean().optional(), elideHistoricToolResultsThresholdChars: z.number().int().min(0).optional(), + elideHistoricImages: z.boolean().optional(), }); const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json"); @@ -42,7 +54,7 @@ const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json"); // Read the elision policy from config/context.json, falling back to defaults // for missing keys or an unreadable file. Read per resolve so a config edit // applies to the next turn without a restart. -export function loadElisionPolicy(): ToolResultElisionPolicy { +export function loadElisionPolicy(): ElisionPolicy { try { if (!fs.existsSync(CONTEXT_CONFIG_PATH)) { return DEFAULT_ELISION_POLICY; @@ -50,12 +62,14 @@ export function loadElisionPolicy(): ToolResultElisionPolicy { const raw = fs.readFileSync(CONTEXT_CONFIG_PATH, "utf-8"); const parsed = ContextConfig.parse(JSON.parse(raw)); return { - enabled: + toolResults: parsed.elideHistoricToolResults ?? - DEFAULT_ELISION_POLICY.enabled, - thresholdChars: + DEFAULT_ELISION_POLICY.toolResults, + toolResultThresholdChars: parsed.elideHistoricToolResultsThresholdChars ?? - DEFAULT_ELISION_POLICY.thresholdChars, + DEFAULT_ELISION_POLICY.toolResultThresholdChars, + images: + parsed.elideHistoricImages ?? DEFAULT_ELISION_POLICY.images, }; } catch { return DEFAULT_ELISION_POLICY; @@ -80,18 +94,48 @@ export function elideHistoricToolResults( }); } +export function elideHistoricImages( + messages: Array>, +): Array> { + return messages.map((message) => { + if (message.role !== "user" || typeof message.content === "string") { + return message; + } + const images = message.content.filter((part) => part.type === "image"); + if (images.length === 0) { + return message; + } + const camera = images.filter((part) => part.source !== "screen").length; + const screen = images.length - camera; + const kinds = [ + ...(camera > 0 ? [`${camera} webcam frame${camera === 1 ? "" : "s"}`] : []), + ...(screen > 0 ? [`${screen} screen-share frame${screen === 1 ? "" : "s"}`] : []), + ].join(" and "); + return { + ...message, + content: [ + ...message.content.filter((part) => part.type !== "image"), + { + type: "text" as const, + text: `[Omitted from history to save context: ${kinds} captured while this message was composed. The assistant saw them when responding to this message.]`, + }, + ], + }; + }); +} + // IContextResolver decorator: applies the elision policy to the materialized // cross-turn prefix. Agent snapshot resolution is delegated untouched. export class ElidingContextResolver implements IContextResolver { private readonly inner: IContextResolver; - private readonly loadPolicy: () => ToolResultElisionPolicy; + private readonly loadPolicy: () => ElisionPolicy; constructor({ inner, loadPolicy, }: { inner: IContextResolver; - loadPolicy?: () => ToolResultElisionPolicy; + loadPolicy?: () => ElisionPolicy; }) { this.inner = inner; this.loadPolicy = loadPolicy ?? loadElisionPolicy; @@ -100,12 +144,18 @@ export class ElidingContextResolver implements IContextResolver { async resolve( context: z.infer, ): Promise>> { - const prefix = await this.inner.resolve(context); + let prefix = await this.inner.resolve(context); const policy = this.loadPolicy(); - if (!policy.enabled) { - return prefix; + if (policy.toolResults) { + prefix = elideHistoricToolResults( + prefix, + policy.toolResultThresholdChars, + ); } - return elideHistoricToolResults(prefix, policy.thresholdChars); + if (policy.images) { + prefix = elideHistoricImages(prefix); + } + return prefix; } resolveAgent(