feat(core): elide historic video-mode frames from model context

Video-mode webcam and screen-share frames matter for the response they
were captured for; afterwards the assistant's own text carries the
takeaway and fresh frames arrive with every new call message, yet each
message's frames (~10k tokens) were resent on every subsequent model
call. The elision decorator now strips image parts from prior-turn user
messages, leaving a text part recording how many frames of each kind
were dropped. The current turn's just-captured frames are always sent
verbatim, and the policy generalizes the existing config
(elideHistoricImages, default on).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

apps/x/packages/core/docs/turn-runtime-design.md

@@ -393,14 +393,18 @@ Rules:
 The app wires the resolver through a decorator (`context-elision.ts`) that
 applies transmit-time elision to the materialized prefix: tool results from
 prior turns above a size threshold are replaced with a short placeholder
-telling the model to re-run the tool if it needs the output. The durable log
+telling the model to re-run the tool if it needs the output, and inline
-is untouched; only the transmitted bytes change. Elision is a pure function
+image parts from prior turns (video-mode webcam and screen-share frames) are
-of each message, so resolved prefixes stay byte-stable across calls (provider
+replaced with a text part recording how many frames of each kind were
-prefix caches keep working), and the current turn's own in-flight tool
+dropped. The durable log is untouched; only the transmitted bytes change.
-results never pass through the resolver, so they are always sent verbatim.
+Elision is a pure function of each message, so resolved prefixes stay
-Policy lives in `config/context.json` (`elideHistoricToolResults`, default
+byte-stable across calls (provider prefix caches keep working), and the
-true; `elideHistoricToolResultsThresholdChars`, default 10000). The inspect
+current turn's own messages never pass through the resolver, so in-flight
-CLI composes through the same decorated resolver.
+tool results and just-captured frames are always sent verbatim. Policy lives
+in `config/context.json` (`elideHistoricToolResults`, default true;
+`elideHistoricToolResultsThresholdChars`, default 10000;
+`elideHistoricImages`, default true). The inspect CLI composes through the
+same decorated resolver.
 
 ### 6.7 Agent snapshot inheritance
 

apps/x/packages/core/src/turns/context-elision.test.ts

@@ -3,6 +3,7 @@ import type { z } from "zod";
 import type { TurnContext, TurnEvent } from "@x/shared/dist/turns.js";
 import {
     ElidingContextResolver,
+    elideHistoricImages,
     elideHistoricToolResults,
 } from "./context-elision.js";
 import { TurnRepoContextResolver } from "./context-resolver.js";
@@ -150,6 +151,16 @@ function toolTurnLog(
     ];
 }
 
+function frame(source: "camera" | "screen") {
+    return {
+        type: "image" as const,
+        data: "aGVsbG8=".repeat(50),
+        mediaType: "image/jpeg",
+        source,
+        capturedAt: "2026-07-02T10:00:00Z",
+    };
+}
+
 const T1 = "2026-07-02T10-00-00Z-0000001-000";
 
 describe("elideHistoricToolResults", () => {
@@ -191,10 +202,56 @@ describe("elideHistoricToolResults", () => {
     });
 });
 
+describe("elideHistoricImages", () => {
+    it("replaces image parts with a labeled placeholder, keeping other parts", () => {
+        const message = {
+            role: "user" as const,
+            content: [
+                { type: "text" as const, text: "how do I look?" },
+                frame("camera"),
+                frame("camera"),
+                frame("screen"),
+            ],
+        };
+        const [elided] = elideHistoricImages([message]);
+        if (typeof elided.content === "string" || elided.role !== "user") {
+            throw new Error("expected user message with parts");
+        }
+        expect(elided.content.filter((p) => p.type === "image")).toHaveLength(0);
+        expect(elided.content[0]).toEqual({ type: "text", text: "how do I look?" });
+        const placeholder = elided.content[elided.content.length - 1];
+        if (placeholder.type !== "text") throw new Error("expected text placeholder");
+        expect(placeholder.text).toContain("2 webcam frames");
+        expect(placeholder.text).toContain("1 screen-share frame");
+    });
+
+    it("leaves string-content and image-free user messages untouched", () => {
+        const messages = [
+            user("plain"),
+            {
+                role: "user" as const,
+                content: [{ type: "text" as const, text: "no images" }],
+            },
+            assistant("a"),
+        ];
+        expect(elideHistoricImages(messages)).toEqual(messages);
+    });
+});
+
+const POLICY_OFF = {
+    toolResults: false,
+    toolResultThresholdChars: 100,
+    images: false,
+};
+
 describe("ElidingContextResolver", () => {
     function resolver(
         repo: InMemoryTurnRepo,
-        policy: { enabled: boolean; thresholdChars: number },
+        policy: {
+            toolResults: boolean;
+            toolResultThresholdChars: number;
+            images: boolean;
+        },
     ) {
         return new ElidingContextResolver({
             inner: new TurnRepoContextResolver({ turnRepo: repo }),
@@ -206,8 +263,8 @@ describe("ElidingContextResolver", () => {
         const repo = new InMemoryTurnRepo();
         repo.seed(toolTurnLog(T1, [], "x".repeat(200)));
         const resolved = await resolver(repo, {
-            enabled: true,
+            ...POLICY_OFF,
-            thresholdChars: 100,
+            toolResults: true,
         }).resolve({ previousTurnId: T1 });
         const tool = resolved.find((m) => m.role === "tool");
         expect(tool?.content).toContain("elided");
@@ -224,10 +281,9 @@ describe("ElidingContextResolver", () => {
         const repo = new InMemoryTurnRepo();
         const output = "x".repeat(200);
         repo.seed(toolTurnLog(T1, [], output));
-        const resolved = await resolver(repo, {
+        const resolved = await resolver(repo, POLICY_OFF).resolve({
-            enabled: false,
+            previousTurnId: T1,
-            thresholdChars: 100,
+        });
-        }).resolve({ previousTurnId: T1 });
         const tool = resolved.find((m) => m.role === "tool");
         expect(tool?.content).toBe(output);
     });
@@ -236,10 +292,39 @@ describe("ElidingContextResolver", () => {
         const repo = new InMemoryTurnRepo();
         repo.seed(toolTurnLog(T1, [], "small"));
         const resolved = await resolver(repo, {
-            enabled: true,
+            ...POLICY_OFF,
-            thresholdChars: 100,
+            toolResults: true,
         }).resolve({ previousTurnId: T1 });
         const tool = resolved.find((m) => m.role === "tool");
         expect(tool?.content).toBe("small");
     });
+
+    it("elides images in the resolved prefix when enabled", async () => {
+        const repo = new InMemoryTurnRepo();
+        const log = toolTurnLog(T1, [], "small").map((event) =>
+            event.type === "turn_created"
+                ? {
+                      ...event,
+                      input: {
+                          role: "user" as const,
+                          content: [
+                              { type: "text" as const, text: "watch me" },
+                              frame("camera"),
+                          ],
+                      },
+                  }
+                : event,
+        );
+        repo.seed(log);
+        const resolved = await resolver(repo, {
+            ...POLICY_OFF,
+            images: true,
+        }).resolve({ previousTurnId: T1 });
+        const input = resolved[0];
+        if (input.role !== "user" || typeof input.content === "string") {
+            throw new Error("expected user message with parts");
+        }
+        expect(input.content.some((p) => p.type === "image")).toBe(false);
+        expect(JSON.stringify(input.content)).toContain("1 webcam frame");
+    });
 });

apps/x/packages/core/src/turns/context-elision.ts

@@ -12,29 +12,41 @@ import type { IContextResolver } from "./context-resolver.js";
 import { TurnRepoContextResolver } from "./context-resolver.js";
 import type { ITurnRepo } from "./repo.js";
 
-// Transmit-time elision of historic tool results (the cross-turn prefix
+// Transmit-time elision of historic context (the cross-turn prefix only —
-// only — the current turn's own messages never pass through the resolver, so
+// the current turn's own messages never pass through the resolver, so
-// in-flight tool results are always sent verbatim). Large tool outputs from
+// in-flight tool results and just-captured frames are always sent verbatim).
-// earlier turns (skill loads, file reads, HTTP fetches) dominate resent
+//
-// context; the model rarely needs them verbatim and can re-run the tool when
+// Two policies, both on by default:
-// it does. Elision is a pure function of each message's content, so resolved
+//   - Tool results: large tool outputs from earlier turns (skill loads, file
-// prefixes stay byte-stable across calls and turns (provider prefix caches
+//     reads, HTTP fetches) dominate resent context; the model rarely needs
-// keep working), and the durable JSONL log is untouched — only the
+//     them verbatim and can re-run the tool when it does.
-// transmitted bytes change.
+//   - Inline images: video-mode webcam and screen-share frames matter for
+//     the response they were captured for; afterwards the assistant's own
+//     text carries the takeaway, and fresh frames arrive with every new call
+//     message. Unlike tool results they cannot be re-fetched, so the
+//     placeholder only records what was there.
+//
+// Elision is a pure function of each message's content, so resolved prefixes
+// stay byte-stable across calls and turns (provider prefix caches keep
+// working), and the durable JSONL log is untouched — only the transmitted
+// bytes change.
 
-export interface ToolResultElisionPolicy {
+export interface ElisionPolicy {
-    enabled: boolean;
+    toolResults: boolean;
-    thresholdChars: number;
+    toolResultThresholdChars: number;
+    images: boolean;
 }
 
-export const DEFAULT_ELISION_POLICY: ToolResultElisionPolicy = {
+export const DEFAULT_ELISION_POLICY: ElisionPolicy = {
-    enabled: true,
+    toolResults: true,
-    thresholdChars: 10_000,
+    toolResultThresholdChars: 10_000,
+    images: true,
 };
 
 const ContextConfig = z.object({
     elideHistoricToolResults: z.boolean().optional(),
     elideHistoricToolResultsThresholdChars: z.number().int().min(0).optional(),
+    elideHistoricImages: z.boolean().optional(),
 });
 
 const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json");
@@ -42,7 +54,7 @@ const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json");
 // Read the elision policy from config/context.json, falling back to defaults
 // for missing keys or an unreadable file. Read per resolve so a config edit
 // applies to the next turn without a restart.
-export function loadElisionPolicy(): ToolResultElisionPolicy {
+export function loadElisionPolicy(): ElisionPolicy {
     try {
         if (!fs.existsSync(CONTEXT_CONFIG_PATH)) {
             return DEFAULT_ELISION_POLICY;
@@ -50,12 +62,14 @@ export function loadElisionPolicy(): ToolResultElisionPolicy {
         const raw = fs.readFileSync(CONTEXT_CONFIG_PATH, "utf-8");
         const parsed = ContextConfig.parse(JSON.parse(raw));
         return {
-            enabled:
+            toolResults:
                 parsed.elideHistoricToolResults ??
-                DEFAULT_ELISION_POLICY.enabled,
+                DEFAULT_ELISION_POLICY.toolResults,
-            thresholdChars:
+            toolResultThresholdChars:
                 parsed.elideHistoricToolResultsThresholdChars ??
-                DEFAULT_ELISION_POLICY.thresholdChars,
+                DEFAULT_ELISION_POLICY.toolResultThresholdChars,
+            images:
+                parsed.elideHistoricImages ?? DEFAULT_ELISION_POLICY.images,
         };
     } catch {
         return DEFAULT_ELISION_POLICY;
@@ -80,18 +94,48 @@ export function elideHistoricToolResults(
     });
 }
 
+export function elideHistoricImages(
+    messages: Array<z.infer<typeof ConversationMessage>>,
+): Array<z.infer<typeof ConversationMessage>> {
+    return messages.map((message) => {
+        if (message.role !== "user" || typeof message.content === "string") {
+            return message;
+        }
+        const images = message.content.filter((part) => part.type === "image");
+        if (images.length === 0) {
+            return message;
+        }
+        const camera = images.filter((part) => part.source !== "screen").length;
+        const screen = images.length - camera;
+        const kinds = [
+            ...(camera > 0 ? [`${camera} webcam frame${camera === 1 ? "" : "s"}`] : []),
+            ...(screen > 0 ? [`${screen} screen-share frame${screen === 1 ? "" : "s"}`] : []),
+        ].join(" and ");
+        return {
+            ...message,
+            content: [
+                ...message.content.filter((part) => part.type !== "image"),
+                {
+                    type: "text" as const,
+                    text: `[Omitted from history to save context: ${kinds} captured while this message was composed. The assistant saw them when responding to this message.]`,
+                },
+            ],
+        };
+    });
+}
+
 // IContextResolver decorator: applies the elision policy to the materialized
 // cross-turn prefix. Agent snapshot resolution is delegated untouched.
 export class ElidingContextResolver implements IContextResolver {
     private readonly inner: IContextResolver;
-    private readonly loadPolicy: () => ToolResultElisionPolicy;
+    private readonly loadPolicy: () => ElisionPolicy;
 
     constructor({
         inner,
         loadPolicy,
     }: {
         inner: IContextResolver;
-        loadPolicy?: () => ToolResultElisionPolicy;
+        loadPolicy?: () => ElisionPolicy;
     }) {
         this.inner = inner;
         this.loadPolicy = loadPolicy ?? loadElisionPolicy;
@@ -100,12 +144,18 @@ export class ElidingContextResolver implements IContextResolver {
     async resolve(
         context: z.infer<typeof TurnContext>,
     ): Promise<Array<z.infer<typeof ConversationMessage>>> {
-        const prefix = await this.inner.resolve(context);
+        let prefix = await this.inner.resolve(context);
         const policy = this.loadPolicy();
-        if (!policy.enabled) {
+        if (policy.toolResults) {
-            return prefix;
+            prefix = elideHistoricToolResults(
+                prefix,
+                policy.toolResultThresholdChars,
+            );
         }
-        return elideHistoricToolResults(prefix, policy.thresholdChars);
+        if (policy.images) {
+            prefix = elideHistoricImages(prefix);
+        }
+        return prefix;
     }
 
     resolveAgent(