apunkt/rowboat
1
0
Fork 0
mirror of https://github.com/rowboatlabs/rowboat.git synced 2026-07-12 21:02:17 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(core): elide historic video-mode frames from model context

Browse source 
Video-mode webcam and screen-share frames matter for the response they
were captured for; afterwards the assistant's own text carries the
takeaway and fresh frames arrive with every new call message, yet each
message's frames (~10k tokens) were resent on every subsequent model
call. The elision decorator now strips image parts from prior-turn user
messages, leaving a text part recording how many frames of each kind
were dropped. The current turn's just-captured frames are always sent
verbatim, and the policy generalizes the existing config
(elideHistoricImages, default on).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Arjun 2026-07-07 03:24:44 +05:30 • committed by Ramnique Singh
parent 99f196e16c
commit 05d7fa41cf
3 changed files with 182 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

20
apps/x/packages/core/docs/turn-runtime-design.md
View file

@ -393,14 +393,18 @@ Rules:
The app wires the resolver through a decorator (`context-elision.ts`) that
applies transmit-time elision to the materialized prefix: tool results from
prior turns above a size threshold are replaced with a short placeholder
telling the model to re-run the tool if it needs the output. The durable log
is untouched; only the transmitted bytes change. Elision is a pure function
of each message, so resolved prefixes stay byte-stable across calls (provider
prefix caches keep working), and the current turn's own in-flight tool
results never pass through the resolver, so they are always sent verbatim.
Policy lives in `config/context.json` (`elideHistoricToolResults`, default
true; `elideHistoricToolResultsThresholdChars`, default 10000). The inspect
CLI composes through the same decorated resolver.
telling the model to re-run the tool if it needs the output, and inline
image parts from prior turns (video-mode webcam and screen-share frames) are
replaced with a text part recording how many frames of each kind were
dropped. The durable log is untouched; only the transmitted bytes change.
Elision is a pure function of each message, so resolved prefixes stay
byte-stable across calls (provider prefix caches keep working), and the
current turn's own messages never pass through the resolver, so in-flight
tool results and just-captured frames are always sent verbatim. Policy lives
in `config/context.json` (`elideHistoricToolResults`, default true;
`elideHistoricToolResultsThresholdChars`, default 10000;
`elideHistoricImages`, default true). The inspect CLI composes through the
same decorated resolver.
### 6.7 Agent snapshot inheritance

103
apps/x/packages/core/src/turns/context-elision.test.ts
View file

@ -3,6 +3,7 @@ import type { z } from "zod";
import type { TurnContext, TurnEvent } from "@x/shared/dist/turns.js";
import {
ElidingContextResolver,
elideHistoricImages,
elideHistoricToolResults,
} from "./context-elision.js";
import { TurnRepoContextResolver } from "./context-resolver.js";
@ -150,6 +151,16 @@ function toolTurnLog(
];
}
function frame(source: "camera" | "screen") {
return {
type: "image" as const,
data: "aGVsbG8=".repeat(50),
mediaType: "image/jpeg",
source,
capturedAt: "2026-07-02T10:00:00Z",
};
}
const T1 = "2026-07-02T10-00-00Z-0000001-000";
describe("elideHistoricToolResults", () => {
@ -191,10 +202,56 @@ describe("elideHistoricToolResults", () => {
});
});
describe("elideHistoricImages", () => {
it("replaces image parts with a labeled placeholder, keeping other parts", () => {
const message = {
role: "user" as const,
content: [
{ type: "text" as const, text: "how do I look?" },
frame("camera"),
frame("camera"),
frame("screen"),
],
};
const [elided] = elideHistoricImages([message]);
if (typeof elided.content === "string" || elided.role !== "user") {
throw new Error("expected user message with parts");
}
expect(elided.content.filter((p) => p.type === "image")).toHaveLength(0);
expect(elided.content[0]).toEqual({ type: "text", text: "how do I look?" });
const placeholder = elided.content[elided.content.length - 1];
if (placeholder.type !== "text") throw new Error("expected text placeholder");
expect(placeholder.text).toContain("2 webcam frames");
expect(placeholder.text).toContain("1 screen-share frame");
});
it("leaves string-content and image-free user messages untouched", () => {
const messages = [
user("plain"),
{
role: "user" as const,
content: [{ type: "text" as const, text: "no images" }],
},
assistant("a"),
];
expect(elideHistoricImages(messages)).toEqual(messages);
});
});
const POLICY_OFF = {
toolResults: false,
toolResultThresholdChars: 100,
images: false,
};
describe("ElidingContextResolver", () => {
function resolver(
repo: InMemoryTurnRepo,
policy: { enabled: boolean; thresholdChars: number },
policy: {
toolResults: boolean;
toolResultThresholdChars: number;
images: boolean;
},
) {
return new ElidingContextResolver({
inner: new TurnRepoContextResolver({ turnRepo: repo }),
@ -206,8 +263,8 @@ describe("ElidingContextResolver", () => {
const repo = new InMemoryTurnRepo();
repo.seed(toolTurnLog(T1, [], "x".repeat(200)));
const resolved = await resolver(repo, {
enabled: true,
thresholdChars: 100,
...POLICY_OFF,
toolResults: true,
}).resolve({ previousTurnId: T1 });
const tool = resolved.find((m) => m.role === "tool");
expect(tool?.content).toContain("elided");
@ -224,10 +281,9 @@ describe("ElidingContextResolver", () => {
const repo = new InMemoryTurnRepo();
const output = "x".repeat(200);
repo.seed(toolTurnLog(T1, [], output));
const resolved = await resolver(repo, {
enabled: false,
thresholdChars: 100,
}).resolve({ previousTurnId: T1 });
const resolved = await resolver(repo, POLICY_OFF).resolve({
previousTurnId: T1,
});
const tool = resolved.find((m) => m.role === "tool");
expect(tool?.content).toBe(output);
});
@ -236,10 +292,39 @@ describe("ElidingContextResolver", () => {
const repo = new InMemoryTurnRepo();
repo.seed(toolTurnLog(T1, [], "small"));
const resolved = await resolver(repo, {
enabled: true,
thresholdChars: 100,
...POLICY_OFF,
toolResults: true,
}).resolve({ previousTurnId: T1 });
const tool = resolved.find((m) => m.role === "tool");
expect(tool?.content).toBe("small");
});
it("elides images in the resolved prefix when enabled", async () => {
const repo = new InMemoryTurnRepo();
const log = toolTurnLog(T1, [], "small").map((event) =>
event.type === "turn_created"
? {
...event,
input: {
role: "user" as const,
content: [
{ type: "text" as const, text: "watch me" },
frame("camera"),
],
},
}
: event,
);
repo.seed(log);
const resolved = await resolver(repo, {
...POLICY_OFF,
images: true,
}).resolve({ previousTurnId: T1 });
const input = resolved[0];
if (input.role !== "user" || typeof input.content === "string") {
throw new Error("expected user message with parts");
}
expect(input.content.some((p) => p.type === "image")).toBe(false);
expect(JSON.stringify(input.content)).toContain("1 webcam frame");
});
});

102
apps/x/packages/core/src/turns/context-elision.ts
View file

@ -12,29 +12,41 @@ import type { IContextResolver } from "./context-resolver.js";
import { TurnRepoContextResolver } from "./context-resolver.js";
import type { ITurnRepo } from "./repo.js";
// Transmit-time elision of historic tool results (the cross-turn prefix
// only — the current turn's own messages never pass through the resolver, so
// in-flight tool results are always sent verbatim). Large tool outputs from
// earlier turns (skill loads, file reads, HTTP fetches) dominate resent
// context; the model rarely needs them verbatim and can re-run the tool when
// it does. Elision is a pure function of each message's content, so resolved
// prefixes stay byte-stable across calls and turns (provider prefix caches
// keep working), and the durable JSONL log is untouched — only the
// transmitted bytes change.
// Transmit-time elision of historic context (the cross-turn prefix only —
// the current turn's own messages never pass through the resolver, so
// in-flight tool results and just-captured frames are always sent verbatim).
//
// Two policies, both on by default:
// - Tool results: large tool outputs from earlier turns (skill loads, file
// reads, HTTP fetches) dominate resent context; the model rarely needs
// them verbatim and can re-run the tool when it does.
// - Inline images: video-mode webcam and screen-share frames matter for
// the response they were captured for; afterwards the assistant's own
// text carries the takeaway, and fresh frames arrive with every new call
// message. Unlike tool results they cannot be re-fetched, so the
// placeholder only records what was there.
//
// Elision is a pure function of each message's content, so resolved prefixes
// stay byte-stable across calls and turns (provider prefix caches keep
// working), and the durable JSONL log is untouched — only the transmitted
// bytes change.
export interface ToolResultElisionPolicy {
enabled: boolean;
thresholdChars: number;
export interface ElisionPolicy {
toolResults: boolean;
toolResultThresholdChars: number;
images: boolean;
}
export const DEFAULT_ELISION_POLICY: ToolResultElisionPolicy = {
enabled: true,
thresholdChars: 10_000,
export const DEFAULT_ELISION_POLICY: ElisionPolicy = {
toolResults: true,
toolResultThresholdChars: 10_000,
images: true,
};
const ContextConfig = z.object({
elideHistoricToolResults: z.boolean().optional(),
elideHistoricToolResultsThresholdChars: z.number().int().min(0).optional(),
elideHistoricImages: z.boolean().optional(),
});
const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json");
@ -42,7 +54,7 @@ const CONTEXT_CONFIG_PATH = path.join(WorkDir, "config", "context.json");
// Read the elision policy from config/context.json, falling back to defaults
// for missing keys or an unreadable file. Read per resolve so a config edit
// applies to the next turn without a restart.
export function loadElisionPolicy(): ToolResultElisionPolicy {
export function loadElisionPolicy(): ElisionPolicy {
try {
if (!fs.existsSync(CONTEXT_CONFIG_PATH)) {
return DEFAULT_ELISION_POLICY;
@ -50,12 +62,14 @@ export function loadElisionPolicy(): ToolResultElisionPolicy {
const raw = fs.readFileSync(CONTEXT_CONFIG_PATH, "utf-8");
const parsed = ContextConfig.parse(JSON.parse(raw));
return {
enabled:
toolResults:
parsed.elideHistoricToolResults ??
DEFAULT_ELISION_POLICY.enabled,
thresholdChars:
DEFAULT_ELISION_POLICY.toolResults,
toolResultThresholdChars:
parsed.elideHistoricToolResultsThresholdChars ??
DEFAULT_ELISION_POLICY.thresholdChars,
DEFAULT_ELISION_POLICY.toolResultThresholdChars,
images:
parsed.elideHistoricImages ?? DEFAULT_ELISION_POLICY.images,
};
} catch {
return DEFAULT_ELISION_POLICY;
@ -80,18 +94,48 @@ export function elideHistoricToolResults(
});
}
export function elideHistoricImages(
messages: Array<z.infer<typeof ConversationMessage>>,
): Array<z.infer<typeof ConversationMessage>> {
return messages.map((message) => {
if (message.role !== "user" || typeof message.content === "string") {
return message;
}
const images = message.content.filter((part) => part.type === "image");
if (images.length === 0) {
return message;
}
const camera = images.filter((part) => part.source !== "screen").length;
const screen = images.length - camera;
const kinds = [
...(camera > 0 ? [`${camera} webcam frame${camera === 1 ? "" : "s"}`] : []),
...(screen > 0 ? [`${screen} screen-share frame${screen === 1 ? "" : "s"}`] : []),
].join(" and ");
return {
...message,
content: [
...message.content.filter((part) => part.type !== "image"),
{
type: "text" as const,
text: `[Omitted from history to save context: ${kinds} captured while this message was composed. The assistant saw them when responding to this message.]`,
},
],
};
});
}
// IContextResolver decorator: applies the elision policy to the materialized
// cross-turn prefix. Agent snapshot resolution is delegated untouched.
export class ElidingContextResolver implements IContextResolver {
private readonly inner: IContextResolver;
private readonly loadPolicy: () => ToolResultElisionPolicy;
private readonly loadPolicy: () => ElisionPolicy;
constructor({
inner,
loadPolicy,
}: {
inner: IContextResolver;
loadPolicy?: () => ToolResultElisionPolicy;
loadPolicy?: () => ElisionPolicy;
}) {
this.inner = inner;
this.loadPolicy = loadPolicy ?? loadElisionPolicy;
@ -100,12 +144,18 @@ export class ElidingContextResolver implements IContextResolver {
async resolve(
context: z.infer<typeof TurnContext>,
): Promise<Array<z.infer<typeof ConversationMessage>>> {
const prefix = await this.inner.resolve(context);
let prefix = await this.inner.resolve(context);
const policy = this.loadPolicy();
if (!policy.enabled) {
return prefix;
if (policy.toolResults) {
prefix = elideHistoricToolResults(
prefix,
policy.toolResultThresholdChars,
);
}
return elideHistoricToolResults(prefix, policy.thresholdChars);
if (policy.images) {
prefix = elideHistoricImages(prefix);
}
return prefix;
}
resolveAgent(