Initial open-source release

scripts/validate-llm-debug-jsonl.test.mjs

@@ -0,0 +1,112 @@
+import { spawnSync } from 'node:child_process';
+import assert from 'node:assert/strict';
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { test } from 'node:test';
+
+const scriptPath = new URL('./validate-llm-debug-jsonl.mjs', import.meta.url).pathname;
+
+function runValidator(args) {
+  return spawnSync(process.execPath, [scriptPath, ...args], {
+    encoding: 'utf8',
+  });
+}
+
+function writeDebugJsonl(records) {
+  const dir = mkdtempSync(join(tmpdir(), 'klo-llm-debug-validator-'));
+  const filePath = join(dir, 'debug.jsonl');
+  writeFileSync(filePath, `${records.map((record) => JSON.stringify(record)).join('\n')}\n`, 'utf8');
+  return filePath;
+}
+
+const validRecord = {
+  operationName: 'ingest-bundle-wu',
+  modelRole: 'candidateExtraction',
+  modelId: 'claude-sonnet-4-6',
+  messageCount: 2,
+  toolNames: ['emit_candidate'],
+  providerOptions: [
+    {
+      target: 'message',
+      index: 0,
+      role: 'system',
+      providerOptions: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '1h' } } },
+    },
+    {
+      target: 'message-part',
+      index: 1,
+      role: 'user',
+      partIndex: 0,
+      providerOptions: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '5m' } } },
+    },
+    {
+      target: 'tool',
+      name: 'emit_candidate',
+      providerOptions: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '1h' } } },
+    },
+  ],
+};
+
+test('prints usage and exits 2 when required arguments are missing', () => {
+  const result = runValidator([]);
+
+  assert.equal(result.status, 2);
+  assert.match(result.stderr, /Usage: node klo\/scripts\/validate-llm-debug-jsonl\.mjs anthropic\|vertex/);
+});
+
+test('accepts sanitized debug JSONL with message, message-part, and tool cache markers', () => {
+  const filePath = writeDebugJsonl([validRecord]);
+  const result = runValidator(['anthropic', filePath]);
+
+  assert.equal(result.status, 0, result.stderr);
+  const parsed = JSON.parse(result.stdout);
+  assert.equal(parsed.backend, 'anthropic');
+  assert.equal(parsed.records, 1);
+  assert.equal(parsed.providerOptionEntries, 3);
+  assert.equal(parsed.cacheMarkerEntries, 3);
+  assert.deepEqual(parsed.markerTargets, ['message', 'message-part', 'tool']);
+  assert.deepEqual(parsed.ttlValues, ['1h', '5m']);
+});
+
+test('rejects debug JSONL that lacks nested message-part cache marker evidence', () => {
+  const filePath = writeDebugJsonl([
+    {
+      ...validRecord,
+      providerOptions: validRecord.providerOptions.filter((entry) => entry.target !== 'message-part'),
+    },
+  ]);
+  const result = runValidator(['anthropic', filePath]);
+
+  assert.notEqual(result.status, 0);
+  assert.match(result.stderr, /missing cacheControl marker target: message-part/);
+});
+
+test('rejects prompt-shaped fields in debug JSONL', () => {
+  const filePath = writeDebugJsonl([{ ...validRecord, text: 'SECRET PROMPT' }]);
+  const result = runValidator(['anthropic', filePath]);
+
+  assert.notEqual(result.status, 0);
+  assert.match(result.stderr, /prompt, schema, credential, or token-shaped field/);
+});
+
+test('rejects direct-Anthropic extended cache beta header in Vertex debug summaries', () => {
+  const filePath = writeDebugJsonl([
+    {
+      ...validRecord,
+      providerOptions: [
+        ...validRecord.providerOptions,
+        {
+          target: 'message',
+          index: 0,
+          role: 'system',
+          providerOptions: { header: 'extended-cache-ttl-2025-04-11' },
+        },
+      ],
+    },
+  ]);
+  const result = runValidator(['vertex', filePath]);
+
+  assert.notEqual(result.status, 0);
+  assert.match(result.stderr, /direct-Anthropic extended cache TTL beta header/);
+});