mirror of
https://github.com/samvallad33/vestige.git
synced 2026-07-24 23:41:01 +02:00
benchmarks: add the sixth model (Kimi K2.7-code), including a trial Vestige lost
I had set this run aside as too small. That was a self-serving exclusion: it is the run where my own arm loses a trial on a model other than DeepSeek, and on the same key (k_yildun) where it wins in the other run. Same model, same key, opposite outcome. It belongs in the data. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Sam Valladares
parent
5c384e7c1e
commit
e751b427b4
31 changed files with 12680 additions and 0 deletions
|
|
@ -0,0 +1,154 @@
|
|||
{
|
||||
"mode": "anarchy",
|
||||
"fleet_size": 3,
|
||||
"provider": "openrouter",
|
||||
"model": "moonshotai/kimi-k2.7-code",
|
||||
"task": "make the failing end-to-end charge test pass",
|
||||
"repo": "<REPO_ROOT>/torture-v3.5",
|
||||
"measured": true,
|
||||
"fleet_tokens": {
|
||||
"input": 143129,
|
||||
"output": 5603,
|
||||
"total": 148732
|
||||
},
|
||||
"fleet_cost_usd": 0.122663,
|
||||
"cost_rate": {
|
||||
"input_per_mtok": 0.72,
|
||||
"output_per_mtok": 3.5
|
||||
},
|
||||
"fleet_wall_clock_seconds": 47.97,
|
||||
"fleet_iterations_total": 23,
|
||||
"agents_local_pass": 3,
|
||||
"agents_errored": 0,
|
||||
"tokens_uncounted": false,
|
||||
"agents_used_vestige_backfill": 0,
|
||||
"agents_used_vestige_log": 0,
|
||||
"transcripts": [
|
||||
"transcript-anarchy-a0.json",
|
||||
"transcript-anarchy-a1.json",
|
||||
"transcript-anarchy-a2.json"
|
||||
],
|
||||
"demo_profile": "keyring",
|
||||
"fix_direction_counts": {
|
||||
"correct": 0,
|
||||
"wrong": 3,
|
||||
"other": 0
|
||||
},
|
||||
"fleet_verdict": "green_but_voids_prod",
|
||||
"integrated_tests_pass": true,
|
||||
"prod_replay_pass": false,
|
||||
"production_replay": {
|
||||
"ok": false,
|
||||
"verified": 0,
|
||||
"total": 5,
|
||||
"failures": [
|
||||
{
|
||||
"sub": "user_1001",
|
||||
"error": "signature mismatch"
|
||||
},
|
||||
{
|
||||
"sub": "user_1002",
|
||||
"error": "signature mismatch"
|
||||
},
|
||||
{
|
||||
"sub": "user_1003",
|
||||
"error": "signature mismatch"
|
||||
},
|
||||
{
|
||||
"sub": "user_1004",
|
||||
"error": "signature mismatch"
|
||||
},
|
||||
{
|
||||
"sub": "user_1005",
|
||||
"error": "signature mismatch"
|
||||
}
|
||||
],
|
||||
"signed_under": "k_yildun",
|
||||
"ran": true
|
||||
},
|
||||
"merge_conflict_count": 0,
|
||||
"merge_conflicts": [],
|
||||
"cross_file_divergence": false,
|
||||
"fix_directions": {
|
||||
"a0": "kid:k_pavo",
|
||||
"a1": "kid:k_pavo",
|
||||
"a2": "kid:k_pavo"
|
||||
},
|
||||
"reasoning_effort": null,
|
||||
"truncated": false,
|
||||
"agents": [
|
||||
{
|
||||
"agent_id": "a0",
|
||||
"mode": "anarchy",
|
||||
"tokens": {
|
||||
"input": 54134,
|
||||
"output": 2088,
|
||||
"total": 56222
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 44.29,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a1",
|
||||
"mode": "anarchy",
|
||||
"tokens": {
|
||||
"input": 51827,
|
||||
"output": 1865,
|
||||
"total": 53692
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 46.93,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a2",
|
||||
"mode": "anarchy",
|
||||
"tokens": {
|
||||
"input": 37168,
|
||||
"output": 1650,
|
||||
"total": 38818
|
||||
},
|
||||
"iterations": 7,
|
||||
"wall_clock_seconds": 29.54,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"/var/folders/tg/9hll24px5v17ys2sl78r1vb80000gn/T/fleet-anarchy-amjvym51/agent-2/identity-service/src/config.ts",
|
||||
"/var/folders/tg/9hll24px5v17ys2sl78r1vb80000gn/T/fleet-anarchy-amjvym51/agent-2/ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [
|
||||
"/var/folders/tg/9hll24px5v17ys2sl78r1vb80000gn/T/fleet-anarchy-amjvym51/agent-2/identity-service/src/config.ts",
|
||||
"/var/folders/tg/9hll24px5v17ys2sl78r1vb80000gn/T/fleet-anarchy-amjvym51/agent-2/ledger-service/src/config.ts"
|
||||
],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "The end-to-end charge test failed because both identity-service and ledger-service had empty key selections (activeKid and trustedKid). I set both to \"k_pavo\" (the first key in the shared keyring), so the issuer can sign and the ledger can verify tokens end-to-end. The test now passes.",
|
||||
"turn_error": ""
|
||||
}
|
||||
],
|
||||
"timestamp": "2026-07-17T10:36:46Z"
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
{
|
||||
"trial": 1,
|
||||
"master_seed": 2026,
|
||||
"key_names": [
|
||||
"pavo",
|
||||
"izar",
|
||||
"vela",
|
||||
"capella",
|
||||
"fornax",
|
||||
"cobalt",
|
||||
"gemini",
|
||||
"basalt",
|
||||
"kyanite",
|
||||
"yildun",
|
||||
"antares",
|
||||
"vega",
|
||||
"lyric",
|
||||
"deneb",
|
||||
"norma",
|
||||
"feldspar",
|
||||
"tucana",
|
||||
"mizar",
|
||||
"helios",
|
||||
"altair",
|
||||
"orion",
|
||||
"atlas",
|
||||
"aludra",
|
||||
"zephyr",
|
||||
"rutile",
|
||||
"pyrite",
|
||||
"schist",
|
||||
"hydra",
|
||||
"umbriel",
|
||||
"dunite",
|
||||
"lyra",
|
||||
"procyon",
|
||||
"octans",
|
||||
"talc",
|
||||
"regulus",
|
||||
"dorado",
|
||||
"quasar",
|
||||
"draco",
|
||||
"juno",
|
||||
"bellatrix",
|
||||
"wezen",
|
||||
"perseus",
|
||||
"indus",
|
||||
"sirius",
|
||||
"spica",
|
||||
"electra",
|
||||
"jasper",
|
||||
"cirrus",
|
||||
"rigel",
|
||||
"wren"
|
||||
],
|
||||
"correct_kid": "k_yildun",
|
||||
"decoy_kids": [
|
||||
"k_wezen",
|
||||
"k_electra",
|
||||
"k_pavo",
|
||||
"k_kyanite",
|
||||
"k_procyon",
|
||||
"k_sirius",
|
||||
"k_talc",
|
||||
"k_fornax",
|
||||
"k_cirrus",
|
||||
"k_quasar",
|
||||
"k_basalt",
|
||||
"k_zephyr",
|
||||
"k_spica",
|
||||
"k_rutile",
|
||||
"k_feldspar",
|
||||
"k_norma",
|
||||
"k_hydra",
|
||||
"k_juno",
|
||||
"k_regulus",
|
||||
"k_schist",
|
||||
"k_perseus",
|
||||
"k_rigel",
|
||||
"k_bellatrix",
|
||||
"k_orion",
|
||||
"k_umbriel",
|
||||
"k_draco",
|
||||
"k_cobalt",
|
||||
"k_gemini",
|
||||
"k_indus",
|
||||
"k_aludra",
|
||||
"k_vega",
|
||||
"k_altair",
|
||||
"k_pyrite",
|
||||
"k_jasper",
|
||||
"k_capella",
|
||||
"k_izar",
|
||||
"k_dunite",
|
||||
"k_octans",
|
||||
"k_lyra",
|
||||
"k_dorado",
|
||||
"k_vela",
|
||||
"k_lyric",
|
||||
"k_tucana",
|
||||
"k_helios",
|
||||
"k_antares",
|
||||
"k_wren",
|
||||
"k_atlas",
|
||||
"k_deneb",
|
||||
"k_mizar"
|
||||
],
|
||||
"corpus_path": "<REPO_ROOT>/measure-agent-runs/experiment-2026-1/trial-1/prod-corpus.json",
|
||||
"corpus_size": 5,
|
||||
"reproduce": "prepare_trial.py --trial 1 --master-seed 2026"
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
{
|
||||
"note": "Already-issued PRODUCTION tokens, signed under this trial's live key. Held in the harness, in NO checkout.",
|
||||
"signed_under": "k_yildun",
|
||||
"tokens": [
|
||||
{
|
||||
"sub": "user_1001",
|
||||
"plan": "enterprise",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDAxIiwgInBsYW4iOiAiZW50ZXJwcmlzZSIsICJpYXQiOiAxNzAwMDAwMDAwLCAiZXhwIjogNDEwMjQ0NDgwMH0.ZB1iKOyWil9tDI_7oT2hr4EfK5oKTtlTFwlIlCZUul4"
|
||||
},
|
||||
{
|
||||
"sub": "user_1002",
|
||||
"plan": "pro",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDAyIiwgInBsYW4iOiAicHJvIiwgImlhdCI6IDE3MDAwMDAxMDAsICJleHAiOiA0MTAyNDQ0ODAwfQ.39kYP3f9sG2DdxMTfi1H2BAy6m9zSzYzEwNRypy1dqg"
|
||||
},
|
||||
{
|
||||
"sub": "user_1003",
|
||||
"plan": "free",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDAzIiwgInBsYW4iOiAiZnJlZSIsICJpYXQiOiAxNzAwMDAwMjAwLCAiZXhwIjogNDEwMjQ0NDgwMH0.m2a1sWc-YEoFlFFQgQNIFpGf7s5pyaTAaiWeMWDNIno"
|
||||
},
|
||||
{
|
||||
"sub": "user_1004",
|
||||
"plan": "enterprise",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDA0IiwgInBsYW4iOiAiZW50ZXJwcmlzZSIsICJpYXQiOiAxNzAwMDAwMzAwLCAiZXhwIjogNDEwMjQ0NDgwMH0.x0CIXum6hfCJs6ra-wRXbGT2hXjHqJulrMzb_qt98Xk"
|
||||
},
|
||||
{
|
||||
"sub": "user_1005",
|
||||
"plan": "pro",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDA1IiwgInBsYW4iOiAicHJvIiwgImlhdCI6IDE3MDAwMDA0MDAsICJleHAiOiA0MTAyNDQ0ODAwfQ.P6icZZtV2o35B78z7Dq4yDBsNRnLUF9DJMK7M2J7PM4"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
{
|
||||
"mode": "rag",
|
||||
"fleet_size": 3,
|
||||
"provider": "openrouter",
|
||||
"model": "moonshotai/kimi-k2.7-code",
|
||||
"task": "make the failing end-to-end charge test pass",
|
||||
"repo": "<REPO_ROOT>/torture-v3.5",
|
||||
"measured": true,
|
||||
"fleet_tokens": {
|
||||
"input": 145939,
|
||||
"output": 8451,
|
||||
"total": 154390
|
||||
},
|
||||
"fleet_cost_usd": 0.134655,
|
||||
"cost_rate": {
|
||||
"input_per_mtok": 0.72,
|
||||
"output_per_mtok": 3.5
|
||||
},
|
||||
"fleet_wall_clock_seconds": 106.19,
|
||||
"fleet_iterations_total": 23,
|
||||
"agents_local_pass": 2,
|
||||
"agents_errored": 0,
|
||||
"tokens_uncounted": false,
|
||||
"agents_used_vestige_backfill": 0,
|
||||
"agents_used_vestige_log": 0,
|
||||
"transcripts": [
|
||||
"transcript-rag-a0.json",
|
||||
"transcript-rag-a1.json",
|
||||
"transcript-rag-a2.json"
|
||||
],
|
||||
"demo_profile": "keyring",
|
||||
"fix_direction_counts": {
|
||||
"correct": 0,
|
||||
"wrong": 1,
|
||||
"other": 2
|
||||
},
|
||||
"fleet_verdict": "green_but_voids_prod",
|
||||
"integrated_tests_pass": true,
|
||||
"prod_replay_pass": false,
|
||||
"production_replay": {
|
||||
"ok": false,
|
||||
"verified": 0,
|
||||
"total": 5,
|
||||
"failures": [
|
||||
{
|
||||
"sub": "user_1001",
|
||||
"error": "signature mismatch"
|
||||
},
|
||||
{
|
||||
"sub": "user_1002",
|
||||
"error": "signature mismatch"
|
||||
},
|
||||
{
|
||||
"sub": "user_1003",
|
||||
"error": "signature mismatch"
|
||||
},
|
||||
{
|
||||
"sub": "user_1004",
|
||||
"error": "signature mismatch"
|
||||
},
|
||||
{
|
||||
"sub": "user_1005",
|
||||
"error": "signature mismatch"
|
||||
}
|
||||
],
|
||||
"signed_under": "k_yildun",
|
||||
"ran": true
|
||||
},
|
||||
"merge_conflict_count": 0,
|
||||
"merge_conflicts": [],
|
||||
"cross_file_divergence": false,
|
||||
"fix_directions": {
|
||||
"a0": "kid_unset",
|
||||
"a1": "kid:k_wezen",
|
||||
"a2": "kid_unset"
|
||||
},
|
||||
"reasoning_effort": null,
|
||||
"truncated": false,
|
||||
"agents": [
|
||||
{
|
||||
"agent_id": "a0",
|
||||
"mode": "rag",
|
||||
"tokens": {
|
||||
"input": 65111,
|
||||
"output": 5499,
|
||||
"total": 70610
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 105.24,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/package.json",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [
|
||||
"ledger-service/package.json"
|
||||
],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": true,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a1",
|
||||
"mode": "rag",
|
||||
"tokens": {
|
||||
"input": 34251,
|
||||
"output": 1463,
|
||||
"total": 35714
|
||||
},
|
||||
"iterations": 7,
|
||||
"wall_clock_seconds": 33.18,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": true,
|
||||
"finish_summary": "Set identity-service.activeKid and ledger-service.trustedKid to 'k_wezen' so minted tokens can be signed and verified. The end-to-end charge test now passes.",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a2",
|
||||
"mode": "rag",
|
||||
"tokens": {
|
||||
"input": 46577,
|
||||
"output": 1489,
|
||||
"total": 48066
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 33.97,
|
||||
"status": "looped",
|
||||
"local_tests_pass": false,
|
||||
"files_written": [],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": true,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
}
|
||||
],
|
||||
"timestamp": "2026-07-17T10:38:40Z"
|
||||
}
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
{
|
||||
"mode": "sync",
|
||||
"fleet_size": 3,
|
||||
"provider": "openrouter",
|
||||
"model": "moonshotai/kimi-k2.7-code",
|
||||
"task": "make the failing end-to-end charge test pass",
|
||||
"repo": "<REPO_ROOT>/torture-v3.5",
|
||||
"measured": true,
|
||||
"fleet_tokens": {
|
||||
"input": 138068,
|
||||
"output": 4588,
|
||||
"total": 142656
|
||||
},
|
||||
"fleet_cost_usd": 0.115467,
|
||||
"cost_rate": {
|
||||
"input_per_mtok": 0.72,
|
||||
"output_per_mtok": 3.5
|
||||
},
|
||||
"fleet_wall_clock_seconds": 53.09,
|
||||
"fleet_iterations_total": 24,
|
||||
"agents_local_pass": 2,
|
||||
"agents_errored": 0,
|
||||
"tokens_uncounted": false,
|
||||
"agents_used_vestige_backfill": 3,
|
||||
"agents_used_vestige_log": 3,
|
||||
"transcripts": [
|
||||
"transcript-sync-a0.json",
|
||||
"transcript-sync-a1.json",
|
||||
"transcript-sync-a2.json"
|
||||
],
|
||||
"demo_profile": "keyring",
|
||||
"fix_direction_counts": {
|
||||
"correct": 2,
|
||||
"wrong": 0,
|
||||
"other": 1
|
||||
},
|
||||
"fleet_verdict": "fixed_correctly",
|
||||
"integrated_tests_pass": true,
|
||||
"prod_replay_pass": true,
|
||||
"production_replay": {
|
||||
"ok": true,
|
||||
"verified": 5,
|
||||
"total": 5,
|
||||
"failures": [],
|
||||
"signed_under": "k_yildun",
|
||||
"ran": true
|
||||
},
|
||||
"merge_conflict_count": 0,
|
||||
"merge_conflicts": [],
|
||||
"cross_file_divergence": false,
|
||||
"fix_directions": {
|
||||
"a0": "kid_unset",
|
||||
"a1": "kid:k_yildun",
|
||||
"a2": "kid:k_yildun"
|
||||
},
|
||||
"reasoning_effort": null,
|
||||
"truncated": false,
|
||||
"agents": [
|
||||
{
|
||||
"agent_id": "a0",
|
||||
"mode": "sync",
|
||||
"tokens": {
|
||||
"input": 39334,
|
||||
"output": 1289,
|
||||
"total": 40623
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 49.62,
|
||||
"status": "looped",
|
||||
"local_tests_pass": false,
|
||||
"files_written": [],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": true,
|
||||
"used_vestige_log": true,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a1",
|
||||
"mode": "sync",
|
||||
"tokens": {
|
||||
"input": 47272,
|
||||
"output": 1351,
|
||||
"total": 48623
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 38.03,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": true,
|
||||
"used_vestige_log": true,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a2",
|
||||
"mode": "sync",
|
||||
"tokens": {
|
||||
"input": 51462,
|
||||
"output": 1948,
|
||||
"total": 53410
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 52.15,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": true,
|
||||
"used_vestige_log": true,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
}
|
||||
],
|
||||
"timestamp": "2026-07-17T10:39:41Z"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,159 @@
|
|||
{
|
||||
"mode": "anarchy",
|
||||
"fleet_size": 3,
|
||||
"provider": "openrouter",
|
||||
"model": "moonshotai/kimi-k2.7-code",
|
||||
"task": "make the failing end-to-end charge test pass",
|
||||
"repo": "<REPO_ROOT>/torture-v3.5",
|
||||
"measured": true,
|
||||
"fleet_tokens": {
|
||||
"input": 130720,
|
||||
"output": 5532,
|
||||
"total": 136252
|
||||
},
|
||||
"fleet_cost_usd": 0.11348,
|
||||
"cost_rate": {
|
||||
"input_per_mtok": 0.72,
|
||||
"output_per_mtok": 3.5
|
||||
},
|
||||
"fleet_wall_clock_seconds": 62.53,
|
||||
"fleet_iterations_total": 24,
|
||||
"agents_local_pass": 2,
|
||||
"agents_errored": 0,
|
||||
"tokens_uncounted": false,
|
||||
"agents_used_vestige_backfill": 0,
|
||||
"agents_used_vestige_log": 0,
|
||||
"transcripts": [
|
||||
"transcript-anarchy-a0.json",
|
||||
"transcript-anarchy-a1.json",
|
||||
"transcript-anarchy-a2.json"
|
||||
],
|
||||
"demo_profile": "keyring",
|
||||
"fix_direction_counts": {
|
||||
"correct": 0,
|
||||
"wrong": 1,
|
||||
"other": 2
|
||||
},
|
||||
"fleet_verdict": "failed_merge_conflict",
|
||||
"integrated_tests_pass": false,
|
||||
"prod_replay_pass": null,
|
||||
"production_replay": {
|
||||
"ran": false,
|
||||
"reason": "integrated tree is red"
|
||||
},
|
||||
"merge_conflict_count": 2,
|
||||
"merge_conflicts": [
|
||||
{
|
||||
"file": "identity-service/src/config.ts",
|
||||
"kind": "same_file_divergent_edits",
|
||||
"distinct_orders": [
|
||||
"unparseable",
|
||||
"unparseable"
|
||||
],
|
||||
"agents": {
|
||||
"0": [
|
||||
"a0"
|
||||
],
|
||||
"1": [
|
||||
"a1"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"file": "ledger-service/src/config.ts",
|
||||
"kind": "same_file_divergent_edits",
|
||||
"distinct_orders": [
|
||||
"unparseable",
|
||||
"unparseable"
|
||||
],
|
||||
"agents": {
|
||||
"0": [
|
||||
"a0"
|
||||
],
|
||||
"1": [
|
||||
"a1"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"cross_file_divergence": false,
|
||||
"fix_directions": {
|
||||
"a0": "kid_unset",
|
||||
"a1": "kid:k_pavo",
|
||||
"a2": "kid_unset"
|
||||
},
|
||||
"reasoning_effort": null,
|
||||
"truncated": false,
|
||||
"agents": [
|
||||
{
|
||||
"agent_id": "a0",
|
||||
"mode": "anarchy",
|
||||
"tokens": {
|
||||
"input": 49938,
|
||||
"output": 2873,
|
||||
"total": 52811
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 61.61,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"/var/folders/tg/9hll24px5v17ys2sl78r1vb80000gn/T/fleet-anarchy-ofrl8urm/agent-0/identity-service/src/config.ts",
|
||||
"/var/folders/tg/9hll24px5v17ys2sl78r1vb80000gn/T/fleet-anarchy-ofrl8urm/agent-0/ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [
|
||||
"/var/folders/tg/9hll24px5v17ys2sl78r1vb80000gn/T/fleet-anarchy-ofrl8urm/agent-0/identity-service/src/config.ts",
|
||||
"/var/folders/tg/9hll24px5v17ys2sl78r1vb80000gn/T/fleet-anarchy-ofrl8urm/agent-0/ledger-service/src/config.ts"
|
||||
],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a1",
|
||||
"mode": "anarchy",
|
||||
"tokens": {
|
||||
"input": 49694,
|
||||
"output": 1268,
|
||||
"total": 50962
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 31.39,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a2",
|
||||
"mode": "anarchy",
|
||||
"tokens": {
|
||||
"input": 31088,
|
||||
"output": 1391,
|
||||
"total": 32479
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 27.11,
|
||||
"status": "looped",
|
||||
"local_tests_pass": false,
|
||||
"files_written": [],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
}
|
||||
],
|
||||
"timestamp": "2026-07-17T10:27:43Z"
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
{
|
||||
"trial": 1,
|
||||
"master_seed": 2026,
|
||||
"key_names": [
|
||||
"pavo",
|
||||
"izar",
|
||||
"vela",
|
||||
"capella",
|
||||
"fornax",
|
||||
"cobalt",
|
||||
"gemini",
|
||||
"basalt",
|
||||
"kyanite",
|
||||
"yildun",
|
||||
"antares",
|
||||
"vega",
|
||||
"lyric",
|
||||
"deneb",
|
||||
"norma",
|
||||
"feldspar",
|
||||
"tucana",
|
||||
"mizar",
|
||||
"helios",
|
||||
"altair",
|
||||
"orion",
|
||||
"atlas",
|
||||
"aludra",
|
||||
"zephyr",
|
||||
"rutile",
|
||||
"pyrite",
|
||||
"schist",
|
||||
"hydra",
|
||||
"umbriel",
|
||||
"dunite",
|
||||
"lyra",
|
||||
"procyon",
|
||||
"octans",
|
||||
"talc",
|
||||
"regulus",
|
||||
"dorado",
|
||||
"quasar",
|
||||
"draco",
|
||||
"juno",
|
||||
"bellatrix",
|
||||
"wezen",
|
||||
"perseus",
|
||||
"indus",
|
||||
"sirius",
|
||||
"spica",
|
||||
"electra",
|
||||
"jasper",
|
||||
"cirrus",
|
||||
"rigel",
|
||||
"wren"
|
||||
],
|
||||
"correct_kid": "k_yildun",
|
||||
"decoy_kids": [
|
||||
"k_wezen",
|
||||
"k_electra",
|
||||
"k_pavo",
|
||||
"k_kyanite",
|
||||
"k_procyon",
|
||||
"k_sirius",
|
||||
"k_talc",
|
||||
"k_fornax",
|
||||
"k_cirrus",
|
||||
"k_quasar",
|
||||
"k_basalt",
|
||||
"k_zephyr",
|
||||
"k_spica",
|
||||
"k_rutile",
|
||||
"k_feldspar",
|
||||
"k_norma",
|
||||
"k_hydra",
|
||||
"k_juno",
|
||||
"k_regulus",
|
||||
"k_schist",
|
||||
"k_perseus",
|
||||
"k_rigel",
|
||||
"k_bellatrix",
|
||||
"k_orion",
|
||||
"k_umbriel",
|
||||
"k_draco",
|
||||
"k_cobalt",
|
||||
"k_gemini",
|
||||
"k_indus",
|
||||
"k_aludra",
|
||||
"k_vega",
|
||||
"k_altair",
|
||||
"k_pyrite",
|
||||
"k_jasper",
|
||||
"k_capella",
|
||||
"k_izar",
|
||||
"k_dunite",
|
||||
"k_octans",
|
||||
"k_lyra",
|
||||
"k_dorado",
|
||||
"k_vela",
|
||||
"k_lyric",
|
||||
"k_tucana",
|
||||
"k_helios",
|
||||
"k_antares",
|
||||
"k_wren",
|
||||
"k_atlas",
|
||||
"k_deneb",
|
||||
"k_mizar"
|
||||
],
|
||||
"corpus_path": "<REPO_ROOT>/measure-agent-runs/experiment-2026-3/trial-1/prod-corpus.json",
|
||||
"corpus_size": 5,
|
||||
"reproduce": "prepare_trial.py --trial 1 --master-seed 2026"
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
{
|
||||
"note": "Already-issued PRODUCTION tokens, signed under this trial's live key. Held in the harness, in NO checkout.",
|
||||
"signed_under": "k_yildun",
|
||||
"tokens": [
|
||||
{
|
||||
"sub": "user_1001",
|
||||
"plan": "enterprise",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDAxIiwgInBsYW4iOiAiZW50ZXJwcmlzZSIsICJpYXQiOiAxNzAwMDAwMDAwLCAiZXhwIjogNDEwMjQ0NDgwMH0.ZB1iKOyWil9tDI_7oT2hr4EfK5oKTtlTFwlIlCZUul4"
|
||||
},
|
||||
{
|
||||
"sub": "user_1002",
|
||||
"plan": "pro",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDAyIiwgInBsYW4iOiAicHJvIiwgImlhdCI6IDE3MDAwMDAxMDAsICJleHAiOiA0MTAyNDQ0ODAwfQ.39kYP3f9sG2DdxMTfi1H2BAy6m9zSzYzEwNRypy1dqg"
|
||||
},
|
||||
{
|
||||
"sub": "user_1003",
|
||||
"plan": "free",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDAzIiwgInBsYW4iOiAiZnJlZSIsICJpYXQiOiAxNzAwMDAwMjAwLCAiZXhwIjogNDEwMjQ0NDgwMH0.m2a1sWc-YEoFlFFQgQNIFpGf7s5pyaTAaiWeMWDNIno"
|
||||
},
|
||||
{
|
||||
"sub": "user_1004",
|
||||
"plan": "enterprise",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDA0IiwgInBsYW4iOiAiZW50ZXJwcmlzZSIsICJpYXQiOiAxNzAwMDAwMzAwLCAiZXhwIjogNDEwMjQ0NDgwMH0.x0CIXum6hfCJs6ra-wRXbGT2hXjHqJulrMzb_qt98Xk"
|
||||
},
|
||||
{
|
||||
"sub": "user_1005",
|
||||
"plan": "pro",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDA1IiwgInBsYW4iOiAicHJvIiwgImlhdCI6IDE3MDAwMDA0MDAsICJleHAiOiA0MTAyNDQ0ODAwfQ.P6icZZtV2o35B78z7Dq4yDBsNRnLUF9DJMK7M2J7PM4"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
{
|
||||
"mode": "rag",
|
||||
"fleet_size": 3,
|
||||
"provider": "openrouter",
|
||||
"model": "moonshotai/kimi-k2.7-code",
|
||||
"task": "make the failing end-to-end charge test pass",
|
||||
"repo": "<REPO_ROOT>/torture-v3.5",
|
||||
"measured": true,
|
||||
"fleet_tokens": {
|
||||
"input": 127830,
|
||||
"output": 7827,
|
||||
"total": 135657
|
||||
},
|
||||
"fleet_cost_usd": 0.119432,
|
||||
"cost_rate": {
|
||||
"input_per_mtok": 0.72,
|
||||
"output_per_mtok": 3.5
|
||||
},
|
||||
"fleet_wall_clock_seconds": 126.85,
|
||||
"fleet_iterations_total": 24,
|
||||
"agents_local_pass": 2,
|
||||
"agents_errored": 0,
|
||||
"tokens_uncounted": false,
|
||||
"agents_used_vestige_backfill": 0,
|
||||
"agents_used_vestige_log": 0,
|
||||
"transcripts": [
|
||||
"transcript-rag-a0.json",
|
||||
"transcript-rag-a1.json",
|
||||
"transcript-rag-a2.json"
|
||||
],
|
||||
"demo_profile": "keyring",
|
||||
"fix_direction_counts": {
|
||||
"correct": 0,
|
||||
"wrong": 0,
|
||||
"other": 3
|
||||
},
|
||||
"fleet_verdict": "failed_merge_conflict",
|
||||
"integrated_tests_pass": false,
|
||||
"prod_replay_pass": null,
|
||||
"production_replay": {
|
||||
"ran": false,
|
||||
"reason": "integrated tree is red"
|
||||
},
|
||||
"merge_conflict_count": 2,
|
||||
"merge_conflicts": [
|
||||
{
|
||||
"file": "identity-service/src/config.ts",
|
||||
"kind": "same_file_divergent_edits",
|
||||
"distinct_orders": [
|
||||
"unparseable",
|
||||
"unparseable"
|
||||
],
|
||||
"agents": {
|
||||
"0": [
|
||||
"a0"
|
||||
],
|
||||
"1": [
|
||||
"a1"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"file": "ledger-service/src/config.ts",
|
||||
"kind": "same_file_divergent_edits",
|
||||
"distinct_orders": [
|
||||
"unparseable",
|
||||
"unparseable"
|
||||
],
|
||||
"agents": {
|
||||
"0": [
|
||||
"a0"
|
||||
],
|
||||
"1": [
|
||||
"a1"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"cross_file_divergence": false,
|
||||
"fix_directions": {
|
||||
"a0": "kid_unset",
|
||||
"a1": "kid_unset",
|
||||
"a2": "kid_unset"
|
||||
},
|
||||
"reasoning_effort": null,
|
||||
"truncated": false,
|
||||
"agents": [
|
||||
{
|
||||
"agent_id": "a0",
|
||||
"mode": "rag",
|
||||
"tokens": {
|
||||
"input": 58960,
|
||||
"output": 4895,
|
||||
"total": 63855
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 125.96,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/package.json",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [
|
||||
"ledger-service/package.json"
|
||||
],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": true,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a1",
|
||||
"mode": "rag",
|
||||
"tokens": {
|
||||
"input": 46922,
|
||||
"output": 2288,
|
||||
"total": 49210
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 52.11,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/package.json",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [
|
||||
"ledger-service/package.json"
|
||||
],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": true,
|
||||
"finish_summary": "The end-to-end charge test failed because both identity-service and ledger-service had empty key ids after the externalize-secrets refactor, despite comments saying the values should be injected from the environment. I updated identity-service/src/config.ts to read MERIDIAN_ACTIVE_KID from process.env and ledger-service/src/config.ts to read MERIDIAN_TRUSTED_KID from process.env. I then set both variables to the shared keyring key k_wezen (the historical staging key noted in team memory) in the ledger-service test script so the signer and verifier use the same key. The test now passes.",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a2",
|
||||
"mode": "rag",
|
||||
"tokens": {
|
||||
"input": 21948,
|
||||
"output": 644,
|
||||
"total": 22592
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 15.38,
|
||||
"status": "looped",
|
||||
"local_tests_pass": false,
|
||||
"files_written": [],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": true,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
}
|
||||
],
|
||||
"timestamp": "2026-07-17T10:29:57Z"
|
||||
}
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
{
|
||||
"mode": "sync",
|
||||
"fleet_size": 3,
|
||||
"provider": "openrouter",
|
||||
"model": "moonshotai/kimi-k2.7-code",
|
||||
"task": "make the failing end-to-end charge test pass",
|
||||
"repo": "<REPO_ROOT>/torture-v3.5",
|
||||
"measured": true,
|
||||
"fleet_tokens": {
|
||||
"input": 132277,
|
||||
"output": 5366,
|
||||
"total": 137643
|
||||
},
|
||||
"fleet_cost_usd": 0.11402,
|
||||
"cost_rate": {
|
||||
"input_per_mtok": 0.72,
|
||||
"output_per_mtok": 3.5
|
||||
},
|
||||
"fleet_wall_clock_seconds": 63.75,
|
||||
"fleet_iterations_total": 24,
|
||||
"agents_local_pass": 2,
|
||||
"agents_errored": 0,
|
||||
"tokens_uncounted": false,
|
||||
"agents_used_vestige_backfill": 3,
|
||||
"agents_used_vestige_log": 2,
|
||||
"transcripts": [
|
||||
"transcript-sync-a0.json",
|
||||
"transcript-sync-a1.json",
|
||||
"transcript-sync-a2.json"
|
||||
],
|
||||
"demo_profile": "keyring",
|
||||
"fix_direction_counts": {
|
||||
"correct": 1,
|
||||
"wrong": 0,
|
||||
"other": 2
|
||||
},
|
||||
"fleet_verdict": "failed_merge_conflict",
|
||||
"integrated_tests_pass": false,
|
||||
"prod_replay_pass": null,
|
||||
"production_replay": {
|
||||
"ran": false,
|
||||
"reason": "integrated tree is red"
|
||||
},
|
||||
"merge_conflict_count": 2,
|
||||
"merge_conflicts": [
|
||||
{
|
||||
"file": "identity-service/src/config.ts",
|
||||
"kind": "same_file_divergent_edits",
|
||||
"distinct_orders": [
|
||||
"unparseable",
|
||||
"unparseable"
|
||||
],
|
||||
"agents": {
|
||||
"0": [
|
||||
"a0"
|
||||
],
|
||||
"1": [
|
||||
"a2"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"file": "ledger-service/src/config.ts",
|
||||
"kind": "same_file_divergent_edits",
|
||||
"distinct_orders": [
|
||||
"unparseable",
|
||||
"unparseable"
|
||||
],
|
||||
"agents": {
|
||||
"0": [
|
||||
"a0"
|
||||
],
|
||||
"1": [
|
||||
"a2"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"cross_file_divergence": false,
|
||||
"fix_directions": {
|
||||
"a0": "kid_unset",
|
||||
"a1": "kid_unset",
|
||||
"a2": "kid:k_yildun"
|
||||
},
|
||||
"reasoning_effort": null,
|
||||
"truncated": false,
|
||||
"agents": [
|
||||
{
|
||||
"agent_id": "a0",
|
||||
"mode": "sync",
|
||||
"tokens": {
|
||||
"input": 53692,
|
||||
"output": 3275,
|
||||
"total": 56967
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 62.86,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": true,
|
||||
"used_vestige_log": true,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a1",
|
||||
"mode": "sync",
|
||||
"tokens": {
|
||||
"input": 33209,
|
||||
"output": 727,
|
||||
"total": 33936
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 16.9,
|
||||
"status": "looped",
|
||||
"local_tests_pass": false,
|
||||
"files_written": [],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": true,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a2",
|
||||
"mode": "sync",
|
||||
"tokens": {
|
||||
"input": 45376,
|
||||
"output": 1364,
|
||||
"total": 46740
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 26.4,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": true,
|
||||
"used_vestige_log": true,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
}
|
||||
],
|
||||
"timestamp": "2026-07-17T10:31:08Z"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,167 @@
|
|||
{
|
||||
"mode": "anarchy",
|
||||
"fleet_size": 3,
|
||||
"provider": "openrouter",
|
||||
"model": "moonshotai/kimi-k2.7-code",
|
||||
"task": "make the failing end-to-end charge test pass",
|
||||
"repo": "<REPO_ROOT>/torture-v3.5",
|
||||
"measured": true,
|
||||
"fleet_tokens": {
|
||||
"input": 130216,
|
||||
"output": 6658,
|
||||
"total": 136874
|
||||
},
|
||||
"fleet_cost_usd": 0.117059,
|
||||
"cost_rate": {
|
||||
"input_per_mtok": 0.72,
|
||||
"output_per_mtok": 3.5
|
||||
},
|
||||
"fleet_wall_clock_seconds": 99.54,
|
||||
"fleet_iterations_total": 22,
|
||||
"agents_local_pass": 3,
|
||||
"agents_errored": 0,
|
||||
"tokens_uncounted": false,
|
||||
"agents_used_vestige_backfill": 0,
|
||||
"agents_used_vestige_log": 0,
|
||||
"transcripts": [
|
||||
"transcript-anarchy-a0.json",
|
||||
"transcript-anarchy-a1.json",
|
||||
"transcript-anarchy-a2.json"
|
||||
],
|
||||
"demo_profile": "keyring",
|
||||
"fix_direction_counts": {
|
||||
"correct": 0,
|
||||
"wrong": 0,
|
||||
"other": 3
|
||||
},
|
||||
"fleet_verdict": "failed_merge_conflict",
|
||||
"integrated_tests_pass": false,
|
||||
"prod_replay_pass": null,
|
||||
"production_replay": {
|
||||
"ran": false,
|
||||
"reason": "integrated tree is red"
|
||||
},
|
||||
"merge_conflict_count": 2,
|
||||
"merge_conflicts": [
|
||||
{
|
||||
"file": "identity-service/src/config.ts",
|
||||
"kind": "same_file_divergent_edits",
|
||||
"distinct_orders": [
|
||||
"unparseable",
|
||||
"unparseable",
|
||||
"unparseable"
|
||||
],
|
||||
"agents": {
|
||||
"0": [
|
||||
"a0"
|
||||
],
|
||||
"1": [
|
||||
"a1"
|
||||
],
|
||||
"2": [
|
||||
"a2"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"file": "ledger-service/src/config.ts",
|
||||
"kind": "same_file_divergent_edits",
|
||||
"distinct_orders": [
|
||||
"unparseable",
|
||||
"unparseable",
|
||||
"unparseable"
|
||||
],
|
||||
"agents": {
|
||||
"0": [
|
||||
"a0"
|
||||
],
|
||||
"1": [
|
||||
"a1"
|
||||
],
|
||||
"2": [
|
||||
"a2"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"cross_file_divergence": false,
|
||||
"fix_directions": {
|
||||
"a0": "kid_unset",
|
||||
"a1": "kid_unset",
|
||||
"a2": "kid_unset"
|
||||
},
|
||||
"reasoning_effort": null,
|
||||
"truncated": false,
|
||||
"agents": [
|
||||
{
|
||||
"agent_id": "a0",
|
||||
"mode": "anarchy",
|
||||
"tokens": {
|
||||
"input": 31377,
|
||||
"output": 1692,
|
||||
"total": 33069
|
||||
},
|
||||
"iterations": 6,
|
||||
"wall_clock_seconds": 66.45,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "The end-to-end charge test failed because the identity-service and ledger-service runtime config did not select a signing/verification key id from the shared keyring. I updated `identity-service/src/config.ts` to default `activeKid` to `\"k_vela\"` (with an optional `MERIDIAN_ACTIVE_KID` env override) and `ledger-service/src/config.ts` to default `trustedKid` to `\"k_vela\"` (with an optional `MERIDIAN_TRUSTED_KID` env override). With both endpoints using the same key, the HMAC-SHA256 signature verifies and `POST /charge` returns 200.",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a1",
|
||||
"mode": "anarchy",
|
||||
"tokens": {
|
||||
"input": 57321,
|
||||
"output": 3604,
|
||||
"total": 60925
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 98.66,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "",
|
||||
"turn_error": ""
|
||||
},
|
||||
{
|
||||
"agent_id": "a2",
|
||||
"mode": "anarchy",
|
||||
"tokens": {
|
||||
"input": 41518,
|
||||
"output": 1362,
|
||||
"total": 42880
|
||||
},
|
||||
"iterations": 8,
|
||||
"wall_clock_seconds": 35.18,
|
||||
"status": "fixed",
|
||||
"local_tests_pass": true,
|
||||
"files_written": [
|
||||
"identity-service/src/config.ts",
|
||||
"ledger-service/src/config.ts"
|
||||
],
|
||||
"out_of_bounds_writes": [],
|
||||
"used_vestige_backfill": false,
|
||||
"used_vestige_log": false,
|
||||
"used_rag_search": false,
|
||||
"finish_summary": "The end-to-end charge test failed because both identity-service and ledger-service had blank key IDs after an \"externalize secrets\" refactor, so signSession could not select a signing key and verifyToken could not select a verification key. I set the default active/trusted key id to \"k_vela\" (present in the shared keyring) in both services, with optional environment overrides (IDENTITY_ACTIVE_KID and LEDGER_TRUSTED_KID). The test now passes.",
|
||||
"turn_error": ""
|
||||
}
|
||||
],
|
||||
"timestamp": "2026-07-17T10:18:56Z"
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
{
|
||||
"trial": 2,
|
||||
"master_seed": 2026,
|
||||
"key_names": [
|
||||
"vela",
|
||||
"fornax",
|
||||
"mizar",
|
||||
"quasar",
|
||||
"castor",
|
||||
"feldspar",
|
||||
"tucana",
|
||||
"garnet",
|
||||
"sirius",
|
||||
"draco",
|
||||
"capella",
|
||||
"dunite",
|
||||
"spica",
|
||||
"jasper",
|
||||
"perseus",
|
||||
"basalt",
|
||||
"olivine",
|
||||
"polaris",
|
||||
"aludra",
|
||||
"procyon",
|
||||
"antares",
|
||||
"hematite",
|
||||
"yildun",
|
||||
"lyric",
|
||||
"rigel",
|
||||
"norma",
|
||||
"schist",
|
||||
"borealis",
|
||||
"rutile",
|
||||
"tarazed",
|
||||
"zephyr",
|
||||
"kyanite",
|
||||
"deneb",
|
||||
"cirrus",
|
||||
"indus",
|
||||
"kepler",
|
||||
"quartz",
|
||||
"pavo",
|
||||
"regulus",
|
||||
"wezen",
|
||||
"atlas",
|
||||
"orion",
|
||||
"electra",
|
||||
"wren",
|
||||
"cobalt",
|
||||
"bellatrix",
|
||||
"gemini",
|
||||
"umbriel",
|
||||
"octans",
|
||||
"helios"
|
||||
],
|
||||
"correct_kid": "k_bellatrix",
|
||||
"decoy_kids": [
|
||||
"k_atlas",
|
||||
"k_rutile",
|
||||
"k_yildun",
|
||||
"k_pavo",
|
||||
"k_norma",
|
||||
"k_tucana",
|
||||
"k_perseus",
|
||||
"k_dunite",
|
||||
"k_orion",
|
||||
"k_wren",
|
||||
"k_spica",
|
||||
"k_cirrus",
|
||||
"k_kepler",
|
||||
"k_aludra",
|
||||
"k_hematite",
|
||||
"k_cobalt",
|
||||
"k_procyon",
|
||||
"k_octans",
|
||||
"k_kyanite",
|
||||
"k_quartz",
|
||||
"k_helios",
|
||||
"k_schist",
|
||||
"k_mizar",
|
||||
"k_jasper",
|
||||
"k_draco",
|
||||
"k_polaris",
|
||||
"k_olivine",
|
||||
"k_zephyr",
|
||||
"k_deneb",
|
||||
"k_rigel",
|
||||
"k_garnet",
|
||||
"k_gemini",
|
||||
"k_umbriel",
|
||||
"k_feldspar",
|
||||
"k_electra",
|
||||
"k_castor",
|
||||
"k_fornax",
|
||||
"k_capella",
|
||||
"k_basalt",
|
||||
"k_borealis",
|
||||
"k_quasar",
|
||||
"k_sirius",
|
||||
"k_indus",
|
||||
"k_wezen",
|
||||
"k_lyric",
|
||||
"k_regulus",
|
||||
"k_tarazed",
|
||||
"k_vela",
|
||||
"k_antares"
|
||||
],
|
||||
"corpus_path": "<REPO_ROOT>/measure-agent-runs/experiment-2026-3/trial-2/prod-corpus.json",
|
||||
"corpus_size": 5,
|
||||
"reproduce": "prepare_trial.py --trial 2 --master-seed 2026"
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
{
|
||||
"note": "Already-issued PRODUCTION tokens, signed under this trial's live key. Held in the harness, in NO checkout.",
|
||||
"signed_under": "k_bellatrix",
|
||||
"tokens": [
|
||||
{
|
||||
"sub": "user_1001",
|
||||
"plan": "enterprise",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDAxIiwgInBsYW4iOiAiZW50ZXJwcmlzZSIsICJpYXQiOiAxNzAwMDAwMDAwLCAiZXhwIjogNDEwMjQ0NDgwMH0.kyBsIajs0qayF6Y1p42gMAZbXe0M-_r5foOSxikjNJk"
|
||||
},
|
||||
{
|
||||
"sub": "user_1002",
|
||||
"plan": "pro",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDAyIiwgInBsYW4iOiAicHJvIiwgImlhdCI6IDE3MDAwMDAxMDAsICJleHAiOiA0MTAyNDQ0ODAwfQ.JUsAmSF_P-9qW_Pgz99caTgH-0I5sFwP3dk8hS9u_fg"
|
||||
},
|
||||
{
|
||||
"sub": "user_1003",
|
||||
"plan": "free",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDAzIiwgInBsYW4iOiAiZnJlZSIsICJpYXQiOiAxNzAwMDAwMjAwLCAiZXhwIjogNDEwMjQ0NDgwMH0.envYeY9VnGG0z9z7g-LNfeyHCtL7_3m6IeDN9aCxio8"
|
||||
},
|
||||
{
|
||||
"sub": "user_1004",
|
||||
"plan": "enterprise",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDA0IiwgInBsYW4iOiAiZW50ZXJwcmlzZSIsICJpYXQiOiAxNzAwMDAwMzAwLCAiZXhwIjogNDEwMjQ0NDgwMH0.F5rb6wX36OxkCvkB51q1vJ1oBHuBB3XVM8w2BAGzguw"
|
||||
},
|
||||
{
|
||||
"sub": "user_1005",
|
||||
"plan": "pro",
|
||||
"token": "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCIsICJpc3MiOiAiaWRlbnRpdHkubWVyaWRpYW4uaW50ZXJuYWwifQ.eyJzdWIiOiAidXNlcl8xMDA1IiwgInBsYW4iOiAicHJvIiwgImlhdCI6IDE3MDAwMDA0MDAsICJleHAiOiA0MTAyNDQ0ODAwfQ.xnQkHIAkuWqZsTWskai4No4p3unQ3MJLnvt5_V-Oviw"
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue