2026-05-21 11:22:13 -05:00
|
|
|
# Eval corpus budget.
|
2026-05-15 19:22:40 -05:00
|
|
|
#
|
2026-05-21 11:22:13 -05:00
|
|
|
# `report.py` enforces these values when `run.sh` or `run_full.sh` pass
|
|
|
|
|
# `--budget`. Each (cap, lang) cell uses the default row unless a specific
|
|
|
|
|
# override appears below.
|
2026-05-15 20:34:53 -05:00
|
|
|
#
|
2026-05-21 11:22:13 -05:00
|
|
|
# Wall-clock cost is measured separately from this per-cell budget.
|
2026-05-15 19:22:40 -05:00
|
|
|
#
|
|
|
|
|
# Schema:
|
|
|
|
|
#
|
|
|
|
|
# [default]
|
2026-05-15 20:34:53 -05:00
|
|
|
# unsupported_rate = 0.20 # max(Unsupported / total) per cell
|
|
|
|
|
# false_confirmed_rate = 0.02 # max(wrong / Confirmed) per cap
|
|
|
|
|
# repro_stability = 0.95 # min(stable / Confirmed) per cell
|
|
|
|
|
# ratchet_deadline = "..." # informational; cells already at headline
|
2026-05-15 19:22:40 -05:00
|
|
|
#
|
|
|
|
|
# [[cell]]
|
2026-05-15 20:34:53 -05:00
|
|
|
# cap = "..."
|
|
|
|
|
# lang = "..."
|
|
|
|
|
# <overrides as above>
|
2026-05-15 19:22:40 -05:00
|
|
|
#
|
2026-05-15 20:34:53 -05:00
|
|
|
# `cap` matches `tabulate.py`'s _CAP_BIT_TABLE / _CAP_RULE_TABLE labels.
|
2026-05-15 19:22:40 -05:00
|
|
|
# `lang` matches the ext_map values (`python`, `javascript`, …).
|
|
|
|
|
# A wildcard `"*"` matches any cell that does not have an exact entry.
|
|
|
|
|
|
|
|
|
|
[default]
|
2026-05-15 20:34:53 -05:00
|
|
|
unsupported_rate = 0.20
|
2026-05-15 19:22:40 -05:00
|
|
|
false_confirmed_rate = 0.02
|
|
|
|
|
repro_stability = 0.95
|
2026-05-15 20:34:53 -05:00
|
|
|
ratchet_deadline = "2026-05-15"
|
