mirror of
https://github.com/ModernRelay/omnigraph.git
synced 2026-07-12 03:12:11 +02:00
bench(engine): append every run to a results log (ts + git_sha)
The harness's JSON lines were vanishing into terminal scrollback — the numbers that justify decisions (the 6.9GB merge crash, the prefilter recall pair) had no system of record, and the v7 baselines are needed for before/after comparison across the Lance 9 bump. Every run now appends its record, stamped with a unix ts and the current git sha, to --out <path> / OMNIGRAPH_BENCH_RESULTS / benches/results.jsonl (gitignored — host-specific, self-describing via host+params).
This commit is contained in:
parent
54dbf2f11c
commit
80db05f16e
3 changed files with 59 additions and 2 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -28,3 +28,4 @@ demo/*.omni/
|
|||
/schema-issues.md
|
||||
/schema-prior-art.md
|
||||
mdrip/
|
||||
crates/omnigraph/benches/results.jsonl
|
||||
|
|
|
|||
|
|
@ -55,6 +55,8 @@ struct Args {
|
|||
/// ANN k (the query's `limit`) for nearest-prefilter.
|
||||
k: usize,
|
||||
memory_cap_mb: Option<u64>,
|
||||
/// Results-log override; see `results_path`.
|
||||
out: Option<String>,
|
||||
/// Run the identical workload but SKIP the measured operation; the
|
||||
/// peak-RSS delta between a normal run and a baseline run isolates the
|
||||
/// measured op's own memory contribution (ru_maxrss spans the whole
|
||||
|
|
@ -75,6 +77,7 @@ impl Args {
|
|||
selectivity: 0.05,
|
||||
k: 10,
|
||||
memory_cap_mb: None,
|
||||
out: None,
|
||||
baseline: false,
|
||||
child: false,
|
||||
};
|
||||
|
|
@ -94,6 +97,7 @@ impl Args {
|
|||
args.selectivity = take("--selectivity").parse().expect("--selectivity")
|
||||
}
|
||||
"--k" => args.k = take("--k").parse().expect("--k"),
|
||||
"--out" => args.out = Some(take("--out")),
|
||||
"--memory-cap-mb" => {
|
||||
args.memory_cap_mb = Some(take("--memory-cap-mb").parse().expect("cap"))
|
||||
}
|
||||
|
|
@ -162,10 +166,53 @@ fn main() {
|
|||
for run in 0..args.runs {
|
||||
let record = run_once(&args, run);
|
||||
println!("{record}");
|
||||
append_result(&args, &record);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
/// Where run records accumulate: `--out <path>`, else `OMNIGRAPH_BENCH_RESULTS`,
|
||||
/// else `benches/results.jsonl` next to this crate (gitignored — results are
|
||||
/// host-specific; each record is self-describing via `host` + `params` +
|
||||
/// `git_sha`). Append-only JSON lines, the harness's system of record.
|
||||
fn results_path(args: &Args) -> std::path::PathBuf {
|
||||
if let Some(ref out) = args.out {
|
||||
return out.into();
|
||||
}
|
||||
if let Ok(env_path) = std::env::var("OMNIGRAPH_BENCH_RESULTS") {
|
||||
if !env_path.trim().is_empty() {
|
||||
return env_path.trim().into();
|
||||
}
|
||||
}
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("benches/results.jsonl")
|
||||
}
|
||||
|
||||
fn append_result(args: &Args, record: &serde_json::Value) {
|
||||
let path = results_path(args);
|
||||
let appended = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&path)
|
||||
.and_then(|mut f| {
|
||||
use std::io::Write as _;
|
||||
writeln!(f, "{record}")
|
||||
});
|
||||
if let Err(e) = appended {
|
||||
eprintln!("WARNING: could not append to {}: {e}", path.display());
|
||||
}
|
||||
}
|
||||
|
||||
fn git_sha() -> Option<String> {
|
||||
let out = std::process::Command::new("git")
|
||||
.args(["rev-parse", "--short", "HEAD"])
|
||||
.current_dir(env!("CARGO_MANIFEST_DIR"))
|
||||
.output()
|
||||
.ok()?;
|
||||
out.status
|
||||
.success()
|
||||
.then(|| String::from_utf8_lossy(&out.stdout).trim().to_string())
|
||||
}
|
||||
|
||||
fn run_once(args: &Args, run: usize) -> serde_json::Value {
|
||||
let exe = std::env::current_exe().expect("current_exe");
|
||||
let mut child = std::process::Command::new(exe)
|
||||
|
|
@ -200,6 +247,11 @@ fn run_once(args: &Args, run: usize) -> serde_json::Value {
|
|||
serde_json::json!({
|
||||
"scenario": args.scenario,
|
||||
"run": run,
|
||||
"ts": std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0),
|
||||
"git_sha": git_sha(),
|
||||
"params": {
|
||||
"rows": args.rows,
|
||||
"dims": args.dims,
|
||||
|
|
|
|||
|
|
@ -114,8 +114,12 @@ The CLI system tests (`system_local.rs`) spawn the workspace-built `omnigraph` a
|
|||
merge-all-changed --rows 20000 --dims 256` (also `nearest-prefilter`;
|
||||
`--baseline` re-runs the identical workload minus the measured op so the
|
||||
peak-RSS delta isolates it; `--memory-cap-mb` applies `RLIMIT_AS` — enforced
|
||||
on Linux, best-effort on macOS). Add new scenarios here rather than new
|
||||
bench targets; keep the JSON-lines/no-assertions contract.
|
||||
on Linux, best-effort on macOS). Every run appends its record (with `ts` +
|
||||
`git_sha`) to a results log — `--out <path>`, else `OMNIGRAPH_BENCH_RESULTS`,
|
||||
else `crates/omnigraph/benches/results.jsonl` (gitignored; host-specific) —
|
||||
so baselines survive across sessions and substrate bumps. Add new scenarios
|
||||
here rather than new bench targets; keep the JSON-lines/no-assertions
|
||||
contract.
|
||||
- Add `benches/` per crate when you ship a perf-driven change, and include the motivating workload with the optimization.
|
||||
|
||||
## Coverage tooling — what's missing
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue