diff --git a/.gitignore b/.gitignore index bcee4b31..74d54125 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ demo/*.omni/ /schema-issues.md /schema-prior-art.md mdrip/ +crates/omnigraph/benches/results.jsonl diff --git a/crates/omnigraph/benches/scenarios.rs b/crates/omnigraph/benches/scenarios.rs index 5f19fa8e..d3855721 100644 --- a/crates/omnigraph/benches/scenarios.rs +++ b/crates/omnigraph/benches/scenarios.rs @@ -55,6 +55,8 @@ struct Args { /// ANN k (the query's `limit`) for nearest-prefilter. k: usize, memory_cap_mb: Option, + /// Results-log override; see `results_path`. + out: Option, /// Run the identical workload but SKIP the measured operation; the /// peak-RSS delta between a normal run and a baseline run isolates the /// measured op's own memory contribution (ru_maxrss spans the whole @@ -75,6 +77,7 @@ impl Args { selectivity: 0.05, k: 10, memory_cap_mb: None, + out: None, baseline: false, child: false, }; @@ -94,6 +97,7 @@ impl Args { args.selectivity = take("--selectivity").parse().expect("--selectivity") } "--k" => args.k = take("--k").parse().expect("--k"), + "--out" => args.out = Some(take("--out")), "--memory-cap-mb" => { args.memory_cap_mb = Some(take("--memory-cap-mb").parse().expect("cap")) } @@ -162,10 +166,53 @@ fn main() { for run in 0..args.runs { let record = run_once(&args, run); println!("{record}"); + append_result(&args, &record); } } #[cfg(unix)] +/// Where run records accumulate: `--out `, else `OMNIGRAPH_BENCH_RESULTS`, +/// else `benches/results.jsonl` next to this crate (gitignored — results are +/// host-specific; each record is self-describing via `host` + `params` + +/// `git_sha`). Append-only JSON lines, the harness's system of record. +fn results_path(args: &Args) -> std::path::PathBuf { + if let Some(ref out) = args.out { + return out.into(); + } + if let Ok(env_path) = std::env::var("OMNIGRAPH_BENCH_RESULTS") { + if !env_path.trim().is_empty() { + return env_path.trim().into(); + } + } + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("benches/results.jsonl") +} + +fn append_result(args: &Args, record: &serde_json::Value) { + let path = results_path(args); + let appended = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .and_then(|mut f| { + use std::io::Write as _; + writeln!(f, "{record}") + }); + if let Err(e) = appended { + eprintln!("WARNING: could not append to {}: {e}", path.display()); + } +} + +fn git_sha() -> Option { + let out = std::process::Command::new("git") + .args(["rev-parse", "--short", "HEAD"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .output() + .ok()?; + out.status + .success() + .then(|| String::from_utf8_lossy(&out.stdout).trim().to_string()) +} + fn run_once(args: &Args, run: usize) -> serde_json::Value { let exe = std::env::current_exe().expect("current_exe"); let mut child = std::process::Command::new(exe) @@ -200,6 +247,11 @@ fn run_once(args: &Args, run: usize) -> serde_json::Value { serde_json::json!({ "scenario": args.scenario, "run": run, + "ts": std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0), + "git_sha": git_sha(), "params": { "rows": args.rows, "dims": args.dims, diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 5682caf9..72dc4abe 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -114,8 +114,12 @@ The CLI system tests (`system_local.rs`) spawn the workspace-built `omnigraph` a merge-all-changed --rows 20000 --dims 256` (also `nearest-prefilter`; `--baseline` re-runs the identical workload minus the measured op so the peak-RSS delta isolates it; `--memory-cap-mb` applies `RLIMIT_AS` — enforced - on Linux, best-effort on macOS). Add new scenarios here rather than new - bench targets; keep the JSON-lines/no-assertions contract. + on Linux, best-effort on macOS). Every run appends its record (with `ts` + + `git_sha`) to a results log — `--out `, else `OMNIGRAPH_BENCH_RESULTS`, + else `crates/omnigraph/benches/results.jsonl` (gitignored; host-specific) — + so baselines survive across sessions and substrate bumps. Add new scenarios + here rather than new bench targets; keep the JSON-lines/no-assertions + contract. - Add `benches/` per crate when you ship a perf-driven change, and include the motivating workload with the optimization. ## Coverage tooling — what's missing