mirror of
https://github.com/elicpeter/nyx.git
synced 2026-06-12 19:55:14 +02:00
refactor(dynamic): introduce build pools for Python, C, C++, Go, Ruby, PHP, and Node.js with shared caching and warming improvements; enhance test coverage with micro-benchmarks
This commit is contained in:
parent
3d710c856d
commit
bd76cd5b9d
20 changed files with 2123 additions and 23 deletions
92
tests/dynamic_c_build_pool.rs
Normal file
92
tests/dynamic_c_build_pool.rs
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
//! Phase 23 / Track O.1 micro-benchmark for the C build pool.
|
||||
//!
|
||||
//! Asserts the hot-build P50 (a `ccache`-fronted recompile, or a bare trivial
|
||||
//! `cc` when ccache is absent) stays ≤ 1s, the compiled-language budget.
|
||||
//! Skips when `cc` is not runnable.
|
||||
|
||||
#![cfg(feature = "dynamic")]
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use nyx_scanner::dynamic::build_pool::BuildPool;
|
||||
use nyx_scanner::dynamic::build_pool::c::CPool;
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
struct PoolDirGuard {
|
||||
_lock: MutexGuard<'static, ()>,
|
||||
prior: Option<String>,
|
||||
_dir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl PoolDirGuard {
|
||||
fn isolated() -> Self {
|
||||
let lock = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let prior = std::env::var("NYX_BUILD_POOL_DIR").ok();
|
||||
unsafe { std::env::set_var("NYX_BUILD_POOL_DIR", dir.path()) };
|
||||
Self {
|
||||
_lock: lock,
|
||||
prior,
|
||||
_dir: dir,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PoolDirGuard {
|
||||
fn drop(&mut self) {
|
||||
match self.prior.take() {
|
||||
Some(v) => unsafe { std::env::set_var("NYX_BUILD_POOL_DIR", v) },
|
||||
None => unsafe { std::env::remove_var("NYX_BUILD_POOL_DIR") },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn median(mut ds: Vec<Duration>) -> Duration {
|
||||
ds.sort();
|
||||
ds[ds.len() / 2]
|
||||
}
|
||||
|
||||
fn write_source(workdir: &Path) {
|
||||
std::fs::write(workdir.join("main.c"), "int main(void) { return 0; }\n").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "real-toolchain perf bench: spawns `cc`. Opt-in so the default suite stays hermetic + fast. Run: cargo nextest run --features dynamic --run-ignored ignored-only -E 'binary(~build_pool) | binary(~compile_pool)'"]
|
||||
fn hot_rebuild_p50_under_one_second() {
|
||||
let _guard = PoolDirGuard::isolated();
|
||||
let pool = match CPool::try_new() {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
eprintln!("skipping c build-pool bench: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let work = tempfile::TempDir::new().unwrap();
|
||||
write_source(work.path());
|
||||
let dest = work.path().join("nyx_harness_out");
|
||||
let args = [dest.to_string_lossy().into_owned(), "dynamic".to_owned()];
|
||||
|
||||
let cold = pool.compile_batch(work.path(), &args);
|
||||
assert!(cold.success, "cold build must succeed: {}", cold.stderr);
|
||||
assert!(dest.exists(), "cold build must emit the binary");
|
||||
|
||||
let mut hot = Vec::new();
|
||||
for _ in 0..5 {
|
||||
let _ = std::fs::remove_file(&dest);
|
||||
let start = Instant::now();
|
||||
let r = pool.compile_batch(work.path(), &args);
|
||||
hot.push(start.elapsed());
|
||||
assert!(r.success, "hot build must succeed: {}", r.stderr);
|
||||
}
|
||||
|
||||
let p50 = median(hot);
|
||||
eprintln!("c build-pool hot P50: {p50:?}");
|
||||
assert!(
|
||||
p50 <= Duration::from_secs(1),
|
||||
"c hot-build P50 {p50:?} exceeds the 1s compiled budget",
|
||||
);
|
||||
}
|
||||
92
tests/dynamic_cpp_build_pool.rs
Normal file
92
tests/dynamic_cpp_build_pool.rs
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
//! Phase 23 / Track O.1 micro-benchmark for the C++ build pool.
|
||||
//!
|
||||
//! Asserts the hot-build P50 (a `ccache`-fronted recompile, or a bare trivial
|
||||
//! `c++` when ccache is absent) stays ≤ 1s, the compiled-language budget.
|
||||
//! Skips when `c++` is not runnable.
|
||||
|
||||
#![cfg(feature = "dynamic")]
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use nyx_scanner::dynamic::build_pool::BuildPool;
|
||||
use nyx_scanner::dynamic::build_pool::cpp::CppPool;
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
struct PoolDirGuard {
|
||||
_lock: MutexGuard<'static, ()>,
|
||||
prior: Option<String>,
|
||||
_dir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl PoolDirGuard {
|
||||
fn isolated() -> Self {
|
||||
let lock = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let prior = std::env::var("NYX_BUILD_POOL_DIR").ok();
|
||||
unsafe { std::env::set_var("NYX_BUILD_POOL_DIR", dir.path()) };
|
||||
Self {
|
||||
_lock: lock,
|
||||
prior,
|
||||
_dir: dir,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PoolDirGuard {
|
||||
fn drop(&mut self) {
|
||||
match self.prior.take() {
|
||||
Some(v) => unsafe { std::env::set_var("NYX_BUILD_POOL_DIR", v) },
|
||||
None => unsafe { std::env::remove_var("NYX_BUILD_POOL_DIR") },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn median(mut ds: Vec<Duration>) -> Duration {
|
||||
ds.sort();
|
||||
ds[ds.len() / 2]
|
||||
}
|
||||
|
||||
fn write_source(workdir: &Path) {
|
||||
std::fs::write(workdir.join("main.cpp"), "int main() { return 0; }\n").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "real-toolchain perf bench: spawns `c++`. Opt-in so the default suite stays hermetic + fast. Run: cargo nextest run --features dynamic --run-ignored ignored-only -E 'binary(~build_pool) | binary(~compile_pool)'"]
|
||||
fn hot_rebuild_p50_under_one_second() {
|
||||
let _guard = PoolDirGuard::isolated();
|
||||
let pool = match CppPool::try_new() {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
eprintln!("skipping cpp build-pool bench: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let work = tempfile::TempDir::new().unwrap();
|
||||
write_source(work.path());
|
||||
let dest = work.path().join("nyx_harness_out");
|
||||
let args = [dest.to_string_lossy().into_owned()];
|
||||
|
||||
let cold = pool.compile_batch(work.path(), &args);
|
||||
assert!(cold.success, "cold build must succeed: {}", cold.stderr);
|
||||
assert!(dest.exists(), "cold build must emit the binary");
|
||||
|
||||
let mut hot = Vec::new();
|
||||
for _ in 0..5 {
|
||||
let _ = std::fs::remove_file(&dest);
|
||||
let start = Instant::now();
|
||||
let r = pool.compile_batch(work.path(), &args);
|
||||
hot.push(start.elapsed());
|
||||
assert!(r.success, "hot build must succeed: {}", r.stderr);
|
||||
}
|
||||
|
||||
let p50 = median(hot);
|
||||
eprintln!("cpp build-pool hot P50: {p50:?}");
|
||||
assert!(
|
||||
p50 <= Duration::from_secs(1),
|
||||
"cpp hot-build P50 {p50:?} exceeds the 1s compiled budget",
|
||||
);
|
||||
}
|
||||
93
tests/dynamic_go_build_pool.rs
Normal file
93
tests/dynamic_go_build_pool.rs
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
//! Phase 23 / Track O.1 micro-benchmark for the Go build pool.
|
||||
//!
|
||||
//! Asserts the hot-build P50 (a warm rebuild through the shared `GOCACHE` /
|
||||
//! `GOMODCACHE`) stays ≤ 1s, the compiled-language budget. Skips when `go`
|
||||
//! is not runnable.
|
||||
|
||||
#![cfg(feature = "dynamic")]
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use nyx_scanner::dynamic::build_pool::BuildPool;
|
||||
use nyx_scanner::dynamic::build_pool::go::GoPool;
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
struct PoolDirGuard {
|
||||
_lock: MutexGuard<'static, ()>,
|
||||
prior: Option<String>,
|
||||
_dir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl PoolDirGuard {
|
||||
fn isolated() -> Self {
|
||||
let lock = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let prior = std::env::var("NYX_BUILD_POOL_DIR").ok();
|
||||
unsafe { std::env::set_var("NYX_BUILD_POOL_DIR", dir.path()) };
|
||||
Self {
|
||||
_lock: lock,
|
||||
prior,
|
||||
_dir: dir,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PoolDirGuard {
|
||||
fn drop(&mut self) {
|
||||
match self.prior.take() {
|
||||
Some(v) => unsafe { std::env::set_var("NYX_BUILD_POOL_DIR", v) },
|
||||
None => unsafe { std::env::remove_var("NYX_BUILD_POOL_DIR") },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn median(mut ds: Vec<Duration>) -> Duration {
|
||||
ds.sort();
|
||||
ds[ds.len() / 2]
|
||||
}
|
||||
|
||||
fn write_project(workdir: &Path) {
|
||||
std::fs::write(workdir.join("go.mod"), "module nyxharness\n\ngo 1.21\n").unwrap();
|
||||
std::fs::write(workdir.join("main.go"), "package main\n\nfunc main() {}\n").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "real-toolchain perf bench: spawns `go build` + `go mod tidy`. Opt-in so the default suite stays hermetic + fast. Run: cargo nextest run --features dynamic --run-ignored ignored-only -E 'binary(~build_pool) | binary(~compile_pool)'"]
|
||||
fn hot_rebuild_p50_under_one_second() {
|
||||
let _guard = PoolDirGuard::isolated();
|
||||
let pool = match GoPool::try_new() {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
eprintln!("skipping go build-pool bench: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let work = tempfile::TempDir::new().unwrap();
|
||||
write_project(work.path());
|
||||
let dest = work.path().join("nyx_harness_out");
|
||||
let args = [dest.to_string_lossy().into_owned()];
|
||||
|
||||
let cold = pool.compile_batch(work.path(), &args);
|
||||
assert!(cold.success, "cold build must succeed: {}", cold.stderr);
|
||||
assert!(dest.exists(), "cold build must emit the binary");
|
||||
|
||||
let mut hot = Vec::new();
|
||||
for _ in 0..5 {
|
||||
let _ = std::fs::remove_file(&dest);
|
||||
let start = Instant::now();
|
||||
let r = pool.compile_batch(work.path(), &args);
|
||||
hot.push(start.elapsed());
|
||||
assert!(r.success, "hot build must succeed: {}", r.stderr);
|
||||
}
|
||||
|
||||
let p50 = median(hot);
|
||||
eprintln!("go build-pool hot P50: {p50:?}");
|
||||
assert!(
|
||||
p50 <= Duration::from_secs(1),
|
||||
"go hot-build P50 {p50:?} exceeds the 1s compiled budget",
|
||||
);
|
||||
}
|
||||
|
|
@ -76,6 +76,7 @@ fn write_harness(workdir: &Path, idx: usize) -> Vec<String> {
|
|||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "real-toolchain perf bench: runs 50 real `javac` compiles. Opt-in so the default suite stays hermetic + fast. Run: cargo nextest run --features dynamic --run-ignored ignored-only -E 'binary(~build_pool) | binary(~compile_pool)'"]
|
||||
fn batch_of_fifty_harness_compiles_meets_perf_target() {
|
||||
if !jdk_available() {
|
||||
eprintln!("skipping: javac / java not available on PATH");
|
||||
|
|
|
|||
136
tests/dynamic_node_build_pool.rs
Normal file
136
tests/dynamic_node_build_pool.rs
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
//! Phase 23 / Track O.1 micro-benchmark for the Node build pool.
|
||||
//!
|
||||
//! Asserts the warm-cache hot path (a `prepare_node` cache hit fronted by the
|
||||
//! shared npm download cache) stays ≤ 200ms, the interpreted-language budget.
|
||||
//! Skips when `npm` is not runnable so a toolchain-less CI image keeps the gate
|
||||
//! green.
|
||||
|
||||
#![cfg(feature = "dynamic")]
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use nyx_scanner::dynamic::build_sandbox::prepare_node;
|
||||
use nyx_scanner::dynamic::spec::{
|
||||
EntryKind, HarnessSpec, JavaToolchain, PayloadSlot, SpecDerivationStrategy,
|
||||
};
|
||||
use nyx_scanner::labels::Cap;
|
||||
use nyx_scanner::symbol::Lang;
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// Isolates `NYX_BUILD_CACHE` + `NYX_BUILD_POOL_DIR` to private tempdirs so the
|
||||
/// benchmark never reads or writes the user-level build cache.
|
||||
struct CacheGuard {
|
||||
_lock: MutexGuard<'static, ()>,
|
||||
prior_cache: Option<String>,
|
||||
prior_pool: Option<String>,
|
||||
_cache: tempfile::TempDir,
|
||||
_pool: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl CacheGuard {
|
||||
fn isolated() -> Self {
|
||||
let lock = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let cache = tempfile::TempDir::new().unwrap();
|
||||
let pool = tempfile::TempDir::new().unwrap();
|
||||
let prior_cache = std::env::var("NYX_BUILD_CACHE").ok();
|
||||
let prior_pool = std::env::var("NYX_BUILD_POOL_DIR").ok();
|
||||
unsafe {
|
||||
std::env::set_var("NYX_BUILD_CACHE", cache.path());
|
||||
std::env::set_var("NYX_BUILD_POOL_DIR", pool.path());
|
||||
}
|
||||
Self {
|
||||
_lock: lock,
|
||||
prior_cache,
|
||||
prior_pool,
|
||||
_cache: cache,
|
||||
_pool: pool,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CacheGuard {
|
||||
fn drop(&mut self) {
|
||||
restore("NYX_BUILD_CACHE", self.prior_cache.take());
|
||||
restore("NYX_BUILD_POOL_DIR", self.prior_pool.take());
|
||||
}
|
||||
}
|
||||
|
||||
fn restore(key: &str, prior: Option<String>) {
|
||||
match prior {
|
||||
Some(v) => unsafe { std::env::set_var(key, v) },
|
||||
None => unsafe { std::env::remove_var(key) },
|
||||
}
|
||||
}
|
||||
|
||||
fn median(mut ds: Vec<Duration>) -> Duration {
|
||||
ds.sort();
|
||||
ds[ds.len() / 2]
|
||||
}
|
||||
|
||||
fn mk_spec() -> HarnessSpec {
|
||||
HarnessSpec {
|
||||
finding_id: "bench".to_owned(),
|
||||
entry_file: "entry".to_owned(),
|
||||
entry_name: "main".to_owned(),
|
||||
entry_kind: EntryKind::Function,
|
||||
lang: Lang::JavaScript,
|
||||
toolchain_id: "bench-node".to_owned(),
|
||||
payload_slot: PayloadSlot::Param(0),
|
||||
expected_cap: Cap::CODE_EXEC,
|
||||
constraint_hints: vec![],
|
||||
sink_file: "sink".to_owned(),
|
||||
sink_line: 1,
|
||||
spec_hash: "0000000000000000".to_owned(),
|
||||
derivation: SpecDerivationStrategy::FromFlowSteps,
|
||||
stubs_required: vec![],
|
||||
framework: None,
|
||||
java_toolchain: JavaToolchain::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_project(workdir: &Path) {
|
||||
// Dependency-free manifest: `npm install` succeeds offline and the warm
|
||||
// cache marker lets every later call short-circuit.
|
||||
std::fs::write(
|
||||
workdir.join("package.json"),
|
||||
"{\"name\":\"nyxbench\",\"version\":\"1.0.0\",\"private\":true}\n",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "real-toolchain perf bench: spawns `npm install`. Opt-in so the default suite stays hermetic + fast. Run: cargo nextest run --features dynamic --run-ignored ignored-only -E 'binary(~build_pool) | binary(~compile_pool)'"]
|
||||
fn warm_prepare_p50_under_200ms() {
|
||||
let _guard = CacheGuard::isolated();
|
||||
let spec = mk_spec();
|
||||
let work = tempfile::TempDir::new().unwrap();
|
||||
write_project(work.path());
|
||||
|
||||
// Cold prep warms the cache; not measured. A toolchain-less host returns
|
||||
// Err here, so skip rather than fail.
|
||||
match prepare_node(&spec, work.path()) {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
eprintln!("skipping node build-pool bench: {e:?}");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let mut hot = Vec::new();
|
||||
for _ in 0..5 {
|
||||
let start = Instant::now();
|
||||
let r = prepare_node(&spec, work.path()).expect("warm prepare must succeed");
|
||||
hot.push(start.elapsed());
|
||||
assert!(r.cache_hit, "warm prepare_node must be a cache hit");
|
||||
}
|
||||
|
||||
let p50 = median(hot);
|
||||
eprintln!("node build-pool warm P50: {p50:?}");
|
||||
assert!(
|
||||
p50 <= Duration::from_millis(200),
|
||||
"node warm-prepare P50 {p50:?} exceeds the 200ms interpreted budget",
|
||||
);
|
||||
}
|
||||
127
tests/dynamic_php_build_pool.rs
Normal file
127
tests/dynamic_php_build_pool.rs
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
//! Phase 23 / Track O.1 micro-benchmark for the PHP build pool.
|
||||
//!
|
||||
//! Asserts the warm-cache hot path (a `prepare_php` cache hit backed by the
|
||||
//! shared Composer download cache + opcache file-cache warm) stays ≤ 200ms,
|
||||
//! the interpreted budget. Skips when `composer` is not runnable.
|
||||
|
||||
#![cfg(feature = "dynamic")]
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use nyx_scanner::dynamic::build_sandbox::prepare_php;
|
||||
use nyx_scanner::dynamic::spec::{
|
||||
EntryKind, HarnessSpec, JavaToolchain, PayloadSlot, SpecDerivationStrategy,
|
||||
};
|
||||
use nyx_scanner::labels::Cap;
|
||||
use nyx_scanner::symbol::Lang;
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
struct CacheGuard {
|
||||
_lock: MutexGuard<'static, ()>,
|
||||
prior_cache: Option<String>,
|
||||
prior_pool: Option<String>,
|
||||
_cache: tempfile::TempDir,
|
||||
_pool: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl CacheGuard {
|
||||
fn isolated() -> Self {
|
||||
let lock = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let cache = tempfile::TempDir::new().unwrap();
|
||||
let pool = tempfile::TempDir::new().unwrap();
|
||||
let prior_cache = std::env::var("NYX_BUILD_CACHE").ok();
|
||||
let prior_pool = std::env::var("NYX_BUILD_POOL_DIR").ok();
|
||||
unsafe {
|
||||
std::env::set_var("NYX_BUILD_CACHE", cache.path());
|
||||
std::env::set_var("NYX_BUILD_POOL_DIR", pool.path());
|
||||
}
|
||||
Self {
|
||||
_lock: lock,
|
||||
prior_cache,
|
||||
prior_pool,
|
||||
_cache: cache,
|
||||
_pool: pool,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CacheGuard {
|
||||
fn drop(&mut self) {
|
||||
restore("NYX_BUILD_CACHE", self.prior_cache.take());
|
||||
restore("NYX_BUILD_POOL_DIR", self.prior_pool.take());
|
||||
}
|
||||
}
|
||||
|
||||
fn restore(key: &str, prior: Option<String>) {
|
||||
match prior {
|
||||
Some(v) => unsafe { std::env::set_var(key, v) },
|
||||
None => unsafe { std::env::remove_var(key) },
|
||||
}
|
||||
}
|
||||
|
||||
fn median(mut ds: Vec<Duration>) -> Duration {
|
||||
ds.sort();
|
||||
ds[ds.len() / 2]
|
||||
}
|
||||
|
||||
fn mk_spec() -> HarnessSpec {
|
||||
HarnessSpec {
|
||||
finding_id: "bench".to_owned(),
|
||||
entry_file: "entry".to_owned(),
|
||||
entry_name: "main".to_owned(),
|
||||
entry_kind: EntryKind::Function,
|
||||
lang: Lang::Php,
|
||||
toolchain_id: "bench-php".to_owned(),
|
||||
payload_slot: PayloadSlot::Param(0),
|
||||
expected_cap: Cap::CODE_EXEC,
|
||||
constraint_hints: vec![],
|
||||
sink_file: "sink".to_owned(),
|
||||
sink_line: 1,
|
||||
spec_hash: "0000000000000000".to_owned(),
|
||||
derivation: SpecDerivationStrategy::FromFlowSteps,
|
||||
stubs_required: vec![],
|
||||
framework: None,
|
||||
java_toolchain: JavaToolchain::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_project(workdir: &Path) {
|
||||
// Dependency-free composer manifest: install succeeds offline and the
|
||||
// `.php_cache_done` marker turns later calls into cache hits.
|
||||
std::fs::write(workdir.join("composer.json"), "{}\n").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "real-toolchain perf bench: spawns `composer install`. Opt-in so the default suite stays hermetic + fast. Run: cargo nextest run --features dynamic --run-ignored ignored-only -E 'binary(~build_pool) | binary(~compile_pool)'"]
|
||||
fn warm_prepare_p50_under_200ms() {
|
||||
let _guard = CacheGuard::isolated();
|
||||
let spec = mk_spec();
|
||||
let work = tempfile::TempDir::new().unwrap();
|
||||
write_project(work.path());
|
||||
|
||||
match prepare_php(&spec, work.path()) {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
eprintln!("skipping php build-pool bench: {e:?}");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let mut hot = Vec::new();
|
||||
for _ in 0..5 {
|
||||
let start = Instant::now();
|
||||
let r = prepare_php(&spec, work.path()).expect("warm prepare must succeed");
|
||||
hot.push(start.elapsed());
|
||||
assert!(r.cache_hit, "warm prepare_php must be a cache hit");
|
||||
}
|
||||
|
||||
let p50 = median(hot);
|
||||
eprintln!("php build-pool warm P50: {p50:?}");
|
||||
assert!(
|
||||
p50 <= Duration::from_millis(200),
|
||||
"php warm-prepare P50 {p50:?} exceeds the 200ms interpreted budget",
|
||||
);
|
||||
}
|
||||
127
tests/dynamic_python_build_pool.rs
Normal file
127
tests/dynamic_python_build_pool.rs
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
//! Phase 23 / Track O.1 micro-benchmark for the Python build pool.
|
||||
//!
|
||||
//! Asserts the warm-cache hot path (a `prepare_python` cache hit backed by the
|
||||
//! shared venv + `compileall` bytecode warm) stays ≤ 200ms, the interpreted
|
||||
//! budget. Skips when `python3` is not runnable.
|
||||
|
||||
#![cfg(feature = "dynamic")]
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use nyx_scanner::dynamic::build_sandbox::prepare_python;
|
||||
use nyx_scanner::dynamic::spec::{
|
||||
EntryKind, HarnessSpec, JavaToolchain, PayloadSlot, SpecDerivationStrategy,
|
||||
};
|
||||
use nyx_scanner::labels::Cap;
|
||||
use nyx_scanner::symbol::Lang;
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
struct CacheGuard {
|
||||
_lock: MutexGuard<'static, ()>,
|
||||
prior_cache: Option<String>,
|
||||
prior_pool: Option<String>,
|
||||
_cache: tempfile::TempDir,
|
||||
_pool: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl CacheGuard {
|
||||
fn isolated() -> Self {
|
||||
let lock = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let cache = tempfile::TempDir::new().unwrap();
|
||||
let pool = tempfile::TempDir::new().unwrap();
|
||||
let prior_cache = std::env::var("NYX_BUILD_CACHE").ok();
|
||||
let prior_pool = std::env::var("NYX_BUILD_POOL_DIR").ok();
|
||||
unsafe {
|
||||
std::env::set_var("NYX_BUILD_CACHE", cache.path());
|
||||
std::env::set_var("NYX_BUILD_POOL_DIR", pool.path());
|
||||
}
|
||||
Self {
|
||||
_lock: lock,
|
||||
prior_cache,
|
||||
prior_pool,
|
||||
_cache: cache,
|
||||
_pool: pool,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CacheGuard {
|
||||
fn drop(&mut self) {
|
||||
restore("NYX_BUILD_CACHE", self.prior_cache.take());
|
||||
restore("NYX_BUILD_POOL_DIR", self.prior_pool.take());
|
||||
}
|
||||
}
|
||||
|
||||
fn restore(key: &str, prior: Option<String>) {
|
||||
match prior {
|
||||
Some(v) => unsafe { std::env::set_var(key, v) },
|
||||
None => unsafe { std::env::remove_var(key) },
|
||||
}
|
||||
}
|
||||
|
||||
fn median(mut ds: Vec<Duration>) -> Duration {
|
||||
ds.sort();
|
||||
ds[ds.len() / 2]
|
||||
}
|
||||
|
||||
fn mk_spec() -> HarnessSpec {
|
||||
HarnessSpec {
|
||||
finding_id: "bench".to_owned(),
|
||||
entry_file: "entry".to_owned(),
|
||||
entry_name: "main".to_owned(),
|
||||
entry_kind: EntryKind::Function,
|
||||
lang: Lang::Python,
|
||||
toolchain_id: "bench-python".to_owned(),
|
||||
payload_slot: PayloadSlot::Param(0),
|
||||
expected_cap: Cap::CODE_EXEC,
|
||||
constraint_hints: vec![],
|
||||
sink_file: "sink".to_owned(),
|
||||
sink_line: 1,
|
||||
spec_hash: "0000000000000000".to_owned(),
|
||||
derivation: SpecDerivationStrategy::FromFlowSteps,
|
||||
stubs_required: vec![],
|
||||
framework: None,
|
||||
java_toolchain: JavaToolchain::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_project(workdir: &Path) {
|
||||
// Empty requirements: venv creation succeeds offline; the cached
|
||||
// `pyvenv.cfg` turns every later call into a cache hit.
|
||||
std::fs::write(workdir.join("requirements.txt"), "").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "real-toolchain perf bench: spawns `python -m venv` + pip. Opt-in so the default suite stays hermetic + fast. Run: cargo nextest run --features dynamic --run-ignored ignored-only -E 'binary(~build_pool) | binary(~compile_pool)'"]
|
||||
fn warm_prepare_p50_under_200ms() {
|
||||
let _guard = CacheGuard::isolated();
|
||||
let spec = mk_spec();
|
||||
let work = tempfile::TempDir::new().unwrap();
|
||||
write_project(work.path());
|
||||
|
||||
match prepare_python(&spec, work.path()) {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
eprintln!("skipping python build-pool bench: {e:?}");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let mut hot = Vec::new();
|
||||
for _ in 0..5 {
|
||||
let start = Instant::now();
|
||||
let r = prepare_python(&spec, work.path()).expect("warm prepare must succeed");
|
||||
hot.push(start.elapsed());
|
||||
assert!(r.cache_hit, "warm prepare_python must be a cache hit");
|
||||
}
|
||||
|
||||
let p50 = median(hot);
|
||||
eprintln!("python build-pool warm P50: {p50:?}");
|
||||
assert!(
|
||||
p50 <= Duration::from_millis(200),
|
||||
"python warm-prepare P50 {p50:?} exceeds the 200ms interpreted budget",
|
||||
);
|
||||
}
|
||||
115
tests/dynamic_ruby_build_pool.rs
Normal file
115
tests/dynamic_ruby_build_pool.rs
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
//! Phase 23 / Track O.1 micro-benchmark for the Ruby build pool.
|
||||
//!
|
||||
//! Asserts the `prepare_ruby` hot path stays ≤ 200ms, the interpreted budget.
|
||||
//!
|
||||
//! A warm Bootsnap/Bundler cache hit needs real gems, which means a network
|
||||
//! fetch — flaky offline. The deterministic, offline-safe hot path is the
|
||||
//! no-`Gemfile` cheap leg `prepare_ruby` takes for gem-free projects, which is
|
||||
//! the path actually exercised most in a scan. We benchmark that.
|
||||
|
||||
#![cfg(feature = "dynamic")]
|
||||
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use nyx_scanner::dynamic::build_sandbox::prepare_ruby;
|
||||
use nyx_scanner::dynamic::spec::{
|
||||
EntryKind, HarnessSpec, JavaToolchain, PayloadSlot, SpecDerivationStrategy,
|
||||
};
|
||||
use nyx_scanner::labels::Cap;
|
||||
use nyx_scanner::symbol::Lang;
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
struct CacheGuard {
|
||||
_lock: MutexGuard<'static, ()>,
|
||||
prior_cache: Option<String>,
|
||||
prior_pool: Option<String>,
|
||||
_cache: tempfile::TempDir,
|
||||
_pool: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl CacheGuard {
|
||||
fn isolated() -> Self {
|
||||
let lock = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let cache = tempfile::TempDir::new().unwrap();
|
||||
let pool = tempfile::TempDir::new().unwrap();
|
||||
let prior_cache = std::env::var("NYX_BUILD_CACHE").ok();
|
||||
let prior_pool = std::env::var("NYX_BUILD_POOL_DIR").ok();
|
||||
unsafe {
|
||||
std::env::set_var("NYX_BUILD_CACHE", cache.path());
|
||||
std::env::set_var("NYX_BUILD_POOL_DIR", pool.path());
|
||||
}
|
||||
Self {
|
||||
_lock: lock,
|
||||
prior_cache,
|
||||
prior_pool,
|
||||
_cache: cache,
|
||||
_pool: pool,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CacheGuard {
|
||||
fn drop(&mut self) {
|
||||
restore("NYX_BUILD_CACHE", self.prior_cache.take());
|
||||
restore("NYX_BUILD_POOL_DIR", self.prior_pool.take());
|
||||
}
|
||||
}
|
||||
|
||||
fn restore(key: &str, prior: Option<String>) {
|
||||
match prior {
|
||||
Some(v) => unsafe { std::env::set_var(key, v) },
|
||||
None => unsafe { std::env::remove_var(key) },
|
||||
}
|
||||
}
|
||||
|
||||
fn median(mut ds: Vec<Duration>) -> Duration {
|
||||
ds.sort();
|
||||
ds[ds.len() / 2]
|
||||
}
|
||||
|
||||
fn mk_spec() -> HarnessSpec {
|
||||
HarnessSpec {
|
||||
finding_id: "bench".to_owned(),
|
||||
entry_file: "entry".to_owned(),
|
||||
entry_name: "main".to_owned(),
|
||||
entry_kind: EntryKind::Function,
|
||||
lang: Lang::Ruby,
|
||||
toolchain_id: "bench-ruby".to_owned(),
|
||||
payload_slot: PayloadSlot::Param(0),
|
||||
expected_cap: Cap::CODE_EXEC,
|
||||
constraint_hints: vec![],
|
||||
sink_file: "sink".to_owned(),
|
||||
sink_line: 1,
|
||||
spec_hash: "0000000000000000".to_owned(),
|
||||
derivation: SpecDerivationStrategy::FromFlowSteps,
|
||||
stubs_required: vec![],
|
||||
framework: None,
|
||||
java_toolchain: JavaToolchain::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "real-toolchain perf bench: spawns `bundle`. Opt-in so the default suite stays hermetic + fast. Run: cargo nextest run --features dynamic --run-ignored ignored-only -E 'binary(~build_pool) | binary(~compile_pool)'"]
|
||||
fn warm_prepare_p50_under_200ms() {
|
||||
let _guard = CacheGuard::isolated();
|
||||
let spec = mk_spec();
|
||||
let work = tempfile::TempDir::new().unwrap();
|
||||
|
||||
prepare_ruby(&spec, work.path()).expect("gem-free prepare_ruby must succeed");
|
||||
|
||||
let mut hot = Vec::new();
|
||||
for _ in 0..5 {
|
||||
let start = Instant::now();
|
||||
prepare_ruby(&spec, work.path()).expect("prepare_ruby must succeed");
|
||||
hot.push(start.elapsed());
|
||||
}
|
||||
|
||||
let p50 = median(hot);
|
||||
eprintln!("ruby build-pool warm P50: {p50:?}");
|
||||
assert!(
|
||||
p50 <= Duration::from_millis(200),
|
||||
"ruby prepare P50 {p50:?} exceeds the 200ms interpreted budget",
|
||||
);
|
||||
}
|
||||
100
tests/dynamic_rust_build_pool.rs
Normal file
100
tests/dynamic_rust_build_pool.rs
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
//! Phase 23 / Track O.1 micro-benchmark for the Rust build pool.
|
||||
//!
|
||||
//! Asserts the hot-build P50 (a warm incremental rebuild through the shared
|
||||
//! `CARGO_TARGET_DIR`) stays ≤ 1s, the compiled-language budget. Skips when
|
||||
//! `cargo` is not runnable so a toolchain-less CI image keeps the gate green.
|
||||
|
||||
#![cfg(feature = "dynamic")]
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use nyx_scanner::dynamic::build_pool::BuildPool;
|
||||
use nyx_scanner::dynamic::build_pool::rust::RustPool;
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
struct PoolDirGuard {
|
||||
_lock: MutexGuard<'static, ()>,
|
||||
prior: Option<String>,
|
||||
_dir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl PoolDirGuard {
|
||||
fn isolated() -> Self {
|
||||
let lock = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let prior = std::env::var("NYX_BUILD_POOL_DIR").ok();
|
||||
unsafe { std::env::set_var("NYX_BUILD_POOL_DIR", dir.path()) };
|
||||
Self {
|
||||
_lock: lock,
|
||||
prior,
|
||||
_dir: dir,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PoolDirGuard {
|
||||
fn drop(&mut self) {
|
||||
match self.prior.take() {
|
||||
Some(v) => unsafe { std::env::set_var("NYX_BUILD_POOL_DIR", v) },
|
||||
None => unsafe { std::env::remove_var("NYX_BUILD_POOL_DIR") },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn median(mut ds: Vec<Duration>) -> Duration {
|
||||
ds.sort();
|
||||
ds[ds.len() / 2]
|
||||
}
|
||||
|
||||
fn write_project(workdir: &Path) {
|
||||
std::fs::write(
|
||||
workdir.join("Cargo.toml"),
|
||||
"[package]\nname = \"nyx_harness\"\nversion = \"0.0.0\"\nedition = \"2021\"\n\n\
|
||||
[[bin]]\nname = \"nyx_harness\"\npath = \"src/main.rs\"\n",
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::create_dir_all(workdir.join("src")).unwrap();
|
||||
std::fs::write(workdir.join("src/main.rs"), "fn main() {}\n").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "real-toolchain perf bench: spawns `cargo build --release`. Opt-in so the default suite stays hermetic + fast. Run: cargo nextest run --features dynamic --run-ignored ignored-only -E 'binary(~build_pool) | binary(~compile_pool)'"]
|
||||
fn hot_rebuild_p50_under_one_second() {
|
||||
let _guard = PoolDirGuard::isolated();
|
||||
let pool = match RustPool::try_new() {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
eprintln!("skipping rust build-pool bench: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let work = tempfile::TempDir::new().unwrap();
|
||||
write_project(work.path());
|
||||
let dest = work.path().join("nyx_harness_out");
|
||||
let args = [dest.to_string_lossy().into_owned()];
|
||||
|
||||
// Cold build warms the shared target dir; not measured.
|
||||
let cold = pool.compile_batch(work.path(), &args);
|
||||
assert!(cold.success, "cold build must succeed: {}", cold.stderr);
|
||||
assert!(dest.exists(), "cold build must emit the binary");
|
||||
|
||||
let mut hot = Vec::new();
|
||||
for _ in 0..5 {
|
||||
let _ = std::fs::remove_file(&dest);
|
||||
let start = Instant::now();
|
||||
let r = pool.compile_batch(work.path(), &args);
|
||||
hot.push(start.elapsed());
|
||||
assert!(r.success, "hot build must succeed: {}", r.stderr);
|
||||
}
|
||||
|
||||
let p50 = median(hot);
|
||||
eprintln!("rust build-pool hot P50: {p50:?}");
|
||||
assert!(
|
||||
p50 <= Duration::from_secs(1),
|
||||
"rust hot-build P50 {p50:?} exceeds the 1s compiled budget",
|
||||
);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue