refactor(dynamic): introduce build pools for Python, C, C++, Go, Ruby, PHP, and Node.js with shared caching and warming improvements; enhance test coverage with micro-benchmarks

This commit is contained in:
elipeter 2026-05-29 10:23:49 -05:00
parent 3d710c856d
commit bd76cd5b9d
20 changed files with 2123 additions and 23 deletions

113
src/dynamic/build_pool/c.rs Normal file
View file

@ -0,0 +1,113 @@
//! C build pool (Phase 23 / Track O.1).
//!
//! Wraps the C compiler in `ccache` (when present) backed by a shared object
//! cache under the pool cache root, so a finding that recompiles a harness
//! whose `main.c` matches a previously-built one gets a cache hit instead of a
//! cold `cc` invocation.
//!
//! `ccache` degrades gracefully: when it is not on `PATH` the pool runs the
//! bare compiler, byte-for-byte the same `cc` invocation the legacy
//! [`crate::dynamic::build_sandbox::prepare_c`] path uses, so success / failure
//! parity holds. The static-link fallback (drop `-static` and retry) mirrors
//! the legacy `run_cc` behaviour for chroot-bound Strict-profile harnesses.
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
use std::path::Path;
use std::time::Instant;
pub struct CPool {
cc_bin: String,
ccache_bin: Option<String>,
}
impl CPool {
pub fn try_new() -> Result<Self, String> {
let cc_bin = std::env::var("NYX_CC_BIN").unwrap_or_else(|_| "cc".to_owned());
if !binary_runnable(&cc_bin, "--version") {
return Err(format!("c-pool: {cc_bin} not runnable"));
}
Ok(CPool {
cc_bin,
ccache_bin: super::detect_ccache(),
})
}
}
impl BuildPool for CPool {
fn name(&self) -> &'static str {
"c"
}
/// `args[0]` = binary destination, `args[1]` = `"static"` or `"dynamic"`.
fn compile_batch(&self, workdir: &Path, args: &[String]) -> PoolCompileResult {
let start = Instant::now();
let dest = match args.first() {
Some(d) => d.clone(),
None => {
return PoolCompileResult {
success: false,
stderr: "c-pool: missing binary destination arg".to_owned(),
duration: start.elapsed(),
};
}
};
let static_link = args.get(1).map(|s| s == "static").unwrap_or(false);
if static_link {
match self.run(workdir, &dest, &["-static", "-O0", "-g"]) {
Ok(()) => {
return PoolCompileResult {
success: true,
stderr: String::new(),
duration: start.elapsed(),
};
}
Err(stderr) => {
unsafe { std::env::set_var("NYX_BUILD_STATIC_FALLBACK", "1") };
eprintln!("nyx: c-pool cc -static failed, retrying without -static: {stderr}");
let _ = std::fs::remove_file(&dest);
}
}
}
match self.run(workdir, &dest, &["-O0", "-g"]) {
Ok(()) => PoolCompileResult {
success: true,
stderr: String::new(),
duration: start.elapsed(),
},
Err(stderr) => PoolCompileResult {
success: false,
stderr,
duration: start.elapsed(),
},
}
}
fn is_healthy(&self) -> bool {
binary_runnable(&self.cc_bin, "--version")
}
}
impl CPool {
/// Run one compile of `main.c`, optionally fronted by `ccache`.
fn run(&self, workdir: &Path, dest: &str, leading_flags: &[&str]) -> Result<(), String> {
let mut cmd = match (&self.ccache_bin, pool_cache_dir("c", "ccache")) {
(Some(ccache), Some(cache_dir)) => {
let mut c = base_command(ccache);
c.arg(&self.cc_bin).env("CCACHE_DIR", cache_dir);
c
}
_ => base_command(&self.cc_bin),
};
cmd.args(leading_flags)
.args(["-o", dest, "main.c"])
.current_dir(workdir);
let output = cmd.output().map_err(|e| format!("c-pool: cc: {e}"))?;
if !output.status.success() {
return Err(String::from_utf8_lossy(&output.stderr).into_owned());
}
Ok(())
}
}

View file

@ -0,0 +1,83 @@
//! C++ build pool (Phase 23 / Track O.1).
//!
//! Same shape as the C pool: front the C++ driver with `ccache` backed by a
//! shared object cache under the pool cache root. Falls back to a bare
//! `c++ -std=c++17` compile — byte-for-byte the legacy
//! [`crate::dynamic::build_sandbox::prepare_cpp`] invocation — when `ccache` is
//! absent.
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
use std::path::Path;
use std::time::Instant;
pub struct CppPool {
cxx_bin: String,
ccache_bin: Option<String>,
}
impl CppPool {
pub fn try_new() -> Result<Self, String> {
let cxx_bin = std::env::var("NYX_CXX_BIN").unwrap_or_else(|_| "c++".to_owned());
if !binary_runnable(&cxx_bin, "--version") {
return Err(format!("cpp-pool: {cxx_bin} not runnable"));
}
Ok(CppPool {
cxx_bin,
ccache_bin: super::detect_ccache(),
})
}
}
impl BuildPool for CppPool {
fn name(&self) -> &'static str {
"cpp"
}
/// `args[0]` = absolute path the compiled `nyx_harness` binary lands at.
fn compile_batch(&self, workdir: &Path, args: &[String]) -> PoolCompileResult {
let start = Instant::now();
let dest = match args.first() {
Some(d) => d.clone(),
None => {
return PoolCompileResult {
success: false,
stderr: "cpp-pool: missing binary destination arg".to_owned(),
duration: start.elapsed(),
};
}
};
let mut cmd = match (&self.ccache_bin, pool_cache_dir("cpp", "ccache")) {
(Some(ccache), Some(cache_dir)) => {
let mut c = base_command(ccache);
c.arg(&self.cxx_bin).env("CCACHE_DIR", cache_dir);
c
}
_ => base_command(&self.cxx_bin),
};
cmd.args(["-O0", "-g", "-std=c++17", "-o", &dest, "main.cpp"])
.current_dir(workdir);
match cmd.output() {
Ok(o) if o.status.success() => PoolCompileResult {
success: true,
stderr: String::new(),
duration: start.elapsed(),
},
Ok(o) => PoolCompileResult {
success: false,
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
duration: start.elapsed(),
},
Err(e) => PoolCompileResult {
success: false,
stderr: format!("cpp-pool: c++: {e}"),
duration: start.elapsed(),
},
}
}
fn is_healthy(&self) -> bool {
binary_runnable(&self.cxx_bin, "--version")
}
}

View file

@ -0,0 +1,147 @@
//! Go build pool (Phase 23 / Track O.1).
//!
//! The legacy [`crate::dynamic::build_sandbox::prepare_go`] gives each finding
//! its own `GOCACHE`/`GOMODCACHE` (default: a per-workdir `.gocache`), so the
//! Go toolchain recompiles the standard library and every module from cold on
//! every harness.
//!
//! [`GoPool`] mounts one shared `GOCACHE` + `GOMODCACHE` under the pool cache
//! root so compiled std-lib + module artefacts are reused across findings, and
//! builds with `-trimpath -buildvcs=false` so the output is reproducible (no
//! absolute workdir paths or VCS stamping baked in, which otherwise defeats the
//! build cache's keying).
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
use std::path::Path;
use std::time::Instant;
pub struct GoPool {
go_bin: String,
}
impl GoPool {
pub fn try_new() -> Result<Self, String> {
let go_bin = std::env::var("NYX_GO_BIN").unwrap_or_else(|_| "go".to_owned());
if !binary_runnable(&go_bin, "version") {
return Err(format!("go-pool: {go_bin} not runnable"));
}
Ok(GoPool { go_bin })
}
}
impl BuildPool for GoPool {
fn name(&self) -> &'static str {
"go"
}
/// `args[0]` = absolute path the compiled `nyx_harness` binary must land
/// at.
fn compile_batch(&self, workdir: &Path, args: &[String]) -> PoolCompileResult {
let start = Instant::now();
let dest = match args.first() {
Some(d) => d.clone(),
None => {
return PoolCompileResult {
success: false,
stderr: "go-pool: missing binary destination arg".to_owned(),
duration: start.elapsed(),
};
}
};
let go_cache = match pool_cache_dir("go", "cache") {
Some(d) => d,
None => {
return PoolCompileResult {
success: false,
stderr: "go-pool: no shared GOCACHE".to_owned(),
duration: start.elapsed(),
};
}
};
let go_mod_cache = match pool_cache_dir("go", "modcache") {
Some(d) => d,
None => {
return PoolCompileResult {
success: false,
stderr: "go-pool: no shared GOMODCACHE".to_owned(),
duration: start.elapsed(),
};
}
};
let go_path = std::env::var("GOPATH").unwrap_or_else(|_| {
std::env::var("HOME")
.map(|h| format!("{h}/go"))
.unwrap_or_else(|_| "/tmp/go".to_owned())
});
// `go mod tidy` resolves imports into the shared module cache.
if workdir.join("go.mod").exists() {
let tidy = base_command(&self.go_bin)
.args(["mod", "tidy"])
.current_dir(workdir)
.env("GOCACHE", &go_cache)
.env("GOPATH", &go_path)
.env("GOMODCACHE", &go_mod_cache)
.output();
match tidy {
Ok(o) if o.status.success() => {}
Ok(o) => {
let mut msg = String::from_utf8_lossy(&o.stderr).into_owned();
if msg.is_empty() {
msg = String::from_utf8_lossy(&o.stdout).into_owned();
}
return PoolCompileResult {
success: false,
stderr: format!("go mod tidy failed: {msg}"),
duration: start.elapsed(),
};
}
Err(e) => {
return PoolCompileResult {
success: false,
stderr: format!("go-pool: go mod tidy: {e}"),
duration: start.elapsed(),
};
}
}
}
let output = base_command(&self.go_bin)
.args([
"build",
"-trimpath",
"-buildvcs=false",
"-o",
&dest,
".",
])
.current_dir(workdir)
.env("GOCACHE", &go_cache)
.env("GOPATH", &go_path)
.env("GOMODCACHE", &go_mod_cache)
.output();
match output {
Ok(o) if o.status.success() => PoolCompileResult {
success: true,
stderr: String::new(),
duration: start.elapsed(),
},
Ok(o) => PoolCompileResult {
success: false,
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
duration: start.elapsed(),
},
Err(e) => PoolCompileResult {
success: false,
stderr: format!("go-pool: go build: {e}"),
duration: start.elapsed(),
},
}
}
fn is_healthy(&self) -> bool {
binary_runnable(&self.go_bin, "version")
}
}

View file

@ -26,10 +26,19 @@
//! [`crate::dynamic::build_sandbox`] reads `NYX_DYNAMIC_BUILD_POOL` and
//! routes each request to the matching pool when enabled.
use std::path::Path;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Duration;
pub mod c;
pub mod cpp;
pub mod go;
pub mod java;
pub mod node;
pub mod php;
pub mod python;
pub mod ruby;
pub mod rust;
/// Outcome of a single batched compile request.
#[derive(Debug)]
@ -66,13 +75,21 @@ pub trait BuildPool: Send + Sync {
fn is_healthy(&self) -> bool;
}
/// Languages that ship a [`BuildPool`] implementation and are therefore
/// enabled by default. Phase 22 shipped `java`; Phase 23 (Track O.1) adds
/// the remaining eight, so every supported language now has a warm fast path
/// unless an operator opts out via `NYX_DYNAMIC_BUILD_POOL=<lang>=0`.
const POOL_ENABLED_LANGS: &[&str] = &[
"java", "node", "python", "php", "ruby", "go", "rust", "c", "cpp",
];
/// Parse the `NYX_DYNAMIC_BUILD_POOL` env var.
///
/// Format is a comma-separated list of `lang=bit` entries: `java=1,node=0`.
/// A missing language returns the default (currently `true` for `java`,
/// `false` for every other language because no other pool ships yet).
/// A missing language returns the default: `true` for every language that
/// ships a pool (see [`POOL_ENABLED_LANGS`]), `false` otherwise.
pub fn is_pool_enabled(lang: &str) -> bool {
let default = matches!(lang, "java");
let default = POOL_ENABLED_LANGS.contains(&lang);
let raw = match std::env::var("NYX_DYNAMIC_BUILD_POOL") {
Ok(v) => v,
Err(_) => return default,
@ -93,6 +110,64 @@ pub fn is_pool_enabled(lang: &str) -> bool {
default
}
/// Shared root for a pool's persistent caches (sccache dir, shared
/// `GOCACHE`, opcache file-cache, Bootsnap cache, shared venvs, …).
///
/// Honours `NYX_BUILD_POOL_DIR` so tests can redirect the cache into a
/// `TempDir`; otherwise falls back to the platform cache dir, mirroring
/// the javac pool's layout under `dynamic/build-pool/`.
///
/// Returns `None` only when neither the env override nor a platform cache
/// dir is available — callers treat that as "pool unavailable" and fall
/// back to the legacy direct-spawn build path.
pub(crate) fn pool_cache_dir(lang: &str, sub: &str) -> Option<PathBuf> {
let base = if let Ok(custom) = std::env::var("NYX_BUILD_POOL_DIR") {
PathBuf::from(custom)
} else {
directories::ProjectDirs::from("dev", "nyx", "nyx")?
.cache_dir()
.join("dynamic")
.join("build-pool")
};
let dir = base.join(lang).join(sub);
std::fs::create_dir_all(&dir).ok()?;
Some(dir)
}
/// Construct a `Command` for `bin` with a scrubbed environment, matching
/// the isolation envelope every legacy `prepare_*` build uses: `env_clear`
/// plus an inherited `PATH` + `HOME` only. Pools layer their cache env
/// (`CARGO_TARGET_DIR`, `CCACHE_DIR`, `GOCACHE`, …) on top of this.
pub(crate) fn base_command(bin: &str) -> Command {
let mut cmd = Command::new(bin);
cmd.env_clear()
.env("PATH", std::env::var("PATH").unwrap_or_default())
.env("HOME", std::env::var("HOME").unwrap_or_default());
cmd
}
/// Detect a runnable `ccache` binary (honouring `NYX_CCACHE_BIN`). Shared
/// by the C and C++ pools to front their compiler with the shared object
/// cache; `None` means "compile bare", preserving legacy parity.
pub(crate) fn detect_ccache() -> Option<String> {
let bin = std::env::var("NYX_CCACHE_BIN").unwrap_or_else(|_| "ccache".to_owned());
binary_runnable(&bin, "--version").then_some(bin)
}
/// Cheap "is this binary runnable" probe used by every pool's
/// [`BuildPool::is_healthy`] / `try_new`. Runs `bin <probe_arg>` with a
/// scrubbed env and reports whether it exited 0.
pub(crate) fn binary_runnable(bin: &str, probe_arg: &str) -> bool {
base_command(bin)
.arg(probe_arg)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
}
#[cfg(test)]
mod tests {
use super::*;
@ -125,12 +200,23 @@ mod tests {
}
#[test]
fn default_enables_java_only() {
fn default_enables_every_shipped_pool() {
let _l = ENV_LOCK.lock().unwrap();
let _g = EnvGuard::set(None);
assert!(is_pool_enabled("java"));
for lang in POOL_ENABLED_LANGS {
assert!(is_pool_enabled(lang), "{lang} pool must default on");
}
// A language with no pool stays off.
assert!(!is_pool_enabled("cobol"));
}
#[test]
fn explicit_override_disables_node() {
let _l = ENV_LOCK.lock().unwrap();
let _g = EnvGuard::set(Some("node=0"));
assert!(!is_pool_enabled("node"));
assert!(!is_pool_enabled("python"));
// Other languages keep their default-on state.
assert!(is_pool_enabled("python"));
}
#[test]

View file

@ -0,0 +1,87 @@
//! Node.js build pool (Phase 23 / Track O.1).
//!
//! `prepare_node` already snapshots `node_modules` per `package.json` hash.
//! What it lacks is a shared npm download cache: a fresh lock hash re-downloads
//! every tarball from cold.
//!
//! [`NodePool`] points `npm_config_cache` at the shared pool root so package
//! tarballs are reused across lock hashes, collapsing a cold `npm install` to
//! an unpack of already-fetched tarballs. TypeScript harnesses that do not
//! need full type checking are run with `--experimental-strip-types` at
//! execution time (the runner reads [`strip_types_flag`]); the pool itself only
//! owns the install step.
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
use std::path::Path;
use std::time::Instant;
pub struct NodePool {
npm_bin: String,
}
impl NodePool {
pub fn try_new() -> Result<Self, String> {
let npm_bin = std::env::var("NYX_NPM_BIN").unwrap_or_else(|_| "npm".to_owned());
if !binary_runnable(&npm_bin, "--version") {
return Err(format!("node-pool: {npm_bin} not runnable"));
}
Ok(NodePool { npm_bin })
}
}
/// The Node flag that lets a TS harness skip a full `tsc` compile when the
/// spec does not need type checking. Surfaced as a free function so the
/// runner can splice it into the harness exec without holding a pool handle.
pub fn strip_types_flag() -> &'static str {
"--experimental-strip-types"
}
impl BuildPool for NodePool {
fn name(&self) -> &'static str {
"node"
}
/// Install dependencies declared by `workdir/package.json` into
/// `workdir/node_modules`. Args are unused.
fn compile_batch(&self, workdir: &Path, _args: &[String]) -> PoolCompileResult {
let start = Instant::now();
let mut cmd = base_command(&self.npm_bin);
cmd.args(["install", "--no-save", "--no-audit", "--no-fund"])
.current_dir(workdir);
if let Some(cache) = pool_cache_dir("node", "npm-cache") {
cmd.env("npm_config_cache", cache);
}
match cmd.output() {
Ok(o) if o.status.success() => PoolCompileResult {
success: true,
stderr: String::new(),
duration: start.elapsed(),
},
Ok(o) => PoolCompileResult {
success: false,
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
duration: start.elapsed(),
},
Err(e) => PoolCompileResult {
success: false,
stderr: format!("node-pool: npm install: {e}"),
duration: start.elapsed(),
},
}
}
fn is_healthy(&self) -> bool {
binary_runnable(&self.npm_bin, "--version")
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn strip_types_flag_is_the_node_native_ts_flag() {
assert_eq!(strip_types_flag(), "--experimental-strip-types");
}
}

View file

@ -0,0 +1,110 @@
//! PHP build pool (Phase 23 / Track O.1).
//!
//! Two warm caches keyed off the Composer lockfile:
//! - `COMPOSER_CACHE_DIR` points at the shared pool root so package downloads
//! are reused across lock hashes, and
//! - an opcache file-cache directory is pre-warmed so the harness `php`
//! process skips re-parsing the vendored sources on first run.
//!
//! Both degrade gracefully: a missing `composer` makes `try_new` fail and the
//! caller falls back to the legacy
//! [`crate::dynamic::build_sandbox::prepare_php`] path; a missing `php` simply
//! skips the opcache warm (the install still succeeds).
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
use std::path::Path;
use std::time::Instant;
pub struct PhpPool {
composer_bin: String,
}
impl PhpPool {
pub fn try_new() -> Result<Self, String> {
let composer_bin =
std::env::var("NYX_COMPOSER_BIN").unwrap_or_else(|_| "composer".to_owned());
if !binary_runnable(&composer_bin, "--version") {
return Err(format!("php-pool: {composer_bin} not runnable"));
}
Ok(PhpPool { composer_bin })
}
}
impl BuildPool for PhpPool {
fn name(&self) -> &'static str {
"php"
}
/// Install `composer.json` deps into `workdir/vendor` then warm the
/// shared opcache file-cache. Args are unused.
fn compile_batch(&self, workdir: &Path, _args: &[String]) -> PoolCompileResult {
let start = Instant::now();
let mut cmd = base_command(&self.composer_bin);
cmd.args(["install", "--no-interaction", "--no-dev", "--prefer-dist"])
.current_dir(workdir)
.env("COMPOSER_ALLOW_SUPERUSER", "1");
if let Some(cache) = pool_cache_dir("php", "composer-cache") {
cmd.env("COMPOSER_CACHE_DIR", cache);
}
match cmd.output() {
Ok(o) if o.status.success() => {}
Ok(o) => {
return PoolCompileResult {
success: false,
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
duration: start.elapsed(),
};
}
Err(e) => {
return PoolCompileResult {
success: false,
stderr: format!("php-pool: composer install: {e}"),
duration: start.elapsed(),
};
}
}
warm_opcache(workdir);
PoolCompileResult {
success: true,
stderr: String::new(),
duration: start.elapsed(),
}
}
fn is_healthy(&self) -> bool {
binary_runnable(&self.composer_bin, "--version")
}
}
/// Best-effort opcache file-cache pre-warm: compile every vendored `.php`
/// into the shared opcache file-cache so the harness `php` process boots with
/// the bytecode already on disk. A missing `php` or partial failure is
/// swallowed — the install already succeeded and opcache is a pure speed win.
fn warm_opcache(workdir: &Path) {
let vendor = workdir.join("vendor");
if !vendor.exists() {
return;
}
let php = std::env::var("NYX_PHP_BIN").unwrap_or_else(|_| "php".to_owned());
let file_cache = match pool_cache_dir("php", "opcache") {
Some(d) => d,
None => return,
};
let _ = base_command(&php)
.arg("-d")
.arg("opcache.enable_cli=1")
.arg("-d")
.arg(format!("opcache.file_cache={}", file_cache.display()))
.arg("-d")
.arg("opcache.file_cache_only=1")
.arg("-r")
.arg(
"foreach(new RecursiveIteratorIterator(new RecursiveDirectoryIterator('vendor')) \
as $f){ if(substr($f,-4)==='.php'){ @opcache_compile_file($f); } }",
)
.current_dir(workdir)
.output();
}

View file

@ -0,0 +1,122 @@
//! Python build pool (Phase 23 / Track O.1).
//!
//! `prepare_python` already keys its venv on the requirements hash, so the
//! venv itself is the "shared venv per `requirements_hash`". What the legacy
//! path lacks is a warm bytecode cache: the first harness to import a package
//! pays the `.py` -> `.pyc` compile.
//!
//! [`PythonPool`] runs `python -m compileall` over the venv's `site-packages`
//! once at venv-creation time so every later harness import is a `__pycache__`
//! hit. The pip download cache is pointed at the shared pool root so repeated
//! installs across requirements hashes reuse wheels.
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
use std::path::Path;
use std::time::Instant;
pub struct PythonPool;
impl PythonPool {
pub fn try_new(python_bin: &str) -> Result<Self, String> {
if !binary_runnable(python_bin, "--version") {
return Err(format!("python-pool: {python_bin} not runnable"));
}
Ok(PythonPool)
}
}
impl BuildPool for PythonPool {
fn name(&self) -> &'static str {
"python"
}
/// `args[0]` = venv path to create, `args[1]` = python interpreter binary.
fn compile_batch(&self, workdir: &Path, args: &[String]) -> PoolCompileResult {
let start = Instant::now();
let venv_path = match args.first() {
Some(v) => Path::new(v),
None => {
return PoolCompileResult {
success: false,
stderr: "python-pool: missing venv path arg".to_owned(),
duration: start.elapsed(),
};
}
};
let python = args.get(1).map(String::as_str).unwrap_or("python3");
// 1. Create the venv.
let create = base_command(python)
.args(["-m", "venv", "--clear"])
.arg(venv_path)
.status();
match create {
Ok(s) if s.success() => {}
Ok(s) => {
return PoolCompileResult {
success: false,
stderr: format!("venv create failed: exit {s}"),
duration: start.elapsed(),
};
}
Err(e) => {
return PoolCompileResult {
success: false,
stderr: format!("python-pool: venv create: {e}"),
duration: start.elapsed(),
};
}
}
// 2. Install requirements with the shared wheel cache.
let req_path = workdir.join("requirements.txt");
if req_path.exists() {
let pip = venv_path.join("bin").join("pip");
let mut cmd = base_command(&pip.to_string_lossy());
cmd.args(["install", "-r"]).arg(&req_path);
if let Some(cache) = pool_cache_dir("python", "pip-cache") {
cmd.env("PIP_CACHE_DIR", cache);
} else {
cmd.arg("--no-cache-dir");
}
match cmd.output() {
Ok(o) if o.status.success() => {}
Ok(o) => {
return PoolCompileResult {
success: false,
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
duration: start.elapsed(),
};
}
Err(e) => {
return PoolCompileResult {
success: false,
stderr: format!("python-pool: pip install: {e}"),
duration: start.elapsed(),
};
}
}
}
// 3. Warm __pycache__ for the whole venv (best-effort: a partial
// failure to byte-compile one module must not fail the build).
let venv_python = venv_path.join("bin").join("python");
let _ = base_command(&venv_python.to_string_lossy())
.args(["-m", "compileall", "-q"])
.arg(venv_path)
.output();
PoolCompileResult {
success: true,
stderr: String::new(),
duration: start.elapsed(),
}
}
fn is_healthy(&self) -> bool {
// The interpreter is resolved per-request via args; treat the pool as
// always healthy and let an unrunnable interpreter surface as a build
// error, which the dispatcher already falls back from.
true
}
}

View file

@ -0,0 +1,107 @@
//! Ruby build pool (Phase 23 / Track O.1).
//!
//! `prepare_ruby` already vendors gems per `Gemfile.lock` hash. What it lacks
//! is a warm Bootsnap cache: the first harness to `require` a gem pays the
//! load-path scan + compile.
//!
//! [`RubyPool`] points `BOOTSNAP_CACHE_DIR` at the shared pool root and runs
//! `bundle install` with the shared gem cache. Bootsnap then persists its
//! compiled require-cache across findings. Falls back to the legacy path when
//! `bundle` is not runnable.
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
use std::path::Path;
use std::time::Instant;
pub struct RubyPool {
bundle_bin: String,
}
impl RubyPool {
pub fn try_new() -> Result<Self, String> {
let bundle_bin = std::env::var("NYX_BUNDLE_BIN").unwrap_or_else(|_| "bundle".to_owned());
if !binary_runnable(&bundle_bin, "--version") {
return Err(format!("ruby-pool: {bundle_bin} not runnable"));
}
Ok(RubyPool { bundle_bin })
}
fn bundle(&self, workdir: &Path) -> std::process::Command {
let mut cmd = base_command(&self.bundle_bin);
cmd.current_dir(workdir);
if let Some(cache) = pool_cache_dir("ruby", "bootsnap") {
cmd.env("BOOTSNAP_CACHE_DIR", cache);
}
cmd
}
}
impl BuildPool for RubyPool {
fn name(&self) -> &'static str {
"ruby"
}
/// Resolve `Gemfile` deps into `workdir/vendor/bundle`. Args are unused.
fn compile_batch(&self, workdir: &Path, _args: &[String]) -> PoolCompileResult {
let start = Instant::now();
// `bundle check` short-circuits when the host already has every gem.
if let Ok(o) = self.bundle(workdir).arg("check").output() {
if o.status.success() {
return PoolCompileResult {
success: true,
stderr: String::new(),
duration: start.elapsed(),
};
}
}
let config = self
.bundle(workdir)
.args(["config", "set", "--local", "path", "vendor/bundle"])
.output();
match config {
Ok(o) if o.status.success() => {}
Ok(o) => {
return PoolCompileResult {
success: false,
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
duration: start.elapsed(),
};
}
Err(e) => {
return PoolCompileResult {
success: false,
stderr: format!("ruby-pool: bundle config: {e}"),
duration: start.elapsed(),
};
}
}
let install = self
.bundle(workdir)
.args(["install", "--jobs", "4", "--retry", "2"])
.output();
match install {
Ok(o) if o.status.success() => PoolCompileResult {
success: true,
stderr: String::new(),
duration: start.elapsed(),
},
Ok(o) => PoolCompileResult {
success: false,
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
duration: start.elapsed(),
},
Err(e) => PoolCompileResult {
success: false,
stderr: format!("ruby-pool: bundle install: {e}"),
duration: start.elapsed(),
},
}
}
fn is_healthy(&self) -> bool {
binary_runnable(&self.bundle_bin, "--version")
}
}

View file

@ -0,0 +1,190 @@
//! Rust build pool (Phase 23 / Track O.1).
//!
//! The legacy [`crate::dynamic::build_sandbox::prepare_rust`] runs a fresh
//! `cargo build --release` per finding with a per-workdir `target/`. Every
//! harness therefore recompiles the (identical) harness scaffold and all of
//! its dependencies from cold.
//!
//! [`RustPool`] keeps two warm caches keyed on the `Cargo.lock` hash:
//! - a shared `CARGO_TARGET_DIR` so incremental artefacts survive across
//! per-finding workdirs, and
//! - `sccache` as `RUSTC_WRAPPER` when it is on `PATH`, which caches the
//! per-crate `rustc` invocations across *different* lock hashes too.
//!
//! Both degrade gracefully: a missing `sccache` simply drops the wrapper and
//! a fresh lock hash gets a fresh (empty) shared target dir. The compile
//! itself is byte-for-byte the same `cargo build --release` the legacy path
//! runs, so success / failure parity holds.
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
use blake3::Hasher;
use std::path::Path;
use std::time::Instant;
pub struct RustPool {
cargo_bin: String,
/// `Some(path)` when an `sccache` binary is runnable. Wired in as
/// `RUSTC_WRAPPER`; `None` falls back to plain `rustc`.
sccache_bin: Option<String>,
}
impl RustPool {
pub fn try_new() -> Result<Self, String> {
let cargo_bin = std::env::var("NYX_CARGO_BIN").unwrap_or_else(|_| "cargo".to_owned());
if !binary_runnable(&cargo_bin, "--version") {
return Err(format!("rust-pool: {cargo_bin} not runnable"));
}
let sccache_bin = detect_sccache();
Ok(RustPool {
cargo_bin,
sccache_bin,
})
}
}
fn detect_sccache() -> Option<String> {
let bin = std::env::var("NYX_SCCACHE_BIN").unwrap_or_else(|_| "sccache".to_owned());
binary_runnable(&bin, "--version").then_some(bin)
}
impl BuildPool for RustPool {
fn name(&self) -> &'static str {
"rust"
}
/// `args[0]` = absolute path the compiled `nyx_harness` binary must land
/// at (the caller's cache slot).
fn compile_batch(&self, workdir: &Path, args: &[String]) -> PoolCompileResult {
let start = Instant::now();
let dest = match args.first() {
Some(d) => Path::new(d),
None => {
return PoolCompileResult {
success: false,
stderr: "rust-pool: missing binary destination arg".to_owned(),
duration: start.elapsed(),
};
}
};
let lock_hash = hash_files(workdir, &["Cargo.lock", "Cargo.toml"]);
let target_dir = match pool_cache_dir("rust", &lock_hash) {
Some(d) => d,
None => {
return PoolCompileResult {
success: false,
stderr: "rust-pool: no shared target dir".to_owned(),
duration: start.elapsed(),
};
}
};
let mut cmd = base_command(&self.cargo_bin);
cmd.args(["build", "--release"])
.current_dir(workdir)
.env(
"CARGO_HOME",
std::env::var("CARGO_HOME")
.unwrap_or_else(|_| default_cargo_home()),
)
.env(
"RUSTUP_HOME",
std::env::var("RUSTUP_HOME").unwrap_or_default(),
)
.env("CARGO_TARGET_DIR", &target_dir);
if let Some(sccache) = &self.sccache_bin {
cmd.env("RUSTC_WRAPPER", sccache);
}
let output = match cmd.output() {
Ok(o) => o,
Err(e) => {
return PoolCompileResult {
success: false,
stderr: format!("rust-pool: cargo build: {e}"),
duration: start.elapsed(),
};
}
};
if !output.status.success() {
return PoolCompileResult {
success: false,
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
duration: start.elapsed(),
};
}
let compiled = target_dir.join("release").join("nyx_harness");
if let Err(e) = std::fs::copy(&compiled, dest) {
return PoolCompileResult {
success: false,
stderr: format!(
"rust-pool: cargo build ok but copy {} -> {} failed: {e}",
compiled.display(),
dest.display(),
),
duration: start.elapsed(),
};
}
PoolCompileResult {
success: true,
stderr: String::new(),
duration: start.elapsed(),
}
}
fn is_healthy(&self) -> bool {
binary_runnable(&self.cargo_bin, "--version")
}
}
fn default_cargo_home() -> String {
std::env::var("HOME")
.map(|h| format!("{h}/.cargo"))
.unwrap_or_else(|_| ".cargo".to_owned())
}
/// Stable short hash of the named manifest files under `workdir`.
fn hash_files(workdir: &Path, files: &[&str]) -> String {
let mut h = Hasher::new();
for fname in files {
if let Ok(content) = std::fs::read(workdir.join(fname)) {
h.update(fname.as_bytes());
h.update(&content);
}
}
let out = h.finalize();
format!(
"{:016x}",
u64::from_le_bytes(out.as_bytes()[..8].try_into().unwrap())
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hash_is_deterministic_and_content_sensitive() {
let dir = tempfile::TempDir::new().unwrap();
let h1 = hash_files(dir.path(), &["Cargo.lock"]);
let h2 = hash_files(dir.path(), &["Cargo.lock"]);
assert_eq!(h1, h2);
std::fs::write(dir.path().join("Cargo.lock"), b"[[package]]\n").unwrap();
let h3 = hash_files(dir.path(), &["Cargo.lock"]);
assert_ne!(h1, h3);
}
#[test]
fn missing_dest_arg_is_an_error_not_a_panic() {
let dir = tempfile::TempDir::new().unwrap();
// Construct without a toolchain probe so the test runs JDK/cargo-free.
let pool = RustPool {
cargo_bin: "cargo".to_owned(),
sccache_bin: None,
};
let r = pool.compile_batch(dir.path(), &[]);
assert!(!r.success);
assert!(r.stderr.contains("missing binary destination"));
}
}

View file

@ -12,7 +12,15 @@
//! Failed-build retry policy (§12 Q4): one retry on `BuildFailed` with
//! backoff (1s, 4s), then `Inconclusive(BuildFailed, attempts: 2)`.
use crate::dynamic::build_pool::c::CPool;
use crate::dynamic::build_pool::cpp::CppPool;
use crate::dynamic::build_pool::go::GoPool;
use crate::dynamic::build_pool::java::JavacPool;
use crate::dynamic::build_pool::node::NodePool;
use crate::dynamic::build_pool::php::PhpPool;
use crate::dynamic::build_pool::python::PythonPool;
use crate::dynamic::build_pool::ruby::RubyPool;
use crate::dynamic::build_pool::rust::RustPool;
use crate::dynamic::build_pool::{BuildPool, is_pool_enabled};
use crate::dynamic::sandbox::ProcessHardeningProfile;
use crate::dynamic::spec::HarnessSpec;
@ -65,7 +73,7 @@ pub fn prepare_rust(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
let _ = std::fs::remove_dir_all(&cache_path);
std::fs::create_dir_all(&cache_path)?;
match try_build_rust_binary(workdir, &binary) {
match build_rust_binary(workdir, &binary) {
Ok(()) => {
return Ok(BuildResult {
venv_path: cache_path,
@ -86,6 +94,27 @@ pub fn prepare_rust(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
})
}
/// Route the Rust harness build through [`RustPool`] when the pool is
/// enabled, falling back to the legacy direct-spawn `cargo build` on a
/// missing toolchain or a crashed pool. A genuine compile error from a
/// healthy pool is surfaced verbatim (no legacy re-run — it would fail the
/// same way).
fn build_rust_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
if is_pool_enabled("rust") {
if let Ok(pool) = RustPool::try_new() {
let pool_args = [binary_dest.to_string_lossy().into_owned()];
let res = pool.compile_batch(workdir, &pool_args);
if res.success {
return Ok(());
}
if pool.is_healthy() {
return Err(res.stderr);
}
}
}
try_build_rust_binary(workdir, binary_dest)
}
fn try_build_rust_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
let cargo = cargo_binary();
@ -242,7 +271,7 @@ pub fn prepare_python(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult,
}
let start = Instant::now();
match try_build_venv(&cache_path, workdir, spec) {
match build_venv(&cache_path, workdir, spec) {
Ok(()) => {
return Ok(BuildResult {
venv_path: cache_path,
@ -264,6 +293,25 @@ pub fn prepare_python(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult,
})
}
/// Route the Python venv build through [`PythonPool`] (shared wheel cache +
/// `compileall` bytecode warm) when enabled, else the legacy path.
fn build_venv(venv_path: &Path, workdir: &Path, spec: &HarnessSpec) -> Result<(), String> {
if is_pool_enabled("python") {
let python = python_binary(spec);
if let Ok(pool) = PythonPool::try_new(&python) {
let pool_args = [venv_path.to_string_lossy().into_owned(), python.clone()];
let res = pool.compile_batch(workdir, &pool_args);
if res.success {
return Ok(());
}
if pool.is_healthy() {
return Err(res.stderr);
}
}
}
try_build_venv(venv_path, workdir, spec)
}
fn try_build_venv(venv_path: &Path, workdir: &Path, spec: &HarnessSpec) -> Result<(), String> {
// Find python binary.
let python = python_binary(spec);
@ -421,7 +469,7 @@ pub fn prepare_ruby(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
if attempt > 0 {
std::thread::sleep(Duration::from_secs(BACKOFF[attempt as usize - 1]));
}
match try_bundle_install(workdir) {
match bundle_install(workdir) {
Ok(()) => {
if let Some(cache_path) = &cache_path {
persist_ruby_bundle(workdir, cache_path);
@ -445,6 +493,23 @@ pub fn prepare_ruby(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
})
}
/// Route Bundler through [`RubyPool`] (shared Bootsnap cache) when enabled,
/// else the legacy `bundle check`/`install` path.
fn bundle_install(workdir: &Path) -> Result<(), String> {
if is_pool_enabled("ruby") {
if let Ok(pool) = RubyPool::try_new() {
let res = pool.compile_batch(workdir, &[]);
if res.success {
return Ok(());
}
if pool.is_healthy() {
return Err(res.stderr);
}
}
}
try_bundle_install(workdir)
}
fn try_bundle_install(workdir: &Path) -> Result<(), String> {
let bundle = std::env::var("NYX_BUNDLE_BIN").unwrap_or_else(|_| "bundle".to_owned());
if bundle_check(&bundle, workdir)? {
@ -572,7 +637,7 @@ pub fn prepare_node(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
BACKOFF[attempt as usize - 1],
));
}
match try_npm_install(workdir) {
match npm_install(workdir) {
Ok(()) => {
// Persist node_modules to cache so future runs with the same
// package.json but a fresh workdir can restore without re-running npm.
@ -599,6 +664,23 @@ pub fn prepare_node(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
})
}
/// Route `npm install` through [`NodePool`] (shared npm download cache) when
/// enabled, else the legacy direct-spawn path.
fn npm_install(workdir: &Path) -> Result<(), String> {
if is_pool_enabled("node") {
if let Ok(pool) = NodePool::try_new() {
let res = pool.compile_batch(workdir, &[]);
if res.success {
return Ok(());
}
if pool.is_healthy() {
return Err(res.stderr);
}
}
}
try_npm_install(workdir)
}
fn try_npm_install(workdir: &Path) -> Result<(), String> {
let npm = std::env::var("NYX_NPM_BIN").unwrap_or_else(|_| "npm".to_owned());
let output = Command::new(&npm)
@ -692,7 +774,7 @@ pub fn prepare_go(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bui
let _ = std::fs::remove_dir_all(&cache_path);
std::fs::create_dir_all(&cache_path)?;
match try_build_go_binary(workdir, &binary) {
match build_go_binary(workdir, &binary) {
Ok(()) => {
return Ok(BuildResult {
venv_path: cache_path,
@ -713,6 +795,25 @@ pub fn prepare_go(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bui
})
}
/// Route the Go harness build through [`GoPool`] (shared `GOCACHE` /
/// `GOMODCACHE`, `-trimpath -buildvcs=false`) when enabled, else the legacy
/// per-workdir-cache path.
fn build_go_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
if is_pool_enabled("go") {
if let Ok(pool) = GoPool::try_new() {
let pool_args = [binary_dest.to_string_lossy().into_owned()];
let res = pool.compile_batch(workdir, &pool_args);
if res.success {
return Ok(());
}
if pool.is_healthy() {
return Err(res.stderr);
}
}
}
try_build_go_binary(workdir, binary_dest)
}
fn try_build_go_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
let go_bin = std::env::var("NYX_GO_BIN").unwrap_or_else(|_| "go".to_owned());
let go_cache = std::env::var("GOCACHE")
@ -1236,7 +1337,7 @@ pub fn prepare_php(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bu
BACKOFF[attempt as usize - 1],
));
}
match try_composer_install(workdir) {
match composer_install(workdir) {
Ok(()) => {
// Persist vendor/ to cache so future runs with the same
// composer.json but a fresh workdir can restore without re-running composer.
@ -1263,6 +1364,23 @@ pub fn prepare_php(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bu
})
}
/// Route Composer through [`PhpPool`] (shared download cache + opcache
/// file-cache warm) when enabled, else the legacy direct-spawn path.
fn composer_install(workdir: &Path) -> Result<(), String> {
if is_pool_enabled("php") {
if let Ok(pool) = PhpPool::try_new() {
let res = pool.compile_batch(workdir, &[]);
if res.success {
return Ok(());
}
if pool.is_healthy() {
return Err(res.stderr);
}
}
}
try_composer_install(workdir)
}
fn try_composer_install(workdir: &Path) -> Result<(), String> {
let composer = std::env::var("NYX_COMPOSER_BIN").unwrap_or_else(|_| "composer".to_owned());
let output = Command::new(&composer)
@ -1337,7 +1455,7 @@ pub fn prepare_c(
let _ = std::fs::remove_dir_all(&cache_path);
std::fs::create_dir_all(&cache_path)?;
match try_build_c_binary(workdir, &binary, static_link) {
match build_c_binary(workdir, &binary, static_link) {
Ok(()) => {
return Ok(BuildResult {
venv_path: cache_path,
@ -1358,6 +1476,29 @@ pub fn prepare_c(
})
}
/// Route the C harness build through [`CPool`] (`ccache` + shared object
/// cache) when enabled, else the legacy direct-spawn `cc` path. The
/// static-link toggle is forwarded so the pool can reproduce the
/// Strict-profile `-static` fallback.
fn build_c_binary(workdir: &Path, binary_dest: &Path, static_link: bool) -> Result<(), String> {
if is_pool_enabled("c") {
if let Ok(pool) = CPool::try_new() {
let pool_args = [
binary_dest.to_string_lossy().into_owned(),
if static_link { "static" } else { "dynamic" }.to_owned(),
];
let res = pool.compile_batch(workdir, &pool_args);
if res.success {
return Ok(());
}
if pool.is_healthy() {
return Err(res.stderr);
}
}
}
try_build_c_binary(workdir, binary_dest, static_link)
}
fn try_build_c_binary(workdir: &Path, binary_dest: &Path, static_link: bool) -> Result<(), String> {
let cc_bin = std::env::var("NYX_CC_BIN").unwrap_or_else(|_| "cc".to_owned());
@ -1484,7 +1625,7 @@ pub fn prepare_cpp(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bu
let _ = std::fs::remove_dir_all(&cache_path);
std::fs::create_dir_all(&cache_path)?;
match try_build_cpp_binary(workdir, &binary) {
match build_cpp_binary(workdir, &binary) {
Ok(()) => {
return Ok(BuildResult {
venv_path: cache_path,
@ -1505,6 +1646,24 @@ pub fn prepare_cpp(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bu
})
}
/// Route the C++ harness build through [`CppPool`] (`ccache` + shared object
/// cache) when enabled, else the legacy direct-spawn `c++` path.
fn build_cpp_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
if is_pool_enabled("cpp") {
if let Ok(pool) = CppPool::try_new() {
let pool_args = [binary_dest.to_string_lossy().into_owned()];
let res = pool.compile_batch(workdir, &pool_args);
if res.success {
return Ok(());
}
if pool.is_healthy() {
return Err(res.stderr);
}
}
}
try_build_cpp_binary(workdir, binary_dest)
}
fn try_build_cpp_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
let cxx_bin = std::env::var("NYX_CXX_BIN").unwrap_or_else(|_| {
// Prefer c++ which resolves to the system default compiler driver.