mirror of
https://github.com/elicpeter/nyx.git
synced 2026-06-09 19:45:13 +02:00
refactor(dynamic): introduce build pools for Python, C, C++, Go, Ruby, PHP, and Node.js with shared caching and warming improvements; enhance test coverage with micro-benchmarks
This commit is contained in:
parent
3d710c856d
commit
bd76cd5b9d
20 changed files with 2123 additions and 23 deletions
113
src/dynamic/build_pool/c.rs
Normal file
113
src/dynamic/build_pool/c.rs
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
//! C build pool (Phase 23 / Track O.1).
|
||||
//!
|
||||
//! Wraps the C compiler in `ccache` (when present) backed by a shared object
|
||||
//! cache under the pool cache root, so a finding that recompiles a harness
|
||||
//! whose `main.c` matches a previously-built one gets a cache hit instead of a
|
||||
//! cold `cc` invocation.
|
||||
//!
|
||||
//! `ccache` degrades gracefully: when it is not on `PATH` the pool runs the
|
||||
//! bare compiler, byte-for-byte the same `cc` invocation the legacy
|
||||
//! [`crate::dynamic::build_sandbox::prepare_c`] path uses, so success / failure
|
||||
//! parity holds. The static-link fallback (drop `-static` and retry) mirrors
|
||||
//! the legacy `run_cc` behaviour for chroot-bound Strict-profile harnesses.
|
||||
|
||||
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct CPool {
|
||||
cc_bin: String,
|
||||
ccache_bin: Option<String>,
|
||||
}
|
||||
|
||||
impl CPool {
|
||||
pub fn try_new() -> Result<Self, String> {
|
||||
let cc_bin = std::env::var("NYX_CC_BIN").unwrap_or_else(|_| "cc".to_owned());
|
||||
if !binary_runnable(&cc_bin, "--version") {
|
||||
return Err(format!("c-pool: {cc_bin} not runnable"));
|
||||
}
|
||||
Ok(CPool {
|
||||
cc_bin,
|
||||
ccache_bin: super::detect_ccache(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl BuildPool for CPool {
|
||||
fn name(&self) -> &'static str {
|
||||
"c"
|
||||
}
|
||||
|
||||
/// `args[0]` = binary destination, `args[1]` = `"static"` or `"dynamic"`.
|
||||
fn compile_batch(&self, workdir: &Path, args: &[String]) -> PoolCompileResult {
|
||||
let start = Instant::now();
|
||||
let dest = match args.first() {
|
||||
Some(d) => d.clone(),
|
||||
None => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: "c-pool: missing binary destination arg".to_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
};
|
||||
let static_link = args.get(1).map(|s| s == "static").unwrap_or(false);
|
||||
|
||||
if static_link {
|
||||
match self.run(workdir, &dest, &["-static", "-O0", "-g"]) {
|
||||
Ok(()) => {
|
||||
return PoolCompileResult {
|
||||
success: true,
|
||||
stderr: String::new(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
Err(stderr) => {
|
||||
unsafe { std::env::set_var("NYX_BUILD_STATIC_FALLBACK", "1") };
|
||||
eprintln!("nyx: c-pool cc -static failed, retrying without -static: {stderr}");
|
||||
let _ = std::fs::remove_file(&dest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match self.run(workdir, &dest, &["-O0", "-g"]) {
|
||||
Ok(()) => PoolCompileResult {
|
||||
success: true,
|
||||
stderr: String::new(),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
Err(stderr) => PoolCompileResult {
|
||||
success: false,
|
||||
stderr,
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn is_healthy(&self) -> bool {
|
||||
binary_runnable(&self.cc_bin, "--version")
|
||||
}
|
||||
}
|
||||
|
||||
impl CPool {
|
||||
/// Run one compile of `main.c`, optionally fronted by `ccache`.
|
||||
fn run(&self, workdir: &Path, dest: &str, leading_flags: &[&str]) -> Result<(), String> {
|
||||
let mut cmd = match (&self.ccache_bin, pool_cache_dir("c", "ccache")) {
|
||||
(Some(ccache), Some(cache_dir)) => {
|
||||
let mut c = base_command(ccache);
|
||||
c.arg(&self.cc_bin).env("CCACHE_DIR", cache_dir);
|
||||
c
|
||||
}
|
||||
_ => base_command(&self.cc_bin),
|
||||
};
|
||||
cmd.args(leading_flags)
|
||||
.args(["-o", dest, "main.c"])
|
||||
.current_dir(workdir);
|
||||
|
||||
let output = cmd.output().map_err(|e| format!("c-pool: cc: {e}"))?;
|
||||
if !output.status.success() {
|
||||
return Err(String::from_utf8_lossy(&output.stderr).into_owned());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
83
src/dynamic/build_pool/cpp.rs
Normal file
83
src/dynamic/build_pool/cpp.rs
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
//! C++ build pool (Phase 23 / Track O.1).
|
||||
//!
|
||||
//! Same shape as the C pool: front the C++ driver with `ccache` backed by a
|
||||
//! shared object cache under the pool cache root. Falls back to a bare
|
||||
//! `c++ -std=c++17` compile — byte-for-byte the legacy
|
||||
//! [`crate::dynamic::build_sandbox::prepare_cpp`] invocation — when `ccache` is
|
||||
//! absent.
|
||||
|
||||
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct CppPool {
|
||||
cxx_bin: String,
|
||||
ccache_bin: Option<String>,
|
||||
}
|
||||
|
||||
impl CppPool {
|
||||
pub fn try_new() -> Result<Self, String> {
|
||||
let cxx_bin = std::env::var("NYX_CXX_BIN").unwrap_or_else(|_| "c++".to_owned());
|
||||
if !binary_runnable(&cxx_bin, "--version") {
|
||||
return Err(format!("cpp-pool: {cxx_bin} not runnable"));
|
||||
}
|
||||
Ok(CppPool {
|
||||
cxx_bin,
|
||||
ccache_bin: super::detect_ccache(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl BuildPool for CppPool {
|
||||
fn name(&self) -> &'static str {
|
||||
"cpp"
|
||||
}
|
||||
|
||||
/// `args[0]` = absolute path the compiled `nyx_harness` binary lands at.
|
||||
fn compile_batch(&self, workdir: &Path, args: &[String]) -> PoolCompileResult {
|
||||
let start = Instant::now();
|
||||
let dest = match args.first() {
|
||||
Some(d) => d.clone(),
|
||||
None => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: "cpp-pool: missing binary destination arg".to_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let mut cmd = match (&self.ccache_bin, pool_cache_dir("cpp", "ccache")) {
|
||||
(Some(ccache), Some(cache_dir)) => {
|
||||
let mut c = base_command(ccache);
|
||||
c.arg(&self.cxx_bin).env("CCACHE_DIR", cache_dir);
|
||||
c
|
||||
}
|
||||
_ => base_command(&self.cxx_bin),
|
||||
};
|
||||
cmd.args(["-O0", "-g", "-std=c++17", "-o", &dest, "main.cpp"])
|
||||
.current_dir(workdir);
|
||||
|
||||
match cmd.output() {
|
||||
Ok(o) if o.status.success() => PoolCompileResult {
|
||||
success: true,
|
||||
stderr: String::new(),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
Ok(o) => PoolCompileResult {
|
||||
success: false,
|
||||
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
Err(e) => PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("cpp-pool: c++: {e}"),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn is_healthy(&self) -> bool {
|
||||
binary_runnable(&self.cxx_bin, "--version")
|
||||
}
|
||||
}
|
||||
147
src/dynamic/build_pool/go.rs
Normal file
147
src/dynamic/build_pool/go.rs
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
//! Go build pool (Phase 23 / Track O.1).
|
||||
//!
|
||||
//! The legacy [`crate::dynamic::build_sandbox::prepare_go`] gives each finding
|
||||
//! its own `GOCACHE`/`GOMODCACHE` (default: a per-workdir `.gocache`), so the
|
||||
//! Go toolchain recompiles the standard library and every module from cold on
|
||||
//! every harness.
|
||||
//!
|
||||
//! [`GoPool`] mounts one shared `GOCACHE` + `GOMODCACHE` under the pool cache
|
||||
//! root so compiled std-lib + module artefacts are reused across findings, and
|
||||
//! builds with `-trimpath -buildvcs=false` so the output is reproducible (no
|
||||
//! absolute workdir paths or VCS stamping baked in, which otherwise defeats the
|
||||
//! build cache's keying).
|
||||
|
||||
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct GoPool {
|
||||
go_bin: String,
|
||||
}
|
||||
|
||||
impl GoPool {
|
||||
pub fn try_new() -> Result<Self, String> {
|
||||
let go_bin = std::env::var("NYX_GO_BIN").unwrap_or_else(|_| "go".to_owned());
|
||||
if !binary_runnable(&go_bin, "version") {
|
||||
return Err(format!("go-pool: {go_bin} not runnable"));
|
||||
}
|
||||
Ok(GoPool { go_bin })
|
||||
}
|
||||
}
|
||||
|
||||
impl BuildPool for GoPool {
|
||||
fn name(&self) -> &'static str {
|
||||
"go"
|
||||
}
|
||||
|
||||
/// `args[0]` = absolute path the compiled `nyx_harness` binary must land
|
||||
/// at.
|
||||
fn compile_batch(&self, workdir: &Path, args: &[String]) -> PoolCompileResult {
|
||||
let start = Instant::now();
|
||||
let dest = match args.first() {
|
||||
Some(d) => d.clone(),
|
||||
None => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: "go-pool: missing binary destination arg".to_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let go_cache = match pool_cache_dir("go", "cache") {
|
||||
Some(d) => d,
|
||||
None => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: "go-pool: no shared GOCACHE".to_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
};
|
||||
let go_mod_cache = match pool_cache_dir("go", "modcache") {
|
||||
Some(d) => d,
|
||||
None => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: "go-pool: no shared GOMODCACHE".to_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
};
|
||||
let go_path = std::env::var("GOPATH").unwrap_or_else(|_| {
|
||||
std::env::var("HOME")
|
||||
.map(|h| format!("{h}/go"))
|
||||
.unwrap_or_else(|_| "/tmp/go".to_owned())
|
||||
});
|
||||
|
||||
// `go mod tidy` resolves imports into the shared module cache.
|
||||
if workdir.join("go.mod").exists() {
|
||||
let tidy = base_command(&self.go_bin)
|
||||
.args(["mod", "tidy"])
|
||||
.current_dir(workdir)
|
||||
.env("GOCACHE", &go_cache)
|
||||
.env("GOPATH", &go_path)
|
||||
.env("GOMODCACHE", &go_mod_cache)
|
||||
.output();
|
||||
match tidy {
|
||||
Ok(o) if o.status.success() => {}
|
||||
Ok(o) => {
|
||||
let mut msg = String::from_utf8_lossy(&o.stderr).into_owned();
|
||||
if msg.is_empty() {
|
||||
msg = String::from_utf8_lossy(&o.stdout).into_owned();
|
||||
}
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("go mod tidy failed: {msg}"),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
Err(e) => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("go-pool: go mod tidy: {e}"),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let output = base_command(&self.go_bin)
|
||||
.args([
|
||||
"build",
|
||||
"-trimpath",
|
||||
"-buildvcs=false",
|
||||
"-o",
|
||||
&dest,
|
||||
".",
|
||||
])
|
||||
.current_dir(workdir)
|
||||
.env("GOCACHE", &go_cache)
|
||||
.env("GOPATH", &go_path)
|
||||
.env("GOMODCACHE", &go_mod_cache)
|
||||
.output();
|
||||
|
||||
match output {
|
||||
Ok(o) if o.status.success() => PoolCompileResult {
|
||||
success: true,
|
||||
stderr: String::new(),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
Ok(o) => PoolCompileResult {
|
||||
success: false,
|
||||
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
Err(e) => PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("go-pool: go build: {e}"),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn is_healthy(&self) -> bool {
|
||||
binary_runnable(&self.go_bin, "version")
|
||||
}
|
||||
}
|
||||
|
|
@ -26,10 +26,19 @@
|
|||
//! [`crate::dynamic::build_sandbox`] reads `NYX_DYNAMIC_BUILD_POOL` and
|
||||
//! routes each request to the matching pool when enabled.
|
||||
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::time::Duration;
|
||||
|
||||
pub mod c;
|
||||
pub mod cpp;
|
||||
pub mod go;
|
||||
pub mod java;
|
||||
pub mod node;
|
||||
pub mod php;
|
||||
pub mod python;
|
||||
pub mod ruby;
|
||||
pub mod rust;
|
||||
|
||||
/// Outcome of a single batched compile request.
|
||||
#[derive(Debug)]
|
||||
|
|
@ -66,13 +75,21 @@ pub trait BuildPool: Send + Sync {
|
|||
fn is_healthy(&self) -> bool;
|
||||
}
|
||||
|
||||
/// Languages that ship a [`BuildPool`] implementation and are therefore
|
||||
/// enabled by default. Phase 22 shipped `java`; Phase 23 (Track O.1) adds
|
||||
/// the remaining eight, so every supported language now has a warm fast path
|
||||
/// unless an operator opts out via `NYX_DYNAMIC_BUILD_POOL=<lang>=0`.
|
||||
const POOL_ENABLED_LANGS: &[&str] = &[
|
||||
"java", "node", "python", "php", "ruby", "go", "rust", "c", "cpp",
|
||||
];
|
||||
|
||||
/// Parse the `NYX_DYNAMIC_BUILD_POOL` env var.
|
||||
///
|
||||
/// Format is a comma-separated list of `lang=bit` entries: `java=1,node=0`.
|
||||
/// A missing language returns the default (currently `true` for `java`,
|
||||
/// `false` for every other language because no other pool ships yet).
|
||||
/// A missing language returns the default: `true` for every language that
|
||||
/// ships a pool (see [`POOL_ENABLED_LANGS`]), `false` otherwise.
|
||||
pub fn is_pool_enabled(lang: &str) -> bool {
|
||||
let default = matches!(lang, "java");
|
||||
let default = POOL_ENABLED_LANGS.contains(&lang);
|
||||
let raw = match std::env::var("NYX_DYNAMIC_BUILD_POOL") {
|
||||
Ok(v) => v,
|
||||
Err(_) => return default,
|
||||
|
|
@ -93,6 +110,64 @@ pub fn is_pool_enabled(lang: &str) -> bool {
|
|||
default
|
||||
}
|
||||
|
||||
/// Shared root for a pool's persistent caches (sccache dir, shared
|
||||
/// `GOCACHE`, opcache file-cache, Bootsnap cache, shared venvs, …).
|
||||
///
|
||||
/// Honours `NYX_BUILD_POOL_DIR` so tests can redirect the cache into a
|
||||
/// `TempDir`; otherwise falls back to the platform cache dir, mirroring
|
||||
/// the javac pool's layout under `dynamic/build-pool/`.
|
||||
///
|
||||
/// Returns `None` only when neither the env override nor a platform cache
|
||||
/// dir is available — callers treat that as "pool unavailable" and fall
|
||||
/// back to the legacy direct-spawn build path.
|
||||
pub(crate) fn pool_cache_dir(lang: &str, sub: &str) -> Option<PathBuf> {
|
||||
let base = if let Ok(custom) = std::env::var("NYX_BUILD_POOL_DIR") {
|
||||
PathBuf::from(custom)
|
||||
} else {
|
||||
directories::ProjectDirs::from("dev", "nyx", "nyx")?
|
||||
.cache_dir()
|
||||
.join("dynamic")
|
||||
.join("build-pool")
|
||||
};
|
||||
let dir = base.join(lang).join(sub);
|
||||
std::fs::create_dir_all(&dir).ok()?;
|
||||
Some(dir)
|
||||
}
|
||||
|
||||
/// Construct a `Command` for `bin` with a scrubbed environment, matching
|
||||
/// the isolation envelope every legacy `prepare_*` build uses: `env_clear`
|
||||
/// plus an inherited `PATH` + `HOME` only. Pools layer their cache env
|
||||
/// (`CARGO_TARGET_DIR`, `CCACHE_DIR`, `GOCACHE`, …) on top of this.
|
||||
pub(crate) fn base_command(bin: &str) -> Command {
|
||||
let mut cmd = Command::new(bin);
|
||||
cmd.env_clear()
|
||||
.env("PATH", std::env::var("PATH").unwrap_or_default())
|
||||
.env("HOME", std::env::var("HOME").unwrap_or_default());
|
||||
cmd
|
||||
}
|
||||
|
||||
/// Detect a runnable `ccache` binary (honouring `NYX_CCACHE_BIN`). Shared
|
||||
/// by the C and C++ pools to front their compiler with the shared object
|
||||
/// cache; `None` means "compile bare", preserving legacy parity.
|
||||
pub(crate) fn detect_ccache() -> Option<String> {
|
||||
let bin = std::env::var("NYX_CCACHE_BIN").unwrap_or_else(|_| "ccache".to_owned());
|
||||
binary_runnable(&bin, "--version").then_some(bin)
|
||||
}
|
||||
|
||||
/// Cheap "is this binary runnable" probe used by every pool's
|
||||
/// [`BuildPool::is_healthy`] / `try_new`. Runs `bin <probe_arg>` with a
|
||||
/// scrubbed env and reports whether it exited 0.
|
||||
pub(crate) fn binary_runnable(bin: &str, probe_arg: &str) -> bool {
|
||||
base_command(bin)
|
||||
.arg(probe_arg)
|
||||
.stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -125,12 +200,23 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn default_enables_java_only() {
|
||||
fn default_enables_every_shipped_pool() {
|
||||
let _l = ENV_LOCK.lock().unwrap();
|
||||
let _g = EnvGuard::set(None);
|
||||
assert!(is_pool_enabled("java"));
|
||||
for lang in POOL_ENABLED_LANGS {
|
||||
assert!(is_pool_enabled(lang), "{lang} pool must default on");
|
||||
}
|
||||
// A language with no pool stays off.
|
||||
assert!(!is_pool_enabled("cobol"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_override_disables_node() {
|
||||
let _l = ENV_LOCK.lock().unwrap();
|
||||
let _g = EnvGuard::set(Some("node=0"));
|
||||
assert!(!is_pool_enabled("node"));
|
||||
assert!(!is_pool_enabled("python"));
|
||||
// Other languages keep their default-on state.
|
||||
assert!(is_pool_enabled("python"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
87
src/dynamic/build_pool/node.rs
Normal file
87
src/dynamic/build_pool/node.rs
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
//! Node.js build pool (Phase 23 / Track O.1).
|
||||
//!
|
||||
//! `prepare_node` already snapshots `node_modules` per `package.json` hash.
|
||||
//! What it lacks is a shared npm download cache: a fresh lock hash re-downloads
|
||||
//! every tarball from cold.
|
||||
//!
|
||||
//! [`NodePool`] points `npm_config_cache` at the shared pool root so package
|
||||
//! tarballs are reused across lock hashes, collapsing a cold `npm install` to
|
||||
//! an unpack of already-fetched tarballs. TypeScript harnesses that do not
|
||||
//! need full type checking are run with `--experimental-strip-types` at
|
||||
//! execution time (the runner reads [`strip_types_flag`]); the pool itself only
|
||||
//! owns the install step.
|
||||
|
||||
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct NodePool {
|
||||
npm_bin: String,
|
||||
}
|
||||
|
||||
impl NodePool {
|
||||
pub fn try_new() -> Result<Self, String> {
|
||||
let npm_bin = std::env::var("NYX_NPM_BIN").unwrap_or_else(|_| "npm".to_owned());
|
||||
if !binary_runnable(&npm_bin, "--version") {
|
||||
return Err(format!("node-pool: {npm_bin} not runnable"));
|
||||
}
|
||||
Ok(NodePool { npm_bin })
|
||||
}
|
||||
}
|
||||
|
||||
/// The Node flag that lets a TS harness skip a full `tsc` compile when the
|
||||
/// spec does not need type checking. Surfaced as a free function so the
|
||||
/// runner can splice it into the harness exec without holding a pool handle.
|
||||
pub fn strip_types_flag() -> &'static str {
|
||||
"--experimental-strip-types"
|
||||
}
|
||||
|
||||
impl BuildPool for NodePool {
|
||||
fn name(&self) -> &'static str {
|
||||
"node"
|
||||
}
|
||||
|
||||
/// Install dependencies declared by `workdir/package.json` into
|
||||
/// `workdir/node_modules`. Args are unused.
|
||||
fn compile_batch(&self, workdir: &Path, _args: &[String]) -> PoolCompileResult {
|
||||
let start = Instant::now();
|
||||
let mut cmd = base_command(&self.npm_bin);
|
||||
cmd.args(["install", "--no-save", "--no-audit", "--no-fund"])
|
||||
.current_dir(workdir);
|
||||
if let Some(cache) = pool_cache_dir("node", "npm-cache") {
|
||||
cmd.env("npm_config_cache", cache);
|
||||
}
|
||||
|
||||
match cmd.output() {
|
||||
Ok(o) if o.status.success() => PoolCompileResult {
|
||||
success: true,
|
||||
stderr: String::new(),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
Ok(o) => PoolCompileResult {
|
||||
success: false,
|
||||
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
Err(e) => PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("node-pool: npm install: {e}"),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn is_healthy(&self) -> bool {
|
||||
binary_runnable(&self.npm_bin, "--version")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn strip_types_flag_is_the_node_native_ts_flag() {
|
||||
assert_eq!(strip_types_flag(), "--experimental-strip-types");
|
||||
}
|
||||
}
|
||||
110
src/dynamic/build_pool/php.rs
Normal file
110
src/dynamic/build_pool/php.rs
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
//! PHP build pool (Phase 23 / Track O.1).
|
||||
//!
|
||||
//! Two warm caches keyed off the Composer lockfile:
|
||||
//! - `COMPOSER_CACHE_DIR` points at the shared pool root so package downloads
|
||||
//! are reused across lock hashes, and
|
||||
//! - an opcache file-cache directory is pre-warmed so the harness `php`
|
||||
//! process skips re-parsing the vendored sources on first run.
|
||||
//!
|
||||
//! Both degrade gracefully: a missing `composer` makes `try_new` fail and the
|
||||
//! caller falls back to the legacy
|
||||
//! [`crate::dynamic::build_sandbox::prepare_php`] path; a missing `php` simply
|
||||
//! skips the opcache warm (the install still succeeds).
|
||||
|
||||
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct PhpPool {
|
||||
composer_bin: String,
|
||||
}
|
||||
|
||||
impl PhpPool {
|
||||
pub fn try_new() -> Result<Self, String> {
|
||||
let composer_bin =
|
||||
std::env::var("NYX_COMPOSER_BIN").unwrap_or_else(|_| "composer".to_owned());
|
||||
if !binary_runnable(&composer_bin, "--version") {
|
||||
return Err(format!("php-pool: {composer_bin} not runnable"));
|
||||
}
|
||||
Ok(PhpPool { composer_bin })
|
||||
}
|
||||
}
|
||||
|
||||
impl BuildPool for PhpPool {
|
||||
fn name(&self) -> &'static str {
|
||||
"php"
|
||||
}
|
||||
|
||||
/// Install `composer.json` deps into `workdir/vendor` then warm the
|
||||
/// shared opcache file-cache. Args are unused.
|
||||
fn compile_batch(&self, workdir: &Path, _args: &[String]) -> PoolCompileResult {
|
||||
let start = Instant::now();
|
||||
let mut cmd = base_command(&self.composer_bin);
|
||||
cmd.args(["install", "--no-interaction", "--no-dev", "--prefer-dist"])
|
||||
.current_dir(workdir)
|
||||
.env("COMPOSER_ALLOW_SUPERUSER", "1");
|
||||
if let Some(cache) = pool_cache_dir("php", "composer-cache") {
|
||||
cmd.env("COMPOSER_CACHE_DIR", cache);
|
||||
}
|
||||
|
||||
match cmd.output() {
|
||||
Ok(o) if o.status.success() => {}
|
||||
Ok(o) => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
Err(e) => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("php-pool: composer install: {e}"),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
warm_opcache(workdir);
|
||||
|
||||
PoolCompileResult {
|
||||
success: true,
|
||||
stderr: String::new(),
|
||||
duration: start.elapsed(),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_healthy(&self) -> bool {
|
||||
binary_runnable(&self.composer_bin, "--version")
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort opcache file-cache pre-warm: compile every vendored `.php`
|
||||
/// into the shared opcache file-cache so the harness `php` process boots with
|
||||
/// the bytecode already on disk. A missing `php` or partial failure is
|
||||
/// swallowed — the install already succeeded and opcache is a pure speed win.
|
||||
fn warm_opcache(workdir: &Path) {
|
||||
let vendor = workdir.join("vendor");
|
||||
if !vendor.exists() {
|
||||
return;
|
||||
}
|
||||
let php = std::env::var("NYX_PHP_BIN").unwrap_or_else(|_| "php".to_owned());
|
||||
let file_cache = match pool_cache_dir("php", "opcache") {
|
||||
Some(d) => d,
|
||||
None => return,
|
||||
};
|
||||
let _ = base_command(&php)
|
||||
.arg("-d")
|
||||
.arg("opcache.enable_cli=1")
|
||||
.arg("-d")
|
||||
.arg(format!("opcache.file_cache={}", file_cache.display()))
|
||||
.arg("-d")
|
||||
.arg("opcache.file_cache_only=1")
|
||||
.arg("-r")
|
||||
.arg(
|
||||
"foreach(new RecursiveIteratorIterator(new RecursiveDirectoryIterator('vendor')) \
|
||||
as $f){ if(substr($f,-4)==='.php'){ @opcache_compile_file($f); } }",
|
||||
)
|
||||
.current_dir(workdir)
|
||||
.output();
|
||||
}
|
||||
122
src/dynamic/build_pool/python.rs
Normal file
122
src/dynamic/build_pool/python.rs
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
//! Python build pool (Phase 23 / Track O.1).
|
||||
//!
|
||||
//! `prepare_python` already keys its venv on the requirements hash, so the
|
||||
//! venv itself is the "shared venv per `requirements_hash`". What the legacy
|
||||
//! path lacks is a warm bytecode cache: the first harness to import a package
|
||||
//! pays the `.py` -> `.pyc` compile.
|
||||
//!
|
||||
//! [`PythonPool`] runs `python -m compileall` over the venv's `site-packages`
|
||||
//! once at venv-creation time so every later harness import is a `__pycache__`
|
||||
//! hit. The pip download cache is pointed at the shared pool root so repeated
|
||||
//! installs across requirements hashes reuse wheels.
|
||||
|
||||
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct PythonPool;
|
||||
|
||||
impl PythonPool {
|
||||
pub fn try_new(python_bin: &str) -> Result<Self, String> {
|
||||
if !binary_runnable(python_bin, "--version") {
|
||||
return Err(format!("python-pool: {python_bin} not runnable"));
|
||||
}
|
||||
Ok(PythonPool)
|
||||
}
|
||||
}
|
||||
|
||||
impl BuildPool for PythonPool {
|
||||
fn name(&self) -> &'static str {
|
||||
"python"
|
||||
}
|
||||
|
||||
/// `args[0]` = venv path to create, `args[1]` = python interpreter binary.
|
||||
fn compile_batch(&self, workdir: &Path, args: &[String]) -> PoolCompileResult {
|
||||
let start = Instant::now();
|
||||
let venv_path = match args.first() {
|
||||
Some(v) => Path::new(v),
|
||||
None => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: "python-pool: missing venv path arg".to_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
};
|
||||
let python = args.get(1).map(String::as_str).unwrap_or("python3");
|
||||
|
||||
// 1. Create the venv.
|
||||
let create = base_command(python)
|
||||
.args(["-m", "venv", "--clear"])
|
||||
.arg(venv_path)
|
||||
.status();
|
||||
match create {
|
||||
Ok(s) if s.success() => {}
|
||||
Ok(s) => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("venv create failed: exit {s}"),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
Err(e) => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("python-pool: venv create: {e}"),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Install requirements with the shared wheel cache.
|
||||
let req_path = workdir.join("requirements.txt");
|
||||
if req_path.exists() {
|
||||
let pip = venv_path.join("bin").join("pip");
|
||||
let mut cmd = base_command(&pip.to_string_lossy());
|
||||
cmd.args(["install", "-r"]).arg(&req_path);
|
||||
if let Some(cache) = pool_cache_dir("python", "pip-cache") {
|
||||
cmd.env("PIP_CACHE_DIR", cache);
|
||||
} else {
|
||||
cmd.arg("--no-cache-dir");
|
||||
}
|
||||
match cmd.output() {
|
||||
Ok(o) if o.status.success() => {}
|
||||
Ok(o) => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
Err(e) => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("python-pool: pip install: {e}"),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Warm __pycache__ for the whole venv (best-effort: a partial
|
||||
// failure to byte-compile one module must not fail the build).
|
||||
let venv_python = venv_path.join("bin").join("python");
|
||||
let _ = base_command(&venv_python.to_string_lossy())
|
||||
.args(["-m", "compileall", "-q"])
|
||||
.arg(venv_path)
|
||||
.output();
|
||||
|
||||
PoolCompileResult {
|
||||
success: true,
|
||||
stderr: String::new(),
|
||||
duration: start.elapsed(),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_healthy(&self) -> bool {
|
||||
// The interpreter is resolved per-request via args; treat the pool as
|
||||
// always healthy and let an unrunnable interpreter surface as a build
|
||||
// error, which the dispatcher already falls back from.
|
||||
true
|
||||
}
|
||||
}
|
||||
107
src/dynamic/build_pool/ruby.rs
Normal file
107
src/dynamic/build_pool/ruby.rs
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
//! Ruby build pool (Phase 23 / Track O.1).
|
||||
//!
|
||||
//! `prepare_ruby` already vendors gems per `Gemfile.lock` hash. What it lacks
|
||||
//! is a warm Bootsnap cache: the first harness to `require` a gem pays the
|
||||
//! load-path scan + compile.
|
||||
//!
|
||||
//! [`RubyPool`] points `BOOTSNAP_CACHE_DIR` at the shared pool root and runs
|
||||
//! `bundle install` with the shared gem cache. Bootsnap then persists its
|
||||
//! compiled require-cache across findings. Falls back to the legacy path when
|
||||
//! `bundle` is not runnable.
|
||||
|
||||
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct RubyPool {
|
||||
bundle_bin: String,
|
||||
}
|
||||
|
||||
impl RubyPool {
|
||||
pub fn try_new() -> Result<Self, String> {
|
||||
let bundle_bin = std::env::var("NYX_BUNDLE_BIN").unwrap_or_else(|_| "bundle".to_owned());
|
||||
if !binary_runnable(&bundle_bin, "--version") {
|
||||
return Err(format!("ruby-pool: {bundle_bin} not runnable"));
|
||||
}
|
||||
Ok(RubyPool { bundle_bin })
|
||||
}
|
||||
|
||||
fn bundle(&self, workdir: &Path) -> std::process::Command {
|
||||
let mut cmd = base_command(&self.bundle_bin);
|
||||
cmd.current_dir(workdir);
|
||||
if let Some(cache) = pool_cache_dir("ruby", "bootsnap") {
|
||||
cmd.env("BOOTSNAP_CACHE_DIR", cache);
|
||||
}
|
||||
cmd
|
||||
}
|
||||
}
|
||||
|
||||
impl BuildPool for RubyPool {
|
||||
fn name(&self) -> &'static str {
|
||||
"ruby"
|
||||
}
|
||||
|
||||
/// Resolve `Gemfile` deps into `workdir/vendor/bundle`. Args are unused.
|
||||
fn compile_batch(&self, workdir: &Path, _args: &[String]) -> PoolCompileResult {
|
||||
let start = Instant::now();
|
||||
|
||||
// `bundle check` short-circuits when the host already has every gem.
|
||||
if let Ok(o) = self.bundle(workdir).arg("check").output() {
|
||||
if o.status.success() {
|
||||
return PoolCompileResult {
|
||||
success: true,
|
||||
stderr: String::new(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
let config = self
|
||||
.bundle(workdir)
|
||||
.args(["config", "set", "--local", "path", "vendor/bundle"])
|
||||
.output();
|
||||
match config {
|
||||
Ok(o) if o.status.success() => {}
|
||||
Ok(o) => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
Err(e) => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("ruby-pool: bundle config: {e}"),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
let install = self
|
||||
.bundle(workdir)
|
||||
.args(["install", "--jobs", "4", "--retry", "2"])
|
||||
.output();
|
||||
match install {
|
||||
Ok(o) if o.status.success() => PoolCompileResult {
|
||||
success: true,
|
||||
stderr: String::new(),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
Ok(o) => PoolCompileResult {
|
||||
success: false,
|
||||
stderr: String::from_utf8_lossy(&o.stderr).into_owned(),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
Err(e) => PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("ruby-pool: bundle install: {e}"),
|
||||
duration: start.elapsed(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn is_healthy(&self) -> bool {
|
||||
binary_runnable(&self.bundle_bin, "--version")
|
||||
}
|
||||
}
|
||||
190
src/dynamic/build_pool/rust.rs
Normal file
190
src/dynamic/build_pool/rust.rs
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
//! Rust build pool (Phase 23 / Track O.1).
|
||||
//!
|
||||
//! The legacy [`crate::dynamic::build_sandbox::prepare_rust`] runs a fresh
|
||||
//! `cargo build --release` per finding with a per-workdir `target/`. Every
|
||||
//! harness therefore recompiles the (identical) harness scaffold and all of
|
||||
//! its dependencies from cold.
|
||||
//!
|
||||
//! [`RustPool`] keeps two warm caches keyed on the `Cargo.lock` hash:
|
||||
//! - a shared `CARGO_TARGET_DIR` so incremental artefacts survive across
|
||||
//! per-finding workdirs, and
|
||||
//! - `sccache` as `RUSTC_WRAPPER` when it is on `PATH`, which caches the
|
||||
//! per-crate `rustc` invocations across *different* lock hashes too.
|
||||
//!
|
||||
//! Both degrade gracefully: a missing `sccache` simply drops the wrapper and
|
||||
//! a fresh lock hash gets a fresh (empty) shared target dir. The compile
|
||||
//! itself is byte-for-byte the same `cargo build --release` the legacy path
|
||||
//! runs, so success / failure parity holds.
|
||||
|
||||
use super::{BuildPool, PoolCompileResult, base_command, binary_runnable, pool_cache_dir};
|
||||
use blake3::Hasher;
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct RustPool {
|
||||
cargo_bin: String,
|
||||
/// `Some(path)` when an `sccache` binary is runnable. Wired in as
|
||||
/// `RUSTC_WRAPPER`; `None` falls back to plain `rustc`.
|
||||
sccache_bin: Option<String>,
|
||||
}
|
||||
|
||||
impl RustPool {
|
||||
pub fn try_new() -> Result<Self, String> {
|
||||
let cargo_bin = std::env::var("NYX_CARGO_BIN").unwrap_or_else(|_| "cargo".to_owned());
|
||||
if !binary_runnable(&cargo_bin, "--version") {
|
||||
return Err(format!("rust-pool: {cargo_bin} not runnable"));
|
||||
}
|
||||
let sccache_bin = detect_sccache();
|
||||
Ok(RustPool {
|
||||
cargo_bin,
|
||||
sccache_bin,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn detect_sccache() -> Option<String> {
|
||||
let bin = std::env::var("NYX_SCCACHE_BIN").unwrap_or_else(|_| "sccache".to_owned());
|
||||
binary_runnable(&bin, "--version").then_some(bin)
|
||||
}
|
||||
|
||||
impl BuildPool for RustPool {
|
||||
fn name(&self) -> &'static str {
|
||||
"rust"
|
||||
}
|
||||
|
||||
/// `args[0]` = absolute path the compiled `nyx_harness` binary must land
|
||||
/// at (the caller's cache slot).
|
||||
fn compile_batch(&self, workdir: &Path, args: &[String]) -> PoolCompileResult {
|
||||
let start = Instant::now();
|
||||
let dest = match args.first() {
|
||||
Some(d) => Path::new(d),
|
||||
None => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: "rust-pool: missing binary destination arg".to_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let lock_hash = hash_files(workdir, &["Cargo.lock", "Cargo.toml"]);
|
||||
let target_dir = match pool_cache_dir("rust", &lock_hash) {
|
||||
Some(d) => d,
|
||||
None => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: "rust-pool: no shared target dir".to_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let mut cmd = base_command(&self.cargo_bin);
|
||||
cmd.args(["build", "--release"])
|
||||
.current_dir(workdir)
|
||||
.env(
|
||||
"CARGO_HOME",
|
||||
std::env::var("CARGO_HOME")
|
||||
.unwrap_or_else(|_| default_cargo_home()),
|
||||
)
|
||||
.env(
|
||||
"RUSTUP_HOME",
|
||||
std::env::var("RUSTUP_HOME").unwrap_or_default(),
|
||||
)
|
||||
.env("CARGO_TARGET_DIR", &target_dir);
|
||||
if let Some(sccache) = &self.sccache_bin {
|
||||
cmd.env("RUSTC_WRAPPER", sccache);
|
||||
}
|
||||
|
||||
let output = match cmd.output() {
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!("rust-pool: cargo build: {e}"),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
};
|
||||
if !output.status.success() {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
|
||||
let compiled = target_dir.join("release").join("nyx_harness");
|
||||
if let Err(e) = std::fs::copy(&compiled, dest) {
|
||||
return PoolCompileResult {
|
||||
success: false,
|
||||
stderr: format!(
|
||||
"rust-pool: cargo build ok but copy {} -> {} failed: {e}",
|
||||
compiled.display(),
|
||||
dest.display(),
|
||||
),
|
||||
duration: start.elapsed(),
|
||||
};
|
||||
}
|
||||
PoolCompileResult {
|
||||
success: true,
|
||||
stderr: String::new(),
|
||||
duration: start.elapsed(),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_healthy(&self) -> bool {
|
||||
binary_runnable(&self.cargo_bin, "--version")
|
||||
}
|
||||
}
|
||||
|
||||
fn default_cargo_home() -> String {
|
||||
std::env::var("HOME")
|
||||
.map(|h| format!("{h}/.cargo"))
|
||||
.unwrap_or_else(|_| ".cargo".to_owned())
|
||||
}
|
||||
|
||||
/// Stable short hash of the named manifest files under `workdir`.
|
||||
fn hash_files(workdir: &Path, files: &[&str]) -> String {
|
||||
let mut h = Hasher::new();
|
||||
for fname in files {
|
||||
if let Ok(content) = std::fs::read(workdir.join(fname)) {
|
||||
h.update(fname.as_bytes());
|
||||
h.update(&content);
|
||||
}
|
||||
}
|
||||
let out = h.finalize();
|
||||
format!(
|
||||
"{:016x}",
|
||||
u64::from_le_bytes(out.as_bytes()[..8].try_into().unwrap())
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn hash_is_deterministic_and_content_sensitive() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let h1 = hash_files(dir.path(), &["Cargo.lock"]);
|
||||
let h2 = hash_files(dir.path(), &["Cargo.lock"]);
|
||||
assert_eq!(h1, h2);
|
||||
std::fs::write(dir.path().join("Cargo.lock"), b"[[package]]\n").unwrap();
|
||||
let h3 = hash_files(dir.path(), &["Cargo.lock"]);
|
||||
assert_ne!(h1, h3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_dest_arg_is_an_error_not_a_panic() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
// Construct without a toolchain probe so the test runs JDK/cargo-free.
|
||||
let pool = RustPool {
|
||||
cargo_bin: "cargo".to_owned(),
|
||||
sccache_bin: None,
|
||||
};
|
||||
let r = pool.compile_batch(dir.path(), &[]);
|
||||
assert!(!r.success);
|
||||
assert!(r.stderr.contains("missing binary destination"));
|
||||
}
|
||||
}
|
||||
|
|
@ -12,7 +12,15 @@
|
|||
//! Failed-build retry policy (§12 Q4): one retry on `BuildFailed` with
|
||||
//! backoff (1s, 4s), then `Inconclusive(BuildFailed, attempts: 2)`.
|
||||
|
||||
use crate::dynamic::build_pool::c::CPool;
|
||||
use crate::dynamic::build_pool::cpp::CppPool;
|
||||
use crate::dynamic::build_pool::go::GoPool;
|
||||
use crate::dynamic::build_pool::java::JavacPool;
|
||||
use crate::dynamic::build_pool::node::NodePool;
|
||||
use crate::dynamic::build_pool::php::PhpPool;
|
||||
use crate::dynamic::build_pool::python::PythonPool;
|
||||
use crate::dynamic::build_pool::ruby::RubyPool;
|
||||
use crate::dynamic::build_pool::rust::RustPool;
|
||||
use crate::dynamic::build_pool::{BuildPool, is_pool_enabled};
|
||||
use crate::dynamic::sandbox::ProcessHardeningProfile;
|
||||
use crate::dynamic::spec::HarnessSpec;
|
||||
|
|
@ -65,7 +73,7 @@ pub fn prepare_rust(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
|
|||
let _ = std::fs::remove_dir_all(&cache_path);
|
||||
std::fs::create_dir_all(&cache_path)?;
|
||||
|
||||
match try_build_rust_binary(workdir, &binary) {
|
||||
match build_rust_binary(workdir, &binary) {
|
||||
Ok(()) => {
|
||||
return Ok(BuildResult {
|
||||
venv_path: cache_path,
|
||||
|
|
@ -86,6 +94,27 @@ pub fn prepare_rust(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
|
|||
})
|
||||
}
|
||||
|
||||
/// Route the Rust harness build through [`RustPool`] when the pool is
|
||||
/// enabled, falling back to the legacy direct-spawn `cargo build` on a
|
||||
/// missing toolchain or a crashed pool. A genuine compile error from a
|
||||
/// healthy pool is surfaced verbatim (no legacy re-run — it would fail the
|
||||
/// same way).
|
||||
fn build_rust_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
|
||||
if is_pool_enabled("rust") {
|
||||
if let Ok(pool) = RustPool::try_new() {
|
||||
let pool_args = [binary_dest.to_string_lossy().into_owned()];
|
||||
let res = pool.compile_batch(workdir, &pool_args);
|
||||
if res.success {
|
||||
return Ok(());
|
||||
}
|
||||
if pool.is_healthy() {
|
||||
return Err(res.stderr);
|
||||
}
|
||||
}
|
||||
}
|
||||
try_build_rust_binary(workdir, binary_dest)
|
||||
}
|
||||
|
||||
fn try_build_rust_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
|
||||
let cargo = cargo_binary();
|
||||
|
||||
|
|
@ -242,7 +271,7 @@ pub fn prepare_python(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult,
|
|||
}
|
||||
|
||||
let start = Instant::now();
|
||||
match try_build_venv(&cache_path, workdir, spec) {
|
||||
match build_venv(&cache_path, workdir, spec) {
|
||||
Ok(()) => {
|
||||
return Ok(BuildResult {
|
||||
venv_path: cache_path,
|
||||
|
|
@ -264,6 +293,25 @@ pub fn prepare_python(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult,
|
|||
})
|
||||
}
|
||||
|
||||
/// Route the Python venv build through [`PythonPool`] (shared wheel cache +
|
||||
/// `compileall` bytecode warm) when enabled, else the legacy path.
|
||||
fn build_venv(venv_path: &Path, workdir: &Path, spec: &HarnessSpec) -> Result<(), String> {
|
||||
if is_pool_enabled("python") {
|
||||
let python = python_binary(spec);
|
||||
if let Ok(pool) = PythonPool::try_new(&python) {
|
||||
let pool_args = [venv_path.to_string_lossy().into_owned(), python.clone()];
|
||||
let res = pool.compile_batch(workdir, &pool_args);
|
||||
if res.success {
|
||||
return Ok(());
|
||||
}
|
||||
if pool.is_healthy() {
|
||||
return Err(res.stderr);
|
||||
}
|
||||
}
|
||||
}
|
||||
try_build_venv(venv_path, workdir, spec)
|
||||
}
|
||||
|
||||
fn try_build_venv(venv_path: &Path, workdir: &Path, spec: &HarnessSpec) -> Result<(), String> {
|
||||
// Find python binary.
|
||||
let python = python_binary(spec);
|
||||
|
|
@ -421,7 +469,7 @@ pub fn prepare_ruby(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
|
|||
if attempt > 0 {
|
||||
std::thread::sleep(Duration::from_secs(BACKOFF[attempt as usize - 1]));
|
||||
}
|
||||
match try_bundle_install(workdir) {
|
||||
match bundle_install(workdir) {
|
||||
Ok(()) => {
|
||||
if let Some(cache_path) = &cache_path {
|
||||
persist_ruby_bundle(workdir, cache_path);
|
||||
|
|
@ -445,6 +493,23 @@ pub fn prepare_ruby(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
|
|||
})
|
||||
}
|
||||
|
||||
/// Route Bundler through [`RubyPool`] (shared Bootsnap cache) when enabled,
|
||||
/// else the legacy `bundle check`/`install` path.
|
||||
fn bundle_install(workdir: &Path) -> Result<(), String> {
|
||||
if is_pool_enabled("ruby") {
|
||||
if let Ok(pool) = RubyPool::try_new() {
|
||||
let res = pool.compile_batch(workdir, &[]);
|
||||
if res.success {
|
||||
return Ok(());
|
||||
}
|
||||
if pool.is_healthy() {
|
||||
return Err(res.stderr);
|
||||
}
|
||||
}
|
||||
}
|
||||
try_bundle_install(workdir)
|
||||
}
|
||||
|
||||
fn try_bundle_install(workdir: &Path) -> Result<(), String> {
|
||||
let bundle = std::env::var("NYX_BUNDLE_BIN").unwrap_or_else(|_| "bundle".to_owned());
|
||||
if bundle_check(&bundle, workdir)? {
|
||||
|
|
@ -572,7 +637,7 @@ pub fn prepare_node(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
|
|||
BACKOFF[attempt as usize - 1],
|
||||
));
|
||||
}
|
||||
match try_npm_install(workdir) {
|
||||
match npm_install(workdir) {
|
||||
Ok(()) => {
|
||||
// Persist node_modules to cache so future runs with the same
|
||||
// package.json but a fresh workdir can restore without re-running npm.
|
||||
|
|
@ -599,6 +664,23 @@ pub fn prepare_node(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, B
|
|||
})
|
||||
}
|
||||
|
||||
/// Route `npm install` through [`NodePool`] (shared npm download cache) when
|
||||
/// enabled, else the legacy direct-spawn path.
|
||||
fn npm_install(workdir: &Path) -> Result<(), String> {
|
||||
if is_pool_enabled("node") {
|
||||
if let Ok(pool) = NodePool::try_new() {
|
||||
let res = pool.compile_batch(workdir, &[]);
|
||||
if res.success {
|
||||
return Ok(());
|
||||
}
|
||||
if pool.is_healthy() {
|
||||
return Err(res.stderr);
|
||||
}
|
||||
}
|
||||
}
|
||||
try_npm_install(workdir)
|
||||
}
|
||||
|
||||
fn try_npm_install(workdir: &Path) -> Result<(), String> {
|
||||
let npm = std::env::var("NYX_NPM_BIN").unwrap_or_else(|_| "npm".to_owned());
|
||||
let output = Command::new(&npm)
|
||||
|
|
@ -692,7 +774,7 @@ pub fn prepare_go(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bui
|
|||
let _ = std::fs::remove_dir_all(&cache_path);
|
||||
std::fs::create_dir_all(&cache_path)?;
|
||||
|
||||
match try_build_go_binary(workdir, &binary) {
|
||||
match build_go_binary(workdir, &binary) {
|
||||
Ok(()) => {
|
||||
return Ok(BuildResult {
|
||||
venv_path: cache_path,
|
||||
|
|
@ -713,6 +795,25 @@ pub fn prepare_go(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bui
|
|||
})
|
||||
}
|
||||
|
||||
/// Route the Go harness build through [`GoPool`] (shared `GOCACHE` /
|
||||
/// `GOMODCACHE`, `-trimpath -buildvcs=false`) when enabled, else the legacy
|
||||
/// per-workdir-cache path.
|
||||
fn build_go_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
|
||||
if is_pool_enabled("go") {
|
||||
if let Ok(pool) = GoPool::try_new() {
|
||||
let pool_args = [binary_dest.to_string_lossy().into_owned()];
|
||||
let res = pool.compile_batch(workdir, &pool_args);
|
||||
if res.success {
|
||||
return Ok(());
|
||||
}
|
||||
if pool.is_healthy() {
|
||||
return Err(res.stderr);
|
||||
}
|
||||
}
|
||||
}
|
||||
try_build_go_binary(workdir, binary_dest)
|
||||
}
|
||||
|
||||
fn try_build_go_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
|
||||
let go_bin = std::env::var("NYX_GO_BIN").unwrap_or_else(|_| "go".to_owned());
|
||||
let go_cache = std::env::var("GOCACHE")
|
||||
|
|
@ -1236,7 +1337,7 @@ pub fn prepare_php(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bu
|
|||
BACKOFF[attempt as usize - 1],
|
||||
));
|
||||
}
|
||||
match try_composer_install(workdir) {
|
||||
match composer_install(workdir) {
|
||||
Ok(()) => {
|
||||
// Persist vendor/ to cache so future runs with the same
|
||||
// composer.json but a fresh workdir can restore without re-running composer.
|
||||
|
|
@ -1263,6 +1364,23 @@ pub fn prepare_php(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bu
|
|||
})
|
||||
}
|
||||
|
||||
/// Route Composer through [`PhpPool`] (shared download cache + opcache
|
||||
/// file-cache warm) when enabled, else the legacy direct-spawn path.
|
||||
fn composer_install(workdir: &Path) -> Result<(), String> {
|
||||
if is_pool_enabled("php") {
|
||||
if let Ok(pool) = PhpPool::try_new() {
|
||||
let res = pool.compile_batch(workdir, &[]);
|
||||
if res.success {
|
||||
return Ok(());
|
||||
}
|
||||
if pool.is_healthy() {
|
||||
return Err(res.stderr);
|
||||
}
|
||||
}
|
||||
}
|
||||
try_composer_install(workdir)
|
||||
}
|
||||
|
||||
fn try_composer_install(workdir: &Path) -> Result<(), String> {
|
||||
let composer = std::env::var("NYX_COMPOSER_BIN").unwrap_or_else(|_| "composer".to_owned());
|
||||
let output = Command::new(&composer)
|
||||
|
|
@ -1337,7 +1455,7 @@ pub fn prepare_c(
|
|||
let _ = std::fs::remove_dir_all(&cache_path);
|
||||
std::fs::create_dir_all(&cache_path)?;
|
||||
|
||||
match try_build_c_binary(workdir, &binary, static_link) {
|
||||
match build_c_binary(workdir, &binary, static_link) {
|
||||
Ok(()) => {
|
||||
return Ok(BuildResult {
|
||||
venv_path: cache_path,
|
||||
|
|
@ -1358,6 +1476,29 @@ pub fn prepare_c(
|
|||
})
|
||||
}
|
||||
|
||||
/// Route the C harness build through [`CPool`] (`ccache` + shared object
|
||||
/// cache) when enabled, else the legacy direct-spawn `cc` path. The
|
||||
/// static-link toggle is forwarded so the pool can reproduce the
|
||||
/// Strict-profile `-static` fallback.
|
||||
fn build_c_binary(workdir: &Path, binary_dest: &Path, static_link: bool) -> Result<(), String> {
|
||||
if is_pool_enabled("c") {
|
||||
if let Ok(pool) = CPool::try_new() {
|
||||
let pool_args = [
|
||||
binary_dest.to_string_lossy().into_owned(),
|
||||
if static_link { "static" } else { "dynamic" }.to_owned(),
|
||||
];
|
||||
let res = pool.compile_batch(workdir, &pool_args);
|
||||
if res.success {
|
||||
return Ok(());
|
||||
}
|
||||
if pool.is_healthy() {
|
||||
return Err(res.stderr);
|
||||
}
|
||||
}
|
||||
}
|
||||
try_build_c_binary(workdir, binary_dest, static_link)
|
||||
}
|
||||
|
||||
fn try_build_c_binary(workdir: &Path, binary_dest: &Path, static_link: bool) -> Result<(), String> {
|
||||
let cc_bin = std::env::var("NYX_CC_BIN").unwrap_or_else(|_| "cc".to_owned());
|
||||
|
||||
|
|
@ -1484,7 +1625,7 @@ pub fn prepare_cpp(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bu
|
|||
let _ = std::fs::remove_dir_all(&cache_path);
|
||||
std::fs::create_dir_all(&cache_path)?;
|
||||
|
||||
match try_build_cpp_binary(workdir, &binary) {
|
||||
match build_cpp_binary(workdir, &binary) {
|
||||
Ok(()) => {
|
||||
return Ok(BuildResult {
|
||||
venv_path: cache_path,
|
||||
|
|
@ -1505,6 +1646,24 @@ pub fn prepare_cpp(spec: &HarnessSpec, workdir: &Path) -> Result<BuildResult, Bu
|
|||
})
|
||||
}
|
||||
|
||||
/// Route the C++ harness build through [`CppPool`] (`ccache` + shared object
|
||||
/// cache) when enabled, else the legacy direct-spawn `c++` path.
|
||||
fn build_cpp_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
|
||||
if is_pool_enabled("cpp") {
|
||||
if let Ok(pool) = CppPool::try_new() {
|
||||
let pool_args = [binary_dest.to_string_lossy().into_owned()];
|
||||
let res = pool.compile_batch(workdir, &pool_args);
|
||||
if res.success {
|
||||
return Ok(());
|
||||
}
|
||||
if pool.is_healthy() {
|
||||
return Err(res.stderr);
|
||||
}
|
||||
}
|
||||
}
|
||||
try_build_cpp_binary(workdir, binary_dest)
|
||||
}
|
||||
|
||||
fn try_build_cpp_binary(workdir: &Path, binary_dest: &Path) -> Result<(), String> {
|
||||
let cxx_bin = std::env::var("NYX_CXX_BIN").unwrap_or_else(|_| {
|
||||
// Prefer c++ which resolves to the system default compiler driver.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue