diff --git a/CHANGELOG.md b/CHANGELOG.md index 83a46c93..76c4071e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,7 @@ A focused release on three fronts: an attack-surface map and chain composer that ### CLI -- **`nyx scan --verify`** (off by default; opt-in for now) and `--backend {process,docker,firecracker}` select the dynamic-verification harness. +- **`nyx scan --verify`** (enabled by default in standard builds) and `--backend {process,docker,firecracker}` select the dynamic-verification harness. - **`nyx scan --verify-all-confidence`** drops the Medium cutoff and re-verifies everything. - **`nyx scan --unsafe-sandbox`** disables hardening (development only, never for CI). - **`nyx scan --verify-feedback`** writes a `feedback_wrong_for_finding` event so wrong verdicts get logged for offline triage. diff --git a/Cargo.toml b/Cargo.toml index b8471be1..8dbdf5b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,14 +41,13 @@ features = ["serve"] rustdoc-args = ["--cfg", "docsrs"] [features] -default = ["serve"] +default = ["serve", "dynamic"] serve = ["dep:axum", "dep:tokio", "dep:tokio-stream", "dep:tower-http"] smt = ["dep:z3", "z3/bundled"] smt-system-z3 = ["dep:z3"] docgen = [] # Dynamic verification layer: builds harnesses from findings, runs them in a -# sandbox, reports back whether the sink fires. Off by default until the -# static side is honest on real corpora (see ROADMAP.md). +# sandbox, reports back whether the sink fires. dynamic = ["dep:tempfile"] # Phase 19 (Track E.3): the `nyx-image-builder` helper binary that builds # and pins per-toolchain Docker images. Gated so it does not bloat the diff --git a/default-nyx.conf b/default-nyx.conf index 81535366..49a14c38 100644 --- a/default-nyx.conf +++ b/default-nyx.conf @@ -69,6 +69,21 @@ enable_state_analysis = true ## Per-language auth overrides live under [analysis.languages..auth]. enable_auth_analysis = true +## Run dynamic verification on Medium/High confidence findings after static analysis. +## Default builds include this support. Use --no-verify or set this false for +## fast static-only scans, or when building with --no-default-features. +verify = true + +## Also verify Low-confidence findings. Slower; intended for payload tuning. +verify_all_confidence = false + +## Dynamic sandbox backend: auto | docker | process | firecracker +## auto uses Docker when available, otherwise the process backend. +verify_backend = "auto" + +## Process-backend hardening profile: standard | strict +harden_profile = "standard" + ## Catch per-file panics during analysis and continue the scan. ## When false (default), a panic in one file's analyser aborts the whole ## scan — useful for catching engine bugs loudly in development. diff --git a/docs/cli.md b/docs/cli.md index 00d2583f..c9256867 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -154,7 +154,7 @@ nyx scan --engine-profile deep --no-smt --explain-engine ### Dynamic verification -Available with `--features dynamic`. See [dynamic.md](dynamic.md) for the full pipeline and verdict semantics. +Available in default builds, or in custom builds with `--features dynamic`. See [dynamic.md](dynamic.md) for the full pipeline and verdict semantics. | Flag | Default | Description | |------|---------|-------------| diff --git a/docs/configuration.md b/docs/configuration.md index ccc8d8a5..bd14b15d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -68,7 +68,7 @@ excluded_extensions = ["foo", "jpg"] | `enable_auth_analysis` | bool | `true` | Enable auth-state analysis within the state engine. When false, only resource lifecycle findings (leak, use-after-close, double-close) are produced. | | `enable_panic_recovery` | bool | `false` | Catch per-file analysis panics as warnings and continue. When false, a panic aborts the scan, preserving the loud-fail behaviour for users debugging engine bugs. | | `enable_auth_as_taint` | bool | `false` | Fold auth analysis into the SSA/taint engine via `Cap::UNAUTHORIZED_ID`. Off while the standalone path still carries stable detection. | -| `verify` | bool | `true` | Run dynamic verification on each `Confidence >= Medium` finding after the static pass. Requires the binary to be built with `--features dynamic`. CLI overrides: `--verify` / `--no-verify`. | +| `verify` | bool | `true` | Run dynamic verification on each `Confidence >= Medium` finding after the static pass. Included in default builds; custom `--no-default-features` builds need `--features dynamic`. CLI overrides: `--verify` / `--no-verify`. | | `verify_all_confidence` | bool | `false` | Extend dynamic verification to findings below `Confidence::Medium`. Intended for corpus-building, not production scans. CLI: `--verify-all-confidence`. | | `verify_backend` | string | `"auto"` | Sandbox backend for dynamic verification. `"auto"` picks docker when available else process; `"docker"` requires docker; `"process"` runs in-process (same as `--unsafe-sandbox`). | | `harden_profile` | string | `"standard"` | Process-backend hardening profile. `"standard"` engages `PR_SET_NO_NEW_PRIVS` + `setrlimit(RLIMIT_AS)` on Linux; `"strict"` adds namespace unshare, chroot to workdir, and a default-deny seccomp filter on Linux, plus `sandbox-exec` wrapping on macOS keyed off the finding's expected cap. | diff --git a/docs/dynamic.md b/docs/dynamic.md index 6ff753a0..99fd68ec 100644 --- a/docs/dynamic.md +++ b/docs/dynamic.md @@ -3,6 +3,9 @@ Nyx re-runs findings in generated harnesses when verification is enabled. By default, `nyx scan` verifies each `Confidence >= Medium` finding, tries payloads in a sandbox, and writes the result to `evidence.dynamic_verdict`. +Default Nyx builds include the `dynamic` feature; custom +`--no-default-features` builds run static-only unless rebuilt with +`--features dynamic`. Dynamic verification is a second signal, not a replacement for review. A confirmed verdict means Nyx triggered the sink in its harness. `NotConfirmed` diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index ffc627c0..063fd4bb 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -26,6 +26,14 @@ export interface VerifyResult { toolchain_match?: string; } +export interface DynamicVerificationSummary { + total: number; + confirmed: number; + not_confirmed: number; + inconclusive: number; + unsupported: number; +} + export interface FlowStep { step: number; kind: FlowStepKind; @@ -351,6 +359,7 @@ export interface ScannerQuality { call_resolution_rate: number; symex_verified_rate: number; symex_breakdown: Record; + dynamic_verification: DynamicVerificationSummary; } export interface IssueCategoryBucket { diff --git a/frontend/src/components/overview/OverviewWidgets.tsx b/frontend/src/components/overview/OverviewWidgets.tsx index 1e6ede1d..4284cbe9 100644 --- a/frontend/src/components/overview/OverviewWidgets.tsx +++ b/frontend/src/components/overview/OverviewWidgets.tsx @@ -241,6 +241,17 @@ export function ScannerQualityPanel({ : quality.files_scanned > 0 ? `${quality.files_scanned.toLocaleString()} freshly indexed` : undefined; + const dynamic = quality.dynamic_verification ?? { + total: 0, + confirmed: 0, + not_confirmed: 0, + inconclusive: 0, + unsupported: 0, + }; + const dynamicDetail = + dynamic.total > 0 + ? `${dynamic.total.toLocaleString()} verdicts · ${dynamic.not_confirmed.toLocaleString()} not confirmed · ${dynamic.inconclusive.toLocaleString()} inconclusive · ${dynamic.unsupported.toLocaleString()} unsupported` + : 'no dynamic verdicts in latest scan'; const rows: Array<{ label: string; @@ -287,6 +298,15 @@ export function ScannerQualityPanel({ ? `${symexAttempted} of ${symexTotal} taint findings` : 'no taint findings', }, + { + label: 'Dynamic verification', + hint: 'Findings re-run in generated harnesses against the dynamic payload corpus.', + value: + dynamic.total > 0 + ? `${dynamic.confirmed.toLocaleString()} confirmed` + : 'not run', + detail: dynamicDetail, + }, ]; return ( diff --git a/src/cli.rs b/src/cli.rs index c116646a..e4b332ac 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -471,8 +471,8 @@ pub enum Commands { /// /// Dynamic verification is on by default. This flag is a no-op when /// verification is already enabled via config. Use `--no-verify` to - /// disable it for a single run. Requires the binary to be built with - /// `--features dynamic`; without that feature this flag is silently ignored. + /// disable it for a single run. Default builds include dynamic support; + /// custom `--no-default-features` builds need `--features dynamic`. #[cfg_attr(not(feature = "dynamic"), arg(hide = true))] #[arg(long, help_heading = "Dynamic", conflicts_with = "no_verify")] verify: bool, diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 3babd6ee..8d2559f2 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -352,13 +352,19 @@ pub fn handle_command( config.scanner.harden_profile = profile.to_owned(); } } - // Without the dynamic feature, --verify / --no-verify / --unsafe-sandbox / - // --backend / --harden are silently accepted (no-op). + // Without the dynamic feature, keep the user's verify toggle in + // the resolved config so the scan command can either suppress the + // warning (`--no-verify`) or explain why verification is static-only. #[cfg(not(feature = "dynamic"))] { - let _ = verify; - let _ = no_verify; - let _ = verify_all_confidence; + if no_verify { + config.scanner.verify = false; + } else if verify { + config.scanner.verify = true; + } + if verify_all_confidence { + config.scanner.verify_all_confidence = true; + } let _ = unsafe_sandbox; let _ = backend; let _ = harden; diff --git a/src/commands/scan.rs b/src/commands/scan.rs index bfdd07f4..91fb50ad 100644 --- a/src/commands/scan.rs +++ b/src/commands/scan.rs @@ -236,6 +236,116 @@ pub fn compute_stable_hash(diag: &Diag) -> u64 { u64::from_le_bytes(bytes[..8].try_into().unwrap()) } +/// Aggregate status counts for dynamic verification verdicts attached to +/// findings. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct DynamicVerificationSummary { + pub total: usize, + pub confirmed: usize, + pub not_confirmed: usize, + pub inconclusive: usize, + pub unsupported: usize, +} + +impl DynamicVerificationSummary { + pub fn from_diags(diags: &[Diag]) -> Self { + let mut summary = Self::default(); + for diag in diags { + let Some(verdict) = diag + .evidence + .as_ref() + .and_then(|ev| ev.dynamic_verdict.as_ref()) + else { + continue; + }; + summary.total += 1; + match verdict.status { + crate::evidence::VerifyStatus::Confirmed => summary.confirmed += 1, + crate::evidence::VerifyStatus::NotConfirmed => summary.not_confirmed += 1, + crate::evidence::VerifyStatus::Inconclusive => summary.inconclusive += 1, + crate::evidence::VerifyStatus::Unsupported => summary.unsupported += 1, + } + } + summary + } + + pub fn is_empty(self) -> bool { + self.total == 0 + } +} + +/// Human-readable dynamic summary used by both CLI and server scan logs. +pub fn format_dynamic_verification_summary(summary: &DynamicVerificationSummary) -> String { + let noun = if summary.total == 1 { + "verdict" + } else { + "verdicts" + }; + format!( + "{} {} ({} confirmed, {} not confirmed, {} inconclusive, {} unsupported)", + summary.total, + noun, + summary.confirmed, + summary.not_confirmed, + summary.inconclusive, + summary.unsupported + ) +} + +/// Apply dynamic verification to a completed scan. +/// +/// Returns the configured verifier options so callers that perform later +/// composite-chain re-verification can reuse preloaded summaries and callgraph +/// context. +#[cfg(feature = "dynamic")] +pub(crate) fn verify_findings_for_scan( + diags: &mut [Diag], + project_name: &str, + db_path: &Path, + scan_path: &Path, + config: &Config, + verbose: bool, + use_index_db: bool, +) -> Option { + if !config.scanner.verify { + return None; + } + + let mut opts = crate::dynamic::verify::VerifyOptions::from_config(config); + // Phase 30 (Track C observability): surface the per-finding + // [`crate::dynamic::trace::VerifyTrace`] on stderr when the operator + // passes `--verbose`. + opts.trace_verbose = verbose; + + if use_index_db && db_path.exists() { + opts.db_path = Some(db_path.to_path_buf()); + // Preload cross-file summaries once so the spec-derivation pipeline + // can resolve the enclosing function and callgraph entry context + // without re-hitting SQLite per finding. Best-effort: a load failure + // logs and falls through to the substring heuristics. + opts.summaries = load_verify_summaries(project_name, db_path, scan_path); + if let Some(ref summaries) = opts.summaries { + opts.callgraph = Some(load_verify_callgraph(summaries)); + } + } + + let telemetry_log = crate::dynamic::telemetry::log_path(); + for diag in diags { + let mut result = crate::dynamic::verify::verify_finding(diag, &opts); + if result.status == crate::dynamic::report::VerifyStatus::Confirmed + && let Some(ref log_path) = telemetry_log + { + result.wrong = + crate::dynamic::telemetry::feedback_wrong_for_finding(log_path, &result.finding_id); + } + if let Some(ref mut ev) = diag.evidence { + ev.dynamic_verdict = Some(result); + } + } + + Some(opts) +} + /// Rollup data for grouped findings (e.g. 38 occurrences of `rs.quality.unwrap`). #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct RollupData { @@ -562,53 +672,23 @@ pub fn handle( // below can reuse the same preloaded summaries / callgraph without // a second SQLite round-trip. #[cfg(feature = "dynamic")] - let verify_opts: Option = if config.scanner.verify { - let mut opts = crate::dynamic::verify::VerifyOptions::from_config(config); - // Phase 30 (Track C observability): surface the per-finding - // [`crate::dynamic::trace::VerifyTrace`] on stderr when the - // operator passes `--verbose`. - opts.trace_verbose = verbose; - // Enable the verdict cache (§12 Q5) when an index DB is in use. - // When index_mode is Off, the DB is never created, so no cache. - if index_mode != IndexMode::Off && db_path.exists() { - opts.db_path = Some(db_path.clone()); - // Preload cross-file summaries once so the spec-derivation - // pipeline can resolve the enclosing function's `FuncSummary` - // (strategy 3) and its static `entry_kind` (strategy 4) - // without re-hitting SQLite per finding. Best-effort: a load - // failure logs and falls through to the substring heuristics. - opts.summaries = load_verify_summaries(&project_name, &db_path, &scan_path); - // Build the whole-program callgraph from the preloaded summaries - // so strategy 4 can walk reverse edges to a route handler / CLI - // entry when the sink lives in a leaf helper. - if let Some(ref s) = opts.summaries { - opts.callgraph = Some(load_verify_callgraph(s)); - } - } - // Phase 29 follow-up: resolve the telemetry events log path once - // per scan so the per-finding `wrong:` stamp is a cheap fs read, - // not a directories-crate lookup each iteration. `None` (no - // log path resolvable on this host) leaves every `wrong` as - // `None` — the eval-corpus tabulator treats that as "no signal." - let telemetry_log = crate::dynamic::telemetry::log_path(); - for diag in &mut diags { - let mut result = crate::dynamic::verify::verify_finding(diag, &opts); - if result.status == crate::dynamic::report::VerifyStatus::Confirmed { - if let Some(ref log_path) = telemetry_log { - result.wrong = crate::dynamic::telemetry::feedback_wrong_for_finding( - log_path, - &result.finding_id, - ); - } - } - if let Some(ref mut ev) = diag.evidence { - ev.dynamic_verdict = Some(result); - } - } - Some(opts) - } else { - None - }; + let verify_opts: Option = verify_findings_for_scan( + &mut diags, + &project_name, + &db_path, + &scan_path, + config, + verbose, + index_mode != IndexMode::Off, + ); + + #[cfg(not(feature = "dynamic"))] + if config.scanner.verify && !suppress_status { + eprintln!( + "{}: dynamic verification is enabled, but this binary was built without dynamic support; running static-only. Rebuild with `cargo build --features dynamic` or set `[scanner] verify = false`.", + style("warning").yellow().bold() + ); + } // ── Baseline write (§M6.5): persist current findings as stripped baseline if let Some(bw_path) = baseline_write { @@ -3473,6 +3553,58 @@ fn apply_suppressions(diags: &mut [Diag]) { } } +// ───────────────────────────────────────────────────────────────────────────── +// dynamic verification summary tests +// ───────────────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod dynamic_summary_tests { + use super::*; + use crate::evidence::{Evidence, VerifyResult, VerifyStatus}; + + fn diag_with_status(status: VerifyStatus) -> Diag { + Diag { + evidence: Some(Evidence { + dynamic_verdict: Some(VerifyResult { + finding_id: "abc123".into(), + status, + triggered_payload: None, + reason: None, + inconclusive_reason: None, + detail: None, + attempts: vec![], + toolchain_match: None, + differential: None, + replay_stable: None, + wrong: None, + hardening_outcome: None, + }), + ..Evidence::default() + }), + ..Diag::default() + } + } + + #[test] + fn dynamic_summary_counts_verdict_statuses() { + let diags = vec![ + diag_with_status(VerifyStatus::Confirmed), + diag_with_status(VerifyStatus::NotConfirmed), + diag_with_status(VerifyStatus::Inconclusive), + diag_with_status(VerifyStatus::Unsupported), + Diag::default(), + ]; + + let summary = DynamicVerificationSummary::from_diags(&diags); + + assert_eq!(summary.total, 4); + assert_eq!(summary.confirmed, 1); + assert_eq!(summary.not_confirmed, 1); + assert_eq!(summary.inconclusive, 1); + assert_eq!(summary.unsupported, 1); + } +} + // ───────────────────────────────────────────────────────────────────────────── // deduplicate_taint_flows tests // ───────────────────────────────────────────────────────────────────────────── diff --git a/src/dynamic/mod.rs b/src/dynamic/mod.rs index e779783e..a9bbd25a 100644 --- a/src/dynamic/mod.rs +++ b/src/dynamic/mod.rs @@ -26,8 +26,8 @@ //! All submodules are read-only consumers of the static engine's output. //! Nothing in this tree mutates SSA, taint, or label state. //! -//! Off by default. Enable with `--features dynamic`. Heavy deps (container -//! runtime client, fuzzer harness) live behind the same gate. +//! Included in default builds. Custom `--no-default-features` builds can enable +//! it with `--features dynamic`. //! //! # Spec derivation strategies //! diff --git a/src/fmt.rs b/src/fmt.rs index aeeba356..a4f30b73 100644 --- a/src/fmt.rs +++ b/src/fmt.rs @@ -52,6 +52,18 @@ pub fn render_console( } } + let dynamic_summary = crate::commands::scan::DynamicVerificationSummary::from_diags(diags); + if !dynamic_summary.is_empty() { + out.push_str(&format!( + "{} {}\n\n", + style("Dynamic verification:").cyan().bold(), + style(crate::commands::scan::format_dynamic_verification_summary( + &dynamic_summary + )) + .dim() + )); + } + let suppressed_count = diags.iter().filter(|d| d.suppressed).count(); let active_count = diags.len() - suppressed_count; diff --git a/src/output/json.rs b/src/output/json.rs index fd9a7ee1..9e9277b8 100644 --- a/src/output/json.rs +++ b/src/output/json.rs @@ -14,7 +14,7 @@ //! pipeline before reaching this layer. use crate::chain::finding::ChainFinding; -use crate::commands::scan::Diag; +use crate::commands::scan::{Diag, DynamicVerificationSummary}; use serde_json::{Value, json}; use std::collections::HashMap; @@ -42,6 +42,7 @@ pub fn build_findings_json( let mut out = json!({ "findings": findings, "chains": chains_array, + "dynamic_verification": DynamicVerificationSummary::from_diags(diags), }); if let Some(diff) = verdict_diff { out["verdict_diff"] = diff.clone(); diff --git a/src/server/jobs.rs b/src/server/jobs.rs index 3e1a14d8..accd7b62 100644 --- a/src/server/jobs.rs +++ b/src/server/jobs.rs @@ -239,7 +239,7 @@ impl JobManager { Some(&log_collector), )?; let pool = Indexer::init(&db_path)?; - scan::scan_with_index_parallel_observer( + let mut diags = scan::scan_with_index_parallel_observer( &project_name, pool, &config, @@ -250,7 +250,23 @@ impl JobManager { Some(&log_collector), None, None, - ) + )?; + for diag in &mut diags { + diag.stable_hash = scan::compute_stable_hash(diag); + } + #[cfg(feature = "dynamic")] + { + let _verify_opts = scan::verify_findings_for_scan( + &mut diags, + &project_name, + &db_path, + &scan_root, + &config, + false, + true, + ); + } + Ok(diags) }); let elapsed = start.elapsed().as_secs_f64(); @@ -274,6 +290,16 @@ impl JobManager { for d in &mut diags { d.stable_hash = scan::compute_stable_hash(d); } + let dynamic_summary = scan::DynamicVerificationSummary::from_diags(&diags); + if !dynamic_summary.is_empty() { + log_collector.info( + format!( + "Dynamic verification: {}", + scan::format_dynamic_verification_summary(&dynamic_summary) + ), + None, + ); + } log_collector.info(format!("Scan completed: {} findings", diags.len()), None); (JobStatus::Completed, Some(Arc::new(diags)), None) } diff --git a/src/server/models.rs b/src/server/models.rs index bbc282c9..a7753aea 100644 --- a/src/server/models.rs +++ b/src/server/models.rs @@ -717,6 +717,8 @@ pub struct ScannerQuality { pub symex_verified_rate: f64, /// Count broken down by symbolic verdict label. pub symex_breakdown: HashMap, + /// Dynamic verifier verdict counts from the latest scan. + pub dynamic_verification: crate::commands::scan::DynamicVerificationSummary, } /// One issue-category bucket (rule-family derived). Broader than OWASP, with diff --git a/src/server/routes/overview.rs b/src/server/routes/overview.rs index 7cf6cd50..55b85866 100644 --- a/src/server/routes/overview.rs +++ b/src/server/routes/overview.rs @@ -837,6 +837,9 @@ fn compute_scanner_quality( call_resolution_rate, symex_verified_rate, symex_breakdown: breakdown, + dynamic_verification: crate::commands::scan::DynamicVerificationSummary::from_diags( + findings, + ), }) } diff --git a/src/server/routes/scans.rs b/src/server/routes/scans.rs index 1f8a225a..a47d17e4 100644 --- a/src/server/routes/scans.rs +++ b/src/server/routes/scans.rs @@ -40,8 +40,8 @@ struct StartScanRequest { /// `false` - force off even if config says on. /// absent - inherit config default. /// - /// Requires `--features dynamic`; `true` returns 400 when the - /// feature is absent. + /// Included in default builds; custom builds without `dynamic` return 400 + /// when verification is requested. verify: Option, /// Also verify `Confidence < Medium` findings. Default false. verify_all_confidence: Option, @@ -126,6 +126,13 @@ async fn start_scan( config.scanner.verify_all_confidence = true; } + #[cfg(not(feature = "dynamic"))] + if config.scanner.verify || config.scanner.verify_all_confidence { + return Err(bad_request( + "dynamic verification is enabled, but this binary was built without dynamic support; rebuild with `cargo build --features dynamic` or skip dynamic verification for this scan", + )); + } + let event_tx = state.event_tx.clone(); let db_pool = state.db_pool.clone(); let database_dir = state.database_dir.clone(); diff --git a/src/utils/config.rs b/src/utils/config.rs index 36447204..693365bf 100644 --- a/src/utils/config.rs +++ b/src/utils/config.rs @@ -256,8 +256,9 @@ pub struct ScannerConfig { /// `Evidence::dynamic_verdict`. Use `--no-verify` (CLI) or set /// `verify = false` in `nyx.toml` to disable. /// - /// Requires the binary to be built with `--features dynamic`; without - /// that feature the setting has no effect. + /// Included in default builds. Custom `--no-default-features` builds need + /// `--features dynamic`; without that feature the CLI warns and runs + /// static-only. /// /// Migration note: existing `nyx.toml` files that already set /// `verify = false` keep the opt-out behaviour; only the inherited