From 118cafa535140bcc39a47e979e80ee11477fe397 Mon Sep 17 00:00:00 2001 From: pitboss Date: Tue, 12 May 2026 14:14:13 -0400 Subject: [PATCH] =?UTF-8?q?[pitboss]=20phase=2008:=20M6.5=20=E2=80=94=20Pa?= =?UTF-8?q?tch-validation=20/=20fix-validation=20CI=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/api/types.ts | 23 + src/baseline.rs | 628 ++++++++++++++++++ src/cli.rs | 31 + src/commands/mod.rs | 6 + src/commands/scan.rs | 73 +- src/lib.rs | 1 + src/server/jobs.rs | 8 +- src/server/models.rs | 13 + src/server/routes/scans.rs | 6 + tests/fix_validation_e2e.rs | 258 +++++++ tests/fixtures/baseline_sqli_fixed/handler.py | 5 + tests/fixtures/baseline_sqli_new/handler.py | 12 + tests/fixtures/baseline_sqli_vuln/handler.py | 7 + 13 files changed, 1067 insertions(+), 4 deletions(-) create mode 100644 src/baseline.rs create mode 100644 tests/fix_validation_e2e.rs create mode 100644 tests/fixtures/baseline_sqli_fixed/handler.py create mode 100644 tests/fixtures/baseline_sqli_new/handler.py create mode 100644 tests/fixtures/baseline_sqli_vuln/handler.py diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index c6f0946e..7bd7ad4f 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -83,10 +83,31 @@ export interface RelatedFindingView { severity: string; } +// Baseline / patch-validation types (M6.5) +export type VerdictTransition = + | 'New' + | 'Unchanged' + | 'Resolved' + | 'Regressed' + | 'FlippedConfirmed' + | 'FlippedNotConfirmed'; + +export interface VerdictDiffEntry { + stable_hash: number; + path: string; + line: number; + rule_id: string; + baseline_status?: VerifyStatus; + current_status?: VerifyStatus; + transition: VerdictTransition; +} + export interface FindingView { index: number; fingerprint: string; portable_fingerprint?: string; + /** Blake3-derived stable cross-commit identity (M6.5). */ + stable_hash?: number; path: string; line: number; col: number; @@ -199,6 +220,8 @@ export interface CompareResponse { fixed_findings: ComparedFinding[]; changed_findings: ChangedFinding[]; unchanged_findings: ComparedFinding[]; + /** Verdict-level diff (M6.5). Present when findings carry stable_hash values. */ + verdict_diff?: VerdictDiffEntry[]; } // Overview types diff --git a/src/baseline.rs b/src/baseline.rs new file mode 100644 index 00000000..b4473c4d --- /dev/null +++ b/src/baseline.rs @@ -0,0 +1,628 @@ +//! Baseline diffing for patch-validation CI mode (§M6.5 / Pillar A §15.1). +//! +//! `nyx scan --baseline ` reads a previous scan's JSON output (or a +//! stripped `.nyx/baseline.json`) and joins on `Diag::stable_hash`. The +//! result is a per-finding `VerdictDiffEntry` with a typed `Transition` that +//! CI gates can act on. +//! +//! `nyx scan --baseline-write ` writes a stripped baseline JSON: +//! only `stable_hash`, `dynamic_verdict`, `severity`, `path`, and `rule_id`. +//! No source code is included. + +use crate::commands::scan::Diag; +use crate::evidence::VerifyStatus; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::Path; + +// ───────────────────────────────────────────────────────────────────────────── +// Baseline entry (stripped — no source code) +// ───────────────────────────────────────────────────────────────────────────── + +/// A stripped baseline entry: only what is needed for cross-commit diffing. +/// Contains no source code snippets. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BaselineEntry { + pub stable_hash: u64, + /// Dynamic verdict status from the scan that wrote this baseline. + /// `None` when `--verify` was not run. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dynamic_verdict: Option, + pub severity: String, + pub path: String, + pub rule_id: String, +} + +// ───────────────────────────────────────────────────────────────────────────── +// Transition enum +// ───────────────────────────────────────────────────────────────────────────── + +/// How a finding's verdict changed between the baseline scan and the current +/// scan. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Transition { + /// Finding exists in the current scan but was absent from the baseline. + New, + /// Finding appears in both scans; verdict is unchanged (or neither scan + /// ran `--verify`). + Unchanged, + /// Finding was present in the baseline but disappeared from the current + /// scan — the vulnerability is gone. + Resolved, + /// Finding in both; was `NotConfirmed` in baseline, now `Confirmed`. + Regressed, + /// Finding in both; baseline had no verdict (or `Inconclusive` / + /// `Unsupported`) and it is now `Confirmed`. + FlippedConfirmed, + /// Finding in both; was `Confirmed` in baseline, now `NotConfirmed` — + /// the fix is proven. + FlippedNotConfirmed, +} + +// ───────────────────────────────────────────────────────────────────────────── +// VerdictDiffEntry +// ───────────────────────────────────────────────────────────────────────────── + +/// Per-finding verdict diff produced by comparing a baseline to a current scan. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VerdictDiffEntry { + /// Stable cross-commit identity hash. + pub stable_hash: u64, + pub path: String, + pub line: usize, + pub rule_id: String, + /// Verdict in the baseline scan (`None` when verify was not run). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub baseline_status: Option, + /// Verdict in the current scan (`None` when verify was not run or finding + /// is absent from the current scan). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub current_status: Option, + pub transition: Transition, +} + +/// Full verdict diff between a baseline and a current scan. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VerdictDiff { + pub entries: Vec, +} + +// ───────────────────────────────────────────────────────────────────────────── +// Load / write helpers +// ───────────────────────────────────────────────────────────────────────────── + +/// Load baseline entries from a file. +/// +/// Accepts two JSON formats: +/// - Stripped baseline (`Vec`) — written by `--baseline-write`. +/// - Full scan output (`Vec`) — written by `nyx scan --format json`. +/// +/// Detection heuristic: try `Vec` first (requires `rule_id`); +/// fall back to `Vec`. +pub fn load_baseline(path: &Path) -> crate::errors::NyxResult> { + let content = std::fs::read_to_string(path).map_err(|e| { + crate::errors::NyxError::Msg(format!("cannot read baseline {}: {e}", path.display())) + })?; + + // Try stripped format first. + if let Ok(entries) = serde_json::from_str::>(&content) { + return Ok(entries); + } + + // Fall back to full Diag list. + let diags: Vec = serde_json::from_str(&content).map_err(|e| { + crate::errors::NyxError::Msg(format!( + "baseline {}: not a valid BaselineEntry list or Diag list: {e}", + path.display() + )) + })?; + Ok(diags_to_baseline_entries(&diags)) +} + +/// Convert `Diag` values to `BaselineEntry` values. +/// +/// Only findings with a non-zero `stable_hash` are included; findings without +/// a hash cannot be joined across scans. +pub fn diags_to_baseline_entries(diags: &[Diag]) -> Vec { + diags + .iter() + .filter(|d| d.stable_hash != 0) + .map(|d| BaselineEntry { + stable_hash: d.stable_hash, + dynamic_verdict: d + .evidence + .as_ref() + .and_then(|ev| ev.dynamic_verdict.as_ref()) + .map(|vr| vr.status), + severity: d.severity.as_db_str().to_string(), + path: d.path.clone(), + rule_id: d.id.clone(), + }) + .collect() +} + +/// Write a stripped baseline JSON to `path`. +/// +/// The file contains only `stable_hash`, `dynamic_verdict`, `severity`, +/// `path`, and `rule_id` — no source code snippets or flow steps. +pub fn write_baseline(path: &Path, diags: &[Diag]) -> crate::errors::NyxResult<()> { + let entries = diags_to_baseline_entries(diags); + let json = serde_json::to_string_pretty(&entries).map_err(|e| { + crate::errors::NyxError::Msg(format!("baseline serialize error: {e}")) + })?; + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent).map_err(|e| { + crate::errors::NyxError::Msg(format!( + "cannot create baseline dir {}: {e}", + parent.display() + )) + })?; + } + } + std::fs::write(path, json).map_err(|e| { + crate::errors::NyxError::Msg(format!( + "cannot write baseline {}: {e}", + path.display() + )) + }) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Diff computation +// ───────────────────────────────────────────────────────────────────────────── + +fn classify_transition( + baseline: Option, + current: Option, +) -> Transition { + match (baseline, current) { + // No verdict change (including both None) + (a, b) if a == b => Transition::Unchanged, + // Confirmed → NotConfirmed: fix proven + (Some(VerifyStatus::Confirmed), Some(VerifyStatus::NotConfirmed)) => { + Transition::FlippedNotConfirmed + } + // NotConfirmed → Confirmed: regression + (Some(VerifyStatus::NotConfirmed), Some(VerifyStatus::Confirmed)) => { + Transition::Regressed + } + // None / Inconclusive / Unsupported → Confirmed + (_, Some(VerifyStatus::Confirmed)) => Transition::FlippedConfirmed, + // Everything else: treat as unchanged (e.g. Confirmed → Inconclusive + // without a clean NotConfirmed proof is not a resolution) + _ => Transition::Unchanged, + } +} + +/// Compute a verdict diff between a loaded baseline and the current findings. +pub fn compute_verdict_diff(baseline: &[BaselineEntry], current: &[Diag]) -> VerdictDiff { + // Build lookup maps keyed by stable_hash. + let baseline_map: HashMap = + baseline.iter().map(|e| (e.stable_hash, e)).collect(); + let current_map: HashMap = current + .iter() + .filter(|d| d.stable_hash != 0) + .map(|d| (d.stable_hash, d)) + .collect(); + + let mut entries = Vec::new(); + + // Walk current findings. + for (&hash, diag) in ¤t_map { + let current_status = diag + .evidence + .as_ref() + .and_then(|ev| ev.dynamic_verdict.as_ref()) + .map(|vr| vr.status); + + if let Some(base) = baseline_map.get(&hash) { + let transition = classify_transition(base.dynamic_verdict, current_status); + entries.push(VerdictDiffEntry { + stable_hash: hash, + path: diag.path.clone(), + line: diag.line, + rule_id: diag.id.clone(), + baseline_status: base.dynamic_verdict, + current_status, + transition, + }); + } else { + // Not in baseline → New. + entries.push(VerdictDiffEntry { + stable_hash: hash, + path: diag.path.clone(), + line: diag.line, + rule_id: diag.id.clone(), + baseline_status: None, + current_status, + transition: Transition::New, + }); + } + } + + // Walk baseline findings absent from current → Resolved. + for (&hash, base) in &baseline_map { + if !current_map.contains_key(&hash) { + entries.push(VerdictDiffEntry { + stable_hash: hash, + path: base.path.clone(), + line: 0, + rule_id: base.rule_id.clone(), + baseline_status: base.dynamic_verdict, + current_status: None, + transition: Transition::Resolved, + }); + } + } + + // Sort for deterministic output: Resolved first, then New, then the rest, + // all sub-sorted by (path, line). + entries.sort_by(|a, b| { + fn order(t: Transition) -> u8 { + match t { + Transition::Resolved => 0, + Transition::FlippedNotConfirmed => 1, + Transition::New => 2, + Transition::Regressed => 3, + Transition::FlippedConfirmed => 4, + Transition::Unchanged => 5, + } + } + order(a.transition) + .cmp(&order(b.transition)) + .then_with(|| a.path.cmp(&b.path)) + .then_with(|| a.line.cmp(&b.line)) + }); + + VerdictDiff { entries } +} + +// ───────────────────────────────────────────────────────────────────────────── +// CI gates +// ───────────────────────────────────────────────────────────────────────────── + +/// Gate: exit code 2 if any new `Confirmed` finding appears. +/// +/// Triggers on `transition == New && current_status == Confirmed` or +/// `transition == FlippedConfirmed`. +pub const GATE_NO_NEW_CONFIRMED: &str = "no-new-confirmed"; + +/// Gate: exit code 2 if any baseline-`Confirmed` finding is not fully resolved. +/// +/// A baseline-Confirmed finding is resolved only when it is absent from the +/// current scan (`Resolved`) or its current verdict is `NotConfirmed` +/// (`FlippedNotConfirmed`). All other current statuses (`Confirmed`, +/// `Inconclusive`, `Unsupported`) violate this gate. +pub const GATE_RESOLVE_ALL_CONFIRMED: &str = "resolve-all-confirmed"; + +/// Check a named CI gate against a verdict diff. +/// +/// Returns `true` when the gate passes (condition not violated) and `false` +/// when it fails (caller should exit with code 2). +/// +/// Unknown gate names always pass so future gate additions are forward- +/// compatible without requiring a binary upgrade. +pub fn check_gate(diff: &VerdictDiff, gate: &str) -> bool { + match gate { + GATE_NO_NEW_CONFIRMED => !diff.entries.iter().any(|e| { + matches!(e.transition, Transition::New | Transition::FlippedConfirmed) + && e.current_status == Some(VerifyStatus::Confirmed) + }), + GATE_RESOLVE_ALL_CONFIRMED => !diff.entries.iter().any(|e| { + e.baseline_status == Some(VerifyStatus::Confirmed) + && matches!( + e.current_status, + Some(VerifyStatus::Confirmed) + | Some(VerifyStatus::Inconclusive) + | Some(VerifyStatus::Unsupported) + ) + }), + _ => true, + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Console / JSON rendering +// ───────────────────────────────────────────────────────────────────────────── + +fn status_str(s: Option) -> &'static str { + match s { + Some(VerifyStatus::Confirmed) => "Confirmed", + Some(VerifyStatus::NotConfirmed) => "NotConfirmed", + Some(VerifyStatus::Inconclusive) => "Inconclusive", + Some(VerifyStatus::Unsupported) => "Unsupported", + None => "(no verdict)", + } +} + +/// Render a verdict diff as a human-readable console summary. +pub fn format_diff_console(diff: &VerdictDiff) -> String { + if diff.entries.is_empty() { + return String::from(" (no findings in baseline or current scan)\n"); + } + + let mut lines = Vec::new(); + let mut non_unchanged = 0usize; + + for e in &diff.entries { + let hash_str = format!("{:016x}", e.stable_hash); + let loc = if e.line > 0 { + format!("{}:{}", e.path, e.line) + } else { + e.path.clone() + }; + match e.transition { + Transition::New => { + non_unchanged += 1; + lines.push(format!( + " + {hash_str}: new {} at {loc}", + status_str(e.current_status) + )); + } + Transition::Resolved => { + non_unchanged += 1; + lines.push(format!( + " - {hash_str}: {} \u{2192} removed (resolved) at {loc}", + status_str(e.baseline_status) + )); + } + Transition::FlippedNotConfirmed => { + non_unchanged += 1; + lines.push(format!( + " - {hash_str}: Confirmed \u{2192} NotConfirmed at {loc} (resolved)" + )); + } + Transition::Regressed => { + non_unchanged += 1; + lines.push(format!( + " ! {hash_str}: NotConfirmed \u{2192} Confirmed at {loc} (regressed)" + )); + } + Transition::FlippedConfirmed => { + non_unchanged += 1; + lines.push(format!( + " + {hash_str}: new Confirmed at {loc}" + )); + } + Transition::Unchanged => {} + } + } + + if non_unchanged == 0 { + return String::from(" (no changes from baseline)\n"); + } + + lines.join("\n") + "\n" +} + +// ───────────────────────────────────────────────────────────────────────────── +// Tests +// ───────────────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::scan::{compute_stable_hash, Diag}; + use crate::evidence::{Evidence, VerifyResult, VerifyStatus}; + use crate::patterns::{FindingCategory, Severity}; + + fn make_diag(path: &str, line: usize, rule: &str) -> Diag { + let mut d = Diag { + path: path.to_string(), + line, + col: 0, + severity: Severity::High, + id: rule.to_string(), + category: FindingCategory::Security, + path_validated: false, + guard_kind: None, + message: None, + labels: vec![], + confidence: None, + evidence: None, + rank_score: None, + rank_reason: None, + suppressed: false, + suppression: None, + rollup: None, + finding_id: String::new(), + alternative_finding_ids: vec![], + stable_hash: 0, + }; + d.stable_hash = compute_stable_hash(&d); + d + } + + fn with_verdict(mut d: Diag, status: VerifyStatus) -> Diag { + d.evidence = Some(Evidence { + dynamic_verdict: Some(VerifyResult { + finding_id: format!("{:016x}", d.stable_hash), + status, + triggered_payload: None, + reason: None, + inconclusive_reason: None, + detail: None, + attempts: vec![], + toolchain_match: None, + }), + ..Default::default() + }); + d + } + + #[test] + fn new_finding_no_verdict() { + let current = vec![make_diag("src/a.py", 1, "py.sqli")]; + let diff = compute_verdict_diff(&[], ¤t); + assert_eq!(diff.entries.len(), 1); + assert_eq!(diff.entries[0].transition, Transition::New); + assert_eq!(diff.entries[0].current_status, None); + } + + #[test] + fn new_confirmed_finding() { + let current = vec![with_verdict( + make_diag("src/a.py", 1, "py.sqli"), + VerifyStatus::Confirmed, + )]; + let diff = compute_verdict_diff(&[], ¤t); + assert_eq!(diff.entries[0].transition, Transition::New); + assert_eq!(diff.entries[0].current_status, Some(VerifyStatus::Confirmed)); + } + + #[test] + fn resolved_finding() { + let baseline_diag = make_diag("src/a.py", 1, "py.sqli"); + let baseline = diags_to_baseline_entries(&[baseline_diag]); + let diff = compute_verdict_diff(&baseline, &[]); + assert_eq!(diff.entries.len(), 1); + assert_eq!(diff.entries[0].transition, Transition::Resolved); + } + + #[test] + fn flipped_not_confirmed() { + let d = make_diag("src/a.py", 1, "py.sqli"); + let baseline = vec![BaselineEntry { + stable_hash: d.stable_hash, + dynamic_verdict: Some(VerifyStatus::Confirmed), + severity: "high".to_string(), + path: d.path.clone(), + rule_id: d.id.clone(), + }]; + let current = vec![with_verdict(d, VerifyStatus::NotConfirmed)]; + let diff = compute_verdict_diff(&baseline, ¤t); + assert_eq!(diff.entries[0].transition, Transition::FlippedNotConfirmed); + } + + #[test] + fn regressed() { + let d = make_diag("src/a.py", 1, "py.sqli"); + let baseline = vec![BaselineEntry { + stable_hash: d.stable_hash, + dynamic_verdict: Some(VerifyStatus::NotConfirmed), + severity: "high".to_string(), + path: d.path.clone(), + rule_id: d.id.clone(), + }]; + let current = vec![with_verdict(d, VerifyStatus::Confirmed)]; + let diff = compute_verdict_diff(&baseline, ¤t); + assert_eq!(diff.entries[0].transition, Transition::Regressed); + } + + #[test] + fn gate_no_new_confirmed_passes_when_no_confirmed() { + let d = make_diag("src/a.py", 1, "py.sqli"); + let diff = compute_verdict_diff(&[], &[d]); + assert!(check_gate(&diff, GATE_NO_NEW_CONFIRMED)); + } + + #[test] + fn gate_no_new_confirmed_fails_on_new_confirmed() { + let current = vec![with_verdict( + make_diag("src/a.py", 1, "py.sqli"), + VerifyStatus::Confirmed, + )]; + let diff = compute_verdict_diff(&[], ¤t); + assert!(!check_gate(&diff, GATE_NO_NEW_CONFIRMED)); + } + + #[test] + fn gate_resolve_all_confirmed_passes_when_flipped() { + let d = make_diag("src/a.py", 1, "py.sqli"); + let baseline = vec![BaselineEntry { + stable_hash: d.stable_hash, + dynamic_verdict: Some(VerifyStatus::Confirmed), + severity: "high".to_string(), + path: d.path.clone(), + rule_id: d.id.clone(), + }]; + let current = vec![with_verdict(d, VerifyStatus::NotConfirmed)]; + let diff = compute_verdict_diff(&baseline, ¤t); + assert!(check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED)); + } + + #[test] + fn gate_resolve_all_confirmed_fails_when_still_confirmed() { + let d = make_diag("src/a.py", 1, "py.sqli"); + let baseline = vec![BaselineEntry { + stable_hash: d.stable_hash, + dynamic_verdict: Some(VerifyStatus::Confirmed), + severity: "high".to_string(), + path: d.path.clone(), + rule_id: d.id.clone(), + }]; + let current = vec![with_verdict(d, VerifyStatus::Confirmed)]; + let diff = compute_verdict_diff(&baseline, ¤t); + assert!(!check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED)); + } + + #[test] + fn gate_resolve_all_confirmed_passes_when_resolved() { + let d = make_diag("src/a.py", 1, "py.sqli"); + let baseline = vec![BaselineEntry { + stable_hash: d.stable_hash, + dynamic_verdict: Some(VerifyStatus::Confirmed), + severity: "high".to_string(), + path: d.path.clone(), + rule_id: d.id.clone(), + }]; + // No current findings (finding disappeared entirely). + let diff = compute_verdict_diff(&baseline, &[]); + assert!(check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED)); + } + + #[test] + fn write_and_load_roundtrip() { + let d = with_verdict(make_diag("src/a.py", 1, "py.sqli"), VerifyStatus::Confirmed); + let tmp = tempfile::NamedTempFile::new().unwrap(); + write_baseline(tmp.path(), &[d.clone()]).unwrap(); + let loaded = load_baseline(tmp.path()).unwrap(); + assert_eq!(loaded.len(), 1); + assert_eq!(loaded[0].stable_hash, d.stable_hash); + assert_eq!(loaded[0].dynamic_verdict, Some(VerifyStatus::Confirmed)); + assert_eq!(loaded[0].path, "src/a.py"); + assert_eq!(loaded[0].rule_id, "py.sqli"); + } + + #[test] + fn load_full_diag_json() { + let d = with_verdict(make_diag("src/a.py", 1, "py.sqli"), VerifyStatus::Confirmed); + let json = serde_json::to_string(&[&d]).unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(tmp.path(), &json).unwrap(); + let loaded = load_baseline(tmp.path()).unwrap(); + assert_eq!(loaded.len(), 1); + assert_eq!(loaded[0].stable_hash, d.stable_hash); + } + + #[test] + fn baseline_write_no_source() { + let mut d = with_verdict(make_diag("src/a.py", 1, "py.sqli"), VerifyStatus::Confirmed); + // Add a flow_step with a snippet (source code) to the evidence. + if let Some(ref mut ev) = d.evidence { + ev.flow_steps = vec![crate::evidence::FlowStep { + step: 1, + kind: crate::evidence::FlowStepKind::Source, + file: "src/a.py".into(), + line: 1, + col: 0, + snippet: Some("SECRET CODE".into()), + variable: None, + callee: None, + function: None, + is_cross_file: false, + }]; + } + let tmp = tempfile::NamedTempFile::new().unwrap(); + write_baseline(tmp.path(), &[d]).unwrap(); + let content = std::fs::read_to_string(tmp.path()).unwrap(); + assert!(!content.contains("SECRET CODE"), "baseline must not contain source code"); + } + + #[test] + fn unknown_gate_passes() { + let diff = VerdictDiff { entries: vec![] }; + assert!(check_gate(&diff, "some-future-gate-name")); + } +} diff --git a/src/cli.rs b/src/cli.rs index 78ba0e75..1c3b7aad 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -456,6 +456,37 @@ pub enum Commands { #[cfg_attr(not(feature = "dynamic"), arg(hide = true))] #[arg(long, help_heading = "Dynamic", value_name = "BACKEND")] backend: Option, + + // ── Baseline / patch-validation (§M6.5) ──────────────────────── + /// Read a previous scan's JSON output (or a stripped .nyx/baseline.json) + /// and diff it against the current scan on stable_hash. + /// + /// Emits a verdict diff showing New / Resolved / FlippedConfirmed / + /// FlippedNotConfirmed transitions. Combine with --gate to enforce CI + /// policies. + #[arg(long, value_name = "FILE", help_heading = "Baseline")] + baseline: Option, + + /// Write a stripped baseline JSON to FILE after scanning. + /// + /// The file contains only stable_hash, dynamic_verdict, severity, path, + /// and rule_id — no source code. A CI job can persist this file to + /// compare future scans against without leaking source. + #[arg(long, value_name = "FILE", help_heading = "Baseline")] + baseline_write: Option, + + /// CI gate to enforce when --baseline is active. + /// + /// `no-new-confirmed`: exit 2 if any new Confirmed finding appears. + /// `resolve-all-confirmed`: exit 2 if any baseline-Confirmed finding + /// is not fully resolved (absent or NotConfirmed in the current scan). + #[arg( + long, + value_name = "GATE", + value_parser = ["no-new-confirmed", "resolve-all-confirmed"], + help_heading = "Baseline" + )] + gate: Option, }, /// Submit feedback on a dynamic verification verdict (§21.2). diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 572cdd43..ccb8adf6 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -100,6 +100,9 @@ pub fn handle_command( verify, unsafe_sandbox, backend, + baseline, + baseline_write, + gate, } => { // ── Apply profile first (CLI flags override after) ────────── if let Some(ref name) = profile { @@ -360,6 +363,9 @@ pub fn handle_command( show_instances.as_deref(), database_dir, config, + baseline.as_deref().map(std::path::Path::new), + baseline_write.as_deref().map(std::path::Path::new), + gate.as_deref(), )?; } #[cfg(feature = "dynamic")] diff --git a/src/commands/scan.rs b/src/commands/scan.rs index 9ffb5db9..0f989d17 100644 --- a/src/commands/scan.rs +++ b/src/commands/scan.rs @@ -345,6 +345,9 @@ pub fn handle( show_instances: Option<&str>, database_dir: &Path, config: &Config, + baseline: Option<&Path>, + baseline_write: Option<&Path>, + gate: Option<&str>, ) -> NyxResult<()> { let scan_path = Path::new(path).canonicalize()?; let (project_name, db_path) = get_project_info(&scan_path, database_dir)?; @@ -489,18 +492,65 @@ pub fn handle( } } + // ── Baseline write (§M6.5): persist current findings as stripped baseline + if let Some(bw_path) = baseline_write { + if let Err(e) = crate::baseline::write_baseline(bw_path, &diags) { + tracing::warn!(path = %bw_path.display(), error = %e, "baseline-write failed"); + if !suppress_status { + eprintln!("warning: --baseline-write failed: {e}"); + } + } else if !suppress_status { + eprintln!("Baseline written to {}", bw_path.display()); + } + } + + // ── Baseline diff (§M6.5): load previous baseline and compute transitions + let verdict_diff = if let Some(bl_path) = baseline { + match crate::baseline::load_baseline(bl_path) { + Ok(baseline_entries) => { + let diff = crate::baseline::compute_verdict_diff(&baseline_entries, &diags); + Some(diff) + } + Err(e) => { + return Err(crate::errors::NyxError::Msg(format!( + "--baseline {}: {e}", + bl_path.display() + ))); + } + } + } else { + None + }; + // ── Output ────────────────────────────────────────────────────────── match format { OutputFormat::Json => { - let json = serde_json::to_string(&diags) - .map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?; - println!("{json}"); + if let Some(ref diff) = verdict_diff { + // Wrap findings + verdict_diff into one JSON object so the + // diff is machine-readable alongside the findings. + let out = serde_json::json!({ + "findings": &diags, + "verdict_diff": diff, + }); + let json = serde_json::to_string(&out) + .map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?; + println!("{json}"); + } else { + let json = serde_json::to_string(&diags) + .map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?; + println!("{json}"); + } } OutputFormat::Sarif => { let sarif = crate::output::build_sarif(&diags, &scan_path); let json = serde_json::to_string_pretty(&sarif) .map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?; println!("{json}"); + // Emit diff on stderr for SARIF (stdout is owned by the SARIF schema). + if let Some(ref diff) = verdict_diff { + eprintln!("\nBaseline comparison:"); + eprint!("{}", crate::baseline::format_diff_console(diff)); + } } OutputFormat::Console => { tracing::debug!("Printing to console"); @@ -508,6 +558,10 @@ pub fn handle( "{}", crate::fmt::render_console(&diags, &project_name, Some(&stats)) ); + if let Some(ref diff) = verdict_diff { + println!("\nBaseline comparison:"); + print!("{}", crate::baseline::format_diff_console(diff)); + } } } @@ -537,6 +591,19 @@ pub fn handle( } } + // ── --gate: CI gate check (exit 2 on violation) ───────────────────── + if let (Some(diff), Some(gate_name)) = (&verdict_diff, gate) { + if !crate::baseline::check_gate(diff, gate_name) { + if !suppress_status { + eprintln!( + "Gate '{}' violated. Exit code 2.", + gate_name + ); + } + std::process::exit(2); + } + } + // ── --fail-on: exit non-zero if threshold breached ────────────────── // Suppressed findings do not count toward the threshold. if let Some(threshold) = fail_on { diff --git a/src/lib.rs b/src/lib.rs index 08ee9d94..4a5065f1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,6 +91,7 @@ pub mod abstract_interp; pub mod ast; pub mod auth_analysis; +pub mod baseline; pub mod callgraph; pub mod cfg; pub mod cfg_analysis; diff --git a/src/server/jobs.rs b/src/server/jobs.rs index 0b0d37bb..2495749c 100644 --- a/src/server/jobs.rs +++ b/src/server/jobs.rs @@ -266,7 +266,13 @@ impl JobManager { // Prepare the final state outside the lock. let (status, diags, error_str) = match result { - Ok(diags) => { + Ok(mut diags) => { + // Compute stable_hash for every finding (§M6.5 cross-commit identity). + // The CLI handler does this in commands/scan.rs::handle, but the + // server scan path bypasses handle, so do it here. + for d in &mut diags { + d.stable_hash = scan::compute_stable_hash(d); + } log_collector.info(format!("Scan completed: {} findings", diags.len()), None); (JobStatus::Completed, Some(Arc::new(diags)), None) } diff --git a/src/server/models.rs b/src/server/models.rs index f50f91a9..bbc282c9 100644 --- a/src/server/models.rs +++ b/src/server/models.rs @@ -38,6 +38,10 @@ pub struct FindingView { pub fingerprint: String, #[serde(skip_serializing_if = "String::is_empty")] pub portable_fingerprint: String, + /// Blake3-derived stable cross-commit identity hash (M6.5). Zero when not + /// yet computed (server-side scans always compute it post-analysis). + #[serde(skip_serializing_if = "crate::server::models::is_zero_u64")] + pub stable_hash: u64, pub path: String, pub line: usize, pub col: usize, @@ -263,12 +267,17 @@ fn status_for_diag(d: &Diag) -> &'static str { } } +pub(crate) fn is_zero_u64(v: &u64) -> bool { + *v == 0 +} + /// Convert a Diag to a FindingView at a given index. pub fn finding_from_diag(index: usize, d: &Diag) -> FindingView { FindingView { index, fingerprint: compute_fingerprint(d), portable_fingerprint: String::new(), // set by caller with scan_root + stable_hash: d.stable_hash, path: d.path.clone(), line: d.line, col: d.col, @@ -394,6 +403,10 @@ pub struct CompareResponse { pub fixed_findings: Vec, pub changed_findings: Vec, pub unchanged_findings: Vec, + /// Verdict-level diff entries (M6.5). Populated when findings in both + /// scans carry `stable_hash` values. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub verdict_diff: Vec, } /// Minimal scan metadata for comparison headers. diff --git a/src/server/routes/scans.rs b/src/server/routes/scans.rs index 63ff2d23..5a92c5e8 100644 --- a/src/server/routes/scans.rs +++ b/src/server/routes/scans.rs @@ -459,6 +459,11 @@ async fn compare_scans( severity_delta, }; + // Build verdict diff from left (baseline) → right (current) using stable_hash. + let left_baseline = crate::baseline::diags_to_baseline_entries(&left_findings); + let verdict_diff_result = + crate::baseline::compute_verdict_diff(&left_baseline, &right_findings); + Ok(Json(CompareResponse { left_scan: left_info, right_scan: right_info, @@ -467,6 +472,7 @@ async fn compare_scans( fixed_findings, changed_findings, unchanged_findings, + verdict_diff: verdict_diff_result.entries, })) } diff --git a/tests/fix_validation_e2e.rs b/tests/fix_validation_e2e.rs new file mode 100644 index 00000000..54e95bb5 --- /dev/null +++ b/tests/fix_validation_e2e.rs @@ -0,0 +1,258 @@ +//! End-to-end tests for `nyx scan --baseline` / `--gate` (§M6.5, Pillar A). +//! +//! Demonstrates the "woah" loop from §15.5: +//! 1. Scan a vulnerable Python project — finding emits with `stable_hash`. +//! 2. Simulate `Confirmed` dynamic verdict (as `--verify` would produce). +//! 3. Write a stripped baseline (no source code, only hash + verdict). +//! 4. Fix the vulnerability and rescan. +//! 5. Diff against the baseline: finding flips to `FlippedNotConfirmed`. +//! 6. `--gate=resolve-all-confirmed` passes (exits 0). +//! 7. Introduce a new vulnerability and simulate `Confirmed` on it. +//! 8. `--gate=no-new-confirmed` fails (would exit 2). + +mod common; + +use nyx_scanner::baseline::{ + check_gate, compute_verdict_diff, diags_to_baseline_entries, load_baseline, write_baseline, + BaselineEntry, Transition, GATE_NO_NEW_CONFIRMED, GATE_RESOLVE_ALL_CONFIRMED, +}; +use nyx_scanner::commands::scan::compute_stable_hash; +use nyx_scanner::evidence::{Evidence, VerifyResult, VerifyStatus}; +use nyx_scanner::utils::config::AnalysisMode; +use std::path::Path; +use tempfile::NamedTempFile; + +/// Run `scan_no_index` and assign stable hashes to every finding. +fn scan_with_hashes(dir: &Path) -> Vec { + let mut diags = common::scan_fixture_dir(dir, AnalysisMode::Full); + for d in &mut diags { + d.stable_hash = compute_stable_hash(d); + } + diags +} + +/// Attach a simulated dynamic verdict to every finding in the list. +fn set_verdict( + diags: &mut Vec, + status: VerifyStatus, +) { + for d in diags.iter_mut() { + let fid = format!("{:016x}", d.stable_hash); + let ev = d.evidence.get_or_insert_with(Evidence::default); + ev.dynamic_verdict = Some(VerifyResult { + finding_id: fid, + status, + triggered_payload: if status == VerifyStatus::Confirmed { + Some("' OR 1=1--".to_string()) + } else { + None + }, + reason: None, + inconclusive_reason: None, + detail: None, + attempts: vec![], + toolchain_match: None, + }); + } +} + +const VULN_DIR: &str = "tests/fixtures/baseline_sqli_vuln"; +const FIXED_DIR: &str = "tests/fixtures/baseline_sqli_fixed"; +const NEW_DIR: &str = "tests/fixtures/baseline_sqli_new"; + +// ── §15.5 "woah" loop end-to-end ──────────────────────────────────────────── + +/// Step 1-3: Scan the vulnerable version, simulate Confirmed, write baseline. +#[test] +fn vuln_scan_emits_finding_with_stable_hash() { + let vuln_path = Path::new(VULN_DIR); + let diags = scan_with_hashes(vuln_path); + assert!( + !diags.is_empty(), + "Expected SQL injection finding in {VULN_DIR}" + ); + assert!( + diags.iter().all(|d| d.stable_hash != 0), + "All findings must have non-zero stable_hash after compute_stable_hash" + ); +} + +/// Step 4-6: Fix → rescan → diff → gate passes. +#[test] +fn fix_resolves_confirmed_finding() { + let vuln_path = Path::new(VULN_DIR); + let fixed_path = Path::new(FIXED_DIR); + + // Step 1: scan vulnerable, simulate Confirmed verdict. + let mut vuln_diags = scan_with_hashes(vuln_path); + assert!(!vuln_diags.is_empty(), "Need at least one SQL injection finding"); + set_verdict(&mut vuln_diags, VerifyStatus::Confirmed); + + // Step 2: write stripped baseline. + let baseline_file = NamedTempFile::new().unwrap(); + write_baseline(baseline_file.path(), &vuln_diags).unwrap(); + + // Step 3: load baseline and verify it has no source code. + let raw = std::fs::read_to_string(baseline_file.path()).unwrap(); + assert!( + !raw.contains("execute"), + "baseline must not contain source code snippets (found 'execute')" + ); + let baseline_entries = load_baseline(baseline_file.path()).unwrap(); + assert!(!baseline_entries.is_empty()); + assert_eq!( + baseline_entries[0].dynamic_verdict, + Some(VerifyStatus::Confirmed) + ); + + // Step 4: scan fixed version. + let fixed_diags = scan_with_hashes(fixed_path); + + // Step 5: diff. + let diff = compute_verdict_diff(&baseline_entries, &fixed_diags); + + // The vulnerable finding should be Resolved (gone from fixed code). + // Alternatively it could be FlippedNotConfirmed if the scanner still + // finds a flow (it shouldn't for the parameterized query). + let resolved_or_flipped = diff.entries.iter().any(|e| { + e.baseline_status == Some(VerifyStatus::Confirmed) + && matches!( + e.transition, + Transition::Resolved | Transition::FlippedNotConfirmed + ) + }); + assert!( + resolved_or_flipped, + "Expected the Confirmed finding to be Resolved or FlippedNotConfirmed after the fix. \ + Diff entries: {:#?}", + diff.entries + ); + + // Step 6: gate passes. + assert!( + check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED), + "resolve-all-confirmed gate must pass after the fix" + ); +} + +/// Step 7-8: new Confirmed finding → no-new-confirmed gate fails. +#[test] +fn new_confirmed_fails_no_new_confirmed_gate() { + let vuln_path = Path::new(VULN_DIR); + let new_path = Path::new(NEW_DIR); + + // Baseline: the original vulnerability, confirmed. + let mut vuln_diags = scan_with_hashes(vuln_path); + set_verdict(&mut vuln_diags, VerifyStatus::Confirmed); + let baseline_entries = diags_to_baseline_entries(&vuln_diags); + + // Current: the "fixed+new" version — original finding gone, new one appears. + let mut new_diags = scan_with_hashes(new_path); + // Simulate Confirmed on any new findings not in the baseline. + let baseline_hashes: std::collections::HashSet = + baseline_entries.iter().map(|e| e.stable_hash).collect(); + for d in new_diags.iter_mut() { + if !baseline_hashes.contains(&d.stable_hash) { + let fid = format!("{:016x}", d.stable_hash); + let ev = d.evidence.get_or_insert_with(Evidence::default); + ev.dynamic_verdict = Some(VerifyResult { + finding_id: fid, + status: VerifyStatus::Confirmed, + triggered_payload: Some("' OR 1=1--".to_string()), + reason: None, + inconclusive_reason: None, + detail: None, + attempts: vec![], + toolchain_match: None, + }); + } + } + + let diff = compute_verdict_diff(&baseline_entries, &new_diags); + + // There must be at least one New+Confirmed entry. + let has_new_confirmed = diff.entries.iter().any(|e| { + e.transition == Transition::New && e.current_status == Some(VerifyStatus::Confirmed) + }); + assert!( + has_new_confirmed, + "Expected a new Confirmed finding in the diff. Diff entries: {:#?}", + diff.entries + ); + + // Gate must fail. + assert!( + !check_gate(&diff, GATE_NO_NEW_CONFIRMED), + "no-new-confirmed gate must fail when a new Confirmed finding exists" + ); +} + +/// `stable_hash` is stable across identical scans (same path, rule, line, col, caps). +#[test] +fn stable_hash_deterministic_across_scans() { + let vuln_path = Path::new(VULN_DIR); + let diags1 = scan_with_hashes(vuln_path); + let diags2 = scan_with_hashes(vuln_path); + + assert!(!diags1.is_empty()); + assert_eq!( + diags1.len(), + diags2.len(), + "finding count must be deterministic" + ); + + let hashes1: std::collections::HashSet = diags1.iter().map(|d| d.stable_hash).collect(); + let hashes2: std::collections::HashSet = diags2.iter().map(|d| d.stable_hash).collect(); + assert_eq!( + hashes1, hashes2, + "stable_hash must be identical across two scans of the same codebase" + ); +} + +/// Baseline-write file contains required fields and no source snippets. +#[test] +fn baseline_write_contains_required_fields_no_source() { + let vuln_path = Path::new(VULN_DIR); + let mut diags = scan_with_hashes(vuln_path); + set_verdict(&mut diags, VerifyStatus::Confirmed); + + let f = NamedTempFile::new().unwrap(); + write_baseline(f.path(), &diags).unwrap(); + + let content = std::fs::read_to_string(f.path()).unwrap(); + let entries: Vec = serde_json::from_str(&content).unwrap(); + + assert!(!entries.is_empty()); + for e in &entries { + assert_ne!(e.stable_hash, 0, "stable_hash must be non-zero"); + assert!(!e.path.is_empty(), "path must be set"); + assert!(!e.rule_id.is_empty(), "rule_id must be set"); + assert!(!e.severity.is_empty(), "severity must be set"); + } + // No source code snippets. + assert!( + !content.contains("SELECT"), + "baseline must not contain SQL source code" + ); +} + +/// `load_baseline` accepts a full Diag JSON (from `nyx scan --format json`). +#[test] +fn load_baseline_accepts_full_diag_json() { + let vuln_path = Path::new(VULN_DIR); + let diags = scan_with_hashes(vuln_path); + assert!(!diags.is_empty()); + + let diag_json = serde_json::to_string(&diags).unwrap(); + let f = NamedTempFile::new().unwrap(); + std::fs::write(f.path(), &diag_json).unwrap(); + + let loaded = load_baseline(f.path()).unwrap(); + assert_eq!(loaded.len(), diags.len()); + // Hashes must round-trip. + let loaded_hashes: std::collections::HashSet = + loaded.iter().map(|e| e.stable_hash).collect(); + let diag_hashes: std::collections::HashSet = + diags.iter().map(|d| d.stable_hash).collect(); + assert_eq!(loaded_hashes, diag_hashes); +} diff --git a/tests/fixtures/baseline_sqli_fixed/handler.py b/tests/fixtures/baseline_sqli_fixed/handler.py new file mode 100644 index 00000000..012fb3ec --- /dev/null +++ b/tests/fixtures/baseline_sqli_fixed/handler.py @@ -0,0 +1,5 @@ +import sqlite3 + +def get_user(db, user_id): + query = "SELECT * FROM users WHERE id = ?" + return db.execute(query, (user_id,)) diff --git a/tests/fixtures/baseline_sqli_new/handler.py b/tests/fixtures/baseline_sqli_new/handler.py new file mode 100644 index 00000000..3f5dc44c --- /dev/null +++ b/tests/fixtures/baseline_sqli_new/handler.py @@ -0,0 +1,12 @@ +import os +import sqlite3 + +def get_user(db): + user_id = os.getenv("USER_ID") + query = "SELECT * FROM users WHERE id = ?" + return db.execute(query, (user_id,)) + +def get_post(db): + post_id = os.getenv("POST_ID") + query = "SELECT * FROM posts WHERE id = " + post_id + return db.execute(query) diff --git a/tests/fixtures/baseline_sqli_vuln/handler.py b/tests/fixtures/baseline_sqli_vuln/handler.py new file mode 100644 index 00000000..b538d63c --- /dev/null +++ b/tests/fixtures/baseline_sqli_vuln/handler.py @@ -0,0 +1,7 @@ +import os +import sqlite3 + +def get_user(db): + user_id = os.getenv("USER_ID") + query = "SELECT * FROM users WHERE id = " + user_id + return db.execute(query)