[pitboss] phase 08: M6.5 — Patch-validation / fix-validation CI mode

This commit is contained in:
pitboss 2026-05-12 14:14:13 -04:00
parent 25e8b0eb0e
commit 118cafa535
13 changed files with 1067 additions and 4 deletions

View file

@ -83,10 +83,31 @@ export interface RelatedFindingView {
severity: string;
}
// Baseline / patch-validation types (M6.5)
export type VerdictTransition =
| 'New'
| 'Unchanged'
| 'Resolved'
| 'Regressed'
| 'FlippedConfirmed'
| 'FlippedNotConfirmed';
export interface VerdictDiffEntry {
stable_hash: number;
path: string;
line: number;
rule_id: string;
baseline_status?: VerifyStatus;
current_status?: VerifyStatus;
transition: VerdictTransition;
}
export interface FindingView {
index: number;
fingerprint: string;
portable_fingerprint?: string;
/** Blake3-derived stable cross-commit identity (M6.5). */
stable_hash?: number;
path: string;
line: number;
col: number;
@ -199,6 +220,8 @@ export interface CompareResponse {
fixed_findings: ComparedFinding[];
changed_findings: ChangedFinding[];
unchanged_findings: ComparedFinding[];
/** Verdict-level diff (M6.5). Present when findings carry stable_hash values. */
verdict_diff?: VerdictDiffEntry[];
}
// Overview types

628
src/baseline.rs Normal file
View file

@ -0,0 +1,628 @@
//! Baseline diffing for patch-validation CI mode (§M6.5 / Pillar A §15.1).
//!
//! `nyx scan --baseline <file>` reads a previous scan's JSON output (or a
//! stripped `.nyx/baseline.json`) and joins on `Diag::stable_hash`. The
//! result is a per-finding `VerdictDiffEntry` with a typed `Transition` that
//! CI gates can act on.
//!
//! `nyx scan --baseline-write <file>` writes a stripped baseline JSON:
//! only `stable_hash`, `dynamic_verdict`, `severity`, `path`, and `rule_id`.
//! No source code is included.
use crate::commands::scan::Diag;
use crate::evidence::VerifyStatus;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::Path;
// ─────────────────────────────────────────────────────────────────────────────
// Baseline entry (stripped — no source code)
// ─────────────────────────────────────────────────────────────────────────────
/// A stripped baseline entry: only what is needed for cross-commit diffing.
/// Contains no source code snippets.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BaselineEntry {
pub stable_hash: u64,
/// Dynamic verdict status from the scan that wrote this baseline.
/// `None` when `--verify` was not run.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dynamic_verdict: Option<VerifyStatus>,
pub severity: String,
pub path: String,
pub rule_id: String,
}
// ─────────────────────────────────────────────────────────────────────────────
// Transition enum
// ─────────────────────────────────────────────────────────────────────────────
/// How a finding's verdict changed between the baseline scan and the current
/// scan.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum Transition {
/// Finding exists in the current scan but was absent from the baseline.
New,
/// Finding appears in both scans; verdict is unchanged (or neither scan
/// ran `--verify`).
Unchanged,
/// Finding was present in the baseline but disappeared from the current
/// scan — the vulnerability is gone.
Resolved,
/// Finding in both; was `NotConfirmed` in baseline, now `Confirmed`.
Regressed,
/// Finding in both; baseline had no verdict (or `Inconclusive` /
/// `Unsupported`) and it is now `Confirmed`.
FlippedConfirmed,
/// Finding in both; was `Confirmed` in baseline, now `NotConfirmed` —
/// the fix is proven.
FlippedNotConfirmed,
}
// ─────────────────────────────────────────────────────────────────────────────
// VerdictDiffEntry
// ─────────────────────────────────────────────────────────────────────────────
/// Per-finding verdict diff produced by comparing a baseline to a current scan.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VerdictDiffEntry {
/// Stable cross-commit identity hash.
pub stable_hash: u64,
pub path: String,
pub line: usize,
pub rule_id: String,
/// Verdict in the baseline scan (`None` when verify was not run).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub baseline_status: Option<VerifyStatus>,
/// Verdict in the current scan (`None` when verify was not run or finding
/// is absent from the current scan).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub current_status: Option<VerifyStatus>,
pub transition: Transition,
}
/// Full verdict diff between a baseline and a current scan.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VerdictDiff {
pub entries: Vec<VerdictDiffEntry>,
}
// ─────────────────────────────────────────────────────────────────────────────
// Load / write helpers
// ─────────────────────────────────────────────────────────────────────────────
/// Load baseline entries from a file.
///
/// Accepts two JSON formats:
/// - Stripped baseline (`Vec<BaselineEntry>`) — written by `--baseline-write`.
/// - Full scan output (`Vec<Diag>`) — written by `nyx scan --format json`.
///
/// Detection heuristic: try `Vec<BaselineEntry>` first (requires `rule_id`);
/// fall back to `Vec<Diag>`.
pub fn load_baseline(path: &Path) -> crate::errors::NyxResult<Vec<BaselineEntry>> {
let content = std::fs::read_to_string(path).map_err(|e| {
crate::errors::NyxError::Msg(format!("cannot read baseline {}: {e}", path.display()))
})?;
// Try stripped format first.
if let Ok(entries) = serde_json::from_str::<Vec<BaselineEntry>>(&content) {
return Ok(entries);
}
// Fall back to full Diag list.
let diags: Vec<Diag> = serde_json::from_str(&content).map_err(|e| {
crate::errors::NyxError::Msg(format!(
"baseline {}: not a valid BaselineEntry list or Diag list: {e}",
path.display()
))
})?;
Ok(diags_to_baseline_entries(&diags))
}
/// Convert `Diag` values to `BaselineEntry` values.
///
/// Only findings with a non-zero `stable_hash` are included; findings without
/// a hash cannot be joined across scans.
pub fn diags_to_baseline_entries(diags: &[Diag]) -> Vec<BaselineEntry> {
diags
.iter()
.filter(|d| d.stable_hash != 0)
.map(|d| BaselineEntry {
stable_hash: d.stable_hash,
dynamic_verdict: d
.evidence
.as_ref()
.and_then(|ev| ev.dynamic_verdict.as_ref())
.map(|vr| vr.status),
severity: d.severity.as_db_str().to_string(),
path: d.path.clone(),
rule_id: d.id.clone(),
})
.collect()
}
/// Write a stripped baseline JSON to `path`.
///
/// The file contains only `stable_hash`, `dynamic_verdict`, `severity`,
/// `path`, and `rule_id` — no source code snippets or flow steps.
pub fn write_baseline(path: &Path, diags: &[Diag]) -> crate::errors::NyxResult<()> {
let entries = diags_to_baseline_entries(diags);
let json = serde_json::to_string_pretty(&entries).map_err(|e| {
crate::errors::NyxError::Msg(format!("baseline serialize error: {e}"))
})?;
if let Some(parent) = path.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent).map_err(|e| {
crate::errors::NyxError::Msg(format!(
"cannot create baseline dir {}: {e}",
parent.display()
))
})?;
}
}
std::fs::write(path, json).map_err(|e| {
crate::errors::NyxError::Msg(format!(
"cannot write baseline {}: {e}",
path.display()
))
})
}
// ─────────────────────────────────────────────────────────────────────────────
// Diff computation
// ─────────────────────────────────────────────────────────────────────────────
fn classify_transition(
baseline: Option<VerifyStatus>,
current: Option<VerifyStatus>,
) -> Transition {
match (baseline, current) {
// No verdict change (including both None)
(a, b) if a == b => Transition::Unchanged,
// Confirmed → NotConfirmed: fix proven
(Some(VerifyStatus::Confirmed), Some(VerifyStatus::NotConfirmed)) => {
Transition::FlippedNotConfirmed
}
// NotConfirmed → Confirmed: regression
(Some(VerifyStatus::NotConfirmed), Some(VerifyStatus::Confirmed)) => {
Transition::Regressed
}
// None / Inconclusive / Unsupported → Confirmed
(_, Some(VerifyStatus::Confirmed)) => Transition::FlippedConfirmed,
// Everything else: treat as unchanged (e.g. Confirmed → Inconclusive
// without a clean NotConfirmed proof is not a resolution)
_ => Transition::Unchanged,
}
}
/// Compute a verdict diff between a loaded baseline and the current findings.
pub fn compute_verdict_diff(baseline: &[BaselineEntry], current: &[Diag]) -> VerdictDiff {
// Build lookup maps keyed by stable_hash.
let baseline_map: HashMap<u64, &BaselineEntry> =
baseline.iter().map(|e| (e.stable_hash, e)).collect();
let current_map: HashMap<u64, &Diag> = current
.iter()
.filter(|d| d.stable_hash != 0)
.map(|d| (d.stable_hash, d))
.collect();
let mut entries = Vec::new();
// Walk current findings.
for (&hash, diag) in &current_map {
let current_status = diag
.evidence
.as_ref()
.and_then(|ev| ev.dynamic_verdict.as_ref())
.map(|vr| vr.status);
if let Some(base) = baseline_map.get(&hash) {
let transition = classify_transition(base.dynamic_verdict, current_status);
entries.push(VerdictDiffEntry {
stable_hash: hash,
path: diag.path.clone(),
line: diag.line,
rule_id: diag.id.clone(),
baseline_status: base.dynamic_verdict,
current_status,
transition,
});
} else {
// Not in baseline → New.
entries.push(VerdictDiffEntry {
stable_hash: hash,
path: diag.path.clone(),
line: diag.line,
rule_id: diag.id.clone(),
baseline_status: None,
current_status,
transition: Transition::New,
});
}
}
// Walk baseline findings absent from current → Resolved.
for (&hash, base) in &baseline_map {
if !current_map.contains_key(&hash) {
entries.push(VerdictDiffEntry {
stable_hash: hash,
path: base.path.clone(),
line: 0,
rule_id: base.rule_id.clone(),
baseline_status: base.dynamic_verdict,
current_status: None,
transition: Transition::Resolved,
});
}
}
// Sort for deterministic output: Resolved first, then New, then the rest,
// all sub-sorted by (path, line).
entries.sort_by(|a, b| {
fn order(t: Transition) -> u8 {
match t {
Transition::Resolved => 0,
Transition::FlippedNotConfirmed => 1,
Transition::New => 2,
Transition::Regressed => 3,
Transition::FlippedConfirmed => 4,
Transition::Unchanged => 5,
}
}
order(a.transition)
.cmp(&order(b.transition))
.then_with(|| a.path.cmp(&b.path))
.then_with(|| a.line.cmp(&b.line))
});
VerdictDiff { entries }
}
// ─────────────────────────────────────────────────────────────────────────────
// CI gates
// ─────────────────────────────────────────────────────────────────────────────
/// Gate: exit code 2 if any new `Confirmed` finding appears.
///
/// Triggers on `transition == New && current_status == Confirmed` or
/// `transition == FlippedConfirmed`.
pub const GATE_NO_NEW_CONFIRMED: &str = "no-new-confirmed";
/// Gate: exit code 2 if any baseline-`Confirmed` finding is not fully resolved.
///
/// A baseline-Confirmed finding is resolved only when it is absent from the
/// current scan (`Resolved`) or its current verdict is `NotConfirmed`
/// (`FlippedNotConfirmed`). All other current statuses (`Confirmed`,
/// `Inconclusive`, `Unsupported`) violate this gate.
pub const GATE_RESOLVE_ALL_CONFIRMED: &str = "resolve-all-confirmed";
/// Check a named CI gate against a verdict diff.
///
/// Returns `true` when the gate passes (condition not violated) and `false`
/// when it fails (caller should exit with code 2).
///
/// Unknown gate names always pass so future gate additions are forward-
/// compatible without requiring a binary upgrade.
pub fn check_gate(diff: &VerdictDiff, gate: &str) -> bool {
match gate {
GATE_NO_NEW_CONFIRMED => !diff.entries.iter().any(|e| {
matches!(e.transition, Transition::New | Transition::FlippedConfirmed)
&& e.current_status == Some(VerifyStatus::Confirmed)
}),
GATE_RESOLVE_ALL_CONFIRMED => !diff.entries.iter().any(|e| {
e.baseline_status == Some(VerifyStatus::Confirmed)
&& matches!(
e.current_status,
Some(VerifyStatus::Confirmed)
| Some(VerifyStatus::Inconclusive)
| Some(VerifyStatus::Unsupported)
)
}),
_ => true,
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Console / JSON rendering
// ─────────────────────────────────────────────────────────────────────────────
fn status_str(s: Option<VerifyStatus>) -> &'static str {
match s {
Some(VerifyStatus::Confirmed) => "Confirmed",
Some(VerifyStatus::NotConfirmed) => "NotConfirmed",
Some(VerifyStatus::Inconclusive) => "Inconclusive",
Some(VerifyStatus::Unsupported) => "Unsupported",
None => "(no verdict)",
}
}
/// Render a verdict diff as a human-readable console summary.
pub fn format_diff_console(diff: &VerdictDiff) -> String {
if diff.entries.is_empty() {
return String::from(" (no findings in baseline or current scan)\n");
}
let mut lines = Vec::new();
let mut non_unchanged = 0usize;
for e in &diff.entries {
let hash_str = format!("{:016x}", e.stable_hash);
let loc = if e.line > 0 {
format!("{}:{}", e.path, e.line)
} else {
e.path.clone()
};
match e.transition {
Transition::New => {
non_unchanged += 1;
lines.push(format!(
" + {hash_str}: new {} at {loc}",
status_str(e.current_status)
));
}
Transition::Resolved => {
non_unchanged += 1;
lines.push(format!(
" - {hash_str}: {} \u{2192} removed (resolved) at {loc}",
status_str(e.baseline_status)
));
}
Transition::FlippedNotConfirmed => {
non_unchanged += 1;
lines.push(format!(
" - {hash_str}: Confirmed \u{2192} NotConfirmed at {loc} (resolved)"
));
}
Transition::Regressed => {
non_unchanged += 1;
lines.push(format!(
" ! {hash_str}: NotConfirmed \u{2192} Confirmed at {loc} (regressed)"
));
}
Transition::FlippedConfirmed => {
non_unchanged += 1;
lines.push(format!(
" + {hash_str}: new Confirmed at {loc}"
));
}
Transition::Unchanged => {}
}
}
if non_unchanged == 0 {
return String::from(" (no changes from baseline)\n");
}
lines.join("\n") + "\n"
}
// ─────────────────────────────────────────────────────────────────────────────
// Tests
// ─────────────────────────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
use super::*;
use crate::commands::scan::{compute_stable_hash, Diag};
use crate::evidence::{Evidence, VerifyResult, VerifyStatus};
use crate::patterns::{FindingCategory, Severity};
fn make_diag(path: &str, line: usize, rule: &str) -> Diag {
let mut d = Diag {
path: path.to_string(),
line,
col: 0,
severity: Severity::High,
id: rule.to_string(),
category: FindingCategory::Security,
path_validated: false,
guard_kind: None,
message: None,
labels: vec![],
confidence: None,
evidence: None,
rank_score: None,
rank_reason: None,
suppressed: false,
suppression: None,
rollup: None,
finding_id: String::new(),
alternative_finding_ids: vec![],
stable_hash: 0,
};
d.stable_hash = compute_stable_hash(&d);
d
}
fn with_verdict(mut d: Diag, status: VerifyStatus) -> Diag {
d.evidence = Some(Evidence {
dynamic_verdict: Some(VerifyResult {
finding_id: format!("{:016x}", d.stable_hash),
status,
triggered_payload: None,
reason: None,
inconclusive_reason: None,
detail: None,
attempts: vec![],
toolchain_match: None,
}),
..Default::default()
});
d
}
#[test]
fn new_finding_no_verdict() {
let current = vec![make_diag("src/a.py", 1, "py.sqli")];
let diff = compute_verdict_diff(&[], &current);
assert_eq!(diff.entries.len(), 1);
assert_eq!(diff.entries[0].transition, Transition::New);
assert_eq!(diff.entries[0].current_status, None);
}
#[test]
fn new_confirmed_finding() {
let current = vec![with_verdict(
make_diag("src/a.py", 1, "py.sqli"),
VerifyStatus::Confirmed,
)];
let diff = compute_verdict_diff(&[], &current);
assert_eq!(diff.entries[0].transition, Transition::New);
assert_eq!(diff.entries[0].current_status, Some(VerifyStatus::Confirmed));
}
#[test]
fn resolved_finding() {
let baseline_diag = make_diag("src/a.py", 1, "py.sqli");
let baseline = diags_to_baseline_entries(&[baseline_diag]);
let diff = compute_verdict_diff(&baseline, &[]);
assert_eq!(diff.entries.len(), 1);
assert_eq!(diff.entries[0].transition, Transition::Resolved);
}
#[test]
fn flipped_not_confirmed() {
let d = make_diag("src/a.py", 1, "py.sqli");
let baseline = vec![BaselineEntry {
stable_hash: d.stable_hash,
dynamic_verdict: Some(VerifyStatus::Confirmed),
severity: "high".to_string(),
path: d.path.clone(),
rule_id: d.id.clone(),
}];
let current = vec![with_verdict(d, VerifyStatus::NotConfirmed)];
let diff = compute_verdict_diff(&baseline, &current);
assert_eq!(diff.entries[0].transition, Transition::FlippedNotConfirmed);
}
#[test]
fn regressed() {
let d = make_diag("src/a.py", 1, "py.sqli");
let baseline = vec![BaselineEntry {
stable_hash: d.stable_hash,
dynamic_verdict: Some(VerifyStatus::NotConfirmed),
severity: "high".to_string(),
path: d.path.clone(),
rule_id: d.id.clone(),
}];
let current = vec![with_verdict(d, VerifyStatus::Confirmed)];
let diff = compute_verdict_diff(&baseline, &current);
assert_eq!(diff.entries[0].transition, Transition::Regressed);
}
#[test]
fn gate_no_new_confirmed_passes_when_no_confirmed() {
let d = make_diag("src/a.py", 1, "py.sqli");
let diff = compute_verdict_diff(&[], &[d]);
assert!(check_gate(&diff, GATE_NO_NEW_CONFIRMED));
}
#[test]
fn gate_no_new_confirmed_fails_on_new_confirmed() {
let current = vec![with_verdict(
make_diag("src/a.py", 1, "py.sqli"),
VerifyStatus::Confirmed,
)];
let diff = compute_verdict_diff(&[], &current);
assert!(!check_gate(&diff, GATE_NO_NEW_CONFIRMED));
}
#[test]
fn gate_resolve_all_confirmed_passes_when_flipped() {
let d = make_diag("src/a.py", 1, "py.sqli");
let baseline = vec![BaselineEntry {
stable_hash: d.stable_hash,
dynamic_verdict: Some(VerifyStatus::Confirmed),
severity: "high".to_string(),
path: d.path.clone(),
rule_id: d.id.clone(),
}];
let current = vec![with_verdict(d, VerifyStatus::NotConfirmed)];
let diff = compute_verdict_diff(&baseline, &current);
assert!(check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED));
}
#[test]
fn gate_resolve_all_confirmed_fails_when_still_confirmed() {
let d = make_diag("src/a.py", 1, "py.sqli");
let baseline = vec![BaselineEntry {
stable_hash: d.stable_hash,
dynamic_verdict: Some(VerifyStatus::Confirmed),
severity: "high".to_string(),
path: d.path.clone(),
rule_id: d.id.clone(),
}];
let current = vec![with_verdict(d, VerifyStatus::Confirmed)];
let diff = compute_verdict_diff(&baseline, &current);
assert!(!check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED));
}
#[test]
fn gate_resolve_all_confirmed_passes_when_resolved() {
let d = make_diag("src/a.py", 1, "py.sqli");
let baseline = vec![BaselineEntry {
stable_hash: d.stable_hash,
dynamic_verdict: Some(VerifyStatus::Confirmed),
severity: "high".to_string(),
path: d.path.clone(),
rule_id: d.id.clone(),
}];
// No current findings (finding disappeared entirely).
let diff = compute_verdict_diff(&baseline, &[]);
assert!(check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED));
}
#[test]
fn write_and_load_roundtrip() {
let d = with_verdict(make_diag("src/a.py", 1, "py.sqli"), VerifyStatus::Confirmed);
let tmp = tempfile::NamedTempFile::new().unwrap();
write_baseline(tmp.path(), &[d.clone()]).unwrap();
let loaded = load_baseline(tmp.path()).unwrap();
assert_eq!(loaded.len(), 1);
assert_eq!(loaded[0].stable_hash, d.stable_hash);
assert_eq!(loaded[0].dynamic_verdict, Some(VerifyStatus::Confirmed));
assert_eq!(loaded[0].path, "src/a.py");
assert_eq!(loaded[0].rule_id, "py.sqli");
}
#[test]
fn load_full_diag_json() {
let d = with_verdict(make_diag("src/a.py", 1, "py.sqli"), VerifyStatus::Confirmed);
let json = serde_json::to_string(&[&d]).unwrap();
let tmp = tempfile::NamedTempFile::new().unwrap();
std::fs::write(tmp.path(), &json).unwrap();
let loaded = load_baseline(tmp.path()).unwrap();
assert_eq!(loaded.len(), 1);
assert_eq!(loaded[0].stable_hash, d.stable_hash);
}
#[test]
fn baseline_write_no_source() {
let mut d = with_verdict(make_diag("src/a.py", 1, "py.sqli"), VerifyStatus::Confirmed);
// Add a flow_step with a snippet (source code) to the evidence.
if let Some(ref mut ev) = d.evidence {
ev.flow_steps = vec![crate::evidence::FlowStep {
step: 1,
kind: crate::evidence::FlowStepKind::Source,
file: "src/a.py".into(),
line: 1,
col: 0,
snippet: Some("SECRET CODE".into()),
variable: None,
callee: None,
function: None,
is_cross_file: false,
}];
}
let tmp = tempfile::NamedTempFile::new().unwrap();
write_baseline(tmp.path(), &[d]).unwrap();
let content = std::fs::read_to_string(tmp.path()).unwrap();
assert!(!content.contains("SECRET CODE"), "baseline must not contain source code");
}
#[test]
fn unknown_gate_passes() {
let diff = VerdictDiff { entries: vec![] };
assert!(check_gate(&diff, "some-future-gate-name"));
}
}

View file

@ -456,6 +456,37 @@ pub enum Commands {
#[cfg_attr(not(feature = "dynamic"), arg(hide = true))]
#[arg(long, help_heading = "Dynamic", value_name = "BACKEND")]
backend: Option<String>,
// ── Baseline / patch-validation (§M6.5) ────────────────────────
/// Read a previous scan's JSON output (or a stripped .nyx/baseline.json)
/// and diff it against the current scan on stable_hash.
///
/// Emits a verdict diff showing New / Resolved / FlippedConfirmed /
/// FlippedNotConfirmed transitions. Combine with --gate to enforce CI
/// policies.
#[arg(long, value_name = "FILE", help_heading = "Baseline")]
baseline: Option<String>,
/// Write a stripped baseline JSON to FILE after scanning.
///
/// The file contains only stable_hash, dynamic_verdict, severity, path,
/// and rule_id — no source code. A CI job can persist this file to
/// compare future scans against without leaking source.
#[arg(long, value_name = "FILE", help_heading = "Baseline")]
baseline_write: Option<String>,
/// CI gate to enforce when --baseline is active.
///
/// `no-new-confirmed`: exit 2 if any new Confirmed finding appears.
/// `resolve-all-confirmed`: exit 2 if any baseline-Confirmed finding
/// is not fully resolved (absent or NotConfirmed in the current scan).
#[arg(
long,
value_name = "GATE",
value_parser = ["no-new-confirmed", "resolve-all-confirmed"],
help_heading = "Baseline"
)]
gate: Option<String>,
},
/// Submit feedback on a dynamic verification verdict (§21.2).

View file

@ -100,6 +100,9 @@ pub fn handle_command(
verify,
unsafe_sandbox,
backend,
baseline,
baseline_write,
gate,
} => {
// ── Apply profile first (CLI flags override after) ──────────
if let Some(ref name) = profile {
@ -360,6 +363,9 @@ pub fn handle_command(
show_instances.as_deref(),
database_dir,
config,
baseline.as_deref().map(std::path::Path::new),
baseline_write.as_deref().map(std::path::Path::new),
gate.as_deref(),
)?;
}
#[cfg(feature = "dynamic")]

View file

@ -345,6 +345,9 @@ pub fn handle(
show_instances: Option<&str>,
database_dir: &Path,
config: &Config,
baseline: Option<&Path>,
baseline_write: Option<&Path>,
gate: Option<&str>,
) -> NyxResult<()> {
let scan_path = Path::new(path).canonicalize()?;
let (project_name, db_path) = get_project_info(&scan_path, database_dir)?;
@ -489,18 +492,65 @@ pub fn handle(
}
}
// ── Baseline write (§M6.5): persist current findings as stripped baseline
if let Some(bw_path) = baseline_write {
if let Err(e) = crate::baseline::write_baseline(bw_path, &diags) {
tracing::warn!(path = %bw_path.display(), error = %e, "baseline-write failed");
if !suppress_status {
eprintln!("warning: --baseline-write failed: {e}");
}
} else if !suppress_status {
eprintln!("Baseline written to {}", bw_path.display());
}
}
// ── Baseline diff (§M6.5): load previous baseline and compute transitions
let verdict_diff = if let Some(bl_path) = baseline {
match crate::baseline::load_baseline(bl_path) {
Ok(baseline_entries) => {
let diff = crate::baseline::compute_verdict_diff(&baseline_entries, &diags);
Some(diff)
}
Err(e) => {
return Err(crate::errors::NyxError::Msg(format!(
"--baseline {}: {e}",
bl_path.display()
)));
}
}
} else {
None
};
// ── Output ──────────────────────────────────────────────────────────
match format {
OutputFormat::Json => {
let json = serde_json::to_string(&diags)
.map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?;
println!("{json}");
if let Some(ref diff) = verdict_diff {
// Wrap findings + verdict_diff into one JSON object so the
// diff is machine-readable alongside the findings.
let out = serde_json::json!({
"findings": &diags,
"verdict_diff": diff,
});
let json = serde_json::to_string(&out)
.map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?;
println!("{json}");
} else {
let json = serde_json::to_string(&diags)
.map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?;
println!("{json}");
}
}
OutputFormat::Sarif => {
let sarif = crate::output::build_sarif(&diags, &scan_path);
let json = serde_json::to_string_pretty(&sarif)
.map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?;
println!("{json}");
// Emit diff on stderr for SARIF (stdout is owned by the SARIF schema).
if let Some(ref diff) = verdict_diff {
eprintln!("\nBaseline comparison:");
eprint!("{}", crate::baseline::format_diff_console(diff));
}
}
OutputFormat::Console => {
tracing::debug!("Printing to console");
@ -508,6 +558,10 @@ pub fn handle(
"{}",
crate::fmt::render_console(&diags, &project_name, Some(&stats))
);
if let Some(ref diff) = verdict_diff {
println!("\nBaseline comparison:");
print!("{}", crate::baseline::format_diff_console(diff));
}
}
}
@ -537,6 +591,19 @@ pub fn handle(
}
}
// ── --gate: CI gate check (exit 2 on violation) ─────────────────────
if let (Some(diff), Some(gate_name)) = (&verdict_diff, gate) {
if !crate::baseline::check_gate(diff, gate_name) {
if !suppress_status {
eprintln!(
"Gate '{}' violated. Exit code 2.",
gate_name
);
}
std::process::exit(2);
}
}
// ── --fail-on: exit non-zero if threshold breached ──────────────────
// Suppressed findings do not count toward the threshold.
if let Some(threshold) = fail_on {

View file

@ -91,6 +91,7 @@
pub mod abstract_interp;
pub mod ast;
pub mod auth_analysis;
pub mod baseline;
pub mod callgraph;
pub mod cfg;
pub mod cfg_analysis;

View file

@ -266,7 +266,13 @@ impl JobManager {
// Prepare the final state outside the lock.
let (status, diags, error_str) = match result {
Ok(diags) => {
Ok(mut diags) => {
// Compute stable_hash for every finding (§M6.5 cross-commit identity).
// The CLI handler does this in commands/scan.rs::handle, but the
// server scan path bypasses handle, so do it here.
for d in &mut diags {
d.stable_hash = scan::compute_stable_hash(d);
}
log_collector.info(format!("Scan completed: {} findings", diags.len()), None);
(JobStatus::Completed, Some(Arc::new(diags)), None)
}

View file

@ -38,6 +38,10 @@ pub struct FindingView {
pub fingerprint: String,
#[serde(skip_serializing_if = "String::is_empty")]
pub portable_fingerprint: String,
/// Blake3-derived stable cross-commit identity hash (M6.5). Zero when not
/// yet computed (server-side scans always compute it post-analysis).
#[serde(skip_serializing_if = "crate::server::models::is_zero_u64")]
pub stable_hash: u64,
pub path: String,
pub line: usize,
pub col: usize,
@ -263,12 +267,17 @@ fn status_for_diag(d: &Diag) -> &'static str {
}
}
pub(crate) fn is_zero_u64(v: &u64) -> bool {
*v == 0
}
/// Convert a Diag to a FindingView at a given index.
pub fn finding_from_diag(index: usize, d: &Diag) -> FindingView {
FindingView {
index,
fingerprint: compute_fingerprint(d),
portable_fingerprint: String::new(), // set by caller with scan_root
stable_hash: d.stable_hash,
path: d.path.clone(),
line: d.line,
col: d.col,
@ -394,6 +403,10 @@ pub struct CompareResponse {
pub fixed_findings: Vec<ComparedFinding>,
pub changed_findings: Vec<ChangedFinding>,
pub unchanged_findings: Vec<ComparedFinding>,
/// Verdict-level diff entries (M6.5). Populated when findings in both
/// scans carry `stable_hash` values.
#[serde(skip_serializing_if = "Vec::is_empty")]
pub verdict_diff: Vec<crate::baseline::VerdictDiffEntry>,
}
/// Minimal scan metadata for comparison headers.

View file

@ -459,6 +459,11 @@ async fn compare_scans(
severity_delta,
};
// Build verdict diff from left (baseline) → right (current) using stable_hash.
let left_baseline = crate::baseline::diags_to_baseline_entries(&left_findings);
let verdict_diff_result =
crate::baseline::compute_verdict_diff(&left_baseline, &right_findings);
Ok(Json(CompareResponse {
left_scan: left_info,
right_scan: right_info,
@ -467,6 +472,7 @@ async fn compare_scans(
fixed_findings,
changed_findings,
unchanged_findings,
verdict_diff: verdict_diff_result.entries,
}))
}

258
tests/fix_validation_e2e.rs Normal file
View file

@ -0,0 +1,258 @@
//! End-to-end tests for `nyx scan --baseline` / `--gate` (§M6.5, Pillar A).
//!
//! Demonstrates the "woah" loop from §15.5:
//! 1. Scan a vulnerable Python project — finding emits with `stable_hash`.
//! 2. Simulate `Confirmed` dynamic verdict (as `--verify` would produce).
//! 3. Write a stripped baseline (no source code, only hash + verdict).
//! 4. Fix the vulnerability and rescan.
//! 5. Diff against the baseline: finding flips to `FlippedNotConfirmed`.
//! 6. `--gate=resolve-all-confirmed` passes (exits 0).
//! 7. Introduce a new vulnerability and simulate `Confirmed` on it.
//! 8. `--gate=no-new-confirmed` fails (would exit 2).
mod common;
use nyx_scanner::baseline::{
check_gate, compute_verdict_diff, diags_to_baseline_entries, load_baseline, write_baseline,
BaselineEntry, Transition, GATE_NO_NEW_CONFIRMED, GATE_RESOLVE_ALL_CONFIRMED,
};
use nyx_scanner::commands::scan::compute_stable_hash;
use nyx_scanner::evidence::{Evidence, VerifyResult, VerifyStatus};
use nyx_scanner::utils::config::AnalysisMode;
use std::path::Path;
use tempfile::NamedTempFile;
/// Run `scan_no_index` and assign stable hashes to every finding.
fn scan_with_hashes(dir: &Path) -> Vec<nyx_scanner::commands::scan::Diag> {
let mut diags = common::scan_fixture_dir(dir, AnalysisMode::Full);
for d in &mut diags {
d.stable_hash = compute_stable_hash(d);
}
diags
}
/// Attach a simulated dynamic verdict to every finding in the list.
fn set_verdict(
diags: &mut Vec<nyx_scanner::commands::scan::Diag>,
status: VerifyStatus,
) {
for d in diags.iter_mut() {
let fid = format!("{:016x}", d.stable_hash);
let ev = d.evidence.get_or_insert_with(Evidence::default);
ev.dynamic_verdict = Some(VerifyResult {
finding_id: fid,
status,
triggered_payload: if status == VerifyStatus::Confirmed {
Some("' OR 1=1--".to_string())
} else {
None
},
reason: None,
inconclusive_reason: None,
detail: None,
attempts: vec![],
toolchain_match: None,
});
}
}
const VULN_DIR: &str = "tests/fixtures/baseline_sqli_vuln";
const FIXED_DIR: &str = "tests/fixtures/baseline_sqli_fixed";
const NEW_DIR: &str = "tests/fixtures/baseline_sqli_new";
// ── §15.5 "woah" loop end-to-end ────────────────────────────────────────────
/// Step 1-3: Scan the vulnerable version, simulate Confirmed, write baseline.
#[test]
fn vuln_scan_emits_finding_with_stable_hash() {
let vuln_path = Path::new(VULN_DIR);
let diags = scan_with_hashes(vuln_path);
assert!(
!diags.is_empty(),
"Expected SQL injection finding in {VULN_DIR}"
);
assert!(
diags.iter().all(|d| d.stable_hash != 0),
"All findings must have non-zero stable_hash after compute_stable_hash"
);
}
/// Step 4-6: Fix → rescan → diff → gate passes.
#[test]
fn fix_resolves_confirmed_finding() {
let vuln_path = Path::new(VULN_DIR);
let fixed_path = Path::new(FIXED_DIR);
// Step 1: scan vulnerable, simulate Confirmed verdict.
let mut vuln_diags = scan_with_hashes(vuln_path);
assert!(!vuln_diags.is_empty(), "Need at least one SQL injection finding");
set_verdict(&mut vuln_diags, VerifyStatus::Confirmed);
// Step 2: write stripped baseline.
let baseline_file = NamedTempFile::new().unwrap();
write_baseline(baseline_file.path(), &vuln_diags).unwrap();
// Step 3: load baseline and verify it has no source code.
let raw = std::fs::read_to_string(baseline_file.path()).unwrap();
assert!(
!raw.contains("execute"),
"baseline must not contain source code snippets (found 'execute')"
);
let baseline_entries = load_baseline(baseline_file.path()).unwrap();
assert!(!baseline_entries.is_empty());
assert_eq!(
baseline_entries[0].dynamic_verdict,
Some(VerifyStatus::Confirmed)
);
// Step 4: scan fixed version.
let fixed_diags = scan_with_hashes(fixed_path);
// Step 5: diff.
let diff = compute_verdict_diff(&baseline_entries, &fixed_diags);
// The vulnerable finding should be Resolved (gone from fixed code).
// Alternatively it could be FlippedNotConfirmed if the scanner still
// finds a flow (it shouldn't for the parameterized query).
let resolved_or_flipped = diff.entries.iter().any(|e| {
e.baseline_status == Some(VerifyStatus::Confirmed)
&& matches!(
e.transition,
Transition::Resolved | Transition::FlippedNotConfirmed
)
});
assert!(
resolved_or_flipped,
"Expected the Confirmed finding to be Resolved or FlippedNotConfirmed after the fix. \
Diff entries: {:#?}",
diff.entries
);
// Step 6: gate passes.
assert!(
check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED),
"resolve-all-confirmed gate must pass after the fix"
);
}
/// Step 7-8: new Confirmed finding → no-new-confirmed gate fails.
#[test]
fn new_confirmed_fails_no_new_confirmed_gate() {
let vuln_path = Path::new(VULN_DIR);
let new_path = Path::new(NEW_DIR);
// Baseline: the original vulnerability, confirmed.
let mut vuln_diags = scan_with_hashes(vuln_path);
set_verdict(&mut vuln_diags, VerifyStatus::Confirmed);
let baseline_entries = diags_to_baseline_entries(&vuln_diags);
// Current: the "fixed+new" version — original finding gone, new one appears.
let mut new_diags = scan_with_hashes(new_path);
// Simulate Confirmed on any new findings not in the baseline.
let baseline_hashes: std::collections::HashSet<u64> =
baseline_entries.iter().map(|e| e.stable_hash).collect();
for d in new_diags.iter_mut() {
if !baseline_hashes.contains(&d.stable_hash) {
let fid = format!("{:016x}", d.stable_hash);
let ev = d.evidence.get_or_insert_with(Evidence::default);
ev.dynamic_verdict = Some(VerifyResult {
finding_id: fid,
status: VerifyStatus::Confirmed,
triggered_payload: Some("' OR 1=1--".to_string()),
reason: None,
inconclusive_reason: None,
detail: None,
attempts: vec![],
toolchain_match: None,
});
}
}
let diff = compute_verdict_diff(&baseline_entries, &new_diags);
// There must be at least one New+Confirmed entry.
let has_new_confirmed = diff.entries.iter().any(|e| {
e.transition == Transition::New && e.current_status == Some(VerifyStatus::Confirmed)
});
assert!(
has_new_confirmed,
"Expected a new Confirmed finding in the diff. Diff entries: {:#?}",
diff.entries
);
// Gate must fail.
assert!(
!check_gate(&diff, GATE_NO_NEW_CONFIRMED),
"no-new-confirmed gate must fail when a new Confirmed finding exists"
);
}
/// `stable_hash` is stable across identical scans (same path, rule, line, col, caps).
#[test]
fn stable_hash_deterministic_across_scans() {
let vuln_path = Path::new(VULN_DIR);
let diags1 = scan_with_hashes(vuln_path);
let diags2 = scan_with_hashes(vuln_path);
assert!(!diags1.is_empty());
assert_eq!(
diags1.len(),
diags2.len(),
"finding count must be deterministic"
);
let hashes1: std::collections::HashSet<u64> = diags1.iter().map(|d| d.stable_hash).collect();
let hashes2: std::collections::HashSet<u64> = diags2.iter().map(|d| d.stable_hash).collect();
assert_eq!(
hashes1, hashes2,
"stable_hash must be identical across two scans of the same codebase"
);
}
/// Baseline-write file contains required fields and no source snippets.
#[test]
fn baseline_write_contains_required_fields_no_source() {
let vuln_path = Path::new(VULN_DIR);
let mut diags = scan_with_hashes(vuln_path);
set_verdict(&mut diags, VerifyStatus::Confirmed);
let f = NamedTempFile::new().unwrap();
write_baseline(f.path(), &diags).unwrap();
let content = std::fs::read_to_string(f.path()).unwrap();
let entries: Vec<BaselineEntry> = serde_json::from_str(&content).unwrap();
assert!(!entries.is_empty());
for e in &entries {
assert_ne!(e.stable_hash, 0, "stable_hash must be non-zero");
assert!(!e.path.is_empty(), "path must be set");
assert!(!e.rule_id.is_empty(), "rule_id must be set");
assert!(!e.severity.is_empty(), "severity must be set");
}
// No source code snippets.
assert!(
!content.contains("SELECT"),
"baseline must not contain SQL source code"
);
}
/// `load_baseline` accepts a full Diag JSON (from `nyx scan --format json`).
#[test]
fn load_baseline_accepts_full_diag_json() {
let vuln_path = Path::new(VULN_DIR);
let diags = scan_with_hashes(vuln_path);
assert!(!diags.is_empty());
let diag_json = serde_json::to_string(&diags).unwrap();
let f = NamedTempFile::new().unwrap();
std::fs::write(f.path(), &diag_json).unwrap();
let loaded = load_baseline(f.path()).unwrap();
assert_eq!(loaded.len(), diags.len());
// Hashes must round-trip.
let loaded_hashes: std::collections::HashSet<u64> =
loaded.iter().map(|e| e.stable_hash).collect();
let diag_hashes: std::collections::HashSet<u64> =
diags.iter().map(|d| d.stable_hash).collect();
assert_eq!(loaded_hashes, diag_hashes);
}

View file

@ -0,0 +1,5 @@
import sqlite3
def get_user(db, user_id):
query = "SELECT * FROM users WHERE id = ?"
return db.execute(query, (user_id,))

View file

@ -0,0 +1,12 @@
import os
import sqlite3
def get_user(db):
user_id = os.getenv("USER_ID")
query = "SELECT * FROM users WHERE id = ?"
return db.execute(query, (user_id,))
def get_post(db):
post_id = os.getenv("POST_ID")
query = "SELECT * FROM posts WHERE id = " + post_id
return db.execute(query)

View file

@ -0,0 +1,7 @@
import os
import sqlite3
def get_user(db):
user_id = os.getenv("USER_ID")
query = "SELECT * FROM users WHERE id = " + user_id
return db.execute(query)