mirror of
https://github.com/elicpeter/nyx.git
synced 2026-06-09 19:45:13 +02:00
[pitboss] phase 08: M6.5 — Patch-validation / fix-validation CI mode
This commit is contained in:
parent
25e8b0eb0e
commit
118cafa535
13 changed files with 1067 additions and 4 deletions
|
|
@ -83,10 +83,31 @@ export interface RelatedFindingView {
|
|||
severity: string;
|
||||
}
|
||||
|
||||
// Baseline / patch-validation types (M6.5)
|
||||
export type VerdictTransition =
|
||||
| 'New'
|
||||
| 'Unchanged'
|
||||
| 'Resolved'
|
||||
| 'Regressed'
|
||||
| 'FlippedConfirmed'
|
||||
| 'FlippedNotConfirmed';
|
||||
|
||||
export interface VerdictDiffEntry {
|
||||
stable_hash: number;
|
||||
path: string;
|
||||
line: number;
|
||||
rule_id: string;
|
||||
baseline_status?: VerifyStatus;
|
||||
current_status?: VerifyStatus;
|
||||
transition: VerdictTransition;
|
||||
}
|
||||
|
||||
export interface FindingView {
|
||||
index: number;
|
||||
fingerprint: string;
|
||||
portable_fingerprint?: string;
|
||||
/** Blake3-derived stable cross-commit identity (M6.5). */
|
||||
stable_hash?: number;
|
||||
path: string;
|
||||
line: number;
|
||||
col: number;
|
||||
|
|
@ -199,6 +220,8 @@ export interface CompareResponse {
|
|||
fixed_findings: ComparedFinding[];
|
||||
changed_findings: ChangedFinding[];
|
||||
unchanged_findings: ComparedFinding[];
|
||||
/** Verdict-level diff (M6.5). Present when findings carry stable_hash values. */
|
||||
verdict_diff?: VerdictDiffEntry[];
|
||||
}
|
||||
|
||||
// Overview types
|
||||
|
|
|
|||
628
src/baseline.rs
Normal file
628
src/baseline.rs
Normal file
|
|
@ -0,0 +1,628 @@
|
|||
//! Baseline diffing for patch-validation CI mode (§M6.5 / Pillar A §15.1).
|
||||
//!
|
||||
//! `nyx scan --baseline <file>` reads a previous scan's JSON output (or a
|
||||
//! stripped `.nyx/baseline.json`) and joins on `Diag::stable_hash`. The
|
||||
//! result is a per-finding `VerdictDiffEntry` with a typed `Transition` that
|
||||
//! CI gates can act on.
|
||||
//!
|
||||
//! `nyx scan --baseline-write <file>` writes a stripped baseline JSON:
|
||||
//! only `stable_hash`, `dynamic_verdict`, `severity`, `path`, and `rule_id`.
|
||||
//! No source code is included.
|
||||
|
||||
use crate::commands::scan::Diag;
|
||||
use crate::evidence::VerifyStatus;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Baseline entry (stripped — no source code)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// A stripped baseline entry: only what is needed for cross-commit diffing.
|
||||
/// Contains no source code snippets.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BaselineEntry {
|
||||
pub stable_hash: u64,
|
||||
/// Dynamic verdict status from the scan that wrote this baseline.
|
||||
/// `None` when `--verify` was not run.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub dynamic_verdict: Option<VerifyStatus>,
|
||||
pub severity: String,
|
||||
pub path: String,
|
||||
pub rule_id: String,
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Transition enum
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// How a finding's verdict changed between the baseline scan and the current
|
||||
/// scan.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Transition {
|
||||
/// Finding exists in the current scan but was absent from the baseline.
|
||||
New,
|
||||
/// Finding appears in both scans; verdict is unchanged (or neither scan
|
||||
/// ran `--verify`).
|
||||
Unchanged,
|
||||
/// Finding was present in the baseline but disappeared from the current
|
||||
/// scan — the vulnerability is gone.
|
||||
Resolved,
|
||||
/// Finding in both; was `NotConfirmed` in baseline, now `Confirmed`.
|
||||
Regressed,
|
||||
/// Finding in both; baseline had no verdict (or `Inconclusive` /
|
||||
/// `Unsupported`) and it is now `Confirmed`.
|
||||
FlippedConfirmed,
|
||||
/// Finding in both; was `Confirmed` in baseline, now `NotConfirmed` —
|
||||
/// the fix is proven.
|
||||
FlippedNotConfirmed,
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// VerdictDiffEntry
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Per-finding verdict diff produced by comparing a baseline to a current scan.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct VerdictDiffEntry {
|
||||
/// Stable cross-commit identity hash.
|
||||
pub stable_hash: u64,
|
||||
pub path: String,
|
||||
pub line: usize,
|
||||
pub rule_id: String,
|
||||
/// Verdict in the baseline scan (`None` when verify was not run).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub baseline_status: Option<VerifyStatus>,
|
||||
/// Verdict in the current scan (`None` when verify was not run or finding
|
||||
/// is absent from the current scan).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub current_status: Option<VerifyStatus>,
|
||||
pub transition: Transition,
|
||||
}
|
||||
|
||||
/// Full verdict diff between a baseline and a current scan.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct VerdictDiff {
|
||||
pub entries: Vec<VerdictDiffEntry>,
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Load / write helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Load baseline entries from a file.
|
||||
///
|
||||
/// Accepts two JSON formats:
|
||||
/// - Stripped baseline (`Vec<BaselineEntry>`) — written by `--baseline-write`.
|
||||
/// - Full scan output (`Vec<Diag>`) — written by `nyx scan --format json`.
|
||||
///
|
||||
/// Detection heuristic: try `Vec<BaselineEntry>` first (requires `rule_id`);
|
||||
/// fall back to `Vec<Diag>`.
|
||||
pub fn load_baseline(path: &Path) -> crate::errors::NyxResult<Vec<BaselineEntry>> {
|
||||
let content = std::fs::read_to_string(path).map_err(|e| {
|
||||
crate::errors::NyxError::Msg(format!("cannot read baseline {}: {e}", path.display()))
|
||||
})?;
|
||||
|
||||
// Try stripped format first.
|
||||
if let Ok(entries) = serde_json::from_str::<Vec<BaselineEntry>>(&content) {
|
||||
return Ok(entries);
|
||||
}
|
||||
|
||||
// Fall back to full Diag list.
|
||||
let diags: Vec<Diag> = serde_json::from_str(&content).map_err(|e| {
|
||||
crate::errors::NyxError::Msg(format!(
|
||||
"baseline {}: not a valid BaselineEntry list or Diag list: {e}",
|
||||
path.display()
|
||||
))
|
||||
})?;
|
||||
Ok(diags_to_baseline_entries(&diags))
|
||||
}
|
||||
|
||||
/// Convert `Diag` values to `BaselineEntry` values.
|
||||
///
|
||||
/// Only findings with a non-zero `stable_hash` are included; findings without
|
||||
/// a hash cannot be joined across scans.
|
||||
pub fn diags_to_baseline_entries(diags: &[Diag]) -> Vec<BaselineEntry> {
|
||||
diags
|
||||
.iter()
|
||||
.filter(|d| d.stable_hash != 0)
|
||||
.map(|d| BaselineEntry {
|
||||
stable_hash: d.stable_hash,
|
||||
dynamic_verdict: d
|
||||
.evidence
|
||||
.as_ref()
|
||||
.and_then(|ev| ev.dynamic_verdict.as_ref())
|
||||
.map(|vr| vr.status),
|
||||
severity: d.severity.as_db_str().to_string(),
|
||||
path: d.path.clone(),
|
||||
rule_id: d.id.clone(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Write a stripped baseline JSON to `path`.
|
||||
///
|
||||
/// The file contains only `stable_hash`, `dynamic_verdict`, `severity`,
|
||||
/// `path`, and `rule_id` — no source code snippets or flow steps.
|
||||
pub fn write_baseline(path: &Path, diags: &[Diag]) -> crate::errors::NyxResult<()> {
|
||||
let entries = diags_to_baseline_entries(diags);
|
||||
let json = serde_json::to_string_pretty(&entries).map_err(|e| {
|
||||
crate::errors::NyxError::Msg(format!("baseline serialize error: {e}"))
|
||||
})?;
|
||||
if let Some(parent) = path.parent() {
|
||||
if !parent.as_os_str().is_empty() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| {
|
||||
crate::errors::NyxError::Msg(format!(
|
||||
"cannot create baseline dir {}: {e}",
|
||||
parent.display()
|
||||
))
|
||||
})?;
|
||||
}
|
||||
}
|
||||
std::fs::write(path, json).map_err(|e| {
|
||||
crate::errors::NyxError::Msg(format!(
|
||||
"cannot write baseline {}: {e}",
|
||||
path.display()
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Diff computation
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn classify_transition(
|
||||
baseline: Option<VerifyStatus>,
|
||||
current: Option<VerifyStatus>,
|
||||
) -> Transition {
|
||||
match (baseline, current) {
|
||||
// No verdict change (including both None)
|
||||
(a, b) if a == b => Transition::Unchanged,
|
||||
// Confirmed → NotConfirmed: fix proven
|
||||
(Some(VerifyStatus::Confirmed), Some(VerifyStatus::NotConfirmed)) => {
|
||||
Transition::FlippedNotConfirmed
|
||||
}
|
||||
// NotConfirmed → Confirmed: regression
|
||||
(Some(VerifyStatus::NotConfirmed), Some(VerifyStatus::Confirmed)) => {
|
||||
Transition::Regressed
|
||||
}
|
||||
// None / Inconclusive / Unsupported → Confirmed
|
||||
(_, Some(VerifyStatus::Confirmed)) => Transition::FlippedConfirmed,
|
||||
// Everything else: treat as unchanged (e.g. Confirmed → Inconclusive
|
||||
// without a clean NotConfirmed proof is not a resolution)
|
||||
_ => Transition::Unchanged,
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute a verdict diff between a loaded baseline and the current findings.
|
||||
pub fn compute_verdict_diff(baseline: &[BaselineEntry], current: &[Diag]) -> VerdictDiff {
|
||||
// Build lookup maps keyed by stable_hash.
|
||||
let baseline_map: HashMap<u64, &BaselineEntry> =
|
||||
baseline.iter().map(|e| (e.stable_hash, e)).collect();
|
||||
let current_map: HashMap<u64, &Diag> = current
|
||||
.iter()
|
||||
.filter(|d| d.stable_hash != 0)
|
||||
.map(|d| (d.stable_hash, d))
|
||||
.collect();
|
||||
|
||||
let mut entries = Vec::new();
|
||||
|
||||
// Walk current findings.
|
||||
for (&hash, diag) in ¤t_map {
|
||||
let current_status = diag
|
||||
.evidence
|
||||
.as_ref()
|
||||
.and_then(|ev| ev.dynamic_verdict.as_ref())
|
||||
.map(|vr| vr.status);
|
||||
|
||||
if let Some(base) = baseline_map.get(&hash) {
|
||||
let transition = classify_transition(base.dynamic_verdict, current_status);
|
||||
entries.push(VerdictDiffEntry {
|
||||
stable_hash: hash,
|
||||
path: diag.path.clone(),
|
||||
line: diag.line,
|
||||
rule_id: diag.id.clone(),
|
||||
baseline_status: base.dynamic_verdict,
|
||||
current_status,
|
||||
transition,
|
||||
});
|
||||
} else {
|
||||
// Not in baseline → New.
|
||||
entries.push(VerdictDiffEntry {
|
||||
stable_hash: hash,
|
||||
path: diag.path.clone(),
|
||||
line: diag.line,
|
||||
rule_id: diag.id.clone(),
|
||||
baseline_status: None,
|
||||
current_status,
|
||||
transition: Transition::New,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Walk baseline findings absent from current → Resolved.
|
||||
for (&hash, base) in &baseline_map {
|
||||
if !current_map.contains_key(&hash) {
|
||||
entries.push(VerdictDiffEntry {
|
||||
stable_hash: hash,
|
||||
path: base.path.clone(),
|
||||
line: 0,
|
||||
rule_id: base.rule_id.clone(),
|
||||
baseline_status: base.dynamic_verdict,
|
||||
current_status: None,
|
||||
transition: Transition::Resolved,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Sort for deterministic output: Resolved first, then New, then the rest,
|
||||
// all sub-sorted by (path, line).
|
||||
entries.sort_by(|a, b| {
|
||||
fn order(t: Transition) -> u8 {
|
||||
match t {
|
||||
Transition::Resolved => 0,
|
||||
Transition::FlippedNotConfirmed => 1,
|
||||
Transition::New => 2,
|
||||
Transition::Regressed => 3,
|
||||
Transition::FlippedConfirmed => 4,
|
||||
Transition::Unchanged => 5,
|
||||
}
|
||||
}
|
||||
order(a.transition)
|
||||
.cmp(&order(b.transition))
|
||||
.then_with(|| a.path.cmp(&b.path))
|
||||
.then_with(|| a.line.cmp(&b.line))
|
||||
});
|
||||
|
||||
VerdictDiff { entries }
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// CI gates
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Gate: exit code 2 if any new `Confirmed` finding appears.
|
||||
///
|
||||
/// Triggers on `transition == New && current_status == Confirmed` or
|
||||
/// `transition == FlippedConfirmed`.
|
||||
pub const GATE_NO_NEW_CONFIRMED: &str = "no-new-confirmed";
|
||||
|
||||
/// Gate: exit code 2 if any baseline-`Confirmed` finding is not fully resolved.
|
||||
///
|
||||
/// A baseline-Confirmed finding is resolved only when it is absent from the
|
||||
/// current scan (`Resolved`) or its current verdict is `NotConfirmed`
|
||||
/// (`FlippedNotConfirmed`). All other current statuses (`Confirmed`,
|
||||
/// `Inconclusive`, `Unsupported`) violate this gate.
|
||||
pub const GATE_RESOLVE_ALL_CONFIRMED: &str = "resolve-all-confirmed";
|
||||
|
||||
/// Check a named CI gate against a verdict diff.
|
||||
///
|
||||
/// Returns `true` when the gate passes (condition not violated) and `false`
|
||||
/// when it fails (caller should exit with code 2).
|
||||
///
|
||||
/// Unknown gate names always pass so future gate additions are forward-
|
||||
/// compatible without requiring a binary upgrade.
|
||||
pub fn check_gate(diff: &VerdictDiff, gate: &str) -> bool {
|
||||
match gate {
|
||||
GATE_NO_NEW_CONFIRMED => !diff.entries.iter().any(|e| {
|
||||
matches!(e.transition, Transition::New | Transition::FlippedConfirmed)
|
||||
&& e.current_status == Some(VerifyStatus::Confirmed)
|
||||
}),
|
||||
GATE_RESOLVE_ALL_CONFIRMED => !diff.entries.iter().any(|e| {
|
||||
e.baseline_status == Some(VerifyStatus::Confirmed)
|
||||
&& matches!(
|
||||
e.current_status,
|
||||
Some(VerifyStatus::Confirmed)
|
||||
| Some(VerifyStatus::Inconclusive)
|
||||
| Some(VerifyStatus::Unsupported)
|
||||
)
|
||||
}),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Console / JSON rendering
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn status_str(s: Option<VerifyStatus>) -> &'static str {
|
||||
match s {
|
||||
Some(VerifyStatus::Confirmed) => "Confirmed",
|
||||
Some(VerifyStatus::NotConfirmed) => "NotConfirmed",
|
||||
Some(VerifyStatus::Inconclusive) => "Inconclusive",
|
||||
Some(VerifyStatus::Unsupported) => "Unsupported",
|
||||
None => "(no verdict)",
|
||||
}
|
||||
}
|
||||
|
||||
/// Render a verdict diff as a human-readable console summary.
|
||||
pub fn format_diff_console(diff: &VerdictDiff) -> String {
|
||||
if diff.entries.is_empty() {
|
||||
return String::from(" (no findings in baseline or current scan)\n");
|
||||
}
|
||||
|
||||
let mut lines = Vec::new();
|
||||
let mut non_unchanged = 0usize;
|
||||
|
||||
for e in &diff.entries {
|
||||
let hash_str = format!("{:016x}", e.stable_hash);
|
||||
let loc = if e.line > 0 {
|
||||
format!("{}:{}", e.path, e.line)
|
||||
} else {
|
||||
e.path.clone()
|
||||
};
|
||||
match e.transition {
|
||||
Transition::New => {
|
||||
non_unchanged += 1;
|
||||
lines.push(format!(
|
||||
" + {hash_str}: new {} at {loc}",
|
||||
status_str(e.current_status)
|
||||
));
|
||||
}
|
||||
Transition::Resolved => {
|
||||
non_unchanged += 1;
|
||||
lines.push(format!(
|
||||
" - {hash_str}: {} \u{2192} removed (resolved) at {loc}",
|
||||
status_str(e.baseline_status)
|
||||
));
|
||||
}
|
||||
Transition::FlippedNotConfirmed => {
|
||||
non_unchanged += 1;
|
||||
lines.push(format!(
|
||||
" - {hash_str}: Confirmed \u{2192} NotConfirmed at {loc} (resolved)"
|
||||
));
|
||||
}
|
||||
Transition::Regressed => {
|
||||
non_unchanged += 1;
|
||||
lines.push(format!(
|
||||
" ! {hash_str}: NotConfirmed \u{2192} Confirmed at {loc} (regressed)"
|
||||
));
|
||||
}
|
||||
Transition::FlippedConfirmed => {
|
||||
non_unchanged += 1;
|
||||
lines.push(format!(
|
||||
" + {hash_str}: new Confirmed at {loc}"
|
||||
));
|
||||
}
|
||||
Transition::Unchanged => {}
|
||||
}
|
||||
}
|
||||
|
||||
if non_unchanged == 0 {
|
||||
return String::from(" (no changes from baseline)\n");
|
||||
}
|
||||
|
||||
lines.join("\n") + "\n"
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::commands::scan::{compute_stable_hash, Diag};
|
||||
use crate::evidence::{Evidence, VerifyResult, VerifyStatus};
|
||||
use crate::patterns::{FindingCategory, Severity};
|
||||
|
||||
fn make_diag(path: &str, line: usize, rule: &str) -> Diag {
|
||||
let mut d = Diag {
|
||||
path: path.to_string(),
|
||||
line,
|
||||
col: 0,
|
||||
severity: Severity::High,
|
||||
id: rule.to_string(),
|
||||
category: FindingCategory::Security,
|
||||
path_validated: false,
|
||||
guard_kind: None,
|
||||
message: None,
|
||||
labels: vec![],
|
||||
confidence: None,
|
||||
evidence: None,
|
||||
rank_score: None,
|
||||
rank_reason: None,
|
||||
suppressed: false,
|
||||
suppression: None,
|
||||
rollup: None,
|
||||
finding_id: String::new(),
|
||||
alternative_finding_ids: vec![],
|
||||
stable_hash: 0,
|
||||
};
|
||||
d.stable_hash = compute_stable_hash(&d);
|
||||
d
|
||||
}
|
||||
|
||||
fn with_verdict(mut d: Diag, status: VerifyStatus) -> Diag {
|
||||
d.evidence = Some(Evidence {
|
||||
dynamic_verdict: Some(VerifyResult {
|
||||
finding_id: format!("{:016x}", d.stable_hash),
|
||||
status,
|
||||
triggered_payload: None,
|
||||
reason: None,
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![],
|
||||
toolchain_match: None,
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
d
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_finding_no_verdict() {
|
||||
let current = vec![make_diag("src/a.py", 1, "py.sqli")];
|
||||
let diff = compute_verdict_diff(&[], ¤t);
|
||||
assert_eq!(diff.entries.len(), 1);
|
||||
assert_eq!(diff.entries[0].transition, Transition::New);
|
||||
assert_eq!(diff.entries[0].current_status, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_confirmed_finding() {
|
||||
let current = vec![with_verdict(
|
||||
make_diag("src/a.py", 1, "py.sqli"),
|
||||
VerifyStatus::Confirmed,
|
||||
)];
|
||||
let diff = compute_verdict_diff(&[], ¤t);
|
||||
assert_eq!(diff.entries[0].transition, Transition::New);
|
||||
assert_eq!(diff.entries[0].current_status, Some(VerifyStatus::Confirmed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolved_finding() {
|
||||
let baseline_diag = make_diag("src/a.py", 1, "py.sqli");
|
||||
let baseline = diags_to_baseline_entries(&[baseline_diag]);
|
||||
let diff = compute_verdict_diff(&baseline, &[]);
|
||||
assert_eq!(diff.entries.len(), 1);
|
||||
assert_eq!(diff.entries[0].transition, Transition::Resolved);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flipped_not_confirmed() {
|
||||
let d = make_diag("src/a.py", 1, "py.sqli");
|
||||
let baseline = vec![BaselineEntry {
|
||||
stable_hash: d.stable_hash,
|
||||
dynamic_verdict: Some(VerifyStatus::Confirmed),
|
||||
severity: "high".to_string(),
|
||||
path: d.path.clone(),
|
||||
rule_id: d.id.clone(),
|
||||
}];
|
||||
let current = vec![with_verdict(d, VerifyStatus::NotConfirmed)];
|
||||
let diff = compute_verdict_diff(&baseline, ¤t);
|
||||
assert_eq!(diff.entries[0].transition, Transition::FlippedNotConfirmed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regressed() {
|
||||
let d = make_diag("src/a.py", 1, "py.sqli");
|
||||
let baseline = vec![BaselineEntry {
|
||||
stable_hash: d.stable_hash,
|
||||
dynamic_verdict: Some(VerifyStatus::NotConfirmed),
|
||||
severity: "high".to_string(),
|
||||
path: d.path.clone(),
|
||||
rule_id: d.id.clone(),
|
||||
}];
|
||||
let current = vec![with_verdict(d, VerifyStatus::Confirmed)];
|
||||
let diff = compute_verdict_diff(&baseline, ¤t);
|
||||
assert_eq!(diff.entries[0].transition, Transition::Regressed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_no_new_confirmed_passes_when_no_confirmed() {
|
||||
let d = make_diag("src/a.py", 1, "py.sqli");
|
||||
let diff = compute_verdict_diff(&[], &[d]);
|
||||
assert!(check_gate(&diff, GATE_NO_NEW_CONFIRMED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_no_new_confirmed_fails_on_new_confirmed() {
|
||||
let current = vec![with_verdict(
|
||||
make_diag("src/a.py", 1, "py.sqli"),
|
||||
VerifyStatus::Confirmed,
|
||||
)];
|
||||
let diff = compute_verdict_diff(&[], ¤t);
|
||||
assert!(!check_gate(&diff, GATE_NO_NEW_CONFIRMED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_resolve_all_confirmed_passes_when_flipped() {
|
||||
let d = make_diag("src/a.py", 1, "py.sqli");
|
||||
let baseline = vec![BaselineEntry {
|
||||
stable_hash: d.stable_hash,
|
||||
dynamic_verdict: Some(VerifyStatus::Confirmed),
|
||||
severity: "high".to_string(),
|
||||
path: d.path.clone(),
|
||||
rule_id: d.id.clone(),
|
||||
}];
|
||||
let current = vec![with_verdict(d, VerifyStatus::NotConfirmed)];
|
||||
let diff = compute_verdict_diff(&baseline, ¤t);
|
||||
assert!(check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_resolve_all_confirmed_fails_when_still_confirmed() {
|
||||
let d = make_diag("src/a.py", 1, "py.sqli");
|
||||
let baseline = vec![BaselineEntry {
|
||||
stable_hash: d.stable_hash,
|
||||
dynamic_verdict: Some(VerifyStatus::Confirmed),
|
||||
severity: "high".to_string(),
|
||||
path: d.path.clone(),
|
||||
rule_id: d.id.clone(),
|
||||
}];
|
||||
let current = vec![with_verdict(d, VerifyStatus::Confirmed)];
|
||||
let diff = compute_verdict_diff(&baseline, ¤t);
|
||||
assert!(!check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_resolve_all_confirmed_passes_when_resolved() {
|
||||
let d = make_diag("src/a.py", 1, "py.sqli");
|
||||
let baseline = vec![BaselineEntry {
|
||||
stable_hash: d.stable_hash,
|
||||
dynamic_verdict: Some(VerifyStatus::Confirmed),
|
||||
severity: "high".to_string(),
|
||||
path: d.path.clone(),
|
||||
rule_id: d.id.clone(),
|
||||
}];
|
||||
// No current findings (finding disappeared entirely).
|
||||
let diff = compute_verdict_diff(&baseline, &[]);
|
||||
assert!(check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_and_load_roundtrip() {
|
||||
let d = with_verdict(make_diag("src/a.py", 1, "py.sqli"), VerifyStatus::Confirmed);
|
||||
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
write_baseline(tmp.path(), &[d.clone()]).unwrap();
|
||||
let loaded = load_baseline(tmp.path()).unwrap();
|
||||
assert_eq!(loaded.len(), 1);
|
||||
assert_eq!(loaded[0].stable_hash, d.stable_hash);
|
||||
assert_eq!(loaded[0].dynamic_verdict, Some(VerifyStatus::Confirmed));
|
||||
assert_eq!(loaded[0].path, "src/a.py");
|
||||
assert_eq!(loaded[0].rule_id, "py.sqli");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_full_diag_json() {
|
||||
let d = with_verdict(make_diag("src/a.py", 1, "py.sqli"), VerifyStatus::Confirmed);
|
||||
let json = serde_json::to_string(&[&d]).unwrap();
|
||||
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
std::fs::write(tmp.path(), &json).unwrap();
|
||||
let loaded = load_baseline(tmp.path()).unwrap();
|
||||
assert_eq!(loaded.len(), 1);
|
||||
assert_eq!(loaded[0].stable_hash, d.stable_hash);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn baseline_write_no_source() {
|
||||
let mut d = with_verdict(make_diag("src/a.py", 1, "py.sqli"), VerifyStatus::Confirmed);
|
||||
// Add a flow_step with a snippet (source code) to the evidence.
|
||||
if let Some(ref mut ev) = d.evidence {
|
||||
ev.flow_steps = vec![crate::evidence::FlowStep {
|
||||
step: 1,
|
||||
kind: crate::evidence::FlowStepKind::Source,
|
||||
file: "src/a.py".into(),
|
||||
line: 1,
|
||||
col: 0,
|
||||
snippet: Some("SECRET CODE".into()),
|
||||
variable: None,
|
||||
callee: None,
|
||||
function: None,
|
||||
is_cross_file: false,
|
||||
}];
|
||||
}
|
||||
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
write_baseline(tmp.path(), &[d]).unwrap();
|
||||
let content = std::fs::read_to_string(tmp.path()).unwrap();
|
||||
assert!(!content.contains("SECRET CODE"), "baseline must not contain source code");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_gate_passes() {
|
||||
let diff = VerdictDiff { entries: vec![] };
|
||||
assert!(check_gate(&diff, "some-future-gate-name"));
|
||||
}
|
||||
}
|
||||
31
src/cli.rs
31
src/cli.rs
|
|
@ -456,6 +456,37 @@ pub enum Commands {
|
|||
#[cfg_attr(not(feature = "dynamic"), arg(hide = true))]
|
||||
#[arg(long, help_heading = "Dynamic", value_name = "BACKEND")]
|
||||
backend: Option<String>,
|
||||
|
||||
// ── Baseline / patch-validation (§M6.5) ────────────────────────
|
||||
/// Read a previous scan's JSON output (or a stripped .nyx/baseline.json)
|
||||
/// and diff it against the current scan on stable_hash.
|
||||
///
|
||||
/// Emits a verdict diff showing New / Resolved / FlippedConfirmed /
|
||||
/// FlippedNotConfirmed transitions. Combine with --gate to enforce CI
|
||||
/// policies.
|
||||
#[arg(long, value_name = "FILE", help_heading = "Baseline")]
|
||||
baseline: Option<String>,
|
||||
|
||||
/// Write a stripped baseline JSON to FILE after scanning.
|
||||
///
|
||||
/// The file contains only stable_hash, dynamic_verdict, severity, path,
|
||||
/// and rule_id — no source code. A CI job can persist this file to
|
||||
/// compare future scans against without leaking source.
|
||||
#[arg(long, value_name = "FILE", help_heading = "Baseline")]
|
||||
baseline_write: Option<String>,
|
||||
|
||||
/// CI gate to enforce when --baseline is active.
|
||||
///
|
||||
/// `no-new-confirmed`: exit 2 if any new Confirmed finding appears.
|
||||
/// `resolve-all-confirmed`: exit 2 if any baseline-Confirmed finding
|
||||
/// is not fully resolved (absent or NotConfirmed in the current scan).
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "GATE",
|
||||
value_parser = ["no-new-confirmed", "resolve-all-confirmed"],
|
||||
help_heading = "Baseline"
|
||||
)]
|
||||
gate: Option<String>,
|
||||
},
|
||||
|
||||
/// Submit feedback on a dynamic verification verdict (§21.2).
|
||||
|
|
|
|||
|
|
@ -100,6 +100,9 @@ pub fn handle_command(
|
|||
verify,
|
||||
unsafe_sandbox,
|
||||
backend,
|
||||
baseline,
|
||||
baseline_write,
|
||||
gate,
|
||||
} => {
|
||||
// ── Apply profile first (CLI flags override after) ──────────
|
||||
if let Some(ref name) = profile {
|
||||
|
|
@ -360,6 +363,9 @@ pub fn handle_command(
|
|||
show_instances.as_deref(),
|
||||
database_dir,
|
||||
config,
|
||||
baseline.as_deref().map(std::path::Path::new),
|
||||
baseline_write.as_deref().map(std::path::Path::new),
|
||||
gate.as_deref(),
|
||||
)?;
|
||||
}
|
||||
#[cfg(feature = "dynamic")]
|
||||
|
|
|
|||
|
|
@ -345,6 +345,9 @@ pub fn handle(
|
|||
show_instances: Option<&str>,
|
||||
database_dir: &Path,
|
||||
config: &Config,
|
||||
baseline: Option<&Path>,
|
||||
baseline_write: Option<&Path>,
|
||||
gate: Option<&str>,
|
||||
) -> NyxResult<()> {
|
||||
let scan_path = Path::new(path).canonicalize()?;
|
||||
let (project_name, db_path) = get_project_info(&scan_path, database_dir)?;
|
||||
|
|
@ -489,18 +492,65 @@ pub fn handle(
|
|||
}
|
||||
}
|
||||
|
||||
// ── Baseline write (§M6.5): persist current findings as stripped baseline
|
||||
if let Some(bw_path) = baseline_write {
|
||||
if let Err(e) = crate::baseline::write_baseline(bw_path, &diags) {
|
||||
tracing::warn!(path = %bw_path.display(), error = %e, "baseline-write failed");
|
||||
if !suppress_status {
|
||||
eprintln!("warning: --baseline-write failed: {e}");
|
||||
}
|
||||
} else if !suppress_status {
|
||||
eprintln!("Baseline written to {}", bw_path.display());
|
||||
}
|
||||
}
|
||||
|
||||
// ── Baseline diff (§M6.5): load previous baseline and compute transitions
|
||||
let verdict_diff = if let Some(bl_path) = baseline {
|
||||
match crate::baseline::load_baseline(bl_path) {
|
||||
Ok(baseline_entries) => {
|
||||
let diff = crate::baseline::compute_verdict_diff(&baseline_entries, &diags);
|
||||
Some(diff)
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(crate::errors::NyxError::Msg(format!(
|
||||
"--baseline {}: {e}",
|
||||
bl_path.display()
|
||||
)));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// ── Output ──────────────────────────────────────────────────────────
|
||||
match format {
|
||||
OutputFormat::Json => {
|
||||
let json = serde_json::to_string(&diags)
|
||||
.map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?;
|
||||
println!("{json}");
|
||||
if let Some(ref diff) = verdict_diff {
|
||||
// Wrap findings + verdict_diff into one JSON object so the
|
||||
// diff is machine-readable alongside the findings.
|
||||
let out = serde_json::json!({
|
||||
"findings": &diags,
|
||||
"verdict_diff": diff,
|
||||
});
|
||||
let json = serde_json::to_string(&out)
|
||||
.map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?;
|
||||
println!("{json}");
|
||||
} else {
|
||||
let json = serde_json::to_string(&diags)
|
||||
.map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?;
|
||||
println!("{json}");
|
||||
}
|
||||
}
|
||||
OutputFormat::Sarif => {
|
||||
let sarif = crate::output::build_sarif(&diags, &scan_path);
|
||||
let json = serde_json::to_string_pretty(&sarif)
|
||||
.map_err(|e| crate::errors::NyxError::Msg(e.to_string()))?;
|
||||
println!("{json}");
|
||||
// Emit diff on stderr for SARIF (stdout is owned by the SARIF schema).
|
||||
if let Some(ref diff) = verdict_diff {
|
||||
eprintln!("\nBaseline comparison:");
|
||||
eprint!("{}", crate::baseline::format_diff_console(diff));
|
||||
}
|
||||
}
|
||||
OutputFormat::Console => {
|
||||
tracing::debug!("Printing to console");
|
||||
|
|
@ -508,6 +558,10 @@ pub fn handle(
|
|||
"{}",
|
||||
crate::fmt::render_console(&diags, &project_name, Some(&stats))
|
||||
);
|
||||
if let Some(ref diff) = verdict_diff {
|
||||
println!("\nBaseline comparison:");
|
||||
print!("{}", crate::baseline::format_diff_console(diff));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -537,6 +591,19 @@ pub fn handle(
|
|||
}
|
||||
}
|
||||
|
||||
// ── --gate: CI gate check (exit 2 on violation) ─────────────────────
|
||||
if let (Some(diff), Some(gate_name)) = (&verdict_diff, gate) {
|
||||
if !crate::baseline::check_gate(diff, gate_name) {
|
||||
if !suppress_status {
|
||||
eprintln!(
|
||||
"Gate '{}' violated. Exit code 2.",
|
||||
gate_name
|
||||
);
|
||||
}
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
// ── --fail-on: exit non-zero if threshold breached ──────────────────
|
||||
// Suppressed findings do not count toward the threshold.
|
||||
if let Some(threshold) = fail_on {
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@
|
|||
pub mod abstract_interp;
|
||||
pub mod ast;
|
||||
pub mod auth_analysis;
|
||||
pub mod baseline;
|
||||
pub mod callgraph;
|
||||
pub mod cfg;
|
||||
pub mod cfg_analysis;
|
||||
|
|
|
|||
|
|
@ -266,7 +266,13 @@ impl JobManager {
|
|||
|
||||
// Prepare the final state outside the lock.
|
||||
let (status, diags, error_str) = match result {
|
||||
Ok(diags) => {
|
||||
Ok(mut diags) => {
|
||||
// Compute stable_hash for every finding (§M6.5 cross-commit identity).
|
||||
// The CLI handler does this in commands/scan.rs::handle, but the
|
||||
// server scan path bypasses handle, so do it here.
|
||||
for d in &mut diags {
|
||||
d.stable_hash = scan::compute_stable_hash(d);
|
||||
}
|
||||
log_collector.info(format!("Scan completed: {} findings", diags.len()), None);
|
||||
(JobStatus::Completed, Some(Arc::new(diags)), None)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,10 @@ pub struct FindingView {
|
|||
pub fingerprint: String,
|
||||
#[serde(skip_serializing_if = "String::is_empty")]
|
||||
pub portable_fingerprint: String,
|
||||
/// Blake3-derived stable cross-commit identity hash (M6.5). Zero when not
|
||||
/// yet computed (server-side scans always compute it post-analysis).
|
||||
#[serde(skip_serializing_if = "crate::server::models::is_zero_u64")]
|
||||
pub stable_hash: u64,
|
||||
pub path: String,
|
||||
pub line: usize,
|
||||
pub col: usize,
|
||||
|
|
@ -263,12 +267,17 @@ fn status_for_diag(d: &Diag) -> &'static str {
|
|||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_zero_u64(v: &u64) -> bool {
|
||||
*v == 0
|
||||
}
|
||||
|
||||
/// Convert a Diag to a FindingView at a given index.
|
||||
pub fn finding_from_diag(index: usize, d: &Diag) -> FindingView {
|
||||
FindingView {
|
||||
index,
|
||||
fingerprint: compute_fingerprint(d),
|
||||
portable_fingerprint: String::new(), // set by caller with scan_root
|
||||
stable_hash: d.stable_hash,
|
||||
path: d.path.clone(),
|
||||
line: d.line,
|
||||
col: d.col,
|
||||
|
|
@ -394,6 +403,10 @@ pub struct CompareResponse {
|
|||
pub fixed_findings: Vec<ComparedFinding>,
|
||||
pub changed_findings: Vec<ChangedFinding>,
|
||||
pub unchanged_findings: Vec<ComparedFinding>,
|
||||
/// Verdict-level diff entries (M6.5). Populated when findings in both
|
||||
/// scans carry `stable_hash` values.
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub verdict_diff: Vec<crate::baseline::VerdictDiffEntry>,
|
||||
}
|
||||
|
||||
/// Minimal scan metadata for comparison headers.
|
||||
|
|
|
|||
|
|
@ -459,6 +459,11 @@ async fn compare_scans(
|
|||
severity_delta,
|
||||
};
|
||||
|
||||
// Build verdict diff from left (baseline) → right (current) using stable_hash.
|
||||
let left_baseline = crate::baseline::diags_to_baseline_entries(&left_findings);
|
||||
let verdict_diff_result =
|
||||
crate::baseline::compute_verdict_diff(&left_baseline, &right_findings);
|
||||
|
||||
Ok(Json(CompareResponse {
|
||||
left_scan: left_info,
|
||||
right_scan: right_info,
|
||||
|
|
@ -467,6 +472,7 @@ async fn compare_scans(
|
|||
fixed_findings,
|
||||
changed_findings,
|
||||
unchanged_findings,
|
||||
verdict_diff: verdict_diff_result.entries,
|
||||
}))
|
||||
}
|
||||
|
||||
|
|
|
|||
258
tests/fix_validation_e2e.rs
Normal file
258
tests/fix_validation_e2e.rs
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
//! End-to-end tests for `nyx scan --baseline` / `--gate` (§M6.5, Pillar A).
|
||||
//!
|
||||
//! Demonstrates the "woah" loop from §15.5:
|
||||
//! 1. Scan a vulnerable Python project — finding emits with `stable_hash`.
|
||||
//! 2. Simulate `Confirmed` dynamic verdict (as `--verify` would produce).
|
||||
//! 3. Write a stripped baseline (no source code, only hash + verdict).
|
||||
//! 4. Fix the vulnerability and rescan.
|
||||
//! 5. Diff against the baseline: finding flips to `FlippedNotConfirmed`.
|
||||
//! 6. `--gate=resolve-all-confirmed` passes (exits 0).
|
||||
//! 7. Introduce a new vulnerability and simulate `Confirmed` on it.
|
||||
//! 8. `--gate=no-new-confirmed` fails (would exit 2).
|
||||
|
||||
mod common;
|
||||
|
||||
use nyx_scanner::baseline::{
|
||||
check_gate, compute_verdict_diff, diags_to_baseline_entries, load_baseline, write_baseline,
|
||||
BaselineEntry, Transition, GATE_NO_NEW_CONFIRMED, GATE_RESOLVE_ALL_CONFIRMED,
|
||||
};
|
||||
use nyx_scanner::commands::scan::compute_stable_hash;
|
||||
use nyx_scanner::evidence::{Evidence, VerifyResult, VerifyStatus};
|
||||
use nyx_scanner::utils::config::AnalysisMode;
|
||||
use std::path::Path;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
/// Run `scan_no_index` and assign stable hashes to every finding.
|
||||
fn scan_with_hashes(dir: &Path) -> Vec<nyx_scanner::commands::scan::Diag> {
|
||||
let mut diags = common::scan_fixture_dir(dir, AnalysisMode::Full);
|
||||
for d in &mut diags {
|
||||
d.stable_hash = compute_stable_hash(d);
|
||||
}
|
||||
diags
|
||||
}
|
||||
|
||||
/// Attach a simulated dynamic verdict to every finding in the list.
|
||||
fn set_verdict(
|
||||
diags: &mut Vec<nyx_scanner::commands::scan::Diag>,
|
||||
status: VerifyStatus,
|
||||
) {
|
||||
for d in diags.iter_mut() {
|
||||
let fid = format!("{:016x}", d.stable_hash);
|
||||
let ev = d.evidence.get_or_insert_with(Evidence::default);
|
||||
ev.dynamic_verdict = Some(VerifyResult {
|
||||
finding_id: fid,
|
||||
status,
|
||||
triggered_payload: if status == VerifyStatus::Confirmed {
|
||||
Some("' OR 1=1--".to_string())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
reason: None,
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![],
|
||||
toolchain_match: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const VULN_DIR: &str = "tests/fixtures/baseline_sqli_vuln";
|
||||
const FIXED_DIR: &str = "tests/fixtures/baseline_sqli_fixed";
|
||||
const NEW_DIR: &str = "tests/fixtures/baseline_sqli_new";
|
||||
|
||||
// ── §15.5 "woah" loop end-to-end ────────────────────────────────────────────
|
||||
|
||||
/// Step 1-3: Scan the vulnerable version, simulate Confirmed, write baseline.
|
||||
#[test]
|
||||
fn vuln_scan_emits_finding_with_stable_hash() {
|
||||
let vuln_path = Path::new(VULN_DIR);
|
||||
let diags = scan_with_hashes(vuln_path);
|
||||
assert!(
|
||||
!diags.is_empty(),
|
||||
"Expected SQL injection finding in {VULN_DIR}"
|
||||
);
|
||||
assert!(
|
||||
diags.iter().all(|d| d.stable_hash != 0),
|
||||
"All findings must have non-zero stable_hash after compute_stable_hash"
|
||||
);
|
||||
}
|
||||
|
||||
/// Step 4-6: Fix → rescan → diff → gate passes.
|
||||
#[test]
|
||||
fn fix_resolves_confirmed_finding() {
|
||||
let vuln_path = Path::new(VULN_DIR);
|
||||
let fixed_path = Path::new(FIXED_DIR);
|
||||
|
||||
// Step 1: scan vulnerable, simulate Confirmed verdict.
|
||||
let mut vuln_diags = scan_with_hashes(vuln_path);
|
||||
assert!(!vuln_diags.is_empty(), "Need at least one SQL injection finding");
|
||||
set_verdict(&mut vuln_diags, VerifyStatus::Confirmed);
|
||||
|
||||
// Step 2: write stripped baseline.
|
||||
let baseline_file = NamedTempFile::new().unwrap();
|
||||
write_baseline(baseline_file.path(), &vuln_diags).unwrap();
|
||||
|
||||
// Step 3: load baseline and verify it has no source code.
|
||||
let raw = std::fs::read_to_string(baseline_file.path()).unwrap();
|
||||
assert!(
|
||||
!raw.contains("execute"),
|
||||
"baseline must not contain source code snippets (found 'execute')"
|
||||
);
|
||||
let baseline_entries = load_baseline(baseline_file.path()).unwrap();
|
||||
assert!(!baseline_entries.is_empty());
|
||||
assert_eq!(
|
||||
baseline_entries[0].dynamic_verdict,
|
||||
Some(VerifyStatus::Confirmed)
|
||||
);
|
||||
|
||||
// Step 4: scan fixed version.
|
||||
let fixed_diags = scan_with_hashes(fixed_path);
|
||||
|
||||
// Step 5: diff.
|
||||
let diff = compute_verdict_diff(&baseline_entries, &fixed_diags);
|
||||
|
||||
// The vulnerable finding should be Resolved (gone from fixed code).
|
||||
// Alternatively it could be FlippedNotConfirmed if the scanner still
|
||||
// finds a flow (it shouldn't for the parameterized query).
|
||||
let resolved_or_flipped = diff.entries.iter().any(|e| {
|
||||
e.baseline_status == Some(VerifyStatus::Confirmed)
|
||||
&& matches!(
|
||||
e.transition,
|
||||
Transition::Resolved | Transition::FlippedNotConfirmed
|
||||
)
|
||||
});
|
||||
assert!(
|
||||
resolved_or_flipped,
|
||||
"Expected the Confirmed finding to be Resolved or FlippedNotConfirmed after the fix. \
|
||||
Diff entries: {:#?}",
|
||||
diff.entries
|
||||
);
|
||||
|
||||
// Step 6: gate passes.
|
||||
assert!(
|
||||
check_gate(&diff, GATE_RESOLVE_ALL_CONFIRMED),
|
||||
"resolve-all-confirmed gate must pass after the fix"
|
||||
);
|
||||
}
|
||||
|
||||
/// Step 7-8: new Confirmed finding → no-new-confirmed gate fails.
|
||||
#[test]
|
||||
fn new_confirmed_fails_no_new_confirmed_gate() {
|
||||
let vuln_path = Path::new(VULN_DIR);
|
||||
let new_path = Path::new(NEW_DIR);
|
||||
|
||||
// Baseline: the original vulnerability, confirmed.
|
||||
let mut vuln_diags = scan_with_hashes(vuln_path);
|
||||
set_verdict(&mut vuln_diags, VerifyStatus::Confirmed);
|
||||
let baseline_entries = diags_to_baseline_entries(&vuln_diags);
|
||||
|
||||
// Current: the "fixed+new" version — original finding gone, new one appears.
|
||||
let mut new_diags = scan_with_hashes(new_path);
|
||||
// Simulate Confirmed on any new findings not in the baseline.
|
||||
let baseline_hashes: std::collections::HashSet<u64> =
|
||||
baseline_entries.iter().map(|e| e.stable_hash).collect();
|
||||
for d in new_diags.iter_mut() {
|
||||
if !baseline_hashes.contains(&d.stable_hash) {
|
||||
let fid = format!("{:016x}", d.stable_hash);
|
||||
let ev = d.evidence.get_or_insert_with(Evidence::default);
|
||||
ev.dynamic_verdict = Some(VerifyResult {
|
||||
finding_id: fid,
|
||||
status: VerifyStatus::Confirmed,
|
||||
triggered_payload: Some("' OR 1=1--".to_string()),
|
||||
reason: None,
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![],
|
||||
toolchain_match: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let diff = compute_verdict_diff(&baseline_entries, &new_diags);
|
||||
|
||||
// There must be at least one New+Confirmed entry.
|
||||
let has_new_confirmed = diff.entries.iter().any(|e| {
|
||||
e.transition == Transition::New && e.current_status == Some(VerifyStatus::Confirmed)
|
||||
});
|
||||
assert!(
|
||||
has_new_confirmed,
|
||||
"Expected a new Confirmed finding in the diff. Diff entries: {:#?}",
|
||||
diff.entries
|
||||
);
|
||||
|
||||
// Gate must fail.
|
||||
assert!(
|
||||
!check_gate(&diff, GATE_NO_NEW_CONFIRMED),
|
||||
"no-new-confirmed gate must fail when a new Confirmed finding exists"
|
||||
);
|
||||
}
|
||||
|
||||
/// `stable_hash` is stable across identical scans (same path, rule, line, col, caps).
|
||||
#[test]
|
||||
fn stable_hash_deterministic_across_scans() {
|
||||
let vuln_path = Path::new(VULN_DIR);
|
||||
let diags1 = scan_with_hashes(vuln_path);
|
||||
let diags2 = scan_with_hashes(vuln_path);
|
||||
|
||||
assert!(!diags1.is_empty());
|
||||
assert_eq!(
|
||||
diags1.len(),
|
||||
diags2.len(),
|
||||
"finding count must be deterministic"
|
||||
);
|
||||
|
||||
let hashes1: std::collections::HashSet<u64> = diags1.iter().map(|d| d.stable_hash).collect();
|
||||
let hashes2: std::collections::HashSet<u64> = diags2.iter().map(|d| d.stable_hash).collect();
|
||||
assert_eq!(
|
||||
hashes1, hashes2,
|
||||
"stable_hash must be identical across two scans of the same codebase"
|
||||
);
|
||||
}
|
||||
|
||||
/// Baseline-write file contains required fields and no source snippets.
|
||||
#[test]
|
||||
fn baseline_write_contains_required_fields_no_source() {
|
||||
let vuln_path = Path::new(VULN_DIR);
|
||||
let mut diags = scan_with_hashes(vuln_path);
|
||||
set_verdict(&mut diags, VerifyStatus::Confirmed);
|
||||
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
write_baseline(f.path(), &diags).unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(f.path()).unwrap();
|
||||
let entries: Vec<BaselineEntry> = serde_json::from_str(&content).unwrap();
|
||||
|
||||
assert!(!entries.is_empty());
|
||||
for e in &entries {
|
||||
assert_ne!(e.stable_hash, 0, "stable_hash must be non-zero");
|
||||
assert!(!e.path.is_empty(), "path must be set");
|
||||
assert!(!e.rule_id.is_empty(), "rule_id must be set");
|
||||
assert!(!e.severity.is_empty(), "severity must be set");
|
||||
}
|
||||
// No source code snippets.
|
||||
assert!(
|
||||
!content.contains("SELECT"),
|
||||
"baseline must not contain SQL source code"
|
||||
);
|
||||
}
|
||||
|
||||
/// `load_baseline` accepts a full Diag JSON (from `nyx scan --format json`).
|
||||
#[test]
|
||||
fn load_baseline_accepts_full_diag_json() {
|
||||
let vuln_path = Path::new(VULN_DIR);
|
||||
let diags = scan_with_hashes(vuln_path);
|
||||
assert!(!diags.is_empty());
|
||||
|
||||
let diag_json = serde_json::to_string(&diags).unwrap();
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
std::fs::write(f.path(), &diag_json).unwrap();
|
||||
|
||||
let loaded = load_baseline(f.path()).unwrap();
|
||||
assert_eq!(loaded.len(), diags.len());
|
||||
// Hashes must round-trip.
|
||||
let loaded_hashes: std::collections::HashSet<u64> =
|
||||
loaded.iter().map(|e| e.stable_hash).collect();
|
||||
let diag_hashes: std::collections::HashSet<u64> =
|
||||
diags.iter().map(|d| d.stable_hash).collect();
|
||||
assert_eq!(loaded_hashes, diag_hashes);
|
||||
}
|
||||
5
tests/fixtures/baseline_sqli_fixed/handler.py
vendored
Normal file
5
tests/fixtures/baseline_sqli_fixed/handler.py
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
import sqlite3
|
||||
|
||||
def get_user(db, user_id):
|
||||
query = "SELECT * FROM users WHERE id = ?"
|
||||
return db.execute(query, (user_id,))
|
||||
12
tests/fixtures/baseline_sqli_new/handler.py
vendored
Normal file
12
tests/fixtures/baseline_sqli_new/handler.py
vendored
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
import os
|
||||
import sqlite3
|
||||
|
||||
def get_user(db):
|
||||
user_id = os.getenv("USER_ID")
|
||||
query = "SELECT * FROM users WHERE id = ?"
|
||||
return db.execute(query, (user_id,))
|
||||
|
||||
def get_post(db):
|
||||
post_id = os.getenv("POST_ID")
|
||||
query = "SELECT * FROM posts WHERE id = " + post_id
|
||||
return db.execute(query)
|
||||
7
tests/fixtures/baseline_sqli_vuln/handler.py
vendored
Normal file
7
tests/fixtures/baseline_sqli_vuln/handler.py
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
import os
|
||||
import sqlite3
|
||||
|
||||
def get_user(db):
|
||||
user_id = os.getenv("USER_ID")
|
||||
query = "SELECT * FROM users WHERE id = " + user_id
|
||||
return db.execute(query)
|
||||
Loading…
Add table
Add a link
Reference in a new issue