//! Real-world vulnerability fixture test suite. //! //! Scans realistic code snippets (20–120 lines) across all 10 supported languages //! and compares findings against `.expect.json` expectation files. //! //! # Environment Variables //! //! - `NYX_TEST_LANG=python` — run only fixtures for one language //! - `NYX_TEST_FIXTURE=cmdi_subprocess` — run only fixtures whose name contains this string //! - `NYX_TEST_VERBOSE=1` — print full diff details for every fixture //! - `NYX_TEST_CATEGORY=taint` — run only one category (taint/cfg/state/mixed) //! //! # Known-failure handling //! //! Expectations with `"must_match": false` are tracked but do not cause test failure. //! A summary of soft misses is always printed at the end. mod common; use common::test_config; use nyx_scanner::commands::scan::Diag; use nyx_scanner::utils::config::AnalysisMode; use serde::Deserialize; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::sync::OnceLock; // ── Expectation schema ─────────────────────────────────────────────────────── #[derive(Debug, Clone, Deserialize)] #[allow(dead_code)] struct RealWorldExpectations { /// Human description of what this fixture tests. #[serde(default)] description: String, /// Tags for coverage matrix (e.g. ["taint", "cmdi", "express"]). #[serde(default)] tags: Vec, /// Which analysis modes this fixture targets. #[serde(default = "default_modes")] modes: Vec, /// Expected findings. expected: Vec, } fn default_modes() -> Vec { vec!["full".to_string()] } #[derive(Debug, Clone, Deserialize)] struct ExpectedFinding { /// Rule ID substring to match (e.g. "taint-" or "js.xss.innerhtml"). rule_id: String, /// Severity (optional, not checked if absent). #[serde(default)] severity: Option, /// If true, missing this finding is a hard failure. If false, it's a soft miss. #[serde(default = "default_must_match")] must_match: bool, /// Line number or range [start, end] where finding should appear. #[serde(default)] line_range: Option<(usize, usize)>, /// Substrings that must appear in message or evidence fields. #[serde(default)] evidence_contains: Vec, /// Human explanation of this expectation. #[serde(default)] notes: String, } fn default_must_match() -> bool { true } // ── Fixture discovery ──────────────────────────────────────────────────────── #[derive(Debug, Clone)] struct Fixture { /// Language slug (rust, c, cpp, java, go, php, python, ruby, typescript, javascript). lang: String, /// Category (taint, cfg, state, mixed). category: String, /// Fixture name (stem of source file). name: String, /// Path to the source fixture file. source_path: PathBuf, /// Parsed expectations. expectations: RealWorldExpectations, } fn discover_fixtures() -> Vec { let base = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/real_world"); let mut fixtures = Vec::new(); let langs = [ "rust", "c", "cpp", "java", "go", "php", "python", "ruby", "typescript", "javascript", ]; let categories = ["taint", "cfg", "state", "mixed"]; for lang in &langs { for category in &categories { let dir = base.join(lang).join(category); if !dir.is_dir() { continue; } // Find all .expect.json files, derive source file from them. let Ok(entries) = std::fs::read_dir(&dir) else { continue; }; for entry in entries.flatten() { let path = entry.path(); let fname = path.file_name().unwrap().to_string_lossy().to_string(); if !fname.ends_with(".expect.json") { continue; } let stem = fname.trim_end_matches(".expect.json"); // Find the corresponding source file (any extension). let source_path = find_source_file(&dir, stem); let Some(source_path) = source_path else { eprintln!( "WARN: no source file for {}/{}/{}/{}", lang, category, stem, fname ); continue; }; let expect_content = std::fs::read_to_string(&path).unwrap_or_else(|e| { panic!("Failed to read {}: {e}", path.display()); }); let expectations: RealWorldExpectations = serde_json::from_str(&expect_content) .unwrap_or_else(|e| { panic!("Failed to parse {}: {e}", path.display()); }); fixtures.push(Fixture { lang: lang.to_string(), category: category.to_string(), name: stem.to_string(), source_path, expectations, }); } } } // Sort for deterministic ordering. fixtures.sort_by(|a, b| { a.lang .cmp(&b.lang) .then(a.category.cmp(&b.category)) .then(a.name.cmp(&b.name)) }); fixtures } fn find_source_file(dir: &Path, stem: &str) -> Option { let extensions = [ "rs", "c", "cpp", "cc", "cxx", "java", "go", "php", "py", "rb", "ts", "tsx", "js", "jsx", ]; for ext in &extensions { let candidate = dir.join(format!("{stem}.{ext}")); if candidate.exists() { return Some(candidate); } } None } // ── Scanning ───────────────────────────────────────────────────────────────── fn scan_fixture(fixture: &Fixture, mode: AnalysisMode) -> Vec { // We scan the parent directory containing just this fixture file. // To isolate, we copy the fixture to a temp dir. let tmp = tempfile::TempDir::with_prefix("nyx_rw_test_").expect("tempdir"); let dest = tmp.path().join(fixture.source_path.file_name().unwrap()); std::fs::copy(&fixture.source_path, &dest).expect("copy fixture"); let cfg = test_config(mode); let mut diags = nyx_scanner::scan_no_index(tmp.path(), &cfg).expect("scan_no_index should succeed"); // Normalize paths to just the filename for comparison. for d in &mut diags { if let Some(fname) = Path::new(&d.path).file_name() { d.path = fname.to_string_lossy().to_string(); } } // Sort deterministically. diags.sort_by(|a, b| { a.path .cmp(&b.path) .then(a.line.cmp(&b.line)) .then(a.id.cmp(&b.id)) .then(a.col.cmp(&b.col)) }); diags } // ── Matching ───────────────────────────────────────────────────────────────── #[derive(Debug)] struct MatchResult { hard_misses: Vec<(ExpectedFinding, String)>, soft_misses: Vec<(ExpectedFinding, String)>, unexpected: Vec, matched: usize, } fn match_expectations( diags: &[Diag], expectations: &[ExpectedFinding], fixture_file: &str, ) -> MatchResult { let mut hard_misses = Vec::new(); let mut soft_misses = Vec::new(); let mut matched_indices: Vec = vec![false; diags.len()]; let mut matched = 0; for exp in expectations { let found = diags.iter().enumerate().any(|(i, d)| { if matched_indices[i] { return false; } if !d.id.contains(&exp.rule_id) { return false; } // Check file if !d.path.contains(fixture_file) && fixture_file != d.path { return false; } // Check severity if specified if let Some(ref sev) = exp.severity && d.severity.as_db_str() != sev.to_uppercase() { return false; } // Check line range if specified if let Some((start, end)) = exp.line_range && (d.line < start || d.line > end) { return false; } // Check evidence substrings for substr in &exp.evidence_contains { let msg = d.message.as_deref().unwrap_or(""); let ev_text = if let Some(ev) = &d.evidence { let mut parts = Vec::new(); if let Some(src) = &ev.source { parts.push(format!( "source: {}", src.snippet.as_deref().unwrap_or(&src.kind) )); } if let Some(snk) = &ev.sink { parts.push(format!( "sink: {}", snk.snippet.as_deref().unwrap_or(&snk.kind) )); } for note in &ev.notes { parts.push(note.clone()); } parts.join(" ") } else { String::new() }; let combined = format!("{msg} {ev_text}"); if !combined.to_lowercase().contains(&substr.to_lowercase()) { return false; } } matched_indices[i] = true; true }); if found { matched += 1; } else { let reason = format!( "rule_id='{}' severity={:?} line_range={:?}", exp.rule_id, exp.severity, exp.line_range ); if exp.must_match { hard_misses.push((exp.clone(), reason)); } else { soft_misses.push((exp.clone(), reason)); } } } // Unexpected = diags not matched by any expectation (informational only). let unexpected: Vec = diags .iter() .enumerate() .filter(|(i, _)| !matched_indices[*i]) .map(|(_, d)| d.clone()) .collect(); MatchResult { hard_misses, soft_misses, unexpected, matched, } } // ── Mode resolution ────────────────────────────────────────────────────────── fn resolve_mode(mode_str: &str) -> AnalysisMode { match mode_str.to_lowercase().as_str() { "ast" => AnalysisMode::Ast, "taint" => AnalysisMode::Taint, "full" => AnalysisMode::Full, _ => AnalysisMode::Full, } } // ── Coverage matrix ────────────────────────────────────────────────────────── fn print_coverage_matrix(fixtures: &[Fixture]) { let mut matrix: BTreeMap> = BTreeMap::new(); let mut tag_counts: BTreeMap = BTreeMap::new(); for f in fixtures { *matrix .entry(f.lang.clone()) .or_default() .entry(f.category.clone()) .or_default() += 1; for tag in &f.expectations.tags { *tag_counts.entry(tag.clone()).or_default() += 1; } } eprintln!("\n╔══════════════════════════════════════════════════════════╗"); eprintln!("║ REAL-WORLD TEST COVERAGE MATRIX ║"); eprintln!("╠══════════════╦════════╦══════╦════════╦════════╦════════╣"); eprintln!("║ Language ║ Taint ║ CFG ║ State ║ Mixed ║ Total ║"); eprintln!("╠══════════════╬════════╬══════╬════════╬════════╬════════╣"); let mut grand_total = 0; for (lang, cats) in &matrix { let t = cats.get("taint").unwrap_or(&0); let c = cats.get("cfg").unwrap_or(&0); let s = cats.get("state").unwrap_or(&0); let m = cats.get("mixed").unwrap_or(&0); let total = t + c + s + m; grand_total += total; eprintln!( "║ {:<12} ║ {:>4} ║ {:>3} ║ {:>4} ║ {:>4} ║ {:>4} ║", lang, t, c, s, m, total ); } eprintln!("╠══════════════╬════════╬══════╬════════╬════════╬════════╣"); eprintln!( "║ TOTAL ║ ║ ║ ║ ║ {:>4} ║", grand_total ); eprintln!("╚══════════════╩════════╩══════╩════════╩════════╩════════╝"); if !tag_counts.is_empty() { eprintln!("\nTag distribution:"); for (tag, count) in &tag_counts { eprintln!(" {tag}: {count}"); } } } // ── Main test ──────────────────────────────────────────────────────────────── static ALL_FIXTURES: OnceLock> = OnceLock::new(); fn get_fixtures() -> &'static [Fixture] { ALL_FIXTURES.get_or_init(discover_fixtures) } fn should_run(fixture: &Fixture) -> bool { if let Ok(lang) = std::env::var("NYX_TEST_LANG") && !fixture.lang.eq_ignore_ascii_case(&lang) { return false; } if let Ok(name) = std::env::var("NYX_TEST_FIXTURE") && !fixture.name.contains(&name) { return false; } if let Ok(cat) = std::env::var("NYX_TEST_CATEGORY") && !fixture.category.eq_ignore_ascii_case(&cat) { return false; } true } fn is_verbose() -> bool { std::env::var("NYX_TEST_VERBOSE").is_ok() } #[test] fn real_world_fixture_suite() { let fixtures = get_fixtures(); let verbose = is_verbose(); let active: Vec<&Fixture> = fixtures.iter().filter(|f| should_run(f)).collect(); if active.is_empty() { eprintln!( "No fixtures matched filters. Total available: {}", fixtures.len() ); print_coverage_matrix(fixtures); return; } eprintln!( "\nRunning {} real-world fixtures (of {} total)\n", active.len(), fixtures.len() ); let mut total_hard_fails = 0; let mut total_soft_misses = 0; let mut total_matched = 0; let mut total_unexpected = 0; let mut failure_details: Vec = Vec::new(); let mut soft_miss_details: Vec = Vec::new(); for fixture in &active { let fixture_label = format!("{}/{}/{}", fixture.lang, fixture.category, fixture.name); for mode_str in &fixture.expectations.modes { let mode = resolve_mode(mode_str); let diags = scan_fixture(fixture, mode); let fixture_file = fixture .source_path .file_name() .unwrap() .to_string_lossy() .to_string(); let result = match_expectations(&diags, &fixture.expectations.expected, &fixture_file); total_matched += result.matched; total_unexpected += result.unexpected.len(); if !result.hard_misses.is_empty() { let mut msg = format!("FAIL {fixture_label} [{mode_str}]:"); for (exp, reason) in &result.hard_misses { msg.push_str(&format!( "\n MISSING (must_match): {} — {}", reason, exp.notes )); } failure_details.push(msg); total_hard_fails += result.hard_misses.len(); } if !result.soft_misses.is_empty() { let mut msg = format!("SOFT {fixture_label} [{mode_str}]:"); for (exp, reason) in &result.soft_misses { msg.push_str(&format!("\n soft miss: {} — {}", reason, exp.notes)); } soft_miss_details.push(msg); total_soft_misses += result.soft_misses.len(); } if verbose { eprintln!( " {fixture_label} [{mode_str}]: {} matched, {} hard misses, {} soft misses, {} unexpected", result.matched, result.hard_misses.len(), result.soft_misses.len(), result.unexpected.len() ); if !result.unexpected.is_empty() { for d in &result.unexpected { eprintln!( " EXTRA: {}:{} [{}] {}", d.path, d.line, d.severity.as_db_str(), d.id ); } } } } } // Print coverage matrix. print_coverage_matrix(fixtures); // Print summary. eprintln!("\n────────────────────────────────────────────────────"); eprintln!( "RESULTS: {} matched, {} hard failures, {} soft misses, {} unexpected", total_matched, total_hard_fails, total_soft_misses, total_unexpected ); eprintln!("────────────────────────────────────────────────────"); if !failure_details.is_empty() { eprintln!("\n=== HARD FAILURES (must_match=true) ==="); for msg in &failure_details { eprintln!("{msg}"); } } if !soft_miss_details.is_empty() { eprintln!("\n=== SOFT MISSES (must_match=false, informational) ==="); for msg in &soft_miss_details { eprintln!("{msg}"); } } // Hard failures cause test failure. assert_eq!( total_hard_fails, 0, "{total_hard_fails} expected findings not found (must_match=true). \ Run with NYX_TEST_VERBOSE=1 for details." ); }