mirror of
https://github.com/elicpeter/nyx.git
synced 2026-06-09 19:45:13 +02:00
[pitboss] phase 05: Track C.2 + Track I.1 quick unlocks — OOB listener wired + golden-verdict fixture runner
This commit is contained in:
pitboss
parent
937eb479e6
commit
cdbc7f2d21
50 changed files with 790 additions and 587 deletions
48
scripts/update_dynamic_goldens.sh
Executable file
48
scripts/update_dynamic_goldens.sh
Executable file
|
|
@ -0,0 +1,48 @@
|
|||
#!/usr/bin/env bash
|
||||
# Regenerate dynamic-fixture golden verdicts.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/update_dynamic_goldens.sh [--test <name>]
|
||||
#
|
||||
# Re-runs the dynamic fixture suites under `NYX_UPDATE_GOLDENS=1` so each
|
||||
# fixture's harness overwrites its `.golden.json` file with the current
|
||||
# verdict. After this script completes, rerun without the env var to
|
||||
# confirm the goldens match.
|
||||
#
|
||||
# Default: refreshes both python_fixtures and rust_fixtures. Pass --test
|
||||
# to refresh only one suite (e.g. `--test python_fixtures`).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
SUITES=(python_fixtures rust_fixtures)
|
||||
if [[ $# -gt 0 ]]; then
|
||||
case "$1" in
|
||||
--test) SUITES=("$2"); shift 2 ;;
|
||||
-h|--help)
|
||||
sed -n '2,12p' "$0"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown arg: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
for suite in "${SUITES[@]}"; do
|
||||
echo "[update-goldens] refreshing $suite ..."
|
||||
NYX_UPDATE_GOLDENS=1 \
|
||||
cargo nextest run --features dynamic --test "$suite" --no-fail-fast
|
||||
done
|
||||
|
||||
echo "[update-goldens] re-running suites without NYX_UPDATE_GOLDENS=1 to verify ..."
|
||||
for suite in "${SUITES[@]}"; do
|
||||
cargo nextest run --features dynamic --test "$suite"
|
||||
done
|
||||
|
||||
echo "[update-goldens] done. Inspect git diff under tests/dynamic_fixtures/ before committing."
|
||||
|
|
@ -5,6 +5,17 @@
|
|||
//! URL path. The lifetime of the listener is per-scan: create one
|
||||
//! [`OobListener`] at scan start, drop it when the scan finishes.
|
||||
//!
|
||||
//! # Wiring
|
||||
//!
|
||||
//! As of Phase 05 the listener is load-bearing: [`crate::dynamic::verify::VerifyOptions::from_config`]
|
||||
//! constructs one per scan via [`OobListener::bind`] and threads it into
|
||||
//! [`crate::dynamic::sandbox::SandboxOptions::oob_listener`]. The runner
|
||||
//! polls [`OobListener::was_nonce_hit`] after each sandbox run (see
|
||||
//! `src/dynamic/runner.rs`) and toggles
|
||||
//! [`crate::dynamic::sandbox::SandboxOutcome::oob_callback_seen`] when a
|
||||
//! probe arrives — that is the only signal that turns an OOB-only sink
|
||||
//! (e.g. blind SSRF) into a `Confirmed` verdict.
|
||||
//!
|
||||
//! # Nonce URL
|
||||
//!
|
||||
//! The caller generates a per-finding nonce (UUID4 hex) and embeds it in
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
use crate::callgraph::CallGraph;
|
||||
use crate::commands::scan::Diag;
|
||||
use crate::dynamic::corpus::{payloads_for, CORPUS_VERSION};
|
||||
use crate::dynamic::oob::OobListener;
|
||||
use crate::dynamic::report::{AttemptSummary, VerifyResult, VerifyStatus};
|
||||
use crate::dynamic::runner::{run_spec, RunError};
|
||||
use crate::dynamic::sandbox::{toolchain_id_with_digest, SandboxOptions};
|
||||
|
|
@ -54,6 +55,17 @@ pub struct VerifyOptions {
|
|||
|
||||
impl VerifyOptions {
|
||||
/// Build `VerifyOptions` from scanner config.
|
||||
///
|
||||
/// Binds a per-scan [`OobListener`] on a free loopback port and attaches
|
||||
/// it to `sandbox.oob_listener`. The listener is held by `Arc` so every
|
||||
/// per-finding clone of `VerifyOptions` shares the same accept thread;
|
||||
/// it is torn down via the `OobListener::Drop` impl once the last
|
||||
/// `Arc` is released at end of scan.
|
||||
///
|
||||
/// If `OobListener::bind` fails (e.g. all loopback ports are in use),
|
||||
/// the field stays `None`; the runner skips OOB-callback payloads
|
||||
/// (`src/dynamic/runner.rs` `oob_nonce_slot` branch) while non-OOB
|
||||
/// payloads continue to run against their existing oracle.
|
||||
pub fn from_config(config: &Config) -> Self {
|
||||
use crate::dynamic::sandbox::SandboxBackend;
|
||||
let backend = match config.scanner.verify_backend.as_str() {
|
||||
|
|
@ -61,9 +73,11 @@ impl VerifyOptions {
|
|||
"process" => SandboxBackend::Process,
|
||||
_ => SandboxBackend::Auto,
|
||||
};
|
||||
let oob_listener = OobListener::bind().ok().map(Arc::new);
|
||||
Self {
|
||||
sandbox: SandboxOptions {
|
||||
backend,
|
||||
oob_listener,
|
||||
..SandboxOptions::default()
|
||||
},
|
||||
project_root: None,
|
||||
|
|
|
|||
249
tests/common/fixture_harness.rs
Normal file
249
tests/common/fixture_harness.rs
Normal file
|
|
@ -0,0 +1,249 @@
|
|||
//! Golden-verdict regression harness for dynamic-verification fixtures.
|
||||
//!
|
||||
//! Replaces the original hand-rolled `assert_eq!(status, Confirmed)` style
|
||||
//! with a "current verdict is the golden" model: each fixture's first run
|
||||
//! (under `NYX_UPDATE_GOLDENS=1`) records its current verdict shape into a
|
||||
//! `.golden.json` file checked in beside the fixture; subsequent runs diff
|
||||
//! against that golden and fail on regression.
|
||||
//!
|
||||
//! The contract is intentionally agnostic to the verdict's polarity. A
|
||||
//! fixture stuck at `Inconclusive(BuildFailed)` because of a missing
|
||||
//! toolchain is locked at that shape until someone consciously refreshes the
|
||||
//! golden via `scripts/update_dynamic_goldens.sh`. A flip to `Confirmed` is
|
||||
//! also a "regression" in the harness's sense and surfaces as a test
|
||||
//! failure, prompting an explicit golden update.
|
||||
|
||||
use nyx_scanner::commands::scan::Diag;
|
||||
use nyx_scanner::dynamic::verify::{verify_finding, VerifyOptions};
|
||||
use nyx_scanner::evidence::{
|
||||
Confidence, Evidence, FlowStep, FlowStepKind, InconclusiveReason, UnsupportedReason,
|
||||
VerifyResult, VerifyStatus,
|
||||
};
|
||||
use nyx_scanner::labels::Cap;
|
||||
use nyx_scanner::patterns::{FindingCategory, Severity};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// Serialise-once lock guarding the process-global env vars
|
||||
/// (`NYX_REPRO_BASE`, `NYX_TELEMETRY_PATH`) and the shared build cache dir.
|
||||
/// Shared across `python_fixtures` / `rust_fixtures` to prevent cross-suite
|
||||
/// races when nextest runs them in parallel within the same test binary.
|
||||
pub static FIXTURE_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// How the fixture source should land relative to the harness's tempdir
|
||||
/// before [`verify_finding`] is invoked. Mirrors the original per-language
|
||||
/// behaviour: Python copies the file beside its sibling-import siblings;
|
||||
/// Rust lays it out as `src/entry.rs` so the Cargo project emitter finds it.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
#[allow(dead_code)] // Each test binary uses only one variant; the other is dead per-crate.
|
||||
pub enum CopyStrategy {
|
||||
/// Copy the fixture to `tempdir/{fixture_basename}`. The synthesised Diag
|
||||
/// points at the copy so the Python harness can import it directly.
|
||||
PreserveName,
|
||||
/// Copy the fixture to `tempdir/src/entry.rs`. The synthesised Diag
|
||||
/// points at the original fixture path (the Rust emitter reads source via
|
||||
/// the absolute Diag path, not via the temp-dir layout).
|
||||
RustEntry,
|
||||
}
|
||||
|
||||
/// Per-fixture specification.
|
||||
pub struct FixtureSpec<'a> {
|
||||
/// Subdirectory under `tests/dynamic_fixtures/` (e.g. `"python"`, `"rust"`).
|
||||
pub lang_dir: &'a str,
|
||||
/// Fixture filename within `lang_dir`.
|
||||
pub fixture: &'a str,
|
||||
/// Entry-point function name passed in the synthesised flow-step.
|
||||
pub func: &'a str,
|
||||
/// Sink capability bits to set on `Evidence.sink_caps`.
|
||||
pub cap: Cap,
|
||||
/// Sink line for the synthesised flow-step. Adversarial fixtures pass a
|
||||
/// line that does not exist in the source (e.g. 999) so the probe cannot
|
||||
/// fire while the oracle marker still prints.
|
||||
pub sink_line: u32,
|
||||
/// Confidence stamp on the Diag. `Confidence::Low` short-circuits to
|
||||
/// `Unsupported(ConfidenceTooLow)` before the harness executes.
|
||||
pub confidence: Confidence,
|
||||
/// File-layout strategy for the temp-dir copy.
|
||||
pub copy: CopyStrategy,
|
||||
}
|
||||
|
||||
/// Trimmed verdict shape persisted in the `.golden.json` file.
|
||||
///
|
||||
/// Captures the fields a regression test must pin: status + typed reasons
|
||||
/// + whether a payload triggered. Excludes machine-dependent fields
|
||||
/// (`finding_id`, `detail`, `attempts`, `toolchain_match`).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct GoldenVerdict {
|
||||
pub status: VerifyStatus,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub reason: Option<UnsupportedReason>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub inconclusive_reason: Option<InconclusiveReason>,
|
||||
#[serde(default)]
|
||||
pub triggered: bool,
|
||||
}
|
||||
|
||||
impl From<&VerifyResult> for GoldenVerdict {
|
||||
fn from(v: &VerifyResult) -> Self {
|
||||
Self {
|
||||
status: v.status,
|
||||
reason: v.reason.clone(),
|
||||
inconclusive_reason: v.inconclusive_reason.clone(),
|
||||
triggered: v.triggered_payload.is_some(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the fixture through `verify_finding` and either compare against the
|
||||
/// stored golden or — when `NYX_UPDATE_GOLDENS=1` — overwrite the golden
|
||||
/// with the current verdict.
|
||||
pub fn run_fixture_and_compare_to_golden(spec: &FixtureSpec<'_>) {
|
||||
let _guard = FIXTURE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
|
||||
let fixture_root = fixture_dir(spec.lang_dir);
|
||||
let fixture_src = fixture_root.join(spec.fixture);
|
||||
let golden_path = fixture_root.join(format!("{}.golden.json", spec.fixture));
|
||||
|
||||
let tmp = TempDir::new().expect("create tempdir");
|
||||
let diag_path = stage_fixture(&fixture_src, &tmp, spec.copy);
|
||||
|
||||
// SAFETY: env mutation is serialised by FIXTURE_LOCK and the vars are
|
||||
// cleared before the lock guard drops at end of function.
|
||||
unsafe {
|
||||
std::env::set_var("NYX_REPRO_BASE", tmp.path().join("repro").to_str().unwrap());
|
||||
std::env::set_var(
|
||||
"NYX_TELEMETRY_PATH",
|
||||
tmp.path().join("events.jsonl").to_str().unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
let mut diag = make_diag(&diag_path, spec.func, spec.cap, spec.sink_line);
|
||||
diag.confidence = Some(spec.confidence);
|
||||
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&diag, &opts);
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("NYX_REPRO_BASE");
|
||||
std::env::remove_var("NYX_TELEMETRY_PATH");
|
||||
}
|
||||
|
||||
let current = GoldenVerdict::from(&result);
|
||||
let mut current_json =
|
||||
serde_json::to_string_pretty(¤t).expect("serialise golden verdict");
|
||||
current_json.push('\n');
|
||||
|
||||
if std::env::var("NYX_UPDATE_GOLDENS").is_ok_and(|v| v == "1") {
|
||||
std::fs::write(&golden_path, ¤t_json).unwrap_or_else(|e| {
|
||||
panic!("write golden {}: {e}", golden_path.display())
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let expected_json = std::fs::read_to_string(&golden_path).unwrap_or_else(|e| {
|
||||
panic!(
|
||||
"missing golden {}: {e}\n\
|
||||
current verdict:\n{current_json}\n\
|
||||
rerun with NYX_UPDATE_GOLDENS=1 ./scripts/update_dynamic_goldens.sh to seed it.",
|
||||
golden_path.display()
|
||||
)
|
||||
});
|
||||
let expected: GoldenVerdict = serde_json::from_str(&expected_json)
|
||||
.unwrap_or_else(|e| panic!("parse golden {}: {e}", golden_path.display()));
|
||||
|
||||
if current != expected {
|
||||
panic!(
|
||||
"golden regression for {}:\n\
|
||||
expected: {expected_json}\n\
|
||||
actual: {current_json}\n\
|
||||
detail: {:?}\n\
|
||||
rerun with NYX_UPDATE_GOLDENS=1 ./scripts/update_dynamic_goldens.sh if intended.",
|
||||
spec.fixture, result.detail
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture_dir(lang_dir: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/dynamic_fixtures")
|
||||
.join(lang_dir)
|
||||
}
|
||||
|
||||
fn stage_fixture(src: &Path, tmp: &TempDir, copy: CopyStrategy) -> PathBuf {
|
||||
match copy {
|
||||
CopyStrategy::PreserveName => {
|
||||
let dst = tmp.path().join(src.file_name().expect("fixture has filename"));
|
||||
std::fs::copy(src, &dst).expect("copy fixture into tempdir");
|
||||
dst
|
||||
}
|
||||
CopyStrategy::RustEntry => {
|
||||
let dst_dir = tmp.path().join("src");
|
||||
std::fs::create_dir_all(&dst_dir).expect("create src/ in tempdir");
|
||||
let dst = dst_dir.join("entry.rs");
|
||||
std::fs::copy(src, &dst).expect("copy fixture into tempdir/src/entry.rs");
|
||||
// The Rust harness emitter reads source via the Diag's absolute path,
|
||||
// not via the temp-dir layout, so the Diag must point at the original
|
||||
// fixture file. The temp-dir copy is only consulted by the harness
|
||||
// builder for the workdir-relative `src/entry.rs` view.
|
||||
src.to_path_buf()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn make_diag(path: &Path, func: &str, cap: Cap, sink_line: u32) -> Diag {
|
||||
let path_str = path.to_string_lossy().into_owned();
|
||||
let evidence = Evidence {
|
||||
flow_steps: vec![
|
||||
FlowStep {
|
||||
step: 1,
|
||||
kind: FlowStepKind::Source,
|
||||
file: path_str.clone(),
|
||||
line: 1,
|
||||
col: 0,
|
||||
snippet: None,
|
||||
variable: Some("payload".into()),
|
||||
callee: None,
|
||||
function: Some(func.to_owned()),
|
||||
is_cross_file: false,
|
||||
},
|
||||
FlowStep {
|
||||
step: 2,
|
||||
kind: FlowStepKind::Sink,
|
||||
file: path_str.clone(),
|
||||
line: sink_line,
|
||||
col: 4,
|
||||
snippet: None,
|
||||
variable: None,
|
||||
callee: None,
|
||||
function: None,
|
||||
is_cross_file: false,
|
||||
},
|
||||
],
|
||||
sink_caps: cap.bits(),
|
||||
..Default::default()
|
||||
};
|
||||
Diag {
|
||||
path: path_str,
|
||||
line: sink_line as usize,
|
||||
col: 0,
|
||||
severity: Severity::High,
|
||||
id: "taint-unsanitised-flow".into(),
|
||||
category: FindingCategory::Security,
|
||||
path_validated: false,
|
||||
guard_kind: None,
|
||||
message: None,
|
||||
labels: vec![],
|
||||
confidence: Some(Confidence::High),
|
||||
evidence: Some(evidence),
|
||||
rank_score: None,
|
||||
rank_reason: None,
|
||||
suppressed: false,
|
||||
suppression: None,
|
||||
rollup: None,
|
||||
finding_id: String::new(),
|
||||
alternative_finding_ids: vec![],
|
||||
stable_hash: 0,
|
||||
}
|
||||
}
|
||||
|
|
@ -2,6 +2,13 @@
|
|||
|
||||
pub mod recall;
|
||||
|
||||
// Only `python_fixtures` and `rust_fixtures` reference these symbols; every
|
||||
// other test binary pulls `mod common` in and would otherwise emit
|
||||
// per-binary `dead_code` warnings for the whole submodule.
|
||||
#[cfg(feature = "dynamic")]
|
||||
#[allow(dead_code)]
|
||||
pub mod fixture_harness;
|
||||
|
||||
use nyx_scanner::commands::scan::Diag;
|
||||
use nyx_scanner::utils::config::{AnalysisMode, Config};
|
||||
use serde::Deserialize;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Inconclusive",
|
||||
"inconclusive_reason": "OracleCollisionSuspected",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "NotConfirmed",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Unsupported",
|
||||
"reason": "ConfidenceTooLow",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Inconclusive",
|
||||
"inconclusive_reason": "OracleCollisionSuspected",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "NotConfirmed",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Unsupported",
|
||||
"reason": "ConfidenceTooLow",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Inconclusive",
|
||||
"inconclusive_reason": "OracleCollisionSuspected",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "NotConfirmed",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Unsupported",
|
||||
"reason": "ConfidenceTooLow",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Inconclusive",
|
||||
"inconclusive_reason": "OracleCollisionSuspected",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "NotConfirmed",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Unsupported",
|
||||
"reason": "ConfidenceTooLow",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Inconclusive",
|
||||
"inconclusive_reason": "OracleCollisionSuspected",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "NotConfirmed",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Unsupported",
|
||||
"reason": "ConfidenceTooLow",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Inconclusive",
|
||||
"inconclusive_reason": "OracleCollisionSuspected",
|
||||
"triggered": false
|
||||
}
|
||||
4
tests/dynamic_fixtures/rust/cmdi_negative.rs.golden.json
Normal file
4
tests/dynamic_fixtures/rust/cmdi_negative.rs.golden.json
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "NotConfirmed",
|
||||
"triggered": false
|
||||
}
|
||||
4
tests/dynamic_fixtures/rust/cmdi_positive.rs.golden.json
Normal file
4
tests/dynamic_fixtures/rust/cmdi_positive.rs.golden.json
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Unsupported",
|
||||
"reason": "ConfidenceTooLow",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Inconclusive",
|
||||
"inconclusive_reason": "OracleCollisionSuspected",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "NotConfirmed",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Unsupported",
|
||||
"reason": "ConfidenceTooLow",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Inconclusive",
|
||||
"inconclusive_reason": "OracleCollisionSuspected",
|
||||
"triggered": false
|
||||
}
|
||||
4
tests/dynamic_fixtures/rust/sqli_negative.rs.golden.json
Normal file
4
tests/dynamic_fixtures/rust/sqli_negative.rs.golden.json
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "NotConfirmed",
|
||||
"triggered": false
|
||||
}
|
||||
4
tests/dynamic_fixtures/rust/sqli_positive.rs.golden.json
Normal file
4
tests/dynamic_fixtures/rust/sqli_positive.rs.golden.json
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Unsupported",
|
||||
"reason": "ConfidenceTooLow",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Inconclusive",
|
||||
"inconclusive_reason": "OracleCollisionSuspected",
|
||||
"triggered": false
|
||||
}
|
||||
4
tests/dynamic_fixtures/rust/ssrf_negative.rs.golden.json
Normal file
4
tests/dynamic_fixtures/rust/ssrf_negative.rs.golden.json
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "NotConfirmed",
|
||||
"triggered": false
|
||||
}
|
||||
4
tests/dynamic_fixtures/rust/ssrf_positive.rs.golden.json
Normal file
4
tests/dynamic_fixtures/rust/ssrf_positive.rs.golden.json
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Unsupported",
|
||||
"reason": "ConfidenceTooLow",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Inconclusive",
|
||||
"inconclusive_reason": "OracleCollisionSuspected",
|
||||
"triggered": false
|
||||
}
|
||||
4
tests/dynamic_fixtures/rust/xss_negative.rs.golden.json
Normal file
4
tests/dynamic_fixtures/rust/xss_negative.rs.golden.json
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "NotConfirmed",
|
||||
"triggered": false
|
||||
}
|
||||
4
tests/dynamic_fixtures/rust/xss_positive.rs.golden.json
Normal file
4
tests/dynamic_fixtures/rust/xss_positive.rs.golden.json
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"status": "Confirmed",
|
||||
"triggered": true
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"status": "Unsupported",
|
||||
"reason": "ConfidenceTooLow",
|
||||
"triggered": false
|
||||
}
|
||||
|
|
@ -1,36 +1,33 @@
|
|||
//! Python fixture integration tests (§15 Pillar B acceptance gate).
|
||||
//!
|
||||
//! Runs the dynamic verification pipeline against each Python fixture and
|
||||
//! asserts the expected verdict. Requires `--features dynamic` and Python3
|
||||
//! to be available on PATH.
|
||||
//! Each fixture is run through the dynamic verification pipeline; its
|
||||
//! verdict is then compared against the per-fixture golden under
|
||||
//! `tests/dynamic_fixtures/python/{name}.golden.json`. Refresh the goldens
|
||||
//! via `NYX_UPDATE_GOLDENS=1 ./scripts/update_dynamic_goldens.sh`.
|
||||
//!
|
||||
//! Verdicts under test:
|
||||
//! - positive → Confirmed
|
||||
//! - negative → NotConfirmed
|
||||
//! - unsupported → Unsupported(ConfidenceTooLow) [spec-level rejection]
|
||||
//! - adversarial → Inconclusive(OracleCollisionSuspected)
|
||||
//!
|
||||
//! Tests are skipped when Python3 is not available.
|
||||
//! Tests that need python3 on PATH skip with an `eprintln!` when it is
|
||||
//! missing; `Confidence::Low` rows do not need python3 because the verifier
|
||||
//! short-circuits before harness execution.
|
||||
|
||||
mod common;
|
||||
|
||||
#[cfg(feature = "dynamic")]
|
||||
mod python_fixture_tests {
|
||||
use crate::common::fixture_harness::{
|
||||
run_fixture_and_compare_to_golden, CopyStrategy, FixtureSpec,
|
||||
};
|
||||
use nyx_scanner::commands::scan::Diag;
|
||||
use nyx_scanner::dynamic::verify::{verify_finding, VerifyOptions};
|
||||
use nyx_scanner::evidence::{
|
||||
Confidence, Evidence, FlowStep, FlowStepKind, InconclusiveReason, UnsupportedReason,
|
||||
VerifyStatus,
|
||||
Confidence, Evidence, FlowStep, FlowStepKind, UnsupportedReason, VerifyStatus,
|
||||
};
|
||||
use nyx_scanner::labels::Cap;
|
||||
use nyx_scanner::patterns::{FindingCategory, Severity};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
use tempfile::TempDir;
|
||||
|
||||
// Serialize all fixture tests to prevent races on process-global state
|
||||
// (NYX_REPRO_BASE and NYX_TELEMETRY_PATH env vars).
|
||||
static FIXTURE_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// Returns `true` if `python3` is available.
|
||||
/// `python3` available on PATH? Tests that need an interpreter return
|
||||
/// early with an `eprintln!` when this is false.
|
||||
fn python3_available() -> bool {
|
||||
std::process::Command::new("python3")
|
||||
.arg("--version")
|
||||
|
|
@ -39,337 +36,204 @@ mod python_fixture_tests {
|
|||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn fixture_path(name: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/dynamic_fixtures/python")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
/// Run a fixture and return the verdict.
|
||||
///
|
||||
/// Acquires `FIXTURE_LOCK` for the full duration to prevent races on the
|
||||
/// process-global NYX_REPRO_BASE / NYX_TELEMETRY_PATH env vars.
|
||||
/// `set_current_dir` is NOT used here: `harness::copy_entry_file` resolves
|
||||
/// the entry file via its absolute path, so CWD is irrelevant.
|
||||
fn run_fixture(fixture: &str, func: &str, cap: Cap, sink_line: u32) -> nyx_scanner::evidence::VerifyResult {
|
||||
let _guard = FIXTURE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
|
||||
let path = fixture_path(fixture);
|
||||
// Copy fixture to a temp dir so the harness can import it.
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dst = tmp.path().join(Path::new(fixture).file_name().unwrap());
|
||||
std::fs::copy(&path, &dst).expect("fixture file must exist");
|
||||
|
||||
// Set up repro and telemetry to temp dirs to avoid side effects.
|
||||
unsafe {
|
||||
std::env::set_var("NYX_REPRO_BASE", tmp.path().join("repro").to_str().unwrap());
|
||||
std::env::set_var("NYX_TELEMETRY_PATH", tmp.path().join("events.jsonl").to_str().unwrap());
|
||||
fn spec(fixture: &'static str, func: &'static str, cap: Cap, sink_line: u32) -> FixtureSpec<'static> {
|
||||
FixtureSpec {
|
||||
lang_dir: "python",
|
||||
fixture,
|
||||
func,
|
||||
cap,
|
||||
sink_line,
|
||||
confidence: Confidence::High,
|
||||
copy: CopyStrategy::PreserveName,
|
||||
}
|
||||
}
|
||||
|
||||
// Use the temp dir copy as the fixture path (absolute — no CWD change needed).
|
||||
let diag = make_diag(&dst, func, cap, sink_line);
|
||||
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&diag, &opts);
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("NYX_REPRO_BASE");
|
||||
std::env::remove_var("NYX_TELEMETRY_PATH");
|
||||
fn low_spec(
|
||||
fixture: &'static str,
|
||||
func: &'static str,
|
||||
cap: Cap,
|
||||
sink_line: u32,
|
||||
) -> FixtureSpec<'static> {
|
||||
FixtureSpec {
|
||||
lang_dir: "python",
|
||||
fixture,
|
||||
func,
|
||||
cap,
|
||||
sink_line,
|
||||
confidence: Confidence::Low,
|
||||
copy: CopyStrategy::PreserveName,
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
// ── SQLi fixtures ────────────────────────────────────────────────────────
|
||||
// ── SQLi ─────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn sqli_positive_is_confirmed() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("sqli_positive.py", "login", Cap::SQL_QUERY, 17);
|
||||
assert_eq!(
|
||||
result.status, VerifyStatus::Confirmed,
|
||||
"sqli_positive must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status, result.detail
|
||||
);
|
||||
fn sqli_positive_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec("sqli_positive.py", "login", Cap::SQL_QUERY, 17));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sqli_negative_is_not_confirmed() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("sqli_negative.py", "login", Cap::SQL_QUERY, 12);
|
||||
assert_eq!(
|
||||
result.status, VerifyStatus::NotConfirmed,
|
||||
"sqli_negative must be NotConfirmed; got {:?}",
|
||||
result.status
|
||||
);
|
||||
fn sqli_negative_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec("sqli_negative.py", "login", Cap::SQL_QUERY, 12));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sqli_unsupported_is_unsupported() {
|
||||
// Low-confidence Diag → Unsupported(ConfidenceTooLow) without execution.
|
||||
let path = fixture_path("sqli_unsupported.py");
|
||||
let mut d = make_diag(&path, "find_user", Cap::SQL_QUERY, 10);
|
||||
d.confidence = Some(Confidence::Low);
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&d, &opts);
|
||||
assert_eq!(result.status, VerifyStatus::Unsupported);
|
||||
assert_eq!(result.reason, Some(UnsupportedReason::ConfidenceTooLow));
|
||||
fn sqli_unsupported_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&low_spec(
|
||||
"sqli_unsupported.py",
|
||||
"find_user",
|
||||
Cap::SQL_QUERY,
|
||||
10,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sqli_adversarial_is_inconclusive_collision() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
// The adversarial fixture prints the oracle marker WITHOUT going through
|
||||
// any SQL sink — so the oracle fires but the probe at the (nonexistent)
|
||||
// SQL execute line does not.
|
||||
// We point the sink line at a line that doesn't exist in the file (999)
|
||||
// so the settrace probe can't fire.
|
||||
let result = run_fixture("sqli_adversarial.py", "get_value", Cap::SQL_QUERY, 999);
|
||||
// Oracle fires (prints "NYX_SQL_CONFIRMED") but probe doesn't (line 999 missing).
|
||||
assert_eq!(
|
||||
result.status, VerifyStatus::Inconclusive,
|
||||
"sqli_adversarial must be Inconclusive; got {:?}",
|
||||
result.status
|
||||
);
|
||||
assert_eq!(
|
||||
result.inconclusive_reason,
|
||||
Some(InconclusiveReason::OracleCollisionSuspected),
|
||||
"adversarial must be OracleCollisionSuspected"
|
||||
);
|
||||
fn sqli_adversarial_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec("sqli_adversarial.py", "get_value", Cap::SQL_QUERY, 999));
|
||||
}
|
||||
|
||||
// ── Command injection fixtures ───────────────────────────────────────────
|
||||
// ── Command injection ────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn cmdi_positive_is_confirmed() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("cmdi_positive.py", "run_ping", Cap::CODE_EXEC, 13);
|
||||
assert_eq!(
|
||||
result.status, VerifyStatus::Confirmed,
|
||||
"cmdi_positive must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status, result.detail
|
||||
);
|
||||
fn cmdi_positive_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec("cmdi_positive.py", "run_ping", Cap::CODE_EXEC, 13));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cmdi_negative_is_not_confirmed() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("cmdi_negative.py", "run_ping", Cap::CODE_EXEC, 17);
|
||||
assert_eq!(
|
||||
result.status, VerifyStatus::NotConfirmed,
|
||||
"cmdi_negative must be NotConfirmed; got {:?}",
|
||||
result.status
|
||||
);
|
||||
fn cmdi_negative_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec("cmdi_negative.py", "run_ping", Cap::CODE_EXEC, 17));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cmdi_unsupported_is_unsupported() {
|
||||
let path = fixture_path("cmdi_unsupported.py");
|
||||
let mut d = make_diag(&path, "process_request", Cap::CODE_EXEC, 9);
|
||||
d.confidence = Some(Confidence::Low);
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&d, &opts);
|
||||
assert_eq!(result.status, VerifyStatus::Unsupported);
|
||||
assert_eq!(result.reason, Some(UnsupportedReason::ConfidenceTooLow));
|
||||
fn cmdi_unsupported_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&low_spec(
|
||||
"cmdi_unsupported.py",
|
||||
"process_request",
|
||||
Cap::CODE_EXEC,
|
||||
9,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cmdi_adversarial_is_inconclusive_collision() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("cmdi_adversarial.py", "process_input", Cap::CODE_EXEC, 999);
|
||||
assert_eq!(result.status, VerifyStatus::Inconclusive);
|
||||
assert_eq!(
|
||||
result.inconclusive_reason,
|
||||
Some(InconclusiveReason::OracleCollisionSuspected)
|
||||
);
|
||||
fn cmdi_adversarial_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec(
|
||||
"cmdi_adversarial.py",
|
||||
"process_input",
|
||||
Cap::CODE_EXEC,
|
||||
999,
|
||||
));
|
||||
}
|
||||
|
||||
// ── File I/O fixtures ────────────────────────────────────────────────────
|
||||
// ── File I/O ─────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn fileio_positive_is_confirmed() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("fileio_positive.py", "read_file", Cap::FILE_IO, 11);
|
||||
assert_eq!(
|
||||
result.status, VerifyStatus::Confirmed,
|
||||
"fileio_positive must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status, result.detail
|
||||
);
|
||||
fn fileio_positive_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec("fileio_positive.py", "read_file", Cap::FILE_IO, 11));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fileio_negative_is_not_confirmed() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("fileio_negative.py", "read_file", Cap::FILE_IO, 18);
|
||||
assert_eq!(
|
||||
result.status, VerifyStatus::NotConfirmed,
|
||||
"fileio_negative must be NotConfirmed; got {:?}",
|
||||
result.status
|
||||
);
|
||||
fn fileio_negative_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec("fileio_negative.py", "read_file", Cap::FILE_IO, 18));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fileio_unsupported_is_unsupported() {
|
||||
let path = fixture_path("fileio_unsupported.py");
|
||||
let mut d = make_diag(&path, "read_config", Cap::FILE_IO, 7);
|
||||
d.confidence = Some(Confidence::Low);
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&d, &opts);
|
||||
assert_eq!(result.status, VerifyStatus::Unsupported);
|
||||
assert_eq!(result.reason, Some(UnsupportedReason::ConfidenceTooLow));
|
||||
fn fileio_unsupported_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&low_spec(
|
||||
"fileio_unsupported.py",
|
||||
"read_config",
|
||||
Cap::FILE_IO,
|
||||
7,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fileio_adversarial_is_inconclusive_collision() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("fileio_adversarial.py", "read_file", Cap::FILE_IO, 999);
|
||||
assert_eq!(result.status, VerifyStatus::Inconclusive);
|
||||
assert_eq!(
|
||||
result.inconclusive_reason,
|
||||
Some(InconclusiveReason::OracleCollisionSuspected)
|
||||
);
|
||||
fn fileio_adversarial_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec("fileio_adversarial.py", "read_file", Cap::FILE_IO, 999));
|
||||
}
|
||||
|
||||
// ── SSRF fixtures ────────────────────────────────────────────────────────
|
||||
// ── SSRF ─────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn ssrf_positive_is_confirmed() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("ssrf_positive.py", "fetch_url", Cap::SSRF, 11);
|
||||
assert_eq!(
|
||||
result.status, VerifyStatus::Confirmed,
|
||||
"ssrf_positive must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status, result.detail
|
||||
);
|
||||
fn ssrf_positive_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec("ssrf_positive.py", "fetch_url", Cap::SSRF, 11));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_negative_is_not_confirmed() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("ssrf_negative.py", "fetch_url", Cap::SSRF, 26);
|
||||
// Blocked by host validation — oracle won't fire.
|
||||
assert_eq!(
|
||||
result.status, VerifyStatus::NotConfirmed,
|
||||
"ssrf_negative must be NotConfirmed; got {:?}",
|
||||
result.status
|
||||
);
|
||||
fn ssrf_negative_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec("ssrf_negative.py", "fetch_url", Cap::SSRF, 26));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_unsupported_is_unsupported() {
|
||||
let path = fixture_path("ssrf_unsupported.py");
|
||||
let mut d = make_diag(&path, "fetch", Cap::SSRF, 9);
|
||||
d.confidence = Some(Confidence::Low);
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&d, &opts);
|
||||
assert_eq!(result.status, VerifyStatus::Unsupported);
|
||||
assert_eq!(result.reason, Some(UnsupportedReason::ConfidenceTooLow));
|
||||
fn ssrf_unsupported_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&low_spec("ssrf_unsupported.py", "fetch", Cap::SSRF, 9));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_adversarial_is_inconclusive_collision() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("ssrf_adversarial.py", "fetch_url", Cap::SSRF, 999);
|
||||
assert_eq!(result.status, VerifyStatus::Inconclusive);
|
||||
assert_eq!(
|
||||
result.inconclusive_reason,
|
||||
Some(InconclusiveReason::OracleCollisionSuspected)
|
||||
);
|
||||
fn ssrf_adversarial_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec("ssrf_adversarial.py", "fetch_url", Cap::SSRF, 999));
|
||||
}
|
||||
|
||||
// ── XSS fixtures ─────────────────────────────────────────────────────────
|
||||
// ── XSS ──────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn xss_positive_is_confirmed() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("xss_positive.py", "render_comment", Cap::HTML_ESCAPE, 9);
|
||||
assert_eq!(
|
||||
result.status, VerifyStatus::Confirmed,
|
||||
"xss_positive must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status, result.detail
|
||||
);
|
||||
fn xss_positive_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec(
|
||||
"xss_positive.py",
|
||||
"render_comment",
|
||||
Cap::HTML_ESCAPE,
|
||||
9,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xss_negative_is_not_confirmed() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("xss_negative.py", "render_comment", Cap::HTML_ESCAPE, 11);
|
||||
assert_eq!(
|
||||
result.status, VerifyStatus::NotConfirmed,
|
||||
"xss_negative must be NotConfirmed; got {:?}",
|
||||
result.status
|
||||
);
|
||||
fn xss_negative_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec(
|
||||
"xss_negative.py",
|
||||
"render_comment",
|
||||
Cap::HTML_ESCAPE,
|
||||
11,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xss_unsupported_is_unsupported() {
|
||||
let path = fixture_path("xss_unsupported.py");
|
||||
let mut d = make_diag(&path, "render", Cap::HTML_ESCAPE, 7);
|
||||
d.confidence = Some(Confidence::Low);
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&d, &opts);
|
||||
assert_eq!(result.status, VerifyStatus::Unsupported);
|
||||
assert_eq!(result.reason, Some(UnsupportedReason::ConfidenceTooLow));
|
||||
fn xss_unsupported_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&low_spec(
|
||||
"xss_unsupported.py",
|
||||
"render",
|
||||
Cap::HTML_ESCAPE,
|
||||
7,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xss_adversarial_is_inconclusive_collision() {
|
||||
if !python3_available() {
|
||||
eprintln!("SKIP: python3 not available");
|
||||
return;
|
||||
}
|
||||
let result = run_fixture("xss_adversarial.py", "render_comment", Cap::HTML_ESCAPE, 999);
|
||||
assert_eq!(result.status, VerifyStatus::Inconclusive);
|
||||
assert_eq!(
|
||||
result.inconclusive_reason,
|
||||
Some(InconclusiveReason::OracleCollisionSuspected)
|
||||
);
|
||||
fn xss_adversarial_matches_golden() {
|
||||
if !python3_available() { eprintln!("SKIP: python3 not available"); return; }
|
||||
run_fixture_and_compare_to_golden(&spec(
|
||||
"xss_adversarial.py",
|
||||
"render_comment",
|
||||
Cap::HTML_ESCAPE,
|
||||
999,
|
||||
));
|
||||
}
|
||||
|
||||
// ── Secrets fixture ───────────────────────────────────────────────────────
|
||||
// ── Cross-cutting tests retained verbatim ────────────────────────────────
|
||||
|
||||
/// Telemetry must not contain literal secret strings from the fixture.
|
||||
/// Independent of the golden contract: it inspects the side-channel.
|
||||
#[test]
|
||||
fn secret_not_in_telemetry_after_verify() {
|
||||
if !python3_available() {
|
||||
|
|
@ -377,7 +241,9 @@ mod python_fixture_tests {
|
|||
return;
|
||||
}
|
||||
|
||||
let _guard = FIXTURE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let _guard = crate::common::fixture_harness::FIXTURE_LOCK
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let telemetry_path = tmp.path().join("events.jsonl");
|
||||
|
|
@ -391,15 +257,12 @@ mod python_fixture_tests {
|
|||
let tmp_fix = tmp.path().join("sqli_positive.py");
|
||||
let _ = std::fs::copy(&fixture, &tmp_fix);
|
||||
|
||||
// No set_current_dir: entry file is absolute, copy_entry_file resolves it directly.
|
||||
let diag = make_diag(&tmp_fix, "login", Cap::SQL_QUERY, 17);
|
||||
let opts = VerifyOptions::default();
|
||||
let _ = verify_finding(&diag, &opts);
|
||||
|
||||
// Check telemetry doesn't contain any secret patterns.
|
||||
if telemetry_path.exists() {
|
||||
let content = std::fs::read_to_string(&telemetry_path).unwrap_or_default();
|
||||
// Telemetry must not contain the fake AWS key.
|
||||
assert!(
|
||||
!content.contains("AKIAFAKETEST00000000"),
|
||||
"telemetry must not contain fake AWS key; got: {content}"
|
||||
|
|
@ -412,15 +275,11 @@ mod python_fixture_tests {
|
|||
}
|
||||
}
|
||||
|
||||
// ── Mount-filter gate ─────────────────────────────────────────────────────
|
||||
|
||||
/// If the entry file itself matches a sensitive-file pattern (e.g. `id_rsa*`),
|
||||
/// verify_finding must return Unsupported(RequiredFileRedactedForSecrets).
|
||||
/// No Python3 needed — the check fires before harness execution.
|
||||
/// Sensitive-filename gate fires before any harness execution; no
|
||||
/// python3 needed.
|
||||
#[test]
|
||||
fn sensitive_entry_file_is_unsupported() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
// "id_rsa.py" matches the id_rsa* sensitive pattern in mount_filter.
|
||||
let entry = tmp.path().join("id_rsa.py");
|
||||
std::fs::write(&entry, "def run(x): pass\n").unwrap();
|
||||
|
||||
|
|
@ -428,12 +287,7 @@ mod python_fixture_tests {
|
|||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&diag, &opts);
|
||||
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::Unsupported,
|
||||
"sensitive entry file must be Unsupported; got {:?}",
|
||||
result.status
|
||||
);
|
||||
assert_eq!(result.status, VerifyStatus::Unsupported);
|
||||
match &result.reason {
|
||||
Some(UnsupportedReason::RequiredFileRedactedForSecrets(_)) => {}
|
||||
other => panic!("expected RequiredFileRedactedForSecrets, got {other:?}"),
|
||||
|
|
|
|||
|
|
@ -1,397 +1,225 @@
|
|||
//! Rust fixture integration tests (Phase 04 acceptance gate).
|
||||
//!
|
||||
//! Runs the dynamic verification pipeline against each Rust fixture and
|
||||
//! asserts the expected verdict. Requires `--features dynamic` and a
|
||||
//! working `cargo` toolchain on PATH.
|
||||
//! Each fixture is run through the dynamic verification pipeline; its
|
||||
//! verdict is then compared against the per-fixture golden under
|
||||
//! `tests/dynamic_fixtures/rust/{name}.golden.json`. Refresh the goldens
|
||||
//! via `NYX_UPDATE_GOLDENS=1 ./scripts/update_dynamic_goldens.sh`.
|
||||
//!
|
||||
//! Fixture entry points follow the convention:
|
||||
//! `pub fn run(payload: &str)` in `tests/dynamic_fixtures/rust/{name}.rs`
|
||||
//!
|
||||
//! The harness emitter wraps each fixture in a generated `src/main.rs` that
|
||||
//! reads `NYX_PAYLOAD` from the environment and calls `entry::run(&payload)`.
|
||||
//!
|
||||
//! Build note: the first run per capability compiles a Cargo project; subsequent
|
||||
//! runs with differing entry files hit the build cache only when Cargo.toml and
|
||||
//! src/entry.rs are identical (the cache key includes the entry file hash).
|
||||
//! Expect 2-4 compilations per full test run (one per unique dependency set).
|
||||
//!
|
||||
//! Run with: `cargo nextest run --features dynamic --test rust_fixtures`
|
||||
//! Run with: `cargo nextest run --features dynamic --test rust_fixtures`.
|
||||
|
||||
mod common;
|
||||
|
||||
#[cfg(feature = "dynamic")]
|
||||
mod rust_fixture_tests {
|
||||
use crate::common::fixture_harness::{
|
||||
run_fixture_and_compare_to_golden, CopyStrategy, FixtureSpec,
|
||||
};
|
||||
use nyx_scanner::commands::scan::Diag;
|
||||
use nyx_scanner::dynamic::verify::{verify_finding, VerifyOptions};
|
||||
use nyx_scanner::evidence::{
|
||||
Confidence, Evidence, FlowStep, FlowStepKind, InconclusiveReason, UnsupportedReason,
|
||||
VerifyStatus,
|
||||
Confidence, Evidence, FlowStep, FlowStepKind,
|
||||
};
|
||||
use nyx_scanner::labels::Cap;
|
||||
use nyx_scanner::patterns::{FindingCategory, Severity};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
use tempfile::TempDir;
|
||||
|
||||
// Serialize all fixture tests: prevents races on process-global env vars
|
||||
// (NYX_REPRO_BASE, NYX_TELEMETRY_PATH) and the shared build cache dir.
|
||||
static FIXTURE_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
fn fixture_path(name: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/dynamic_fixtures/rust")
|
||||
.join(name)
|
||||
fn spec(fixture: &'static str, func: &'static str, cap: Cap, sink_line: u32) -> FixtureSpec<'static> {
|
||||
FixtureSpec {
|
||||
lang_dir: "rust",
|
||||
fixture,
|
||||
func,
|
||||
cap,
|
||||
sink_line,
|
||||
confidence: Confidence::High,
|
||||
copy: CopyStrategy::RustEntry,
|
||||
}
|
||||
}
|
||||
|
||||
/// Run a Rust fixture through the full dynamic verification pipeline.
|
||||
///
|
||||
/// The fixture file is copied to a temp dir as `src/entry.rs`.
|
||||
/// `NYX_REPRO_BASE` and `NYX_TELEMETRY_PATH` are redirected to temp dirs.
|
||||
fn run_fixture(
|
||||
fixture: &str,
|
||||
func: &str,
|
||||
fn low_spec(
|
||||
fixture: &'static str,
|
||||
func: &'static str,
|
||||
cap: Cap,
|
||||
sink_line: u32,
|
||||
) -> nyx_scanner::evidence::VerifyResult {
|
||||
let _guard = FIXTURE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
|
||||
let path = fixture_path(fixture);
|
||||
|
||||
let tmp = TempDir::new().unwrap();
|
||||
// Rust fixtures live at src/entry.rs inside the harness workdir;
|
||||
// the Diag's entry_file points to the fixture source on disk.
|
||||
let dst_dir = tmp.path().join("src");
|
||||
std::fs::create_dir_all(&dst_dir).unwrap();
|
||||
let dst = dst_dir.join("entry.rs");
|
||||
std::fs::copy(&path, &dst).expect("fixture file must exist");
|
||||
|
||||
unsafe {
|
||||
std::env::set_var("NYX_REPRO_BASE", tmp.path().join("repro").to_str().unwrap());
|
||||
std::env::set_var(
|
||||
"NYX_TELEMETRY_PATH",
|
||||
tmp.path().join("events.jsonl").to_str().unwrap(),
|
||||
);
|
||||
) -> FixtureSpec<'static> {
|
||||
FixtureSpec {
|
||||
lang_dir: "rust",
|
||||
fixture,
|
||||
func,
|
||||
cap,
|
||||
sink_line,
|
||||
confidence: Confidence::Low,
|
||||
copy: CopyStrategy::RustEntry,
|
||||
}
|
||||
|
||||
// Point the Diag at the original fixture path (absolute), not the copy.
|
||||
// The harness emitter reads the file at entry_file to extract source.
|
||||
let diag = make_diag(&path, func, cap, sink_line);
|
||||
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&diag, &opts);
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("NYX_REPRO_BASE");
|
||||
std::env::remove_var("NYX_TELEMETRY_PATH");
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
// ── SQLi fixtures ────────────────────────────────────────────────────────
|
||||
// ── SQLi ─────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn sqli_positive_is_confirmed() {
|
||||
let result = run_fixture("sqli_positive.rs", "run", Cap::SQL_QUERY, 18);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::Confirmed,
|
||||
"sqli_positive must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status,
|
||||
result.detail
|
||||
);
|
||||
assert!(
|
||||
result.triggered_payload.is_some(),
|
||||
"Confirmed result must have triggered_payload"
|
||||
);
|
||||
fn sqli_positive_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("sqli_positive.rs", "run", Cap::SQL_QUERY, 18));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sqli_negative_is_not_confirmed() {
|
||||
let result = run_fixture("sqli_negative.rs", "run", Cap::SQL_QUERY, 22);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::NotConfirmed,
|
||||
"sqli_negative must be NotConfirmed; got {:?} (detail: {:?})",
|
||||
result.status,
|
||||
result.detail
|
||||
);
|
||||
fn sqli_negative_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("sqli_negative.rs", "run", Cap::SQL_QUERY, 22));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sqli_unsupported_is_unsupported() {
|
||||
let path = fixture_path("sqli_unsupported.rs");
|
||||
let mut d = make_diag(&path, "find_user", Cap::SQL_QUERY, 10);
|
||||
d.confidence = Some(Confidence::Low);
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&d, &opts);
|
||||
assert_eq!(result.status, VerifyStatus::Unsupported);
|
||||
assert_eq!(result.reason, Some(UnsupportedReason::ConfidenceTooLow));
|
||||
fn sqli_unsupported_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&low_spec(
|
||||
"sqli_unsupported.rs",
|
||||
"find_user",
|
||||
Cap::SQL_QUERY,
|
||||
10,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sqli_adversarial_is_inconclusive_collision() {
|
||||
// Adversarial prints oracle marker without __NYX_SINK_HIT__:
|
||||
// oracle_fired = true, sink_hit = false → OracleCollisionSuspected.
|
||||
let result = run_fixture("sqli_adversarial.rs", "run", Cap::SQL_QUERY, 999);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::Inconclusive,
|
||||
"sqli_adversarial must be Inconclusive; got {:?}",
|
||||
result.status
|
||||
);
|
||||
assert_eq!(
|
||||
result.inconclusive_reason,
|
||||
Some(InconclusiveReason::OracleCollisionSuspected),
|
||||
"adversarial must be OracleCollisionSuspected"
|
||||
);
|
||||
fn sqli_adversarial_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("sqli_adversarial.rs", "run", Cap::SQL_QUERY, 999));
|
||||
}
|
||||
|
||||
// ── Command injection fixtures ───────────────────────────────────────────
|
||||
// ── Command injection ────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn cmdi_positive_is_confirmed() {
|
||||
let result = run_fixture("cmdi_positive.rs", "run", Cap::CODE_EXEC, 17);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::Confirmed,
|
||||
"cmdi_positive must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status,
|
||||
result.detail
|
||||
);
|
||||
fn cmdi_positive_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("cmdi_positive.rs", "run", Cap::CODE_EXEC, 17));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cmdi_negative_is_not_confirmed() {
|
||||
let result = run_fixture("cmdi_negative.rs", "run", Cap::CODE_EXEC, 17);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::NotConfirmed,
|
||||
"cmdi_negative must be NotConfirmed; got {:?}",
|
||||
result.status
|
||||
);
|
||||
fn cmdi_negative_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("cmdi_negative.rs", "run", Cap::CODE_EXEC, 17));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cmdi_unsupported_is_unsupported() {
|
||||
let path = fixture_path("cmdi_unsupported.rs");
|
||||
let mut d = make_diag(&path, "execute", Cap::CODE_EXEC, 9);
|
||||
d.confidence = Some(Confidence::Low);
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&d, &opts);
|
||||
assert_eq!(result.status, VerifyStatus::Unsupported);
|
||||
assert_eq!(result.reason, Some(UnsupportedReason::ConfidenceTooLow));
|
||||
fn cmdi_unsupported_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&low_spec(
|
||||
"cmdi_unsupported.rs",
|
||||
"execute",
|
||||
Cap::CODE_EXEC,
|
||||
9,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cmdi_adversarial_is_inconclusive_collision() {
|
||||
let result = run_fixture("cmdi_adversarial.rs", "run", Cap::CODE_EXEC, 999);
|
||||
assert_eq!(result.status, VerifyStatus::Inconclusive);
|
||||
assert_eq!(
|
||||
result.inconclusive_reason,
|
||||
Some(InconclusiveReason::OracleCollisionSuspected)
|
||||
);
|
||||
fn cmdi_adversarial_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("cmdi_adversarial.rs", "run", Cap::CODE_EXEC, 999));
|
||||
}
|
||||
|
||||
// ── File I/O fixtures ────────────────────────────────────────────────────
|
||||
// ── File I/O ─────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn fileio_positive_is_confirmed() {
|
||||
let result = run_fixture("fileio_positive.rs", "run", Cap::FILE_IO, 7);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::Confirmed,
|
||||
"fileio_positive must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status,
|
||||
result.detail
|
||||
);
|
||||
fn fileio_positive_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("fileio_positive.rs", "run", Cap::FILE_IO, 7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fileio_negative_is_not_confirmed() {
|
||||
let result = run_fixture("fileio_negative.rs", "run", Cap::FILE_IO, 17);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::NotConfirmed,
|
||||
"fileio_negative must be NotConfirmed; got {:?}",
|
||||
result.status
|
||||
);
|
||||
fn fileio_negative_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("fileio_negative.rs", "run", Cap::FILE_IO, 17));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fileio_unsupported_is_unsupported() {
|
||||
let path = fixture_path("fileio_unsupported.rs");
|
||||
let mut d = make_diag(&path, "read", Cap::FILE_IO, 8);
|
||||
d.confidence = Some(Confidence::Low);
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&d, &opts);
|
||||
assert_eq!(result.status, VerifyStatus::Unsupported);
|
||||
assert_eq!(result.reason, Some(UnsupportedReason::ConfidenceTooLow));
|
||||
fn fileio_unsupported_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&low_spec(
|
||||
"fileio_unsupported.rs",
|
||||
"read",
|
||||
Cap::FILE_IO,
|
||||
8,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fileio_adversarial_is_inconclusive_collision() {
|
||||
let result = run_fixture("fileio_adversarial.rs", "run", Cap::FILE_IO, 999);
|
||||
assert_eq!(result.status, VerifyStatus::Inconclusive);
|
||||
assert_eq!(
|
||||
result.inconclusive_reason,
|
||||
Some(InconclusiveReason::OracleCollisionSuspected)
|
||||
);
|
||||
fn fileio_adversarial_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("fileio_adversarial.rs", "run", Cap::FILE_IO, 999));
|
||||
}
|
||||
|
||||
// ── SSRF fixtures ────────────────────────────────────────────────────────
|
||||
// ── SSRF ─────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn ssrf_positive_is_confirmed() {
|
||||
let result = run_fixture("ssrf_positive.rs", "run", Cap::SSRF, 7);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::Confirmed,
|
||||
"ssrf_positive must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status,
|
||||
result.detail
|
||||
);
|
||||
fn ssrf_positive_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("ssrf_positive.rs", "run", Cap::SSRF, 7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_negative_is_not_confirmed() {
|
||||
let result = run_fixture("ssrf_negative.rs", "run", Cap::SSRF, 13);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::NotConfirmed,
|
||||
"ssrf_negative must be NotConfirmed; got {:?}",
|
||||
result.status
|
||||
);
|
||||
fn ssrf_negative_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("ssrf_negative.rs", "run", Cap::SSRF, 13));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_unsupported_is_unsupported() {
|
||||
let path = fixture_path("ssrf_unsupported.rs");
|
||||
let mut d = make_diag(&path, "get", Cap::SSRF, 8);
|
||||
d.confidence = Some(Confidence::Low);
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&d, &opts);
|
||||
assert_eq!(result.status, VerifyStatus::Unsupported);
|
||||
assert_eq!(result.reason, Some(UnsupportedReason::ConfidenceTooLow));
|
||||
fn ssrf_unsupported_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&low_spec("ssrf_unsupported.rs", "get", Cap::SSRF, 8));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_adversarial_is_inconclusive_collision() {
|
||||
let result = run_fixture("ssrf_adversarial.rs", "run", Cap::SSRF, 999);
|
||||
assert_eq!(result.status, VerifyStatus::Inconclusive);
|
||||
assert_eq!(
|
||||
result.inconclusive_reason,
|
||||
Some(InconclusiveReason::OracleCollisionSuspected)
|
||||
);
|
||||
fn ssrf_adversarial_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("ssrf_adversarial.rs", "run", Cap::SSRF, 999));
|
||||
}
|
||||
|
||||
// ── XSS fixtures ─────────────────────────────────────────────────────────
|
||||
// ── XSS ──────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn xss_positive_is_confirmed() {
|
||||
let result = run_fixture("xss_positive.rs", "run", Cap::HTML_ESCAPE, 11);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::Confirmed,
|
||||
"xss_positive must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status,
|
||||
result.detail
|
||||
);
|
||||
assert!(
|
||||
result.triggered_payload.is_some(),
|
||||
"Confirmed result must have triggered_payload"
|
||||
);
|
||||
fn xss_positive_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("xss_positive.rs", "run", Cap::HTML_ESCAPE, 11));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xss_negative_is_not_confirmed() {
|
||||
let result = run_fixture("xss_negative.rs", "run", Cap::HTML_ESCAPE, 15);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::NotConfirmed,
|
||||
"xss_negative must be NotConfirmed; got {:?} (detail: {:?})",
|
||||
result.status,
|
||||
result.detail
|
||||
);
|
||||
fn xss_negative_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("xss_negative.rs", "run", Cap::HTML_ESCAPE, 15));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xss_unsupported_is_unsupported() {
|
||||
let path = fixture_path("xss_unsupported.rs");
|
||||
let mut d = make_diag(&path, "render", Cap::HTML_ESCAPE, 14);
|
||||
d.confidence = Some(Confidence::Low);
|
||||
let opts = VerifyOptions::default();
|
||||
let result = verify_finding(&d, &opts);
|
||||
assert_eq!(result.status, VerifyStatus::Unsupported);
|
||||
assert_eq!(result.reason, Some(UnsupportedReason::ConfidenceTooLow));
|
||||
fn xss_unsupported_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&low_spec(
|
||||
"xss_unsupported.rs",
|
||||
"render",
|
||||
Cap::HTML_ESCAPE,
|
||||
14,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xss_adversarial_is_inconclusive_collision() {
|
||||
let result = run_fixture("xss_adversarial.rs", "run", Cap::HTML_ESCAPE, 999);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::Inconclusive,
|
||||
"xss_adversarial must be Inconclusive; got {:?}",
|
||||
result.status
|
||||
);
|
||||
assert_eq!(
|
||||
result.inconclusive_reason,
|
||||
Some(InconclusiveReason::OracleCollisionSuspected),
|
||||
"adversarial must be OracleCollisionSuspected"
|
||||
);
|
||||
fn xss_adversarial_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec(
|
||||
"xss_adversarial.rs",
|
||||
"run",
|
||||
Cap::HTML_ESCAPE,
|
||||
999,
|
||||
));
|
||||
}
|
||||
|
||||
// ── Variant fixtures (smoke-test second positive paths) ──────────────────
|
||||
// ── Smoke-test second positive paths ─────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn cmdi_positive2_is_confirmed() {
|
||||
let result = run_fixture("cmdi_positive2.rs", "run", Cap::CODE_EXEC, 17);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::Confirmed,
|
||||
"cmdi_positive2 must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status,
|
||||
result.detail
|
||||
);
|
||||
fn cmdi_positive2_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("cmdi_positive2.rs", "run", Cap::CODE_EXEC, 17));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fileio_positive2_is_confirmed() {
|
||||
let result = run_fixture("fileio_positive2.rs", "run", Cap::FILE_IO, 11);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::Confirmed,
|
||||
"fileio_positive2 must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status,
|
||||
result.detail
|
||||
);
|
||||
fn fileio_positive2_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("fileio_positive2.rs", "run", Cap::FILE_IO, 11));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_positive2_is_confirmed() {
|
||||
let result = run_fixture("ssrf_positive2.rs", "run", Cap::SSRF, 7);
|
||||
assert_eq!(
|
||||
result.status,
|
||||
VerifyStatus::Confirmed,
|
||||
"ssrf_positive2 must be Confirmed; got {:?} (detail: {:?})",
|
||||
result.status,
|
||||
result.detail
|
||||
);
|
||||
fn ssrf_positive2_matches_golden() {
|
||||
run_fixture_and_compare_to_golden(&spec("ssrf_positive2.rs", "run", Cap::SSRF, 7));
|
||||
}
|
||||
|
||||
// ── Harness architecture: non-Python-specific gate ───────────────────────
|
||||
// ── Pipeline non-panic gate ──────────────────────────────────────────────
|
||||
|
||||
/// Rust fixture must produce a VerifyResult (not panic or ICE).
|
||||
/// This is the Phase 04 acceptance gate: the dynamic pipeline handles
|
||||
/// a compiled-language finding without Python-specific assumptions.
|
||||
/// Confirms the Rust pipeline produces a VerifyResult (not a panic/ICE).
|
||||
/// Independent of the golden contract: this is a structural assertion.
|
||||
#[test]
|
||||
fn rust_pipeline_does_not_panic() {
|
||||
let result = run_fixture("sqli_positive.rs", "run", Cap::SQL_QUERY, 18);
|
||||
// Any verdict is acceptable; the test asserts non-panic only.
|
||||
let _ = result;
|
||||
let _guard = crate::common::fixture_harness::FIXTURE_LOCK
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/dynamic_fixtures/rust/sqli_positive.rs");
|
||||
let diag = make_diag(&path, "run", Cap::SQL_QUERY, 18);
|
||||
let opts = VerifyOptions::default();
|
||||
let _ = verify_finding(&diag, &opts);
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
fn make_diag(path: &Path, func: &str, cap: Cap, sink_line: u32) -> Diag {
|
||||
let path_str = path.to_string_lossy().into_owned();
|
||||
let evidence = Evidence {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue