nyx/tests/repro_determinism.rs

//! Repro determinism test (§18.2).
//!
//! For every `Confirmed` fixture: the repro artifact `expected/outcome.json`
//! produced during verification must be byte-identical when regenerated from
//! the repro bundle.
//!
//! Tests are gated on `#[cfg(feature = "dynamic")]` and Python availability.
//! They are also skipped if no `Confirmed` fixtures have been produced yet
//! (trivially passes — zero assertions).

#[cfg(feature = "dynamic")]
mod repro_determinism_tests {
    use nyx_scanner::dynamic::repro;
    use nyx_scanner::dynamic::sandbox::{SandboxOptions, SandboxOutcome};
    use nyx_scanner::dynamic::spec::{EntryKind, HarnessSpec, PayloadSlot};
    use nyx_scanner::evidence::{AttemptSummary, VerifyResult, VerifyStatus};
    use nyx_scanner::labels::Cap;
    use nyx_scanner::symbol::Lang;
    use std::time::Duration;
    use tempfile::TempDir;

    fn make_confirmed_spec(spec_hash: &str) -> HarnessSpec {
        HarnessSpec {
            finding_id: "determinism00001".into(),
            entry_file: "app.py".into(),
            entry_name: "login".into(),
            entry_kind: EntryKind::Function,
            lang: Lang::Python,
            toolchain_id: "python-3".into(),
            payload_slot: PayloadSlot::Param(0),
            expected_cap: Cap::SQL_QUERY,
            constraint_hints: vec![],
            sink_file: "app.py".into(),
            sink_line: 10,
            spec_hash: spec_hash.to_owned(),
        }
    }

    fn make_confirmed_outcome() -> SandboxOutcome {
        SandboxOutcome {
            exit_code: Some(0),
            stdout: b"NYX_SQL_CONFIRMED\nsome extra output".to_vec(),
            stderr: vec![],
            timed_out: false,
            oob_callback_seen: false,
            sink_hit: true,
            duration: Duration::from_millis(150),
        }
    }

    fn make_confirmed_verdict(finding_id: &str) -> VerifyResult {
        VerifyResult {
            finding_id: finding_id.to_owned(),
            status: VerifyStatus::Confirmed,
            triggered_payload: Some("sqli-union-nyx".into()),
            reason: None,
            inconclusive_reason: None,
            detail: None,
            attempts: vec![AttemptSummary {
                payload_label: "sqli-union-nyx".into(),
                exit_code: Some(0),
                timed_out: false,
                triggered: true,
                sink_hit: true,
            }],
            toolchain_match: Some("exact".into()),
        }
    }

    /// Write a repro bundle and verify it round-trips correctly.
    #[test]
    fn confirmed_repro_is_deterministic() {
        let dir = TempDir::new().unwrap();
        // Override repro base to temp dir.
        unsafe { std::env::set_var("NYX_REPRO_BASE", dir.path().to_str().unwrap()) };

        let spec = make_confirmed_spec("determ0000000001");
        let opts = SandboxOptions::default();
        let outcome = make_confirmed_outcome();
        let verdict = make_confirmed_verdict("determinism00001");

        // Write repro bundle (first time).
        let artifact1 = repro::write(
            &spec, &opts, &outcome, &verdict,
            "# harness source v1\n",
            "def login(x): pass\n",
            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
            "sqli-union-nyx",
            None,
        ).expect("first repro write must succeed");

        let outcome_json_1 =
            std::fs::read_to_string(artifact1.root.join("expected/outcome.json"))
            .expect("outcome.json must exist after first write");

        // Write repro bundle (second time, same inputs).
        // Remove existing dir first (simulate fresh run).
        std::fs::remove_dir_all(&artifact1.root).unwrap();

        let artifact2 = repro::write(
            &spec, &opts, &outcome, &verdict,
            "# harness source v1\n",
            "def login(x): pass\n",
            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
            "sqli-union-nyx",
            None,
        ).expect("second repro write must succeed");

        let outcome_json_2 =
            std::fs::read_to_string(artifact2.root.join("expected/outcome.json"))
            .expect("outcome.json must exist after second write");

        assert_eq!(
            outcome_json_1, outcome_json_2,
            "outcome.json must be byte-identical across two runs with the same inputs"
        );

        unsafe { std::env::remove_var("NYX_REPRO_BASE") };
    }

    /// Verify that redacted outcome.json does not contain the secret.
    #[test]
    fn outcome_json_secrets_are_redacted() {
        let dir = TempDir::new().unwrap();
        unsafe { std::env::set_var("NYX_REPRO_BASE", dir.path().to_str().unwrap()) };

        let spec = make_confirmed_spec("determ0000000002");
        let opts = SandboxOptions::default();
        let mut outcome = make_confirmed_outcome();
        // Inject a fake AWS key into stdout.
        outcome.stdout = b"AKIAFAKETEST00000000 result ok NYX_SQL_CONFIRMED".to_vec();
        let verdict = make_confirmed_verdict("determinism00002");

        let artifact = repro::write(
            &spec, &opts, &outcome, &verdict,
            "# harness", "# entry", b"payload", "label", None,
        ).expect("repro write must succeed");

        let outcome_json =
            std::fs::read_to_string(artifact.root.join("expected/outcome.json")).unwrap();

        assert!(
            !outcome_json.contains("AKIAFAKETEST00000000"),
            "AWS key must be redacted from outcome.json; got: {outcome_json}"
        );

        unsafe { std::env::remove_var("NYX_REPRO_BASE") };
    }

    /// Verify verdict.json is correctly structured.
    #[test]
    fn verdict_json_is_valid() {
        let dir = TempDir::new().unwrap();
        unsafe { std::env::set_var("NYX_REPRO_BASE", dir.path().to_str().unwrap()) };

        let spec = make_confirmed_spec("determ0000000003");
        let opts = SandboxOptions::default();
        let outcome = make_confirmed_outcome();
        let verdict = make_confirmed_verdict("determinism00003");

        let artifact = repro::write(
            &spec, &opts, &outcome, &verdict,
            "# harness", "# entry", b"payload", "label", None,
        ).expect("repro write must succeed");

        let verdict_json =
            std::fs::read_to_string(artifact.root.join("expected/verdict.json")).unwrap();
        let parsed: serde_json::Value = serde_json::from_str(&verdict_json).unwrap();

        assert_eq!(parsed["status"], "Confirmed");
        assert_eq!(parsed["finding_id"], "determinism00003");

        unsafe { std::env::remove_var("NYX_REPRO_BASE") };
    }
}