Dynamic (#77)

tests/repro_determinism.rs

@@ -0,0 +1,636 @@
+//! Repro determinism test (§18.2).
+//!
+//! For every `Confirmed` fixture: the repro artifact `expected/outcome.json`
+//! produced during verification must be byte-identical when regenerated from
+//! the repro bundle.
+//!
+//! Tests are gated on `#[cfg(feature = "dynamic")]` and Python availability.
+//! They are also skipped if no `Confirmed` fixtures have been produced yet
+//! (trivially passes — zero assertions).
+
+#[cfg(feature = "dynamic")]
+mod repro_determinism_tests {
+    use nyx_scanner::dynamic::repro;
+    use nyx_scanner::dynamic::sandbox::{SandboxOptions, SandboxOutcome};
+    use nyx_scanner::dynamic::spec::{EntryKind, HarnessSpec, PayloadSlot};
+    use nyx_scanner::evidence::{AttemptSummary, VerifyResult, VerifyStatus};
+    use nyx_scanner::labels::Cap;
+    use nyx_scanner::symbol::Lang;
+    use std::path::Path;
+    use std::sync::{Mutex, MutexGuard};
+    use std::time::Duration;
+    use tempfile::TempDir;
+
+    static REPRO_ENV_LOCK: Mutex<()> = Mutex::new(());
+
+    struct ReproEnvGuard {
+        _lock: MutexGuard<'static, ()>,
+        prior: Option<String>,
+    }
+
+    impl ReproEnvGuard {
+        fn set(base: &Path) -> Self {
+            let lock = REPRO_ENV_LOCK
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner());
+            let prior = std::env::var("NYX_REPRO_BASE").ok();
+            unsafe { std::env::set_var("NYX_REPRO_BASE", base) };
+            Self { _lock: lock, prior }
+        }
+    }
+
+    impl Drop for ReproEnvGuard {
+        fn drop(&mut self) {
+            match self.prior.take() {
+                Some(value) => unsafe { std::env::set_var("NYX_REPRO_BASE", value) },
+                None => unsafe { std::env::remove_var("NYX_REPRO_BASE") },
+            }
+        }
+    }
+
+    fn make_confirmed_spec(spec_hash: &str) -> HarnessSpec {
+        HarnessSpec {
+            finding_id: "determinism00001".into(),
+            entry_file: "app.py".into(),
+            entry_name: "login".into(),
+            entry_kind: EntryKind::Function,
+            lang: Lang::Python,
+            toolchain_id: "python-3".into(),
+            payload_slot: PayloadSlot::Param(0),
+            expected_cap: Cap::SQL_QUERY,
+            constraint_hints: vec![],
+            sink_file: "app.py".into(),
+            sink_line: 10,
+            spec_hash: spec_hash.to_owned(),
+            derivation: nyx_scanner::dynamic::spec::SpecDerivationStrategy::FromFlowSteps,
+            stubs_required: vec![],
+            framework: None,
+            java_toolchain: nyx_scanner::dynamic::spec::JavaToolchain::default(),
+        }
+    }
+
+    fn make_confirmed_outcome() -> SandboxOutcome {
+        SandboxOutcome {
+            exit_code: Some(0),
+            stdout: b"NYX_SQL_CONFIRMED\nsome extra output".to_vec(),
+            stderr: vec![],
+            timed_out: false,
+            oob_callback_seen: false,
+            sink_hit: true,
+            duration: Duration::from_millis(150),
+            hardening_outcome: None,
+        }
+    }
+
+    fn make_confirmed_verdict(finding_id: &str) -> VerifyResult {
+        VerifyResult {
+            finding_id: finding_id.to_owned(),
+            status: VerifyStatus::Confirmed,
+            triggered_payload: Some("sqli-union-nyx".into()),
+            reason: None,
+            inconclusive_reason: None,
+            detail: None,
+            attempts: vec![AttemptSummary {
+                payload_label: "sqli-union-nyx".into(),
+                exit_code: Some(0),
+                timed_out: false,
+                triggered: true,
+                sink_hit: true,
+            }],
+            toolchain_match: Some("exact".into()),
+            differential: None,
+            replay_stable: None,
+            wrong: None,
+            hardening_outcome: None,
+        }
+    }
+
+    /// Write a repro bundle and verify it round-trips correctly.
+    #[test]
+    fn confirmed_repro_is_deterministic() {
+        let dir = TempDir::new().unwrap();
+        let _env = ReproEnvGuard::set(dir.path());
+
+        let spec = make_confirmed_spec("determ0000000001");
+        let opts = SandboxOptions::default();
+        let outcome = make_confirmed_outcome();
+        let verdict = make_confirmed_verdict("determinism00001");
+
+        // Write repro bundle (first time).
+        let artifact1 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "# harness source v1\n",
+            "def login(x): pass\n",
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("first repro write must succeed");
+
+        let outcome_json_1 = std::fs::read_to_string(artifact1.root.join("expected/outcome.json"))
+            .expect("outcome.json must exist after first write");
+
+        // Write repro bundle (second time, same inputs).
+        // Remove existing dir first (simulate fresh run).
+        std::fs::remove_dir_all(&artifact1.root).unwrap();
+
+        let artifact2 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "# harness source v1\n",
+            "def login(x): pass\n",
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("second repro write must succeed");
+
+        let outcome_json_2 = std::fs::read_to_string(artifact2.root.join("expected/outcome.json"))
+            .expect("outcome.json must exist after second write");
+
+        assert_eq!(
+            outcome_json_1, outcome_json_2,
+            "outcome.json must be byte-identical across two runs with the same inputs"
+        );
+    }
+
+    /// Verify that redacted outcome.json does not contain the secret.
+    #[test]
+    fn outcome_json_secrets_are_redacted() {
+        let dir = TempDir::new().unwrap();
+        let _env = ReproEnvGuard::set(dir.path());
+
+        let spec = make_confirmed_spec("determ0000000002");
+        let opts = SandboxOptions::default();
+        let mut outcome = make_confirmed_outcome();
+        // Inject a fake AWS key into stdout.
+        outcome.stdout = b"AKIAFAKETEST00000000 result ok NYX_SQL_CONFIRMED".to_vec();
+        let verdict = make_confirmed_verdict("determinism00002");
+
+        let artifact = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "# harness",
+            "# entry",
+            b"payload",
+            "label",
+            None,
+        )
+        .expect("repro write must succeed");
+
+        let outcome_json =
+            std::fs::read_to_string(artifact.root.join("expected/outcome.json")).unwrap();
+
+        assert!(
+            !outcome_json.contains("AKIAFAKETEST00000000"),
+            "AWS key must be redacted from outcome.json; got: {outcome_json}"
+        );
+    }
+
+    // ── Rust repro tests ─────────────────────────────────────────────────────
+
+    fn make_confirmed_rust_spec(spec_hash: &str) -> HarnessSpec {
+        HarnessSpec {
+            finding_id: "rust_determ00001".into(),
+            entry_file: "src/entry.rs".into(),
+            entry_name: "run".into(),
+            entry_kind: EntryKind::Function,
+            lang: Lang::Rust,
+            toolchain_id: "rust-stable".into(),
+            payload_slot: PayloadSlot::Param(0),
+            expected_cap: Cap::SQL_QUERY,
+            constraint_hints: vec![],
+            sink_file: "src/entry.rs".into(),
+            sink_line: 18,
+            spec_hash: spec_hash.to_owned(),
+            derivation: nyx_scanner::dynamic::spec::SpecDerivationStrategy::FromFlowSteps,
+            stubs_required: vec![],
+            framework: None,
+            java_toolchain: nyx_scanner::dynamic::spec::JavaToolchain::default(),
+        }
+    }
+
+    fn make_confirmed_rust_harness_source() -> String {
+        r#"mod entry;
+fn main() {
+    let payload = std::env::var("NYX_PAYLOAD").unwrap_or_default();
+    entry::run(&payload);
+}
+"#
+        .into()
+    }
+
+    /// Rust repro bundle has the correct layout.
+    ///
+    /// For Rust, harness is at `harness/src/main.rs` and `harness/Cargo.toml`
+    /// is also written (unlike Python which uses `harness/harness.py`).
+    #[test]
+    fn rust_repro_layout_is_correct() {
+        let dir = TempDir::new().unwrap();
+        let _env = ReproEnvGuard::set(dir.path());
+
+        let spec = make_confirmed_rust_spec("rust_determ00001");
+        let opts = SandboxOptions::default();
+        let outcome = make_confirmed_outcome();
+        let verdict = make_confirmed_verdict("rust_determ00001");
+        let harness_src = make_confirmed_rust_harness_source();
+
+        let artifact = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            &harness_src,
+            "pub fn run(payload: &str) { println!(\"{}\", payload); }\n",
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("Rust repro write must succeed");
+
+        // Rust-specific layout: harness lives under harness/src/main.rs.
+        assert!(
+            artifact.root.join("harness/src/main.rs").exists(),
+            "Rust harness must be at harness/src/main.rs"
+        );
+        assert!(
+            artifact.root.join("harness/Cargo.toml").exists(),
+            "Rust harness must include harness/Cargo.toml"
+        );
+        // Common layout.
+        assert!(artifact.root.join("manifest.json").exists());
+        assert!(artifact.root.join("entry/extracted_source.rs").exists());
+        assert!(artifact.root.join("payload/payload.bin").exists());
+        assert!(artifact.root.join("expected/outcome.json").exists());
+        assert!(artifact.root.join("expected/verdict.json").exists());
+        assert!(artifact.root.join("reproduce.sh").exists());
+    }
+
+    /// Rust repro outcome.json is byte-identical across two writes.
+    #[test]
+    fn rust_repro_outcome_is_deterministic() {
+        let dir = TempDir::new().unwrap();
+        let _env = ReproEnvGuard::set(dir.path());
+
+        let spec = make_confirmed_rust_spec("rust_determ00002");
+        let opts = SandboxOptions::default();
+        let outcome = make_confirmed_outcome();
+        let verdict = make_confirmed_verdict("rust_determ00002");
+        let harness_src = make_confirmed_rust_harness_source();
+        let entry_src = "pub fn run(payload: &str) { println!(\"{}\", payload); }\n";
+
+        let artifact1 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            &harness_src,
+            entry_src,
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("first Rust repro write");
+        let json1 = std::fs::read_to_string(artifact1.root.join("expected/outcome.json")).unwrap();
+
+        std::fs::remove_dir_all(&artifact1.root).unwrap();
+
+        let artifact2 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            &harness_src,
+            entry_src,
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("second Rust repro write");
+        let json2 = std::fs::read_to_string(artifact2.root.join("expected/outcome.json")).unwrap();
+
+        assert_eq!(
+            json1, json2,
+            "Rust outcome.json must be byte-identical across two writes"
+        );
+    }
+
+    // ── JS repro tests ───────────────────────────────────────────────────────
+
+    fn make_confirmed_js_spec(spec_hash: &str) -> HarnessSpec {
+        HarnessSpec {
+            finding_id: "js_determ000001".into(),
+            entry_file: "tests/dynamic_fixtures/js/sqli_positive.js".into(),
+            entry_name: "login".into(),
+            entry_kind: EntryKind::Function,
+            lang: Lang::JavaScript,
+            toolchain_id: "node-20".into(),
+            payload_slot: PayloadSlot::Param(0),
+            expected_cap: Cap::SQL_QUERY,
+            constraint_hints: vec![],
+            sink_file: "tests/dynamic_fixtures/js/sqli_positive.js".into(),
+            sink_line: 8,
+            spec_hash: spec_hash.to_owned(),
+            derivation: nyx_scanner::dynamic::spec::SpecDerivationStrategy::FromFlowSteps,
+            stubs_required: vec![],
+            framework: None,
+            java_toolchain: nyx_scanner::dynamic::spec::JavaToolchain::default(),
+        }
+    }
+
+    #[test]
+    fn js_repro_outcome_is_deterministic() {
+        let dir = TempDir::new().unwrap();
+        let _env = ReproEnvGuard::set(dir.path());
+
+        let spec = make_confirmed_js_spec("js_determ000001a");
+        let opts = SandboxOptions::default();
+        let outcome = make_confirmed_outcome();
+        let verdict = make_confirmed_verdict("js_determ000001");
+        let entry_src = "function login(username) { console.log(username); }\n";
+
+        let artifact1 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "// harness js\n",
+            entry_src,
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("first JS repro write");
+        let json1 = std::fs::read_to_string(artifact1.root.join("expected/outcome.json")).unwrap();
+
+        std::fs::remove_dir_all(&artifact1.root).unwrap();
+
+        let artifact2 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "// harness js\n",
+            entry_src,
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("second JS repro write");
+        let json2 = std::fs::read_to_string(artifact2.root.join("expected/outcome.json")).unwrap();
+
+        assert_eq!(
+            json1, json2,
+            "JS outcome.json must be byte-identical across two writes"
+        );
+    }
+
+    // ── Go repro tests ───────────────────────────────────────────────────────
+
+    fn make_confirmed_go_spec(spec_hash: &str) -> HarnessSpec {
+        HarnessSpec {
+            finding_id: "go_determ000001".into(),
+            entry_file: "tests/dynamic_fixtures/go/sqli_positive.go".into(),
+            entry_name: "Login".into(),
+            entry_kind: EntryKind::Function,
+            lang: Lang::Go,
+            toolchain_id: "go-1.21".into(),
+            payload_slot: PayloadSlot::Param(0),
+            expected_cap: Cap::SQL_QUERY,
+            constraint_hints: vec![],
+            sink_file: "tests/dynamic_fixtures/go/sqli_positive.go".into(),
+            sink_line: 12,
+            spec_hash: spec_hash.to_owned(),
+            derivation: nyx_scanner::dynamic::spec::SpecDerivationStrategy::FromFlowSteps,
+            stubs_required: vec![],
+            framework: None,
+            java_toolchain: nyx_scanner::dynamic::spec::JavaToolchain::default(),
+        }
+    }
+
+    #[test]
+    fn go_repro_outcome_is_deterministic() {
+        let dir = TempDir::new().unwrap();
+        let _env = ReproEnvGuard::set(dir.path());
+
+        let spec = make_confirmed_go_spec("go_determ000001a");
+        let opts = SandboxOptions::default();
+        let outcome = make_confirmed_outcome();
+        let verdict = make_confirmed_verdict("go_determ000001");
+        let entry_src = "package entry\nfunc Login(username string) {}\n";
+
+        let artifact1 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "// harness go\n",
+            entry_src,
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("first Go repro write");
+        let json1 = std::fs::read_to_string(artifact1.root.join("expected/outcome.json")).unwrap();
+
+        std::fs::remove_dir_all(&artifact1.root).unwrap();
+
+        let artifact2 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "// harness go\n",
+            entry_src,
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("second Go repro write");
+        let json2 = std::fs::read_to_string(artifact2.root.join("expected/outcome.json")).unwrap();
+
+        assert_eq!(
+            json1, json2,
+            "Go outcome.json must be byte-identical across two writes"
+        );
+    }
+
+    // ── Java repro tests ─────────────────────────────────────────────────────
+
+    fn make_confirmed_java_spec(spec_hash: &str) -> HarnessSpec {
+        HarnessSpec {
+            finding_id: "java_determ00001".into(),
+            entry_file: "tests/dynamic_fixtures/java/sqli_positive.java".into(),
+            entry_name: "login".into(),
+            entry_kind: EntryKind::Function,
+            lang: Lang::Java,
+            toolchain_id: "java-21".into(),
+            payload_slot: PayloadSlot::Param(0),
+            expected_cap: Cap::SQL_QUERY,
+            constraint_hints: vec![],
+            sink_file: "tests/dynamic_fixtures/java/sqli_positive.java".into(),
+            sink_line: 9,
+            spec_hash: spec_hash.to_owned(),
+            derivation: nyx_scanner::dynamic::spec::SpecDerivationStrategy::FromFlowSteps,
+            stubs_required: vec![],
+            framework: None,
+            java_toolchain: nyx_scanner::dynamic::spec::JavaToolchain::default(),
+        }
+    }
+
+    #[test]
+    fn java_repro_outcome_is_deterministic() {
+        let dir = TempDir::new().unwrap();
+        let _env = ReproEnvGuard::set(dir.path());
+
+        let spec = make_confirmed_java_spec("java_determ00001a");
+        let opts = SandboxOptions::default();
+        let outcome = make_confirmed_outcome();
+        let verdict = make_confirmed_verdict("java_determ00001");
+        let entry_src = "public class Entry { public static void login(String u) {} }\n";
+
+        let artifact1 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "// NyxHarness.java\n",
+            entry_src,
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("first Java repro write");
+        let json1 = std::fs::read_to_string(artifact1.root.join("expected/outcome.json")).unwrap();
+
+        std::fs::remove_dir_all(&artifact1.root).unwrap();
+
+        let artifact2 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "// NyxHarness.java\n",
+            entry_src,
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("second Java repro write");
+        let json2 = std::fs::read_to_string(artifact2.root.join("expected/outcome.json")).unwrap();
+
+        assert_eq!(
+            json1, json2,
+            "Java outcome.json must be byte-identical across two writes"
+        );
+    }
+
+    // ── PHP repro tests ──────────────────────────────────────────────────────
+
+    fn make_confirmed_php_spec(spec_hash: &str) -> HarnessSpec {
+        HarnessSpec {
+            finding_id: "php_determ000001".into(),
+            entry_file: "tests/dynamic_fixtures/php/sqli_positive.php".into(),
+            entry_name: "login".into(),
+            entry_kind: EntryKind::Function,
+            lang: Lang::Php,
+            toolchain_id: "php-8".into(),
+            payload_slot: PayloadSlot::Param(0),
+            expected_cap: Cap::SQL_QUERY,
+            constraint_hints: vec![],
+            sink_file: "tests/dynamic_fixtures/php/sqli_positive.php".into(),
+            sink_line: 9,
+            spec_hash: spec_hash.to_owned(),
+            derivation: nyx_scanner::dynamic::spec::SpecDerivationStrategy::FromFlowSteps,
+            stubs_required: vec![],
+            framework: None,
+            java_toolchain: nyx_scanner::dynamic::spec::JavaToolchain::default(),
+        }
+    }
+
+    #[test]
+    fn php_repro_outcome_is_deterministic() {
+        let dir = TempDir::new().unwrap();
+        let _env = ReproEnvGuard::set(dir.path());
+
+        let spec = make_confirmed_php_spec("php_determ000001a");
+        let opts = SandboxOptions::default();
+        let outcome = make_confirmed_outcome();
+        let verdict = make_confirmed_verdict("php_determ000001");
+        let entry_src = "<?php\nfunction login($username) {}\n";
+
+        let artifact1 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "<?php // harness\n",
+            entry_src,
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("first PHP repro write");
+        let json1 = std::fs::read_to_string(artifact1.root.join("expected/outcome.json")).unwrap();
+
+        std::fs::remove_dir_all(&artifact1.root).unwrap();
+
+        let artifact2 = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "<?php // harness\n",
+            entry_src,
+            b"' UNION SELECT 'NYX_SQL_CONFIRMED'--",
+            "sqli-union-nyx",
+            None,
+        )
+        .expect("second PHP repro write");
+        let json2 = std::fs::read_to_string(artifact2.root.join("expected/outcome.json")).unwrap();
+
+        assert_eq!(
+            json1, json2,
+            "PHP outcome.json must be byte-identical across two writes"
+        );
+    }
+
+    /// Verify verdict.json is correctly structured.
+    #[test]
+    fn verdict_json_is_valid() {
+        let dir = TempDir::new().unwrap();
+        let _env = ReproEnvGuard::set(dir.path());
+
+        let spec = make_confirmed_spec("determ0000000003");
+        let opts = SandboxOptions::default();
+        let outcome = make_confirmed_outcome();
+        let verdict = make_confirmed_verdict("determinism00003");
+
+        let artifact = repro::write(
+            &spec,
+            &opts,
+            &outcome,
+            &verdict,
+            "# harness",
+            "# entry",
+            b"payload",
+            "label",
+            None,
+        )
+        .expect("repro write must succeed");
+
+        let verdict_json =
+            std::fs::read_to_string(artifact.root.join("expected/verdict.json")).unwrap();
+        let parsed: serde_json::Value = serde_json::from_str(&verdict_json).unwrap();
+
+        assert_eq!(parsed["status"], "Confirmed");
+        assert_eq!(parsed["finding_id"], "determinism00003");
+    }
+}