mirror of
https://github.com/elicpeter/nyx.git
synced 2026-06-09 19:45:13 +02:00
[pitboss] sweep after phase 07: 6 deferred items resolved
This commit is contained in:
parent
bfdfcb9d1a
commit
25e8b0eb0e
6 changed files with 730 additions and 8 deletions
|
|
@ -705,10 +705,12 @@ function HowToFix({ finding }: { finding: FindingView }) {
|
|||
|
||||
// ── Dynamic Verification Panel ──────────────────────────────────────────────
|
||||
|
||||
function DynamicVerdictSection({ verdict }: { verdict: VerifyResult }) {
|
||||
export function DynamicVerdictSection({ verdict }: { verdict: VerifyResult }) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
const reproPath = `~/.cache/nyx/dynamic/repro/${verdict.finding_id}/`;
|
||||
const reproCmd = './reproduce.sh';
|
||||
// The repro bundle is keyed by spec_hash (not finding_id) inside the Nyx
|
||||
// cache. Rather than showing a path that may not match, surface the CLI
|
||||
// command that locates and opens the bundle regardless of the hash.
|
||||
const reproCmd = `nyx repro --finding ${verdict.finding_id}`;
|
||||
|
||||
const copyCmd = () => {
|
||||
navigator.clipboard.writeText(reproCmd).then(() => {
|
||||
|
|
@ -733,11 +735,8 @@ function DynamicVerdictSection({ verdict }: { verdict: VerifyResult }) {
|
|||
|
||||
{verdict.status === 'Confirmed' && (
|
||||
<div className="repro-panel" data-testid="repro-panel">
|
||||
<div className="repro-path-row">
|
||||
<span className="repro-label">Repro artifact:</span>
|
||||
<code className="repro-path">{reproPath}</code>
|
||||
</div>
|
||||
<div className="repro-cmd-row">
|
||||
<span className="repro-label">Reproduce:</span>
|
||||
<code className="repro-cmd">{reproCmd}</code>
|
||||
<button
|
||||
type="button"
|
||||
|
|
|
|||
118
frontend/src/test/components/dynamicVerdictSection.test.tsx
Normal file
118
frontend/src/test/components/dynamicVerdictSection.test.tsx
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { render, screen, fireEvent } from '@testing-library/react';
|
||||
import { DynamicVerdictSection } from '@/pages/FindingDetailPage';
|
||||
import type { VerifyResult } from '@/api/types';
|
||||
|
||||
function makeVerdict(
|
||||
status: VerifyResult['status'],
|
||||
extras: Partial<VerifyResult> = {},
|
||||
): VerifyResult {
|
||||
return {
|
||||
finding_id: 'test-finding-id-abc',
|
||||
status,
|
||||
attempts: [],
|
||||
...extras,
|
||||
};
|
||||
}
|
||||
|
||||
// Mock navigator.clipboard before each test.
|
||||
beforeEach(() => {
|
||||
Object.defineProperty(navigator, 'clipboard', {
|
||||
value: { writeText: vi.fn().mockResolvedValue(undefined) },
|
||||
configurable: true,
|
||||
writable: true,
|
||||
});
|
||||
});
|
||||
|
||||
describe('DynamicVerdictSection', () => {
|
||||
it('renders Confirmed badge', () => {
|
||||
render(
|
||||
<DynamicVerdictSection
|
||||
verdict={makeVerdict('Confirmed', { triggered_payload: 'sqli-tautology' })}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByTestId('verdict-badge-confirmed')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders NotConfirmed badge', () => {
|
||||
render(<DynamicVerdictSection verdict={makeVerdict('NotConfirmed')} />);
|
||||
expect(screen.getByTestId('verdict-badge-notconfirmed')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders Unsupported badge', () => {
|
||||
render(
|
||||
<DynamicVerdictSection
|
||||
verdict={makeVerdict('Unsupported', { reason: 'NoPayloadsForCap' })}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByTestId('verdict-badge-unsupported')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders Inconclusive badge', () => {
|
||||
render(
|
||||
<DynamicVerdictSection
|
||||
verdict={makeVerdict('Inconclusive', { inconclusive_reason: 'BuildFailed' })}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByTestId('verdict-badge-inconclusive')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows repro panel only for Confirmed status', () => {
|
||||
const { unmount } = render(
|
||||
<DynamicVerdictSection verdict={makeVerdict('Confirmed')} />,
|
||||
);
|
||||
expect(screen.getByTestId('repro-panel')).toBeInTheDocument();
|
||||
unmount();
|
||||
|
||||
for (const status of ['NotConfirmed', 'Unsupported', 'Inconclusive'] as const) {
|
||||
const { unmount: u } = render(
|
||||
<DynamicVerdictSection verdict={makeVerdict(status)} />,
|
||||
);
|
||||
expect(screen.queryByTestId('repro-panel')).toBeNull();
|
||||
u();
|
||||
}
|
||||
});
|
||||
|
||||
it('repro-panel contains the finding_id in the CLI command', () => {
|
||||
render(
|
||||
<DynamicVerdictSection
|
||||
verdict={makeVerdict('Confirmed', { finding_id: 'cafecafe12345678' })}
|
||||
/>,
|
||||
);
|
||||
const panel = screen.getByTestId('repro-panel');
|
||||
expect(panel.textContent).toContain('cafecafe12345678');
|
||||
expect(panel.textContent).toContain('nyx repro');
|
||||
});
|
||||
|
||||
it('Copy button triggers clipboard writeText with the repro command', async () => {
|
||||
const findingId = 'test-finding-id-abc';
|
||||
render(<DynamicVerdictSection verdict={makeVerdict('Confirmed')} />);
|
||||
|
||||
const copyBtn = screen.getByRole('button', { name: /copy/i });
|
||||
fireEvent.click(copyBtn);
|
||||
|
||||
expect(navigator.clipboard.writeText).toHaveBeenCalledOnce();
|
||||
const calledWith = (navigator.clipboard.writeText as ReturnType<typeof vi.fn>).mock
|
||||
.calls[0][0] as string;
|
||||
expect(calledWith).toContain(findingId);
|
||||
expect(calledWith).toContain('nyx repro');
|
||||
});
|
||||
|
||||
it('shows exact toolchain match label when toolchain_match is exact', () => {
|
||||
render(
|
||||
<DynamicVerdictSection
|
||||
verdict={makeVerdict('Confirmed', { toolchain_match: 'exact' })}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText('exact toolchain')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows approximate toolchain match label when toolchain_match is drift', () => {
|
||||
render(
|
||||
<DynamicVerdictSection
|
||||
verdict={makeVerdict('Confirmed', { toolchain_match: 'drift' })}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText('approximate toolchain')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -118,6 +118,57 @@ pub fn log_path() -> Option<std::path::PathBuf> {
|
|||
events_log_path()
|
||||
}
|
||||
|
||||
// ── Rank delta telemetry ──────────────────────────────────────────────────────
|
||||
|
||||
/// One telemetry event per ranked finding that carries a dynamic verdict delta.
|
||||
///
|
||||
/// Emitted by `rank::rank_diags` for every diag whose dynamic verdict shifts
|
||||
/// its rank score (delta != 0). Used by the M7 calibration pipeline to tune
|
||||
/// the N/M boost/penalty constants from real-world verdict distributions.
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
pub struct RankDeltaEvent {
|
||||
pub ts: String,
|
||||
/// Always `"rank_delta"` — distinguishes from verdict events in the log.
|
||||
pub event_type: &'static str,
|
||||
pub finding_id: String,
|
||||
/// `"Confirmed"`, `"NotConfirmed"`, etc.
|
||||
pub status: String,
|
||||
/// Signed delta applied to the rank score (+N for Confirmed, -M for NotConfirmed).
|
||||
pub delta: f64,
|
||||
}
|
||||
|
||||
/// Write a rank-delta telemetry event to the events log.
|
||||
///
|
||||
/// Silently no-ops under the same conditions as [`emit`]:
|
||||
/// `NYX_NO_TELEMETRY=1`, unresolvable log dir, or write failure.
|
||||
pub fn emit_rank_delta(event: RankDeltaEvent) {
|
||||
if std::env::var("NYX_NO_TELEMETRY").as_deref() == Ok("1") {
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(path) = events_log_path() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let Ok(line) = serde_json::to_string(&event) else {
|
||||
return;
|
||||
};
|
||||
|
||||
let _ = (|| -> std::io::Result<()> {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(parent, fs::Permissions::from_mode(0o700))?;
|
||||
}
|
||||
}
|
||||
let mut f = OpenOptions::new().create(true).append(true).open(&path)?;
|
||||
writeln!(f, "{line}")?;
|
||||
Ok(())
|
||||
})();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
298
src/rank.rs
298
src/rank.rs
|
|
@ -206,6 +206,27 @@ pub fn rank_diags(diags: &mut [Diag]) {
|
|||
if !rank.components.is_empty() {
|
||||
d.rank_reason = Some(rank.components.clone());
|
||||
}
|
||||
// Emit rank-delta telemetry for M7 calibration (§21 / deferred M7 hook).
|
||||
// Only fires when the dynamic verdict shifted the score; benign verdicts
|
||||
// (Unsupported, Inconclusive, no verdict) produce delta = None and are
|
||||
// skipped — emitting them would add noise without calibration value.
|
||||
#[cfg(feature = "dynamic")]
|
||||
if let Some(delta) = dynamic_verdict_delta(d) {
|
||||
use crate::dynamic::telemetry::{self, RankDeltaEvent};
|
||||
let status = d
|
||||
.evidence
|
||||
.as_ref()
|
||||
.and_then(|ev| ev.dynamic_verdict.as_ref())
|
||||
.map(|dv| format!("{:?}", dv.status))
|
||||
.unwrap_or_default();
|
||||
telemetry::emit_rank_delta(RankDeltaEvent {
|
||||
ts: chrono::Utc::now().to_rfc3339(),
|
||||
event_type: "rank_delta",
|
||||
finding_id: d.finding_id.clone(),
|
||||
status,
|
||||
delta,
|
||||
});
|
||||
}
|
||||
}
|
||||
diags.sort_by(|a, b| {
|
||||
let sa = a.rank_score.unwrap_or(0.0);
|
||||
|
|
@ -225,6 +246,15 @@ pub fn rank_diags(diags: &mut [Diag]) {
|
|||
/// Returns `None` when there is no verdict (static-only scan) or the verdict
|
||||
/// does not change the score (Unsupported, Inconclusive).
|
||||
///
|
||||
/// Design note (§deferred M7 payload_corpus_complete): the spec originally
|
||||
/// distinguished `NotConfirmed` + `payload_corpus_complete == true` → `-M`
|
||||
/// from `NotConfirmed` + `NoPayloadsForCap` → no change. In practice the
|
||||
/// `NoPayloadsForCap` path always produces `Unsupported`, never `NotConfirmed`,
|
||||
/// so the two cases are already disjoint in the type. The heuristic
|
||||
/// `!dv.attempts.is_empty()` (corpus was actually tried) is equivalent to
|
||||
/// `payload_corpus_complete == true` for all reachable states — no extra
|
||||
/// field is needed. See also §deferred decision in `.pitboss/play/deferred.md`.
|
||||
///
|
||||
/// TODO(M7): N=20 and M=5 are placeholders; calibrate from telemetry.
|
||||
fn dynamic_verdict_delta(diag: &Diag) -> Option<f64> {
|
||||
use crate::evidence::VerifyStatus;
|
||||
|
|
@ -234,7 +264,8 @@ fn dynamic_verdict_delta(diag: &Diag) -> Option<f64> {
|
|||
// Apply penalty only when the corpus was actually exhausted (attempts
|
||||
// were made); a NotConfirmed with zero attempts means something went
|
||||
// wrong before payload execution, which is an Inconclusive path, not
|
||||
// a meaningful negative signal.
|
||||
// a meaningful negative signal. This is equivalent to the spec's
|
||||
// `payload_corpus_complete == true` condition (see design note above).
|
||||
VerifyStatus::NotConfirmed if !dv.attempts.is_empty() => Some(-5.0),
|
||||
_ => None,
|
||||
}
|
||||
|
|
@ -1083,4 +1114,269 @@ mod tests {
|
|||
"Bail ({s_bail}) must rank at or below UnderReport ({s_under})"
|
||||
);
|
||||
}
|
||||
|
||||
// ── Dynamic verdict delta tests ────────────────────────────────────────
|
||||
|
||||
use crate::evidence::{AttemptSummary, Evidence, VerifyResult, VerifyStatus};
|
||||
|
||||
fn make_diag_with_verdict(verdict: Option<VerifyResult>) -> Diag {
|
||||
let mut d = make_diag(
|
||||
Severity::High,
|
||||
"taint-unsanitised-flow (source 1:1)",
|
||||
"src/main.rs",
|
||||
10,
|
||||
vec![("Source".into(), "stdin at 1:1".into())],
|
||||
false,
|
||||
);
|
||||
d.finding_id = "test_finding_id".into();
|
||||
if let Some(v) = verdict {
|
||||
d.evidence = Some(Evidence {
|
||||
dynamic_verdict: Some(v),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
d
|
||||
}
|
||||
|
||||
fn confirmed_verdict() -> VerifyResult {
|
||||
VerifyResult {
|
||||
finding_id: "test_finding_id".into(),
|
||||
status: VerifyStatus::Confirmed,
|
||||
triggered_payload: Some("sqli-tautology".into()),
|
||||
reason: None,
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![AttemptSummary {
|
||||
payload_label: "sqli-tautology".into(),
|
||||
exit_code: Some(0),
|
||||
timed_out: false,
|
||||
triggered: true,
|
||||
sink_hit: true,
|
||||
}],
|
||||
toolchain_match: Some("exact".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn not_confirmed_with_attempts() -> VerifyResult {
|
||||
VerifyResult {
|
||||
finding_id: "test_finding_id".into(),
|
||||
status: VerifyStatus::NotConfirmed,
|
||||
triggered_payload: None,
|
||||
reason: None,
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![AttemptSummary {
|
||||
payload_label: "sqli-tautology".into(),
|
||||
exit_code: Some(0),
|
||||
timed_out: false,
|
||||
triggered: false,
|
||||
sink_hit: false,
|
||||
}],
|
||||
toolchain_match: Some("exact".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn not_confirmed_no_attempts() -> VerifyResult {
|
||||
VerifyResult {
|
||||
finding_id: "test_finding_id".into(),
|
||||
status: VerifyStatus::NotConfirmed,
|
||||
triggered_payload: None,
|
||||
reason: None,
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![],
|
||||
toolchain_match: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn unsupported_verdict() -> VerifyResult {
|
||||
VerifyResult {
|
||||
finding_id: "test_finding_id".into(),
|
||||
status: VerifyStatus::Unsupported,
|
||||
triggered_payload: None,
|
||||
reason: Some(crate::evidence::UnsupportedReason::NoPayloadsForCap),
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![],
|
||||
toolchain_match: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn inconclusive_verdict() -> VerifyResult {
|
||||
VerifyResult {
|
||||
finding_id: "test_finding_id".into(),
|
||||
status: VerifyStatus::Inconclusive,
|
||||
triggered_payload: None,
|
||||
reason: None,
|
||||
inconclusive_reason: Some(crate::evidence::InconclusiveReason::BuildFailed),
|
||||
detail: None,
|
||||
attempts: vec![],
|
||||
toolchain_match: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_confirmed_delta_is_positive() {
|
||||
let d = make_diag_with_verdict(Some(confirmed_verdict()));
|
||||
assert_eq!(
|
||||
dynamic_verdict_delta(&d),
|
||||
Some(20.0),
|
||||
"Confirmed must produce +20 delta"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_not_confirmed_with_attempts_delta_is_negative() {
|
||||
let d = make_diag_with_verdict(Some(not_confirmed_with_attempts()));
|
||||
assert_eq!(
|
||||
dynamic_verdict_delta(&d),
|
||||
Some(-5.0),
|
||||
"NotConfirmed with attempts must produce -5 delta"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_not_confirmed_no_attempts_no_delta() {
|
||||
let d = make_diag_with_verdict(Some(not_confirmed_no_attempts()));
|
||||
assert_eq!(
|
||||
dynamic_verdict_delta(&d),
|
||||
None,
|
||||
"NotConfirmed with zero attempts must produce no delta"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_unsupported_no_delta() {
|
||||
let d = make_diag_with_verdict(Some(unsupported_verdict()));
|
||||
assert_eq!(
|
||||
dynamic_verdict_delta(&d),
|
||||
None,
|
||||
"Unsupported must produce no delta"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_inconclusive_no_delta() {
|
||||
let d = make_diag_with_verdict(Some(inconclusive_verdict()));
|
||||
assert_eq!(
|
||||
dynamic_verdict_delta(&d),
|
||||
None,
|
||||
"Inconclusive must produce no delta"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_no_verdict_no_delta() {
|
||||
let d = make_diag_with_verdict(None);
|
||||
assert_eq!(
|
||||
dynamic_verdict_delta(&d),
|
||||
None,
|
||||
"No verdict must produce no delta"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_confirmed_ranks_above_no_verdict() {
|
||||
let confirmed = make_diag_with_verdict(Some(confirmed_verdict()));
|
||||
let no_verdict = make_diag_with_verdict(None);
|
||||
|
||||
let s_confirmed = compute_attack_rank(&confirmed).score;
|
||||
let s_none = compute_attack_rank(&no_verdict).score;
|
||||
assert!(
|
||||
s_confirmed > s_none,
|
||||
"Confirmed ({s_confirmed}) must rank above no-verdict ({s_none})"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_no_verdict_ranks_above_not_confirmed_with_attempts() {
|
||||
let no_verdict = make_diag_with_verdict(None);
|
||||
let not_confirmed = make_diag_with_verdict(Some(not_confirmed_with_attempts()));
|
||||
|
||||
let s_none = compute_attack_rank(&no_verdict).score;
|
||||
let s_nc = compute_attack_rank(¬_confirmed).score;
|
||||
assert!(
|
||||
s_none > s_nc,
|
||||
"No-verdict ({s_none}) must rank above NotConfirmed-with-attempts ({s_nc})"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_unsupported_same_as_no_verdict() {
|
||||
let no_verdict = make_diag_with_verdict(None);
|
||||
let unsupported = make_diag_with_verdict(Some(unsupported_verdict()));
|
||||
|
||||
let s_none = compute_attack_rank(&no_verdict).score;
|
||||
let s_uns = compute_attack_rank(&unsupported).score;
|
||||
// Unsupported carries a 4-field Evidence struct so evidence_strength
|
||||
// differs slightly from a None evidence diag. What matters is that
|
||||
// the *delta component* is zero — both deltas must agree.
|
||||
assert_eq!(
|
||||
dynamic_verdict_delta(&no_verdict),
|
||||
dynamic_verdict_delta(&unsupported),
|
||||
"Unsupported and no-verdict must both produce None delta"
|
||||
);
|
||||
// Same base inputs → scores differ only by evidence_strength bonus
|
||||
// from the Evidence wrapper. Verify no "dynamic_verdict" component
|
||||
// in rank_reason.
|
||||
let rank = compute_attack_rank(&unsupported);
|
||||
assert!(
|
||||
!rank.components.iter().any(|(k, _)| k == "dynamic_verdict"),
|
||||
"Unsupported must not appear in rank_reason components"
|
||||
);
|
||||
let _ = s_none;
|
||||
let _ = s_uns;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_inconclusive_same_delta_as_no_verdict() {
|
||||
let no_verdict = make_diag_with_verdict(None);
|
||||
let inconclusive = make_diag_with_verdict(Some(inconclusive_verdict()));
|
||||
|
||||
assert_eq!(
|
||||
dynamic_verdict_delta(&no_verdict),
|
||||
dynamic_verdict_delta(&inconclusive),
|
||||
"Inconclusive and no-verdict must both produce None delta"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_confirmed_rank_reason_contains_component() {
|
||||
let d = make_diag_with_verdict(Some(confirmed_verdict()));
|
||||
let rank = compute_attack_rank(&d);
|
||||
assert!(
|
||||
rank.components.iter().any(|(k, _)| k == "dynamic_verdict"),
|
||||
"Confirmed verdict must appear in rank_reason components"
|
||||
);
|
||||
let dv_component = rank
|
||||
.components
|
||||
.iter()
|
||||
.find(|(k, _)| k == "dynamic_verdict")
|
||||
.unwrap();
|
||||
assert!(
|
||||
dv_component.1.starts_with('+'),
|
||||
"Confirmed delta must be positive in rank_reason: {:?}",
|
||||
dv_component.1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_verdict_not_confirmed_rank_reason_contains_negative_component() {
|
||||
let d = make_diag_with_verdict(Some(not_confirmed_with_attempts()));
|
||||
let rank = compute_attack_rank(&d);
|
||||
assert!(
|
||||
rank.components.iter().any(|(k, _)| k == "dynamic_verdict"),
|
||||
"NotConfirmed-with-attempts must appear in rank_reason components"
|
||||
);
|
||||
let dv_component = rank
|
||||
.components
|
||||
.iter()
|
||||
.find(|(k, _)| k == "dynamic_verdict")
|
||||
.unwrap();
|
||||
assert!(
|
||||
dv_component.1.starts_with('-'),
|
||||
"NotConfirmed delta must be negative in rank_reason: {:?}",
|
||||
dv_component.1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
//! | `src/commands/scan.rs` | enrichment loop lives here |
|
||||
//! | `src/commands/mod.rs` | `verify-feedback` subcommand (§21.2) |
|
||||
//! | `src/server/` (any file) | server start_scan verify wiring |
|
||||
//! | `src/rank.rs` | M7 rank-delta telemetry hook (§21 / M7) |
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
|
@ -28,6 +29,7 @@ const ALLOWED: &[&str] = &[
|
|||
"commands/scan.rs",
|
||||
"commands/mod.rs",
|
||||
"server/",
|
||||
"rank.rs",
|
||||
// The dynamic module itself is obviously allowed.
|
||||
"dynamic/",
|
||||
];
|
||||
|
|
|
|||
256
tests/sarif_dynamic_verdict_tests.rs
Normal file
256
tests/sarif_dynamic_verdict_tests.rs
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
//! SARIF output tests for the dynamic verification vendor extension (§5.4).
|
||||
//!
|
||||
//! Acceptance criterion: SARIF output contains both
|
||||
//! `partialFingerprints.dynamic_verdict_status` and
|
||||
//! `properties.nyx_dynamic_verdict` for every `VerifyStatus` variant, and
|
||||
//! both keys are absent when no dynamic verdict is attached.
|
||||
|
||||
use nyx_scanner::commands::scan::Diag;
|
||||
use nyx_scanner::evidence::{
|
||||
AttemptSummary, Evidence, InconclusiveReason, UnsupportedReason, VerifyResult, VerifyStatus,
|
||||
};
|
||||
use nyx_scanner::output::build_sarif;
|
||||
use nyx_scanner::patterns::{FindingCategory, Severity};
|
||||
use std::path::Path;
|
||||
|
||||
fn base_diag() -> Diag {
|
||||
Diag {
|
||||
path: "/scan_root/src/main.rs".into(),
|
||||
line: 10,
|
||||
col: 5,
|
||||
severity: Severity::High,
|
||||
id: "taint-unsanitised-flow".into(),
|
||||
category: FindingCategory::Security,
|
||||
path_validated: false,
|
||||
guard_kind: None,
|
||||
message: None,
|
||||
labels: vec![],
|
||||
confidence: None,
|
||||
evidence: None,
|
||||
rank_score: None,
|
||||
rank_reason: None,
|
||||
suppressed: false,
|
||||
suppression: None,
|
||||
rollup: None,
|
||||
finding_id: "deadbeef01234567".into(),
|
||||
alternative_finding_ids: Vec::new(),
|
||||
stable_hash: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn diag_with_verdict(verdict: VerifyResult) -> Diag {
|
||||
let mut d = base_diag();
|
||||
d.evidence = Some(Evidence {
|
||||
dynamic_verdict: Some(verdict),
|
||||
..Default::default()
|
||||
});
|
||||
d
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
fn sarif_result(diag: Diag) -> serde_json::Value {
|
||||
let sarif = build_sarif(&[diag], Path::new("/scan_root"));
|
||||
sarif["runs"][0]["results"][0].clone()
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn sarif_confirmed_verdict_sets_partial_fingerprint() {
|
||||
let verdict = VerifyResult {
|
||||
finding_id: "deadbeef01234567".into(),
|
||||
status: VerifyStatus::Confirmed,
|
||||
triggered_payload: Some("sqli-tautology".into()),
|
||||
reason: None,
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![AttemptSummary {
|
||||
payload_label: "sqli-tautology".into(),
|
||||
exit_code: Some(0),
|
||||
timed_out: false,
|
||||
triggered: true,
|
||||
sink_hit: true,
|
||||
}],
|
||||
toolchain_match: Some("exact".into()),
|
||||
};
|
||||
|
||||
let result = sarif_result(diag_with_verdict(verdict));
|
||||
|
||||
assert_eq!(
|
||||
result["partialFingerprints"]["dynamic_verdict_status"],
|
||||
"Confirmed",
|
||||
"partialFingerprints.dynamic_verdict_status must be 'Confirmed'"
|
||||
);
|
||||
assert!(
|
||||
result["properties"]["nyx_dynamic_verdict"].is_object(),
|
||||
"properties.nyx_dynamic_verdict must be an object: {}",
|
||||
result["properties"]["nyx_dynamic_verdict"]
|
||||
);
|
||||
assert_eq!(
|
||||
result["properties"]["nyx_dynamic_verdict"]["status"],
|
||||
"Confirmed",
|
||||
"nyx_dynamic_verdict.status must be 'Confirmed'"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sarif_not_confirmed_verdict_sets_partial_fingerprint() {
|
||||
let verdict = VerifyResult {
|
||||
finding_id: "deadbeef01234567".into(),
|
||||
status: VerifyStatus::NotConfirmed,
|
||||
triggered_payload: None,
|
||||
reason: None,
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![],
|
||||
toolchain_match: Some("exact".into()),
|
||||
};
|
||||
|
||||
let result = sarif_result(diag_with_verdict(verdict));
|
||||
|
||||
assert_eq!(
|
||||
result["partialFingerprints"]["dynamic_verdict_status"],
|
||||
"NotConfirmed",
|
||||
"partialFingerprints.dynamic_verdict_status must be 'NotConfirmed'"
|
||||
);
|
||||
assert!(
|
||||
result["properties"]["nyx_dynamic_verdict"].is_object(),
|
||||
"properties.nyx_dynamic_verdict must be an object"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sarif_unsupported_verdict_sets_partial_fingerprint() {
|
||||
let verdict = VerifyResult {
|
||||
finding_id: "deadbeef01234567".into(),
|
||||
status: VerifyStatus::Unsupported,
|
||||
triggered_payload: None,
|
||||
reason: Some(UnsupportedReason::NoPayloadsForCap),
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![],
|
||||
toolchain_match: None,
|
||||
};
|
||||
|
||||
let result = sarif_result(diag_with_verdict(verdict));
|
||||
|
||||
assert_eq!(
|
||||
result["partialFingerprints"]["dynamic_verdict_status"],
|
||||
"Unsupported",
|
||||
"partialFingerprints.dynamic_verdict_status must be 'Unsupported'"
|
||||
);
|
||||
assert!(
|
||||
result["properties"]["nyx_dynamic_verdict"].is_object(),
|
||||
"properties.nyx_dynamic_verdict must be an object"
|
||||
);
|
||||
assert_eq!(
|
||||
result["properties"]["nyx_dynamic_verdict"]["reason"],
|
||||
"NoPayloadsForCap",
|
||||
"nyx_dynamic_verdict must carry the unsupported reason"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sarif_inconclusive_verdict_sets_partial_fingerprint() {
|
||||
let verdict = VerifyResult {
|
||||
finding_id: "deadbeef01234567".into(),
|
||||
status: VerifyStatus::Inconclusive,
|
||||
triggered_payload: None,
|
||||
reason: None,
|
||||
inconclusive_reason: Some(InconclusiveReason::BuildFailed),
|
||||
detail: Some("build failed after 3 attempts".into()),
|
||||
attempts: vec![],
|
||||
toolchain_match: None,
|
||||
};
|
||||
|
||||
let result = sarif_result(diag_with_verdict(verdict));
|
||||
|
||||
assert_eq!(
|
||||
result["partialFingerprints"]["dynamic_verdict_status"],
|
||||
"Inconclusive",
|
||||
"partialFingerprints.dynamic_verdict_status must be 'Inconclusive'"
|
||||
);
|
||||
assert!(
|
||||
result["properties"]["nyx_dynamic_verdict"].is_object(),
|
||||
"properties.nyx_dynamic_verdict must be an object"
|
||||
);
|
||||
assert_eq!(
|
||||
result["properties"]["nyx_dynamic_verdict"]["inconclusive_reason"],
|
||||
"BuildFailed",
|
||||
"nyx_dynamic_verdict must carry the inconclusive reason"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sarif_no_dynamic_verdict_omits_both_keys() {
|
||||
let diag = base_diag();
|
||||
let result = sarif_result(diag);
|
||||
|
||||
assert!(
|
||||
result["partialFingerprints"].is_null() || result["partialFingerprints"] == serde_json::Value::Null,
|
||||
"partialFingerprints must be absent when no dynamic verdict: {}",
|
||||
result["partialFingerprints"]
|
||||
);
|
||||
assert!(
|
||||
result["properties"]["nyx_dynamic_verdict"].is_null() || result["properties"]["nyx_dynamic_verdict"] == serde_json::Value::Null,
|
||||
"properties.nyx_dynamic_verdict must be absent when no dynamic verdict"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sarif_confirmed_verdict_nyx_dynamic_verdict_contains_triggered_payload() {
|
||||
let verdict = VerifyResult {
|
||||
finding_id: "deadbeef01234567".into(),
|
||||
status: VerifyStatus::Confirmed,
|
||||
triggered_payload: Some("cmd-injection-semicolon".into()),
|
||||
reason: None,
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![],
|
||||
toolchain_match: Some("exact".into()),
|
||||
};
|
||||
|
||||
let result = sarif_result(diag_with_verdict(verdict));
|
||||
|
||||
assert_eq!(
|
||||
result["properties"]["nyx_dynamic_verdict"]["triggered_payload"],
|
||||
"cmd-injection-semicolon",
|
||||
"triggered_payload must appear in nyx_dynamic_verdict"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sarif_all_four_statuses_produce_partial_fingerprint() {
|
||||
let statuses = [
|
||||
(VerifyStatus::Confirmed, "Confirmed"),
|
||||
(VerifyStatus::NotConfirmed, "NotConfirmed"),
|
||||
(VerifyStatus::Unsupported, "Unsupported"),
|
||||
(VerifyStatus::Inconclusive, "Inconclusive"),
|
||||
];
|
||||
|
||||
for (status, expected_str) in statuses {
|
||||
let verdict = VerifyResult {
|
||||
finding_id: "deadbeef01234567".into(),
|
||||
status,
|
||||
triggered_payload: None,
|
||||
reason: None,
|
||||
inconclusive_reason: None,
|
||||
detail: None,
|
||||
attempts: vec![],
|
||||
toolchain_match: None,
|
||||
};
|
||||
|
||||
let result = sarif_result(diag_with_verdict(verdict));
|
||||
|
||||
assert_eq!(
|
||||
result["partialFingerprints"]["dynamic_verdict_status"],
|
||||
expected_str,
|
||||
"status {expected_str}: partialFingerprints.dynamic_verdict_status mismatch"
|
||||
);
|
||||
assert!(
|
||||
result["properties"]["nyx_dynamic_verdict"].is_object(),
|
||||
"status {expected_str}: properties.nyx_dynamic_verdict must be an object"
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue