[pitboss] phase 07: M6 — Evidence consumers: formatters, ranking, UI

This commit is contained in:
pitboss 2026-05-12 13:26:52 -04:00
parent 6f8a645077
commit bfdfcb9d1a
18 changed files with 3208 additions and 46 deletions

View file

@ -404,20 +404,7 @@ jobs:
toolchain: stable
cache: true
- uses: actions/setup-node@v6
with:
node-version: 20
cache: npm
cache-dependency-path: frontend/package-lock.json
- name: Build frontend
working-directory: frontend
run: |
npm ci
npm run build
- name: Corpus unit tests (no_marker_collisions, all_payloads_have_fixture_paths)
run: cargo nextest run --lib -p nyx-scanner --test-threads=4 2>/dev/null || \
cargo nextest run --lib -p nyx-scanner
run: cargo nextest run --lib -p nyx-scanner dynamic::corpus
env:
RUST_LOG: error

View file

@ -106,38 +106,46 @@ jobs:
# Stage candidate files into fuzz-discovered (already there).
# The PR body provides the reviewer with everything they need.
# Build PR body.
body=$(cat <<'EOF'
## Corpus Promotion Proposal
# Build PR body into a temp file to avoid shell re-interpolation of
# sidecar JSON content (which may contain backticks or $(...) sequences).
body_file=$(mktemp)
This PR was generated automatically by the weekly corpus-promote workflow.
It does **not** auto-merge — a human reviewer must approve each candidate
before it can land in `src/dynamic/corpus.rs` (§16.4).
cat > "$body_file" <<'PREAMBLE'
## Corpus Promotion Proposal
### Candidates
This PR was generated automatically by the weekly corpus-promote workflow.
It does **not** auto-merge — a human reviewer must approve each candidate
before it can land in `src/dynamic/corpus.rs` (§16.4).
The following payloads were discovered by the internal mutation fuzzer and
confirmed via `sink_hit && oracle_fired` against instrumented fixtures:
### Candidates
EOF
)
The following payloads were discovered by the internal mutation fuzzer and
confirmed via `sink_hit && oracle_fired` against instrumented fixtures:
PREAMBLE
for f in $CANDIDATE_FILES; do
sidecar="${f}.json"
printf -- '- `%s`\n' "$f" >> "$body_file"
if [ -f "$sidecar" ]; then
body="$body\n- \`$f\`\n \`\`\`json\n$(cat "$sidecar")\n \`\`\`\n"
else
body="$body\n- \`$f\`\n"
printf ' ```json\n' >> "$body_file"
cat "$sidecar" >> "$body_file"
printf '\n ```\n' >> "$body_file"
fi
done
body="$body\n### Review checklist\n"
body="$body\n- [ ] Bytes are a genuine attack vector, not a fixture artifact\n"
body="$body\n- [ ] Oracle marker is unique (no collision with other caps)\n"
body="$body\n- [ ] \`fixture_paths\` updated in \`src/dynamic/corpus.rs\`\n"
body="$body\n- [ ] \`since_corpus_version\` set to next version\n"
body="$body\n- [ ] \`CORPUS_VERSION\` bumped and bump history updated\n"
body="$body\n\n_Generated by corpus_promote.yml — do not auto-merge._\n"
cat >> "$body_file" <<'CHECKLIST'
### Review checklist
- [ ] Bytes are a genuine attack vector, not a fixture artifact
- [ ] Oracle marker is unique (no collision with other caps)
- [ ] `fixture_paths` updated in `src/dynamic/corpus.rs`
- [ ] `since_corpus_version` set to next version
- [ ] `CORPUS_VERSION` bumped and bump history updated
_Generated by corpus_promote.yml — do not auto-merge._
CHECKLIST
git add fuzz-discovered/ || true
git diff --cached --quiet || git commit -m "chore: add ${CANDIDATE_COUNT} fuzzer-discovered corpus candidates"
@ -146,10 +154,12 @@ jobs:
gh pr create \
--title "chore(corpus): promote ${CANDIDATE_COUNT} fuzzer-discovered payload(s)" \
--body "$(printf '%b' "$body")" \
--body "$(cat "$body_file")" \
--base master \
--label "corpus-promotion" || true
rm -f "$body_file"
- name: Dry run summary
if: github.event.inputs.dry_run == 'true'
run: |

1
.gitignore vendored
View file

@ -1,6 +1,7 @@
/target
/fuzz/target
/fuzz/corpus
/fuzz/dynamic_corpus/target
/fuzz/artifacts
/.idea
/frontend/node_modules

View file

@ -2,6 +2,30 @@
export type Confidence = 'Low' | 'Medium' | 'High';
export type FlowStepKind = 'source' | 'assignment' | 'call' | 'phi' | 'sink';
// Dynamic verification types (from src/evidence.rs VerifyStatus / VerifyResult)
export type VerifyStatus = 'Confirmed' | 'NotConfirmed' | 'Inconclusive' | 'Unsupported';
export interface AttemptSummary {
payload_label: string;
exit_code?: number;
timed_out: boolean;
triggered: boolean;
sink_hit?: boolean;
}
export interface VerifyResult {
finding_id: string;
status: VerifyStatus;
triggered_payload?: string;
/** Typed UnsupportedReason (PascalCase string) */
reason?: string;
/** Typed InconclusiveReason (PascalCase string) */
inconclusive_reason?: string;
detail?: string;
attempts: AttemptSummary[];
toolchain_match?: string;
}
export interface FlowStep {
step: number;
kind: FlowStepKind;
@ -40,6 +64,8 @@ export interface Evidence {
flow_steps: FlowStep[];
explanation?: string;
confidence_limiters: string[];
/** Dynamic verification result; present only when --verify was active. */
dynamic_verdict?: VerifyResult;
}
// Finding types

View file

@ -0,0 +1,57 @@
import type { VerifyResult, VerifyStatus } from '../api/types';
const STATUS_LABELS: Record<VerifyStatus, string> = {
Confirmed: 'Confirmed',
NotConfirmed: 'Not confirmed',
Inconclusive: 'Inconclusive',
Unsupported: 'Unsupported',
};
function verdictTooltip(verdict: VerifyResult): string {
const { status, triggered_payload, reason, inconclusive_reason, detail } =
verdict;
switch (status) {
case 'Confirmed':
return triggered_payload
? `Confirmed via payload: ${triggered_payload}`
: 'Dynamically confirmed exploitable';
case 'NotConfirmed':
return verdict.attempts.length > 0
? `Not confirmed after ${verdict.attempts.length} payload attempt(s)`
: 'Not confirmed';
case 'Unsupported':
return reason ? `Unsupported: ${reason}` : 'Dynamic verification not supported';
case 'Inconclusive':
return inconclusive_reason
? `Inconclusive: ${inconclusive_reason}${detail ? `: ${detail}` : ''}`
: detail || 'Inconclusive';
}
}
interface VerdictBadgeProps {
verdict: VerifyResult | undefined;
/** Show full label (default) or compact icon-only mode */
compact?: boolean;
}
export function VerdictBadge({ verdict, compact = false }: VerdictBadgeProps) {
if (!verdict) {
return <span style={{ color: 'var(--text-tertiary)' }}>-</span>;
}
const { status } = verdict;
const label = STATUS_LABELS[status] ?? status;
const tooltip = verdictTooltip(verdict);
const flame = status === 'Confirmed' ? '🔥 ' : '';
return (
<span
className={`badge badge-dyn-${status.toLowerCase()}`}
title={tooltip}
data-testid={`verdict-badge-${status.toLowerCase()}`}
>
{flame}
{compact ? status.charAt(0) : label}
</span>
);
}

View file

@ -8,6 +8,7 @@ import { escapeHtml, highlightSyntax } from '../utils/syntaxHighlight';
import { parseNoteText } from '../utils/parseNote';
import { findingToMarkdown } from '../utils/findingMarkdown';
import { CopyMarkdownButton } from '../components/CopyMarkdownButton';
import { VerdictBadge } from '../components/VerdictBadge';
import { Dropdown, DropdownItem } from '../components/ui/Dropdown';
import { CodeViewerModal } from '../modals/CodeViewerModal';
import type {
@ -16,6 +17,7 @@ import type {
FlowStep,
SpanEvidence,
RelatedFindingView,
VerifyResult,
} from '../api/types';
// ── Helpers ─────────────────────────────────────────────────────────────────
@ -701,6 +703,97 @@ function HowToFix({ finding }: { finding: FindingView }) {
);
}
// ── Dynamic Verification Panel ──────────────────────────────────────────────
function DynamicVerdictSection({ verdict }: { verdict: VerifyResult }) {
const [copied, setCopied] = useState(false);
const reproPath = `~/.cache/nyx/dynamic/repro/${verdict.finding_id}/`;
const reproCmd = './reproduce.sh';
const copyCmd = () => {
navigator.clipboard.writeText(reproCmd).then(() => {
setCopied(true);
setTimeout(() => setCopied(false), 2000);
});
};
return (
<div className="dynamic-verdict-section">
<div className="dynamic-verdict-badge-row">
<VerdictBadge verdict={verdict} />
{verdict.toolchain_match && (
<span
className="dynamic-toolchain-match"
title={`Toolchain match: ${verdict.toolchain_match}`}
>
{verdict.toolchain_match === 'exact' ? 'exact toolchain' : 'approximate toolchain'}
</span>
)}
</div>
{verdict.status === 'Confirmed' && (
<div className="repro-panel" data-testid="repro-panel">
<div className="repro-path-row">
<span className="repro-label">Repro artifact:</span>
<code className="repro-path">{reproPath}</code>
</div>
<div className="repro-cmd-row">
<code className="repro-cmd">{reproCmd}</code>
<button
type="button"
className="btn btn-sm repro-copy-btn"
onClick={copyCmd}
>
{copied ? 'Copied!' : 'Copy'}
</button>
</div>
</div>
)}
{(verdict.reason || verdict.inconclusive_reason || verdict.detail) && (
<div className="dynamic-verdict-detail">
{verdict.reason && (
<div>
<strong>Reason:</strong> {verdict.reason}
</div>
)}
{verdict.inconclusive_reason && (
<div>
<strong>Inconclusive reason:</strong> {verdict.inconclusive_reason}
</div>
)}
{verdict.detail && (
<div className="dynamic-verdict-detail-text">{verdict.detail}</div>
)}
</div>
)}
{verdict.attempts.length > 0 && (
<div className="dynamic-attempts">
<strong>Payload attempts:</strong>
<ul className="dynamic-attempt-list">
{verdict.attempts.map((a, i) => (
<li key={i} className={`attempt-row ${a.triggered ? 'triggered' : ''}`}>
<code>{a.payload_label}</code>
<span className="attempt-outcome">
{a.triggered
? 'triggered'
: a.timed_out
? 'timeout'
: 'no hit'}
</span>
{a.exit_code != null && (
<span className="attempt-exit-code">exit {a.exit_code}</span>
)}
</li>
))}
</ul>
</div>
)}
</div>
);
}
// ── Status Control ──────────────────────────────────────────────────────────
function StatusControl({
@ -1017,6 +1110,13 @@ export function FindingDetailPage() {
</CollapsibleSection>
)}
{/* Dynamic Verification */}
{evidence?.dynamic_verdict && (
<CollapsibleSection title="Dynamic Verification">
<DynamicVerdictSection verdict={evidence.dynamic_verdict} />
</CollapsibleSection>
)}
{/* Code Preview */}
{hasCode && (
<CollapsibleSection title="Code Preview" defaultOpen={false}>

View file

@ -17,6 +17,7 @@ import { Dropdown, DropdownItem } from '../components/ui/Dropdown';
import { LoadingState } from '../components/ui/LoadingState';
import { ErrorState } from '../components/ui/ErrorState';
import { CopyMarkdownButton } from '../components/CopyMarkdownButton';
import { VerdictBadge } from '../components/VerdictBadge';
import { truncPath } from '../utils/truncPath';
import { findingsToMarkdown } from '../utils/findingMarkdown';
import { ApiError } from '../api/client';
@ -711,6 +712,7 @@ export function FindingsPage() {
currentDir={state.sort_dir}
onSort={handleSort}
/>
<th>Verified</th>
</tr>
</thead>
<tbody>
@ -760,6 +762,12 @@ export function FindingsPage() {
{formatTriageState(f.triage_state || f.status)}
</span>
</td>
<td>
<VerdictBadge
verdict={f.evidence?.dynamic_verdict}
compact
/>
</td>
</tr>
))}
</tbody>

View file

@ -0,0 +1,110 @@
import { describe, it, expect } from 'vitest';
import { render, screen } from '@testing-library/react';
import { VerdictBadge } from '@/components/VerdictBadge';
import type { VerifyResult } from '@/api/types';
function makeVerdict(
status: VerifyResult['status'],
extras: Partial<VerifyResult> = {},
): VerifyResult {
return {
finding_id: 'test-finding-id',
status,
attempts: [],
...extras,
};
}
describe('VerdictBadge', () => {
it('renders dash when verdict is undefined', () => {
render(<VerdictBadge verdict={undefined} />);
expect(screen.getByText('-')).toBeInTheDocument();
});
it('renders Confirmed badge with flame and correct class', () => {
render(
<VerdictBadge
verdict={makeVerdict('Confirmed', { triggered_payload: 'sqli-tautology' })}
/>,
);
const badge = screen.getByTestId('verdict-badge-confirmed');
expect(badge).toBeInTheDocument();
expect(badge.className).toContain('badge-dyn-confirmed');
expect(badge.textContent).toContain('🔥');
});
it('renders NotConfirmed badge with correct class', () => {
render(<VerdictBadge verdict={makeVerdict('NotConfirmed')} />);
const badge = screen.getByTestId('verdict-badge-notconfirmed');
expect(badge).toBeInTheDocument();
expect(badge.className).toContain('badge-dyn-notconfirmed');
expect(badge.textContent).not.toContain('🔥');
});
it('renders Unsupported badge with correct class', () => {
render(
<VerdictBadge
verdict={makeVerdict('Unsupported', { reason: 'NoPayloadsForCap' })}
/>,
);
const badge = screen.getByTestId('verdict-badge-unsupported');
expect(badge).toBeInTheDocument();
expect(badge.className).toContain('badge-dyn-unsupported');
});
it('renders Inconclusive badge with amber class', () => {
render(
<VerdictBadge
verdict={makeVerdict('Inconclusive', {
inconclusive_reason: 'BuildFailed',
})}
/>,
);
const badge = screen.getByTestId('verdict-badge-inconclusive');
expect(badge).toBeInTheDocument();
expect(badge.className).toContain('badge-dyn-inconclusive');
});
it('tooltip contains payload for Confirmed', () => {
render(
<VerdictBadge
verdict={makeVerdict('Confirmed', { triggered_payload: 'sqli-payload' })}
/>,
);
const badge = screen.getByTestId('verdict-badge-confirmed');
expect(badge.getAttribute('title')).toContain('sqli-payload');
});
it('tooltip contains reason for Unsupported', () => {
render(
<VerdictBadge
verdict={makeVerdict('Unsupported', { reason: 'ConfidenceTooLow' })}
/>,
);
const badge = screen.getByTestId('verdict-badge-unsupported');
expect(badge.getAttribute('title')).toContain('ConfidenceTooLow');
});
it('compact mode renders single character', () => {
render(<VerdictBadge verdict={makeVerdict('Confirmed')} compact />);
const badge = screen.getByTestId('verdict-badge-confirmed');
// Compact: first char of status + flame emoji
expect(badge.textContent?.replace('🔥 ', '')).toBe('C');
});
it('renders all four VerifyStatus variants without crashing', () => {
const statuses: VerifyResult['status'][] = [
'Confirmed',
'NotConfirmed',
'Unsupported',
'Inconclusive',
];
for (const status of statuses) {
const { unmount } = render(<VerdictBadge verdict={makeVerdict(status)} />);
expect(
screen.getByTestId(`verdict-badge-${status.toLowerCase()}`),
).toBeInTheDocument();
unmount();
}
});
});

2352
fuzz/dynamic_corpus/Cargo.lock generated Normal file

File diff suppressed because it is too large Load diff

View file

@ -25,7 +25,6 @@ use nyx_scanner::dynamic::corpus::{
};
use nyx_scanner::labels::Cap;
use std::collections::HashSet;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::time::SystemTime;

View file

@ -16,7 +16,6 @@ Usage:
import argparse
import json
import os
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path

View file

@ -80,6 +80,24 @@ impl OobListener {
.map(|h| h.contains(nonce))
.unwrap_or(false)
}
/// Polls until `nonce` is recorded or `timeout` elapses.
///
/// Returns immediately on hit; polls every 5 ms otherwise.
/// Prefer this over a fixed sleep + `was_nonce_hit` at call sites.
pub fn wait_for_nonce(&self, nonce: &str, timeout: Duration) -> bool {
let deadline = std::time::Instant::now() + timeout;
loop {
if self.was_nonce_hit(nonce) {
return true;
}
let remaining = deadline.saturating_duration_since(std::time::Instant::now());
if remaining.is_zero() {
return false;
}
std::thread::sleep(remaining.min(Duration::from_millis(5)));
}
}
}
impl Drop for OobListener {

View file

@ -8,7 +8,7 @@
use crate::dynamic::build_sandbox;
use crate::dynamic::corpus::{benign_payload_for, materialise_bytes, payloads_for, Oracle, Payload};
use crate::dynamic::harness::{self, HarnessError};
use crate::dynamic::sandbox::{self, SandboxError, SandboxOptions, SandboxOutcome};
use crate::dynamic::sandbox::{self, SandboxBackend, SandboxError, SandboxOptions, SandboxOutcome};
use crate::dynamic::spec::HarnessSpec;
use crate::symbol::Lang;
@ -214,7 +214,11 @@ pub fn run_spec(spec: &HarnessSpec, opts: &SandboxOptions) -> Result<RunOutcome,
let (oob_nonce, effective_bytes) = if payload.oob_nonce_slot {
if let Some(ref listener) = opts.oob_listener {
let nonce = generate_nonce();
let url = listener.nonce_url(&nonce);
let url = if uses_docker_backend(opts) {
listener.nonce_url_for_host("host-gateway", &nonce)
} else {
listener.nonce_url(&nonce)
};
let bytes = url.into_bytes();
(Some(nonce), bytes)
} else {
@ -229,12 +233,10 @@ pub fn run_spec(spec: &HarnessSpec, opts: &SandboxOptions) -> Result<RunOutcome,
// For OOB payloads, check the nonce listener and update the outcome flag.
if let (Some(nonce), Some(listener)) = (&oob_nonce, &opts.oob_listener) {
// Give the harness a brief window to complete the callback before we check.
// The sandbox run already waited for process exit, so the callback should
// have arrived. A short sleep handles edge cases where the OS hasn't yet
// delivered the TCP segment to the listener thread.
std::thread::sleep(std::time::Duration::from_millis(50));
if listener.was_nonce_hit(nonce) {
// Poll until the nonce arrives or the budget expires. The sandbox run
// already waited for process exit so the callback should arrive quickly;
// 200 ms covers OS TCP delivery jitter without burning wall-clock at scale.
if listener.wait_for_nonce(nonce, std::time::Duration::from_millis(200)) {
outcome.oob_callback_seen = true;
}
}
@ -287,6 +289,18 @@ pub fn run_spec(spec: &HarnessSpec, opts: &SandboxOptions) -> Result<RunOutcome,
})
}
/// Returns true when the active backend will use Docker for execution.
///
/// Used at URL-generation time so Docker runs embed `host-gateway` rather than
/// `127.0.0.1` (the container's loopback ≠ the host's loopback).
fn uses_docker_backend(opts: &SandboxOptions) -> bool {
match opts.backend {
SandboxBackend::Docker => true,
SandboxBackend::Auto => sandbox::docker_available(),
SandboxBackend::Process => false,
}
}
fn oracle_fired(oracle: &Oracle, outcome: &SandboxOutcome) -> bool {
match oracle {
Oracle::OutputContains(needle) => {

View file

@ -424,6 +424,14 @@ fn render_diag(d: &Diag, width: usize) -> String {
));
}
// ── Dynamic verification annotation ──────────────────────────────
if let Some(ev) = d.evidence.as_ref() {
if let Some(ref dv) = ev.dynamic_verdict {
let annotation = format_dynamic_verdict_annotation(dv);
out.push_str(&format!("{indent_str}{}\n", style(&annotation).dim()));
}
}
out
}
@ -453,6 +461,67 @@ fn state_remediation_hint(rule_id: &str) -> Option<&'static str> {
}
}
/// Format a dynamic verification annotation line.
///
/// Spec §5.4: `[DYN: confirmed via {payload}]` / `[DYN: not confirmed]` /
/// `[DYN: unsupported ({reason})]` / `[DYN: inconclusive ({reason})]`
fn format_dynamic_verdict_annotation(dv: &crate::evidence::VerifyResult) -> String {
use crate::evidence::VerifyStatus;
match dv.status {
VerifyStatus::Confirmed => {
let pid = dv.triggered_payload.as_deref().unwrap_or("unknown");
format!("[DYN: confirmed via {pid}]")
}
VerifyStatus::NotConfirmed => "[DYN: not confirmed]".to_string(),
VerifyStatus::Unsupported => {
let reason = dv
.reason
.as_ref()
.map(format_unsupported_reason)
.unwrap_or_else(|| "unknown".to_string());
format!("[DYN: unsupported ({reason})]")
}
VerifyStatus::Inconclusive => {
let reason = dv
.inconclusive_reason
.map(format_inconclusive_reason)
.unwrap_or_else(|| {
dv.detail
.as_deref()
.map(|d| d.chars().take(40).collect())
.unwrap_or_else(|| "unknown".to_string())
});
format!("[DYN: inconclusive ({reason})]")
}
}
}
fn format_unsupported_reason(r: &crate::evidence::UnsupportedReason) -> String {
use crate::evidence::UnsupportedReason;
match r {
UnsupportedReason::BackendUnavailable => "backend unavailable".to_string(),
UnsupportedReason::EntryKindUnsupported => "entry kind not supported".to_string(),
UnsupportedReason::ConfidenceTooLow => "confidence too low".to_string(),
UnsupportedReason::NoFlowSteps => "no flow steps".to_string(),
UnsupportedReason::NoPayloadsForCap => "no payloads for cap".to_string(),
UnsupportedReason::SpecDerivationFailed => "spec derivation failed".to_string(),
UnsupportedReason::RequiredFileRedactedForSecrets(_) => {
"file redacted for secrets".to_string()
}
UnsupportedReason::LangUnsupported => "language not supported".to_string(),
}
}
fn format_inconclusive_reason(r: crate::evidence::InconclusiveReason) -> String {
use crate::evidence::InconclusiveReason;
match r {
InconclusiveReason::OracleCollisionSuspected => "oracle collision".to_string(),
InconclusiveReason::NonReproducible => "non-reproducible".to_string(),
InconclusiveReason::BuildFailed => "build failed".to_string(),
InconclusiveReason::SandboxError => "sandbox error".to_string(),
}
}
/// Colored severity tag with icon. The tag is the visual anchor of each finding.
///
/// - HIGH: bold red

View file

@ -282,6 +282,21 @@ pub fn build_sarif(diags: &[Diag], scan_root: &Path) -> Value {
}
}
// Dynamic verification vendor extension (§5.4).
// `partialFingerprints.dynamic_verdict_status` is a stable string
// consumers can key on without parsing the full verdict object.
// `properties.nyx_dynamic_verdict` carries the full VerifyResult.
if let Some(dv) = d.evidence.as_ref().and_then(|ev| ev.dynamic_verdict.as_ref()) {
result["partialFingerprints"] = json!({
"dynamic_verdict_status": serde_json::to_value(dv.status)
.unwrap_or(Value::Null)
});
props.insert(
"nyx_dynamic_verdict".into(),
serde_json::to_value(dv).unwrap_or(Value::Null),
);
}
// Add rollup data if present
if let Some(ref rollup) = d.rollup {
props.insert(

View file

@ -90,6 +90,22 @@ pub fn compute_attack_rank(diag: &Diag) -> AttackRank {
}
}
// ── 7a. Dynamic verification delta ─────────────────────────────
//
// `Confirmed` findings are verified exploitable — boost rank so they
// surface above equivalent static-only findings.
// `NotConfirmed` findings where all available payloads were tried
// (corpus exhausted) receive a mild downward nudge.
// All other verdicts (Unsupported, Inconclusive, no verdict) are
// unaffected: no data is better than speculative data.
//
// TODO(M7): calibrate N (boost) and M (penalty) from telemetry
// collected here. Placeholder values: N=20, M=5.
if let Some(delta) = dynamic_verdict_delta(diag) {
score += delta;
components.push(("dynamic_verdict".into(), format!("{delta:+}")));
}
// ── 7. Completeness penalty (engine provenance notes) ────────────
//
// When the analysis engine hit a cap, widening, or lowering bail,
@ -204,6 +220,26 @@ pub fn rank_diags(diags: &mut [Diag]) {
// Scoring helpers
// ─────────────────────────────────────────────────────────────────────────────
/// Rank delta from the dynamic verification verdict.
///
/// Returns `None` when there is no verdict (static-only scan) or the verdict
/// does not change the score (Unsupported, Inconclusive).
///
/// TODO(M7): N=20 and M=5 are placeholders; calibrate from telemetry.
fn dynamic_verdict_delta(diag: &Diag) -> Option<f64> {
use crate::evidence::VerifyStatus;
let dv = diag.evidence.as_ref()?.dynamic_verdict.as_ref()?;
match dv.status {
VerifyStatus::Confirmed => Some(20.0),
// Apply penalty only when the corpus was actually exhausted (attempts
// were made); a NotConfirmed with zero attempts means something went
// wrong before payload execution, which is an Inconclusive path, not
// a meaningful negative signal.
VerifyStatus::NotConfirmed if !dv.attempts.is_empty() => Some(-5.0),
_ => None,
}
}
/// Bonus based on analysis kind inferred from rule ID + evidence.
fn analysis_kind_bonus(rule_id: &str, evidence: Option<&Evidence>) -> f64 {
if rule_id.starts_with("taint-data-exfiltration") {

188
tests/console_snapshot.rs Normal file
View file

@ -0,0 +1,188 @@
//! Snapshot-style tests for the `[DYN: ...]` annotation in console output.
//!
//! Each `VerifyStatus` variant must produce the correct dim annotation line
//! beneath the finding block when `evidence.dynamic_verdict` is set.
use nyx_scanner::commands::scan::Diag;
use nyx_scanner::evidence::{
AttemptSummary, Evidence, InconclusiveReason, UnsupportedReason, VerifyResult, VerifyStatus,
};
use nyx_scanner::fmt::render_console;
use nyx_scanner::patterns::{FindingCategory, Severity};
// ── Helper ───────────────────────────────────────────────────────────────────
fn strip_ansi(s: &str) -> String {
let mut out = String::new();
let mut in_escape = false;
for ch in s.chars() {
if ch == '\x1b' {
in_escape = true;
} else if in_escape {
if ch == 'm' {
in_escape = false;
}
} else {
out.push(ch);
}
}
out
}
fn base_diag() -> Diag {
Diag {
path: "src/main.rs".into(),
line: 42,
col: 5,
severity: Severity::High,
id: "taint-unsanitised-flow".into(),
category: FindingCategory::Security,
path_validated: false,
guard_kind: None,
message: Some("unsanitised input flows to exec".into()),
labels: vec![],
confidence: None,
evidence: None,
rank_score: None,
rank_reason: None,
suppressed: false,
suppression: None,
rollup: None,
finding_id: String::new(),
alternative_finding_ids: Vec::new(),
stable_hash: 0,
}
}
fn diag_with_verdict(status: VerifyStatus) -> Diag {
let verdict = match status {
VerifyStatus::Confirmed => VerifyResult {
finding_id: "abc123".into(),
status,
triggered_payload: Some("sqli-tautology".into()),
reason: None,
inconclusive_reason: None,
detail: None,
attempts: vec![AttemptSummary {
payload_label: "sqli-tautology".into(),
exit_code: Some(0),
timed_out: false,
triggered: true,
sink_hit: true,
}],
toolchain_match: Some("exact".into()),
},
VerifyStatus::NotConfirmed => VerifyResult {
finding_id: "abc123".into(),
status,
triggered_payload: None,
reason: None,
inconclusive_reason: None,
detail: None,
attempts: vec![AttemptSummary {
payload_label: "sqli-tautology".into(),
exit_code: Some(0),
timed_out: false,
triggered: false,
sink_hit: false,
}],
toolchain_match: Some("exact".into()),
},
VerifyStatus::Unsupported => VerifyResult {
finding_id: "abc123".into(),
status,
triggered_payload: None,
reason: Some(UnsupportedReason::NoPayloadsForCap),
inconclusive_reason: None,
detail: None,
attempts: vec![],
toolchain_match: None,
},
VerifyStatus::Inconclusive => VerifyResult {
finding_id: "abc123".into(),
status,
triggered_payload: None,
reason: None,
inconclusive_reason: Some(InconclusiveReason::BuildFailed),
detail: Some("build failed after 3 attempts: linker error".into()),
attempts: vec![],
toolchain_match: None,
},
};
let mut d = base_diag();
d.evidence = Some(Evidence {
dynamic_verdict: Some(verdict),
..Default::default()
});
d
}
// ── Tests ────────────────────────────────────────────────────────────────────
#[test]
fn console_confirmed_shows_payload_id() {
let diag = diag_with_verdict(VerifyStatus::Confirmed);
let output = render_console(&[diag], "proj", None);
let stripped = strip_ansi(&output);
assert!(
stripped.contains("[DYN: confirmed via sqli-tautology]"),
"expected DYN confirmed annotation, got:\n{stripped}"
);
}
#[test]
fn console_not_confirmed_shows_annotation() {
let diag = diag_with_verdict(VerifyStatus::NotConfirmed);
let output = render_console(&[diag], "proj", None);
let stripped = strip_ansi(&output);
assert!(
stripped.contains("[DYN: not confirmed]"),
"expected DYN not-confirmed annotation, got:\n{stripped}"
);
}
#[test]
fn console_unsupported_shows_reason() {
let diag = diag_with_verdict(VerifyStatus::Unsupported);
let output = render_console(&[diag], "proj", None);
let stripped = strip_ansi(&output);
assert!(
stripped.contains("[DYN: unsupported (no payloads for cap)]"),
"expected DYN unsupported annotation, got:\n{stripped}"
);
}
#[test]
fn console_inconclusive_shows_reason() {
let diag = diag_with_verdict(VerifyStatus::Inconclusive);
let output = render_console(&[diag], "proj", None);
let stripped = strip_ansi(&output);
assert!(
stripped.contains("[DYN: inconclusive (build failed)]"),
"expected DYN inconclusive annotation, got:\n{stripped}"
);
}
#[test]
fn console_no_annotation_when_no_dynamic_verdict() {
let diag = base_diag();
let output = render_console(&[diag], "proj", None);
let stripped = strip_ansi(&output);
assert!(
!stripped.contains("[DYN:"),
"expected no DYN annotation when evidence is None:\n{stripped}"
);
}
#[test]
fn console_no_annotation_when_evidence_has_no_verdict() {
let mut diag = base_diag();
diag.evidence = Some(Evidence::default());
let output = render_console(&[diag], "proj", None);
let stripped = strip_ansi(&output);
assert!(
!stripped.contains("[DYN:"),
"expected no DYN annotation when dynamic_verdict is None:\n{stripped}"
);
}

173
tests/json_snapshot.rs Normal file
View file

@ -0,0 +1,173 @@
//! Snapshot-style tests for `evidence.dynamic_verdict` in JSON output.
//!
//! When `--verify` is active and produces a verdict, the serialized `Diag`
//! must carry `evidence.dynamic_verdict` with the correct status string and
//! all other fields. When no verdict is set the key must be absent (due to
//! `skip_serializing_if = "Option::is_none"`).
use nyx_scanner::commands::scan::Diag;
use nyx_scanner::evidence::{
AttemptSummary, Evidence, VerifyResult, VerifyStatus,
};
use nyx_scanner::patterns::{FindingCategory, Severity};
fn base_diag() -> Diag {
Diag {
path: "src/main.rs".into(),
line: 10,
col: 5,
severity: Severity::High,
id: "taint-unsanitised-flow".into(),
category: FindingCategory::Security,
path_validated: false,
guard_kind: None,
message: None,
labels: vec![],
confidence: None,
evidence: None,
rank_score: None,
rank_reason: None,
suppressed: false,
suppression: None,
rollup: None,
finding_id: String::new(),
alternative_finding_ids: Vec::new(),
stable_hash: 0,
}
}
// ── Tests ────────────────────────────────────────────────────────────────────
#[test]
fn json_dynamic_verdict_confirmed_serialises_correctly() {
let mut diag = base_diag();
diag.evidence = Some(Evidence {
dynamic_verdict: Some(VerifyResult {
finding_id: "deadbeef01234567".into(),
status: VerifyStatus::Confirmed,
triggered_payload: Some("sqli-tautology".into()),
reason: None,
inconclusive_reason: None,
detail: None,
attempts: vec![AttemptSummary {
payload_label: "sqli-tautology".into(),
exit_code: Some(0),
timed_out: false,
triggered: true,
sink_hit: true,
}],
toolchain_match: Some("exact".into()),
}),
..Default::default()
});
let json = serde_json::to_string(&diag).expect("serialisation must succeed");
assert!(
json.contains("\"dynamic_verdict\""),
"JSON must contain dynamic_verdict key: {json}"
);
assert!(
json.contains("\"Confirmed\""),
"JSON must contain Confirmed status: {json}"
);
assert!(
json.contains("\"sqli-tautology\""),
"JSON must contain triggered payload: {json}"
);
assert!(
json.contains("\"finding_id\""),
"JSON must contain finding_id: {json}"
);
}
#[test]
fn json_dynamic_verdict_not_confirmed_serialises_correctly() {
let mut diag = base_diag();
diag.evidence = Some(Evidence {
dynamic_verdict: Some(VerifyResult {
finding_id: "abcd1234abcd1234".into(),
status: VerifyStatus::NotConfirmed,
triggered_payload: None,
reason: None,
inconclusive_reason: None,
detail: None,
attempts: vec![],
toolchain_match: Some("exact".into()),
}),
..Default::default()
});
let json = serde_json::to_string(&diag).expect("serialisation must succeed");
assert!(
json.contains("\"NotConfirmed\""),
"JSON must contain NotConfirmed status: {json}"
);
// triggered_payload is None → must not appear (skip_serializing_if)
assert!(
!json.contains("\"triggered_payload\""),
"triggered_payload None must be omitted: {json}"
);
}
#[test]
fn json_no_dynamic_verdict_when_not_set() {
let mut diag = base_diag();
diag.evidence = Some(Evidence::default());
let json = serde_json::to_string(&diag).expect("serialisation must succeed");
// dynamic_verdict is None → must not appear (skip_serializing_if)
assert!(
!json.contains("dynamic_verdict"),
"dynamic_verdict must be absent when not set: {json}"
);
}
#[test]
fn json_no_evidence_no_dynamic_verdict() {
let diag = base_diag();
let json = serde_json::to_string(&diag).expect("serialisation must succeed");
assert!(
!json.contains("evidence"),
"evidence must be absent when None: {json}"
);
assert!(
!json.contains("dynamic_verdict"),
"dynamic_verdict must be absent when evidence is None: {json}"
);
}
#[test]
fn json_unsupported_verdict_has_reason() {
use nyx_scanner::evidence::UnsupportedReason;
let mut diag = base_diag();
diag.evidence = Some(Evidence {
dynamic_verdict: Some(VerifyResult {
finding_id: "0000000000000000".into(),
status: VerifyStatus::Unsupported,
triggered_payload: None,
reason: Some(UnsupportedReason::ConfidenceTooLow),
inconclusive_reason: None,
detail: None,
attempts: vec![],
toolchain_match: None,
}),
..Default::default()
});
let json = serde_json::to_string(&diag).expect("serialisation must succeed");
assert!(
json.contains("\"Unsupported\""),
"JSON must contain Unsupported status: {json}"
);
assert!(
json.contains("\"ConfidenceTooLow\""),
"JSON must contain typed reason: {json}"
);
}