mirror of
https://github.com/elicpeter/nyx.git
synced 2026-06-06 19:35:13 +02:00
Dynamic (#77)
This commit is contained in:
parent
55247b7fcd
commit
991c84a1eb
1464 changed files with 225448 additions and 1985 deletions
106
scripts/check_corpus_sync.py
Normal file
106
scripts/check_corpus_sync.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
#!/usr/bin/env python3
|
||||
# Usage: python3 scripts/check_corpus_sync.py
|
||||
# Run from repo root or any subdirectory; the script relocates to repo root.
|
||||
# Exits 0 if scripts/corpus_dashboard.py reads the same CORPUS_VERSION and
|
||||
# payload identities as the canonical Rust registry.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
REPO_ROOT = SCRIPT_DIR.parent
|
||||
os.chdir(REPO_ROOT)
|
||||
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
import corpus_dashboard # noqa: E402
|
||||
|
||||
CORPUS_RS = REPO_ROOT / "src" / "dynamic" / "corpus.rs"
|
||||
CORPUS_DIR = REPO_ROOT / "src" / "dynamic" / "corpus"
|
||||
|
||||
|
||||
def parse_corpus_rs_version(path: Path) -> int | None:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
version_match = re.search(r"pub const CORPUS_VERSION:\s*u32\s*=\s*(\d+);", text)
|
||||
return int(version_match.group(1)) if version_match else None
|
||||
|
||||
|
||||
def payload_identities(payloads: list[corpus_dashboard.PayloadEntry]) -> set[tuple[str, str, str]]:
|
||||
return {(p.cap, p.lang, p.label) for p in payloads}
|
||||
|
||||
|
||||
def count_raw_payload_blocks(path: Path = CORPUS_DIR) -> int:
|
||||
count = 0
|
||||
for source in path.rglob("*.rs"):
|
||||
if source.name in {"audit.rs", "mod.rs", "registry.rs"}:
|
||||
continue
|
||||
text = source.read_text(encoding="utf-8")
|
||||
count += len(re.findall(r"\bCuratedPayload\s*\{", text))
|
||||
return count
|
||||
|
||||
|
||||
def fmt_identity(identity: tuple[str, str, str]) -> str:
|
||||
cap, lang, label = identity
|
||||
return f"{cap}/{lang}/{label}"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
rs_version = parse_corpus_rs_version(CORPUS_RS)
|
||||
dashboard_version = corpus_dashboard.CORPUS_VERSION
|
||||
registry_payloads = corpus_dashboard.load_payloads()
|
||||
raw_payload_count = count_raw_payload_blocks()
|
||||
|
||||
ok = True
|
||||
|
||||
if rs_version is None:
|
||||
print("ERROR: CORPUS_VERSION not found in corpus.rs")
|
||||
ok = False
|
||||
elif rs_version == dashboard_version:
|
||||
print(f"CORPUS_VERSION: {rs_version} [match]")
|
||||
else:
|
||||
print(
|
||||
"CORPUS_VERSION mismatch: "
|
||||
f"corpus.rs={rs_version} corpus_dashboard.py={dashboard_version}"
|
||||
)
|
||||
ok = False
|
||||
|
||||
registry_ids = payload_identities(registry_payloads)
|
||||
dashboard_ids = payload_identities(corpus_dashboard.PAYLOADS)
|
||||
only_in_registry = registry_ids - dashboard_ids
|
||||
only_in_dashboard = dashboard_ids - registry_ids
|
||||
shared = registry_ids & dashboard_ids
|
||||
|
||||
print(f"Payload identities in both: {len(shared)}")
|
||||
if only_in_registry:
|
||||
print(f"Payload identities only in Rust registry: {len(only_in_registry)}")
|
||||
for identity in sorted(only_in_registry):
|
||||
print(f" + {fmt_identity(identity)}")
|
||||
ok = False
|
||||
if only_in_dashboard:
|
||||
print(f"Payload identities only in dashboard: {len(only_in_dashboard)}")
|
||||
for identity in sorted(only_in_dashboard):
|
||||
print(f" - {fmt_identity(identity)}")
|
||||
ok = False
|
||||
|
||||
if len(corpus_dashboard.PAYLOADS) == raw_payload_count:
|
||||
print(f"CuratedPayload blocks covered: {raw_payload_count} [match]")
|
||||
else:
|
||||
print(
|
||||
"CuratedPayload block count mismatch: "
|
||||
f"source_tree={raw_payload_count} dashboard={len(corpus_dashboard.PAYLOADS)}"
|
||||
)
|
||||
ok = False
|
||||
|
||||
if ok:
|
||||
print("Corpus sync: OK")
|
||||
return 0
|
||||
|
||||
print("Corpus sync: FAIL - update corpus_dashboard.py to match the Rust registry")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
104
scripts/check_no_unseeded_rand.sh
Executable file
104
scripts/check_no_unseeded_rand.sh
Executable file
|
|
@ -0,0 +1,104 @@
|
|||
#!/usr/bin/env bash
|
||||
# Phase 30 — Track C: determinism audit gate.
|
||||
#
|
||||
# Greps `src/dynamic/` for non-deterministic RNG APIs. Anything inside
|
||||
# the dynamic verifier must route through `crate::dynamic::rand::SpecRng`
|
||||
# so identical inputs produce identical sandbox runs; the Phase 27
|
||||
# `events.jsonl` replay invariant and the Phase 28 repro bundle
|
||||
# hermeticity contract both depend on it.
|
||||
#
|
||||
# Exits 0 on a clean tree, 1 when any banned API surfaces. CI wires
|
||||
# this into the dynamic workflow so a regression fails the build before
|
||||
# it ships.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
DYN_DIR="$ROOT/src/dynamic"
|
||||
FUZZ_DIR="$ROOT/fuzz/dynamic_corpus/src"
|
||||
|
||||
if [[ ! -d "$DYN_DIR" ]]; then
|
||||
echo "audit: src/dynamic/ missing at $DYN_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# The dynamic-corpus mutation fuzzer is also audited: it routes every
|
||||
# randomness draw through `SpecRng::seeded(&spec.spec_hash)` so two
|
||||
# runs against the same fixture produce identical candidate streams,
|
||||
# matching the determinism contract of the verifier it feeds.
|
||||
if [[ ! -d "$FUZZ_DIR" ]]; then
|
||||
# Soft warn — the fuzzer is optional during early bootstrap.
|
||||
echo "audit: fuzz/dynamic_corpus/src/ missing at $FUZZ_DIR (skipping)" >&2
|
||||
fi
|
||||
|
||||
# Banned patterns: any real call site of a non-deterministic RNG API.
|
||||
#
|
||||
# Each pattern is a Rust-token shape we expect to never appear inside
|
||||
# src/dynamic/ once Phase 30 lands. The seccomp policy file (which
|
||||
# names the "getrandom" syscall as a string literal) is excluded
|
||||
# because its mention is a syscall name, not a Rust API call — the
|
||||
# string-literal regex below matches the bare token, and the seccomp
|
||||
# files spell it inside quotes that look identical, so we exclude the
|
||||
# seccomp subtree explicitly.
|
||||
PATTERNS=(
|
||||
'rand::thread_rng'
|
||||
'thread_rng\s*\('
|
||||
'rand::random'
|
||||
'OsRng'
|
||||
'from_entropy'
|
||||
'getrandom::getrandom'
|
||||
'Uuid::new_v4'
|
||||
'uuid::Uuid::new_v4'
|
||||
'fastrand'
|
||||
'nanoid'
|
||||
)
|
||||
|
||||
EXCLUDE_PATHS=(
|
||||
"$DYN_DIR/sandbox/seccomp"
|
||||
"$DYN_DIR/rand.rs"
|
||||
)
|
||||
|
||||
# Use `git grep` when inside a git repo (respects .gitignore), fall
|
||||
# back to `grep -r` otherwise. Either way the exclusion list is
|
||||
# applied via a post-filter so the audit catches new files even
|
||||
# before they are tracked.
|
||||
if git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
HITS="$(git -C "$ROOT" grep -nE "$(IFS='|'; echo "${PATTERNS[*]}")" \
|
||||
-- 'src/dynamic/**/*.rs' 'src/dynamic/*.rs' \
|
||||
'fuzz/dynamic_corpus/src/**/*.rs' 'fuzz/dynamic_corpus/src/*.rs' \
|
||||
|| true)"
|
||||
else
|
||||
HITS="$(grep -rnE "$(IFS='|'; echo "${PATTERNS[*]}")" --include='*.rs' \
|
||||
"$DYN_DIR" ${FUZZ_DIR:+"$FUZZ_DIR"} || true)"
|
||||
fi
|
||||
|
||||
if [[ -z "$HITS" ]]; then
|
||||
echo "audit: src/dynamic/ is free of unseeded RNG APIs"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
FILTERED=""
|
||||
while IFS= read -r line; do
|
||||
[[ -z "$line" ]] && continue
|
||||
path="${line%%:*}"
|
||||
skip=0
|
||||
for ex in "${EXCLUDE_PATHS[@]}"; do
|
||||
case "$path" in
|
||||
"$ex"*|"${ex#$ROOT/}"*) skip=1; break ;;
|
||||
esac
|
||||
done
|
||||
if [[ $skip -eq 0 ]]; then
|
||||
FILTERED+="$line"$'\n'
|
||||
fi
|
||||
done <<< "$HITS"
|
||||
|
||||
if [[ -z "${FILTERED//[$' \t\n\r']/}" ]]; then
|
||||
echo "audit: src/dynamic/ is free of unseeded RNG APIs"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "audit: banned RNG APIs surfaced inside src/dynamic/" >&2
|
||||
echo "$FILTERED" >&2
|
||||
echo >&2
|
||||
echo "Replace with crate::dynamic::rand::SpecRng::seeded(&spec.spec_hash)." >&2
|
||||
exit 1
|
||||
569
scripts/corpus_dashboard.py
Executable file
569
scripts/corpus_dashboard.py
Executable file
|
|
@ -0,0 +1,569 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Corpus health report for the Rust dynamic payload registry.
|
||||
|
||||
Produces:
|
||||
- Per-cap coverage table (payload count, benign controls, OOB slots)
|
||||
- Per-payload last-confirmed timestamp (from repro artifacts if present)
|
||||
- CVE reference count
|
||||
- Marker collision audit
|
||||
|
||||
Exit code 0 = healthy. Non-zero = collision or missing coverage.
|
||||
|
||||
Usage:
|
||||
python3 scripts/corpus_dashboard.py [--repro-dir REPRO_DIR] [--json]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
REPO_ROOT = SCRIPT_DIR.parent
|
||||
CORPUS_RS = REPO_ROOT / "src" / "dynamic" / "corpus.rs"
|
||||
CORPUS_DIR = REPO_ROOT / "src" / "dynamic" / "corpus"
|
||||
REGISTRY_RS = CORPUS_DIR / "registry.rs"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RegistryEntry:
|
||||
cap: str
|
||||
lang: str
|
||||
module_path: str
|
||||
source_path: Path
|
||||
|
||||
|
||||
@dataclass
|
||||
class PayloadEntry:
|
||||
cap: str
|
||||
lang: str
|
||||
label: str
|
||||
bytes_repr: str
|
||||
oracle_kind: str
|
||||
oracle_value: Optional[str]
|
||||
is_benign: bool
|
||||
provenance: str
|
||||
since_corpus_version: int
|
||||
deprecated_at_corpus_version: Optional[int]
|
||||
fixture_paths: list[str]
|
||||
oob_nonce_slot: bool
|
||||
source_path: str
|
||||
cve_refs: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# Rust source helpers ---------------------------------------------------------
|
||||
|
||||
|
||||
def load_corpus_version(path: Path = CORPUS_RS) -> int:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
match = re.search(r"pub const CORPUS_VERSION:\s*u32\s*=\s*(\d+);", text)
|
||||
if not match:
|
||||
raise ValueError(f"CORPUS_VERSION not found in {path}")
|
||||
return int(match.group(1))
|
||||
|
||||
|
||||
def parse_registry_entries(path: Path = REGISTRY_RS) -> list[RegistryEntry]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
entries: list[RegistryEntry] = []
|
||||
pattern = re.compile(
|
||||
r"\(\s*Cap::([A-Z0-9_]+)\s*,\s*Lang::([A-Za-z0-9_]+)\s*,"
|
||||
r"\s*([A-Za-z0-9_:]+)::PAYLOADS\s*,?\s*\)",
|
||||
re.DOTALL,
|
||||
)
|
||||
for match in pattern.finditer(text):
|
||||
cap, lang, module_path = match.groups()
|
||||
source_path = CORPUS_DIR / f"{module_path.replace('::', '/')}.rs"
|
||||
entries.append(RegistryEntry(cap, lang, module_path, source_path))
|
||||
if not entries:
|
||||
raise ValueError(f"No registry entries found in {path}")
|
||||
return entries
|
||||
|
||||
|
||||
def _raw_string_bounds(text: str, index: int) -> Optional[tuple[int, int, int]]:
|
||||
if text.startswith("br", index):
|
||||
marker_index = index + 2
|
||||
elif text.startswith("r", index):
|
||||
marker_index = index + 1
|
||||
else:
|
||||
return None
|
||||
|
||||
cursor = marker_index
|
||||
while cursor < len(text) and text[cursor] == "#":
|
||||
cursor += 1
|
||||
if cursor >= len(text) or text[cursor] != '"':
|
||||
return None
|
||||
|
||||
hashes = text[marker_index:cursor]
|
||||
body_start = cursor + 1
|
||||
terminator = '"' + hashes
|
||||
body_end = text.find(terminator, body_start)
|
||||
if body_end < 0:
|
||||
raise ValueError("unterminated Rust raw string literal")
|
||||
return body_start, body_end, body_end + len(terminator)
|
||||
|
||||
|
||||
def _quoted_literal_end(text: str, index: int) -> Optional[int]:
|
||||
raw = _raw_string_bounds(text, index)
|
||||
if raw:
|
||||
return raw[2]
|
||||
|
||||
if text.startswith('b"', index):
|
||||
quote = '"'
|
||||
cursor = index + 2
|
||||
elif text[index:index + 1] == '"':
|
||||
quote = '"'
|
||||
cursor = index + 1
|
||||
elif (
|
||||
text[index:index + 1] == "'"
|
||||
and index + 1 < len(text)
|
||||
and not (text[index + 1].isalpha() or text[index + 1] == "_")
|
||||
):
|
||||
quote = "'"
|
||||
cursor = index + 1
|
||||
else:
|
||||
return None
|
||||
|
||||
while cursor < len(text):
|
||||
char = text[cursor]
|
||||
if char == "\\":
|
||||
cursor += 2
|
||||
continue
|
||||
if char == quote:
|
||||
return cursor + 1
|
||||
cursor += 1
|
||||
raise ValueError("unterminated Rust quoted literal")
|
||||
|
||||
|
||||
def _skip_ignored(text: str, index: int) -> int:
|
||||
if text.startswith("//", index):
|
||||
newline = text.find("\n", index + 2)
|
||||
return len(text) if newline < 0 else newline + 1
|
||||
|
||||
if text.startswith("/*", index):
|
||||
depth = 1
|
||||
cursor = index + 2
|
||||
while cursor < len(text) and depth:
|
||||
if text.startswith("/*", cursor):
|
||||
depth += 1
|
||||
cursor += 2
|
||||
elif text.startswith("*/", cursor):
|
||||
depth -= 1
|
||||
cursor += 2
|
||||
else:
|
||||
cursor += 1
|
||||
if depth:
|
||||
raise ValueError("unterminated Rust block comment")
|
||||
return cursor
|
||||
|
||||
literal_end = _quoted_literal_end(text, index)
|
||||
return literal_end if literal_end is not None else index
|
||||
|
||||
|
||||
def _find_matching(text: str, open_index: int, open_char: str, close_char: str) -> int:
|
||||
depth = 1
|
||||
cursor = open_index + 1
|
||||
while cursor < len(text):
|
||||
skipped = _skip_ignored(text, cursor)
|
||||
if skipped != cursor:
|
||||
cursor = skipped
|
||||
continue
|
||||
|
||||
char = text[cursor]
|
||||
if char == open_char:
|
||||
depth += 1
|
||||
elif char == close_char:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return cursor
|
||||
cursor += 1
|
||||
raise ValueError(f"unterminated {open_char}{close_char} block")
|
||||
|
||||
|
||||
def _payload_blocks(text: str) -> list[str]:
|
||||
blocks: list[str] = []
|
||||
for match in re.finditer(r"\bCuratedPayload\s*\{", text):
|
||||
open_index = match.end() - 1
|
||||
close_index = _find_matching(text, open_index, "{", "}")
|
||||
blocks.append(text[open_index + 1:close_index])
|
||||
return blocks
|
||||
|
||||
|
||||
def _add_field(segment: str, fields: dict[str, str]) -> None:
|
||||
match = re.search(r"(^|\n)\s*([A-Za-z_][A-Za-z0-9_]*)\s*:", segment)
|
||||
if not match:
|
||||
return
|
||||
fields[match.group(2)] = segment[match.end():].strip()
|
||||
|
||||
|
||||
def _split_top_level_fields(block: str) -> dict[str, str]:
|
||||
fields: dict[str, str] = {}
|
||||
start = 0
|
||||
cursor = 0
|
||||
brace_depth = 0
|
||||
bracket_depth = 0
|
||||
paren_depth = 0
|
||||
|
||||
while cursor < len(block):
|
||||
skipped = _skip_ignored(block, cursor)
|
||||
if skipped != cursor:
|
||||
cursor = skipped
|
||||
continue
|
||||
|
||||
char = block[cursor]
|
||||
if char == "{":
|
||||
brace_depth += 1
|
||||
elif char == "}":
|
||||
brace_depth -= 1
|
||||
elif char == "[":
|
||||
bracket_depth += 1
|
||||
elif char == "]":
|
||||
bracket_depth -= 1
|
||||
elif char == "(":
|
||||
paren_depth += 1
|
||||
elif char == ")":
|
||||
paren_depth -= 1
|
||||
elif (
|
||||
char == ","
|
||||
and brace_depth == 0
|
||||
and bracket_depth == 0
|
||||
and paren_depth == 0
|
||||
):
|
||||
_add_field(block[start:cursor], fields)
|
||||
start = cursor + 1
|
||||
cursor += 1
|
||||
|
||||
_add_field(block[start:], fields)
|
||||
return fields
|
||||
|
||||
|
||||
def _parse_rust_string_literal(text: str, index: int) -> Optional[tuple[str, int]]:
|
||||
raw = _raw_string_bounds(text, index)
|
||||
if raw:
|
||||
body_start, body_end, literal_end = raw
|
||||
return text[body_start:body_end], literal_end
|
||||
|
||||
if text.startswith('b"', index):
|
||||
cursor = index + 2
|
||||
elif text[index:index + 1] == '"':
|
||||
cursor = index + 1
|
||||
else:
|
||||
return None
|
||||
|
||||
while cursor < len(text):
|
||||
char = text[cursor]
|
||||
if char == "\\":
|
||||
cursor += 2
|
||||
continue
|
||||
if char == '"':
|
||||
literal = text[index:cursor + 1]
|
||||
value = ast.literal_eval(literal)
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("latin-1"), cursor + 1
|
||||
return str(value), cursor + 1
|
||||
cursor += 1
|
||||
raise ValueError("unterminated Rust string literal")
|
||||
|
||||
|
||||
def _rust_string_literals(expr: str) -> list[str]:
|
||||
strings: list[str] = []
|
||||
cursor = 0
|
||||
while cursor < len(expr):
|
||||
if expr.startswith("//", cursor) or expr.startswith("/*", cursor):
|
||||
cursor = _skip_ignored(expr, cursor)
|
||||
continue
|
||||
|
||||
parsed = _parse_rust_string_literal(expr, cursor)
|
||||
if parsed:
|
||||
value, cursor = parsed
|
||||
strings.append(value)
|
||||
continue
|
||||
|
||||
cursor += 1
|
||||
return strings
|
||||
|
||||
|
||||
def _parse_string_constants(text: str) -> dict[str, str]:
|
||||
constants: dict[str, str] = {}
|
||||
pattern = re.compile(r"(?:pub\s+)?const\s+([A-Z][A-Z0-9_]*):\s*&str\s*=\s*([^;]+);")
|
||||
for match in pattern.finditer(text):
|
||||
strings = _rust_string_literals(match.group(2))
|
||||
if strings:
|
||||
constants[match.group(1)] = strings[0]
|
||||
return constants
|
||||
|
||||
|
||||
def _required(fields: dict[str, str], name: str, source_path: Path) -> str:
|
||||
if name not in fields:
|
||||
rel = source_path.relative_to(REPO_ROOT)
|
||||
raise ValueError(f"missing field {name!r} in payload from {rel}")
|
||||
return fields[name]
|
||||
|
||||
|
||||
def _string_expr(expr: str, constants: dict[str, str]) -> str:
|
||||
expr = expr.strip()
|
||||
if expr in constants:
|
||||
return constants[expr]
|
||||
strings = _rust_string_literals(expr)
|
||||
if strings:
|
||||
return strings[0]
|
||||
return expr
|
||||
|
||||
|
||||
def _bool_expr(expr: str) -> bool:
|
||||
value = expr.strip()
|
||||
if value == "true":
|
||||
return True
|
||||
if value == "false":
|
||||
return False
|
||||
raise ValueError(f"expected Rust bool literal, got {value!r}")
|
||||
|
||||
|
||||
def _int_expr(expr: str) -> int:
|
||||
match = re.search(r"\d+", expr)
|
||||
if not match:
|
||||
raise ValueError(f"expected integer literal, got {expr!r}")
|
||||
return int(match.group(0))
|
||||
|
||||
|
||||
def _optional_int_expr(expr: str) -> Optional[int]:
|
||||
expr = expr.strip()
|
||||
if expr == "None":
|
||||
return None
|
||||
match = re.fullmatch(r"Some\(\s*(\d+)\s*\)", expr)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
raise ValueError(f"expected Rust Option<u32> literal, got {expr!r}")
|
||||
|
||||
|
||||
def _oracle_expr(expr: str, constants: dict[str, str]) -> tuple[str, Optional[str]]:
|
||||
expr = expr.strip()
|
||||
if expr.startswith("Oracle::OutputContains"):
|
||||
open_index = expr.find("(")
|
||||
close_index = _find_matching(expr, open_index, "(", ")")
|
||||
marker = _string_expr(expr[open_index + 1:close_index], constants)
|
||||
return "OutputContains", marker
|
||||
|
||||
if expr.startswith("Oracle::OobCallback"):
|
||||
strings = _rust_string_literals(expr)
|
||||
return "OobCallback", f"host={strings[0]}" if strings else None
|
||||
|
||||
if expr.startswith("Oracle::SinkCrash"):
|
||||
return "SinkCrash", "signals=all"
|
||||
|
||||
if expr.startswith("Oracle::SinkProbe"):
|
||||
predicates = list(dict.fromkeys(re.findall(r"ProbePredicate::([A-Za-z0-9_]+)", expr)))
|
||||
return "SinkProbe", ",".join(predicates) if predicates else None
|
||||
|
||||
return expr.split("{", 1)[0].split("(", 1)[0].strip(), None
|
||||
|
||||
|
||||
def _payload_from_block(
|
||||
entry: RegistryEntry,
|
||||
block: str,
|
||||
constants: dict[str, str],
|
||||
) -> PayloadEntry:
|
||||
fields = _split_top_level_fields(block)
|
||||
source_path = entry.source_path
|
||||
oracle_kind, oracle_value = _oracle_expr(_required(fields, "oracle", source_path), constants)
|
||||
rel_source = str(source_path.relative_to(REPO_ROOT))
|
||||
|
||||
return PayloadEntry(
|
||||
cap=entry.cap,
|
||||
lang=entry.lang,
|
||||
label=_string_expr(_required(fields, "label", source_path), constants),
|
||||
bytes_repr=_string_expr(_required(fields, "bytes", source_path), constants),
|
||||
oracle_kind=oracle_kind,
|
||||
oracle_value=oracle_value,
|
||||
is_benign=_bool_expr(_required(fields, "is_benign", source_path)),
|
||||
provenance=_required(fields, "provenance", source_path)
|
||||
.strip()
|
||||
.removeprefix("PayloadProvenance::"),
|
||||
since_corpus_version=_int_expr(_required(fields, "since_corpus_version", source_path)),
|
||||
deprecated_at_corpus_version=_optional_int_expr(
|
||||
_required(fields, "deprecated_at_corpus_version", source_path)
|
||||
),
|
||||
fixture_paths=_rust_string_literals(_required(fields, "fixture_paths", source_path)),
|
||||
oob_nonce_slot=_bool_expr(_required(fields, "oob_nonce_slot", source_path)),
|
||||
source_path=rel_source,
|
||||
cve_refs=sorted(set(re.findall(r"CVE-\d{4}-\d{4,7}", block))),
|
||||
)
|
||||
|
||||
|
||||
def load_payloads() -> list[PayloadEntry]:
|
||||
payloads: list[PayloadEntry] = []
|
||||
for entry in parse_registry_entries():
|
||||
if not entry.source_path.exists():
|
||||
rel = entry.source_path.relative_to(REPO_ROOT)
|
||||
raise FileNotFoundError(f"registry entry points at missing payload file: {rel}")
|
||||
|
||||
text = entry.source_path.read_text(encoding="utf-8")
|
||||
constants = _parse_string_constants(text)
|
||||
blocks = _payload_blocks(text)
|
||||
if not blocks:
|
||||
rel = entry.source_path.relative_to(REPO_ROOT)
|
||||
raise ValueError(f"no CuratedPayload entries found in {rel}")
|
||||
|
||||
for block in blocks:
|
||||
payloads.append(_payload_from_block(entry, block, constants))
|
||||
|
||||
return payloads
|
||||
|
||||
|
||||
CORPUS_VERSION = load_corpus_version()
|
||||
PAYLOADS: list[PayloadEntry] = load_payloads()
|
||||
ALL_CAPS = list(dict.fromkeys(p.cap for p in PAYLOADS))
|
||||
|
||||
|
||||
# Marker collision audit ------------------------------------------------------
|
||||
|
||||
|
||||
def audit_marker_collisions(payloads: list[PayloadEntry] = PAYLOADS) -> list[tuple[str, str, str]]:
|
||||
collisions = []
|
||||
for payload in payloads:
|
||||
if payload.is_benign or payload.oracle_kind != "OutputContains":
|
||||
continue
|
||||
marker = payload.oracle_value or ""
|
||||
if not marker:
|
||||
continue
|
||||
|
||||
for other in payloads:
|
||||
if other.cap == payload.cap:
|
||||
continue
|
||||
if other.is_benign or other.oob_nonce_slot:
|
||||
continue
|
||||
if marker in other.bytes_repr:
|
||||
collisions.append((payload.cap, payload.label, other.cap))
|
||||
return collisions
|
||||
|
||||
|
||||
# Coverage table --------------------------------------------------------------
|
||||
|
||||
|
||||
def build_coverage_table(payloads: list[PayloadEntry] = PAYLOADS) -> dict:
|
||||
result = {}
|
||||
for cap in ALL_CAPS:
|
||||
cap_payloads = [payload for payload in payloads if payload.cap == cap]
|
||||
result[cap] = {
|
||||
"total": len(cap_payloads),
|
||||
"vuln": sum(1 for p in cap_payloads if not p.is_benign),
|
||||
"benign": sum(1 for p in cap_payloads if p.is_benign),
|
||||
"oob_slots": sum(1 for p in cap_payloads if p.oob_nonce_slot),
|
||||
"has_fixture_paths": all(len(p.fixture_paths) > 0 for p in cap_payloads),
|
||||
"payloads": [p.label for p in cap_payloads],
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
# Repro artifact timestamps ---------------------------------------------------
|
||||
|
||||
|
||||
def scan_last_confirmed(repro_dir: Path) -> dict[str, str]:
|
||||
"""Return {payload_label: iso_timestamp} from repro artifact metadata."""
|
||||
timestamps: dict[str, str] = {}
|
||||
if not repro_dir.exists():
|
||||
return timestamps
|
||||
for meta_file in repro_dir.rglob("*.json"):
|
||||
try:
|
||||
data = json.loads(meta_file.read_text())
|
||||
label = data.get("payload_label", "")
|
||||
ts = data.get("confirmed_at", "")
|
||||
if label and ts:
|
||||
if label not in timestamps or ts > timestamps[label]:
|
||||
timestamps[label] = ts
|
||||
except (json.JSONDecodeError, KeyError):
|
||||
pass
|
||||
return timestamps
|
||||
|
||||
|
||||
# fuzz-discovered count -------------------------------------------------------
|
||||
|
||||
|
||||
def count_discovered(discovered_dir: Path) -> int:
|
||||
if not discovered_dir.exists():
|
||||
return 0
|
||||
return sum(
|
||||
1 for path in discovered_dir.rglob("*")
|
||||
if path.is_file() and not path.name.endswith(".json") and path.name != ".gitkeep"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Nyx corpus health dashboard")
|
||||
parser.add_argument("--repro-dir", default="repro", help="Path to repro artifacts")
|
||||
parser.add_argument(
|
||||
"--discovered-dir",
|
||||
default="fuzz-discovered",
|
||||
help="Path to fuzz-discovered/ directory",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="Output JSON instead of text")
|
||||
args = parser.parse_args()
|
||||
|
||||
os.chdir(REPO_ROOT)
|
||||
|
||||
collisions = audit_marker_collisions()
|
||||
coverage = build_coverage_table()
|
||||
timestamps = scan_last_confirmed(Path(args.repro_dir))
|
||||
discovered_count = count_discovered(Path(args.discovered_dir))
|
||||
|
||||
report = {
|
||||
"corpus_version": CORPUS_VERSION,
|
||||
"registry_entries": len(parse_registry_entries()),
|
||||
"total_payloads": len(PAYLOADS),
|
||||
"coverage": coverage,
|
||||
"marker_collisions": collisions,
|
||||
"last_confirmed": timestamps,
|
||||
"cve_reference_count": sum(len(p.cve_refs) for p in PAYLOADS),
|
||||
"fuzz_discovered_pending": discovered_count,
|
||||
"healthy": len(collisions) == 0,
|
||||
}
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(report, indent=2))
|
||||
return 0 if report["healthy"] else 1
|
||||
|
||||
print(f"Nyx Corpus Dashboard (corpus_version={CORPUS_VERSION})")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
print("Per-cap coverage:")
|
||||
hdr = f" {'Cap':<22} {'Total':>5} {'Vuln':>5} {'Benign':>6} {'OOB':>4} {'Fixtures':>8}"
|
||||
print(hdr)
|
||||
print(" " + "-" * 56)
|
||||
for cap, info in coverage.items():
|
||||
fixture_ok = "ok" if info["has_fixture_paths"] else "MISSING"
|
||||
print(
|
||||
f" {cap:<22} {info['total']:>5} {info['vuln']:>5} "
|
||||
f"{info['benign']:>6} {info['oob_slots']:>4} {fixture_ok:>8}"
|
||||
)
|
||||
print()
|
||||
|
||||
if timestamps:
|
||||
print("Last confirmed timestamps:")
|
||||
for label, ts in sorted(timestamps.items()):
|
||||
print(f" {label:<35} {ts}")
|
||||
print()
|
||||
|
||||
print(f"Registry entries: {report['registry_entries']}")
|
||||
print(f"CVE references: {report['cve_reference_count']}")
|
||||
print(f"Fuzz-discovered pending promotion: {discovered_count}")
|
||||
print()
|
||||
|
||||
if collisions:
|
||||
print("FAIL: Marker collisions detected (section 16.3):")
|
||||
for cap, label, other_cap in collisions:
|
||||
print(f" {cap}/{label} marker appears in {other_cap} payload bytes")
|
||||
return 1
|
||||
|
||||
print("OK: No marker collisions detected.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
626
scripts/m7_ship_gate.sh
Executable file
626
scripts/m7_ship_gate.sh
Executable file
|
|
@ -0,0 +1,626 @@
|
|||
#!/usr/bin/env bash
|
||||
# m7_ship_gate.sh — milestone-7 ship gates.
|
||||
#
|
||||
# Each gate runs as an isolated function so CI can call a subset:
|
||||
#
|
||||
# scripts/m7_ship_gate.sh # every gate
|
||||
# scripts/m7_ship_gate.sh --gates 3,6 # only gates 3 + 6
|
||||
# scripts/m7_ship_gate.sh --sets owasp # Java OWASP corpus only
|
||||
# scripts/m7_ship_gate.sh --sets jsts # NodeGoat + Juice Shop only
|
||||
# scripts/m7_ship_gate.sh --sets nodegoat # one JS/TS corpus only
|
||||
# scripts/m7_ship_gate.sh --sets polyglot # RailsGoat+DVWA+DVPWA+gosec+RustSec
|
||||
# scripts/m7_ship_gate.sh --sets railsgoat # one polyglot corpus only
|
||||
#
|
||||
# Gate map (kept in sync with .pitboss/play/plan.md track M.7):
|
||||
# Gate 1: Static-only scan is green on `tests/benchmark/corpus`.
|
||||
# Gate 2: `cargo nextest run --no-fail-fast --features dynamic` is green.
|
||||
# Gate 3: With-verify / static-only wall-clock ratio ≤ 1.5× on
|
||||
# `benches/fixtures/`. Phase 22 had relaxed this to ≤ 2×
|
||||
# while only `javac` had a warm daemon; Phase 23 lands the
|
||||
# cross-lang build pools (shared caches for Node/Python/PHP/
|
||||
# Ruby/Go/Rust/C/C++), so the bar is tightened back to ≤ 1.5×.
|
||||
# Gate 4: SARIF schema validation on every dynamic verdict variant.
|
||||
# Gate 5: Layering boundary test green.
|
||||
# Gate 6: Java OWASP Benchmark v1.2 `--verify` acceptance. Wall-clock
|
||||
# ≤ 15 min on CI / ≤ 10 min on the dev reference machine; and,
|
||||
# per OWASP cap backed by a sound runtime oracle, confirmed-rate
|
||||
# ≥ 40%, precision ≥ 0.85, recall ≥ 0.40, plus the per-(cap,lang)
|
||||
# budget in tests/eval_corpus/budget.toml. Added Phase 22 as the
|
||||
# headline acceptance for the warm `javac` daemon; Phase 27 (Track
|
||||
# R.0) added the precision/recall/budget ratchet. The corpus is
|
||||
# *not* checked into the repo; the gate skips with a clear message
|
||||
# when `NYX_OWASP_CORPUS` does not point at a real checkout.
|
||||
# Gate 7: JS/TS real-corpus acceptance (Track R.1 / Phase 28). OWASP
|
||||
# NodeGoat (Express, .js) + OWASP Juice Shop (TypeScript, .ts)
|
||||
# `--verify` against the committed ground truth. Same shape as
|
||||
# Gate 6: wall-clock budget + the per-(cap,lang) budget in
|
||||
# tests/eval_corpus/budget.toml hard-enforced; per-cap
|
||||
# confirmed-rate / precision / recall published report-only
|
||||
# (NYX_JSTS_FLOOR_CAPS empty by default). Each corpus row
|
||||
# self-skips unless its NYX_NODEGOAT_CORPUS / NYX_JUICESHOP_CORPUS
|
||||
# points at a real checkout.
|
||||
# Gate 8: Polyglot real-corpus acceptance (Track R.2 / Phase 29). OWASP
|
||||
# RailsGoat (Rails, .rb), DVWA (PHP), DVPWA (aiohttp, .py), gosec
|
||||
# (Go) and the RustSec advisory-db (Rust negative control), one
|
||||
# row per corpus. Same shape as Gate 7: wall-clock budget + the
|
||||
# per-(cap,lang) budget hard-enforced; per-cap confirmed/precision/
|
||||
# recall report-only (NYX_POLYGLOT_FLOOR_CAPS empty by default).
|
||||
# Each row self-skips unless its NYX_<NAME>_CORPUS points at a real
|
||||
# checkout.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
# Demote the per-cell Unsupported-rate budget (Gates 6/7/8 -> report.py) to
|
||||
# report-only in CI. Dynamic confirmation is environment-constrained on the
|
||||
# unprivileged CI runners (no oracle infrastructure for several caps), so the
|
||||
# Unsupported budget — calibrated on a dev box where confirmation runs fully —
|
||||
# would fail vacuously there; the precision (false-Confirmed) and confirmed-rate
|
||||
# ratchets stay HARD. Local runs leave it unset, so coverage stays gated. Set
|
||||
# here rather than in eval.yml so the standalone tabulate regression-test step
|
||||
# (which asserts the hard behaviour) never inherits it.
|
||||
if [[ -n "${CI:-}" ]]; then
|
||||
export NYX_EVAL_SOFT_UNSUPPORTED=1
|
||||
fi
|
||||
|
||||
GATES="1,2,3,4,5,6,7,8"
|
||||
SETS=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--gates)
|
||||
GATES="$2"
|
||||
shift 2
|
||||
;;
|
||||
--sets)
|
||||
SETS="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h | --help)
|
||||
sed -n '2,/^$/p' "${BASH_SOURCE[0]}"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown flag: $1" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# `--sets` lets CI run a single real-corpus gate. `owasp` -> Gate 6;
|
||||
# `jsts` (both JS/TS corpora) / `nodegoat` / `juiceshop` -> Gate 7, with the
|
||||
# corpus name passed through so Gate 7 runs only the requested row.
|
||||
case "${SETS}" in
|
||||
owasp) GATES="6" ;;
|
||||
jsts|nodegoat|juiceshop) GATES="7" ;;
|
||||
polyglot|railsgoat|dvwa|dvpwa|gosec|rustsec) GATES="8" ;;
|
||||
"") ;; # no --sets: run the requested --gates
|
||||
*) echo "unknown --sets: ${SETS}" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
want_gate() {
|
||||
[[ ",${GATES}," == *",$1,"* ]]
|
||||
}
|
||||
|
||||
# ── Gate 1 ────────────────────────────────────────────────────────────────────
|
||||
|
||||
gate_1_static_corpus() {
|
||||
echo "── Gate 1: static-only scan on tests/benchmark/corpus ──"
|
||||
if [[ ! -d "${REPO_ROOT}/tests/benchmark/corpus" ]]; then
|
||||
echo " SKIP: tests/benchmark/corpus not present"
|
||||
return 0
|
||||
fi
|
||||
cargo run --release --quiet -- scan \
|
||||
--format json \
|
||||
"${REPO_ROOT}/tests/benchmark/corpus" > /tmp/m7_gate1.json
|
||||
echo " PASS: static scan completed"
|
||||
}
|
||||
|
||||
# ── Gate 2 ────────────────────────────────────────────────────────────────────
|
||||
|
||||
gate_2_dynamic_tests() {
|
||||
echo "── Gate 2: cargo nextest run --no-fail-fast --features dynamic ──"
|
||||
cargo nextest run --no-fail-fast --features dynamic
|
||||
# The real-toolchain build-pool perf benches (dynamic_*_build_pool +
|
||||
# dynamic_java_compile_pool) are #[ignore]d so the default inner-loop
|
||||
# suite stays hermetic + fast: no cargo/go/cc/c++/npm/pip/composer/
|
||||
# bundle/javac spawns. Run them explicitly here so CI still exercises
|
||||
# the warm-pool compile path end to end. They self-skip when a
|
||||
# toolchain is missing, so a toolchain-less CI row stays green.
|
||||
cargo nextest run --no-fail-fast --features dynamic --run-ignored ignored-only \
|
||||
-E 'binary(~build_pool) | binary(~compile_pool)'
|
||||
echo " PASS: dynamic test suite green"
|
||||
}
|
||||
|
||||
# ── Gate 3: with-verify / static-only ratio ───────────────────────────────────
|
||||
|
||||
# Phase 23 target: ratio ≤ 1.5×, now that the cross-lang build pools
|
||||
# give every shipped language a warm cache (was ≤ 2× under Phase 22).
|
||||
GATE3_RATIO_TARGET="${GATE3_RATIO_TARGET:-1.5}"
|
||||
|
||||
gate_3_verify_ratio() {
|
||||
echo "── Gate 3: with-verify / static-only ratio on benches/fixtures/ ──"
|
||||
local fixtures="${REPO_ROOT}/benches/fixtures"
|
||||
if [[ ! -d "${fixtures}" ]]; then
|
||||
echo " SKIP: ${fixtures} not present"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Phase 23: the warm build pools are what buy the ≤ 1.5× ratio, so
|
||||
# make sure they are on for both scans even if the caller's env
|
||||
# disabled them. Default is already ON for every shipped language.
|
||||
export NYX_DYNAMIC_BUILD_POOL="java=1,node=1,python=1,php=1,ruby=1,go=1,rust=1,c=1,cpp=1"
|
||||
|
||||
local static_seconds verify_seconds
|
||||
static_seconds="$(time_scan "${fixtures}" 0)"
|
||||
verify_seconds="$(time_scan "${fixtures}" 1)"
|
||||
local ratio
|
||||
ratio="$(awk -v v="${verify_seconds}" -v s="${static_seconds}" \
|
||||
'BEGIN { if (s <= 0) { print "inf"; exit } printf "%.3f", v / s }')"
|
||||
|
||||
echo " static-only wall-clock: ${static_seconds}s"
|
||||
echo " with-verify wall-clock: ${verify_seconds}s"
|
||||
echo " ratio: ${ratio} (target ≤ ${GATE3_RATIO_TARGET})"
|
||||
|
||||
awk -v r="${ratio}" -v t="${GATE3_RATIO_TARGET}" \
|
||||
'BEGIN { if (r+0 > t+0) exit 1 }' \
|
||||
|| { echo " FAIL: ratio exceeds target"; return 1; }
|
||||
echo " PASS"
|
||||
}
|
||||
|
||||
# Print wall-clock seconds for a single scan run.
|
||||
# $1 = path to scan
|
||||
# $2 = 0 for static-only, 1 for --verify
|
||||
time_scan() {
|
||||
local path="$1" verify="$2"
|
||||
local args=("--format" "json")
|
||||
if [[ "${verify}" == "1" ]]; then
|
||||
args+=("--verify")
|
||||
fi
|
||||
args+=("${path}")
|
||||
local start end
|
||||
start="$(python3 -c 'import time;print(time.monotonic())')"
|
||||
cargo run --release --quiet --features dynamic -- scan "${args[@]}" > /dev/null
|
||||
end="$(python3 -c 'import time;print(time.monotonic())')"
|
||||
awk -v a="${start}" -v b="${end}" 'BEGIN { printf "%.3f", b - a }'
|
||||
}
|
||||
|
||||
# ── Gate 4 ────────────────────────────────────────────────────────────────────
|
||||
|
||||
gate_4_sarif_schema() {
|
||||
echo "── Gate 4: SARIF schema validation ──"
|
||||
cargo nextest run --no-fail-fast --features dynamic --test sarif_dynamic_verdict_tests
|
||||
echo " PASS"
|
||||
}
|
||||
|
||||
# ── Gate 5 ────────────────────────────────────────────────────────────────────
|
||||
|
||||
gate_5_layering() {
|
||||
echo "── Gate 5: dynamic layering boundary ──"
|
||||
cargo nextest run --no-fail-fast --features dynamic --test dynamic_layering
|
||||
echo " PASS"
|
||||
}
|
||||
|
||||
# ── Gate 6: Java OWASP-scale ratio ────────────────────────────────────────────
|
||||
|
||||
# Phase 22 + Phase 27 jointly own this gate. The wall-clock budgets
|
||||
# are split: 10 min on the dev reference (M1 macOS w/ JDK 21) and 15
|
||||
# min in CI. Override `NYX_OWASP_WALLCLOCK_BUDGET_SECONDS` to tighten.
|
||||
GATE6_WALLCLOCK_BUDGET="${NYX_OWASP_WALLCLOCK_BUDGET_SECONDS:-900}"
|
||||
GATE6_CONFIRMED_RATE_TARGET="${NYX_OWASP_CONFIRMED_RATE_TARGET:-0.40}"
|
||||
# Phase 27 acceptance: per-cap precision >= 0.85, recall >= 0.40.
|
||||
GATE6_PRECISION_TARGET="${NYX_OWASP_PRECISION_TARGET:-0.85}"
|
||||
GATE6_RECALL_TARGET="${NYX_OWASP_RECALL_TARGET:-0.40}"
|
||||
# Per-cap confirmation floors (confirmed-rate / precision / recall) are
|
||||
# HARD-enforced only for the caps named here; every cap is still measured and
|
||||
# its numbers published either way. Empty = report-only (publish the per-cap
|
||||
# table, fail nothing on those three metrics) while the verifier still cannot
|
||||
# Confirm OWASP findings end to end: today every BenchmarkTest servlet harness
|
||||
# lands in Inconclusive(BuildFailed) or Inconclusive(SpecDerivationFailed)
|
||||
# (Java servlet entry + classpath are Track L.12 / Track O.0 work), so 0 caps
|
||||
# meet the 40% / 85% / 40% headline. The gate therefore enforces what the
|
||||
# verifier already satisfies — wall-clock, no false confirms, the per-cell
|
||||
# budget — and publishes the unmet detection/confirmation numbers as the
|
||||
# ratchet's destination. Set NYX_OWASP_FLOOR_CAPS (e.g. "sqli,cmdi") to
|
||||
# hard-gate a cap the moment it starts Confirming.
|
||||
GATE6_FLOOR_CAPS="${NYX_OWASP_FLOOR_CAPS:-}"
|
||||
GATE6_BUDGET="${NYX_OWASP_BUDGET:-${REPO_ROOT}/tests/eval_corpus/budget.toml}"
|
||||
|
||||
gate_6_owasp_scale() {
|
||||
echo "── Gate 6: Java OWASP Benchmark v1.2 verify wall-clock + confirmed-rate ──"
|
||||
local corpus="${NYX_OWASP_CORPUS:-}"
|
||||
if [[ -z "${corpus}" || ! -d "${corpus}" ]]; then
|
||||
echo " SKIP: set NYX_OWASP_CORPUS to a v1.2 checkout to run this gate."
|
||||
echo " (Gate 6 is Phase 22's headline acceptance for the warm javac daemon.)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local scan_report="/tmp/m7_gate6_scan.json"
|
||||
local results_report="/tmp/m7_gate6_results.json"
|
||||
local wallclock_report="/tmp/m7_gate6_wallclock.txt"
|
||||
local gate_home="${TMPDIR:-/tmp}/nyx_m7_gate6_home"
|
||||
local gate_build_pool="${TMPDIR:-/tmp}/nyx_m7_gate6_build_pool"
|
||||
local wallclock
|
||||
|
||||
cargo build --release --quiet --features dynamic
|
||||
mkdir -p "${gate_home}" "${gate_build_pool}"
|
||||
rm -f "${scan_report}" "${results_report}" "${wallclock_report}"
|
||||
|
||||
set +e
|
||||
HOME="${gate_home}" \
|
||||
NYX_BUILD_POOL_DIR="${gate_build_pool}" \
|
||||
python3 - "${GATE6_WALLCLOCK_BUDGET}" "${scan_report}" "${wallclock_report}" \
|
||||
"${REPO_ROOT}/target/release/nyx" scan \
|
||||
--verify \
|
||||
--index off \
|
||||
--format json \
|
||||
--quiet \
|
||||
"${corpus}" <<'PY'
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
budget = float(sys.argv[1])
|
||||
scan_report = sys.argv[2]
|
||||
wallclock_report = sys.argv[3]
|
||||
cmd = sys.argv[4:]
|
||||
start = time.monotonic()
|
||||
rc = 0
|
||||
try:
|
||||
with open(scan_report, "wb") as out:
|
||||
completed = subprocess.run(cmd, stdout=out, timeout=budget)
|
||||
rc = completed.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
rc = 124
|
||||
finally:
|
||||
elapsed = time.monotonic() - start
|
||||
with open(wallclock_report, "w") as f:
|
||||
f.write(f"{elapsed:.1f}\n")
|
||||
sys.exit(rc)
|
||||
PY
|
||||
local nyx_exit=$?
|
||||
set -e
|
||||
wallclock="$(cat "${wallclock_report}" 2>/dev/null || printf "%s" "${GATE6_WALLCLOCK_BUDGET}")"
|
||||
|
||||
echo " OWASP verify wall-clock: ${wallclock}s (budget ${GATE6_WALLCLOCK_BUDGET}s)"
|
||||
|
||||
if [[ ${nyx_exit} -eq 124 ]]; then
|
||||
echo " FAIL: nyx scan exceeded wall-clock budget"
|
||||
return 1
|
||||
fi
|
||||
if [[ ${nyx_exit} -ne 0 && ${nyx_exit} -ne 1 ]]; then
|
||||
echo " FAIL: nyx scan exited ${nyx_exit}"
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -s "${scan_report}" ]]; then
|
||||
echo " FAIL: nyx scan produced no JSON report"
|
||||
return 1
|
||||
fi
|
||||
|
||||
awk -v w="${wallclock}" -v b="${GATE6_WALLCLOCK_BUDGET}" \
|
||||
'BEGIN { if (w+0 > b+0) exit 1 }' \
|
||||
|| { echo " FAIL: wall-clock exceeds budget"; return 1; }
|
||||
|
||||
echo "[]" > "${results_report}"
|
||||
# --static buckets a command-injection finding that carries only the
|
||||
# SHELL_ESCAPE sink cap (the static, unconfirmed cmdi class for every
|
||||
# language) as `cmdi` instead of `other`. Without a dynamic Confirm the
|
||||
# SHELL_ESCAPE→CODE_EXEC remap never runs (Java servlet harnesses build-
|
||||
# fail in CI), so the default lens leaves every cmdi finding in `other`
|
||||
# and reads the cmdi cell as 0/0/N; the static lens is the correct
|
||||
# bucketing for an unconfirmed scan and is appended at lowest priority so
|
||||
# no higher-priority cap cell changes.
|
||||
python3 "${REPO_ROOT}/tests/eval_corpus/tabulate.py" \
|
||||
--static \
|
||||
--label owasp \
|
||||
--scan "${scan_report}" \
|
||||
--ground-truth "${REPO_ROOT}/tests/eval_corpus/ground_truth/owasp_benchmark_v1.2.json" \
|
||||
--append "${results_report}" \
|
||||
|| { echo " FAIL: OWASP result tabulation failed"; return 1; }
|
||||
|
||||
local -a report_args=(
|
||||
--results "${results_report}"
|
||||
--budget "${GATE6_BUDGET}"
|
||||
)
|
||||
if [[ -n "${GATE6_FLOOR_CAPS}" ]]; then
|
||||
report_args+=(
|
||||
--floor-caps "${GATE6_FLOOR_CAPS}"
|
||||
--min-confirmed-rate "${GATE6_CONFIRMED_RATE_TARGET}"
|
||||
--min-precision "${GATE6_PRECISION_TARGET}"
|
||||
--min-recall "${GATE6_RECALL_TARGET}"
|
||||
)
|
||||
echo " enforcing per-cap floors (confirmed >= ${GATE6_CONFIRMED_RATE_TARGET}, precision >= ${GATE6_PRECISION_TARGET}, recall >= ${GATE6_RECALL_TARGET}) on: ${GATE6_FLOOR_CAPS}"
|
||||
else
|
||||
echo " per-cap confirmed/precision/recall: report-only (NYX_OWASP_FLOOR_CAPS unset; no cap Confirms OWASP yet)"
|
||||
fi
|
||||
python3 "${REPO_ROOT}/tests/eval_corpus/report.py" "${report_args[@]}" \
|
||||
|| { echo " FAIL: OWASP per-cell budget exceeded or a gated per-cap floor missed"; return 1; }
|
||||
echo " PASS"
|
||||
}
|
||||
|
||||
# ── Shared real-corpus acceptance runner (Gates 7 + 8) ────────────────────────
|
||||
|
||||
# Run one real-corpus `--verify` row: scan under a wall-clock guard,
|
||||
# tabulate against the committed ground truth, enforce the per-cell budget,
|
||||
# publish (or, when floor caps are set, enforce) the per-cap floors. Every
|
||||
# random source nyx uses is seeded from spec_hash, so reruns are
|
||||
# deterministic. Generic across gates — all gate-specific knobs are passed
|
||||
# in so Gate 7 (JS/TS) and Gate 8 (polyglot) share one code path.
|
||||
# $1 label $2 corpus dir $3 ground-truth json
|
||||
# $4 wallclock(s) $5 budget.toml $6 floor caps (may be empty)
|
||||
# $7 confirmed target $8 precision target $9 recall target
|
||||
# $10 floor-unset hint (e.g. "NYX_POLYGLOT_FLOOR_CAPS unset")
|
||||
# $11 lang filter (may be empty) — scope tabulation to one language so
|
||||
# incidental other-language assets (vendored JS in a Rails/aiohttp app)
|
||||
# do not pollute the corpus's per-cap metrics
|
||||
# Returns 0 on pass, 1 on fail. Caller decides skip.
|
||||
_run_corpus_acceptance() {
|
||||
local label="$1" corpus="$2" gt="$3" wallclock_budget="$4" budget_file="$5"
|
||||
local floor_caps="$6" confirmed_target="$7" precision_target="$8"
|
||||
local recall_target="$9" floor_hint="${10}" lang_filter="${11:-}"
|
||||
local scan_report="/tmp/m7_corpus_${label}_scan.json"
|
||||
local results_report="/tmp/m7_corpus_${label}_results.json"
|
||||
local wallclock_report="/tmp/m7_corpus_${label}_wallclock.txt"
|
||||
local gate_home="${TMPDIR:-/tmp}/nyx_m7_corpus_${label}_home"
|
||||
local gate_build_pool="${TMPDIR:-/tmp}/nyx_m7_corpus_${label}_build_pool"
|
||||
local wallclock
|
||||
|
||||
mkdir -p "${gate_home}" "${gate_build_pool}"
|
||||
rm -f "${scan_report}" "${results_report}" "${wallclock_report}"
|
||||
|
||||
set +e
|
||||
HOME="${gate_home}" \
|
||||
NYX_BUILD_POOL_DIR="${gate_build_pool}" \
|
||||
python3 - "${wallclock_budget}" "${scan_report}" "${wallclock_report}" \
|
||||
"${REPO_ROOT}/target/release/nyx" scan \
|
||||
--verify \
|
||||
--index off \
|
||||
--format json \
|
||||
--quiet \
|
||||
"${corpus}" <<'PY'
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
budget = float(sys.argv[1])
|
||||
scan_report = sys.argv[2]
|
||||
wallclock_report = sys.argv[3]
|
||||
cmd = sys.argv[4:]
|
||||
start = time.monotonic()
|
||||
rc = 0
|
||||
try:
|
||||
with open(scan_report, "wb") as out:
|
||||
completed = subprocess.run(cmd, stdout=out, timeout=budget)
|
||||
rc = completed.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
rc = 124
|
||||
finally:
|
||||
elapsed = time.monotonic() - start
|
||||
with open(wallclock_report, "w") as f:
|
||||
f.write(f"{elapsed:.1f}\n")
|
||||
sys.exit(rc)
|
||||
PY
|
||||
local nyx_exit=$?
|
||||
set -e
|
||||
wallclock="$(cat "${wallclock_report}" 2>/dev/null || printf "%s" "${wallclock_budget}")"
|
||||
|
||||
echo " ${label} verify wall-clock: ${wallclock}s (budget ${wallclock_budget}s)"
|
||||
|
||||
if [[ ${nyx_exit} -eq 124 ]]; then
|
||||
echo " FAIL: ${label} scan exceeded wall-clock budget"
|
||||
return 1
|
||||
fi
|
||||
if [[ ${nyx_exit} -ne 0 && ${nyx_exit} -ne 1 ]]; then
|
||||
echo " FAIL: ${label} scan exited ${nyx_exit}"
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -s "${scan_report}" ]]; then
|
||||
echo " FAIL: ${label} scan produced no JSON report"
|
||||
return 1
|
||||
fi
|
||||
awk -v w="${wallclock}" -v b="${wallclock_budget}" \
|
||||
'BEGIN { if (w+0 > b+0) exit 1 }' \
|
||||
|| { echo " FAIL: ${label} wall-clock exceeds budget"; return 1; }
|
||||
|
||||
echo "[]" > "${results_report}"
|
||||
# --static: bucket SHELL_ESCAPE-only command-injection findings as `cmdi`
|
||||
# (see the Gate 6 note) so the per-cap table reflects the engine's real
|
||||
# static classification in CI where no dynamic Confirm runs the
|
||||
# SHELL_ESCAPE→CODE_EXEC remap. Appended at lowest priority; no other cap
|
||||
# cell changes.
|
||||
local -a tabulate_args=(
|
||||
--static
|
||||
--label "${label}"
|
||||
--scan "${scan_report}"
|
||||
--ground-truth "${gt}"
|
||||
--append "${results_report}"
|
||||
)
|
||||
if [[ -n "${lang_filter}" ]]; then
|
||||
tabulate_args+=(--lang "${lang_filter}")
|
||||
echo " scoping tabulation to language(s): ${lang_filter}"
|
||||
fi
|
||||
python3 "${REPO_ROOT}/tests/eval_corpus/tabulate.py" "${tabulate_args[@]}" \
|
||||
|| { echo " FAIL: ${label} result tabulation failed"; return 1; }
|
||||
|
||||
local -a report_args=(
|
||||
--results "${results_report}"
|
||||
--budget "${budget_file}"
|
||||
)
|
||||
if [[ -n "${floor_caps}" ]]; then
|
||||
report_args+=(
|
||||
--floor-caps "${floor_caps}"
|
||||
--min-confirmed-rate "${confirmed_target}"
|
||||
--min-precision "${precision_target}"
|
||||
--min-recall "${recall_target}"
|
||||
)
|
||||
echo " enforcing per-cap floors (confirmed >= ${confirmed_target}, precision >= ${precision_target}, recall >= ${recall_target}) on: ${floor_caps}"
|
||||
else
|
||||
echo " per-cap confirmed/precision/recall: report-only (${floor_hint})"
|
||||
fi
|
||||
python3 "${REPO_ROOT}/tests/eval_corpus/report.py" "${report_args[@]}" \
|
||||
|| { echo " FAIL: ${label} per-cell budget exceeded or a gated per-cap floor missed"; return 1; }
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Gate 7: JS/TS real-corpus acceptance (NodeGoat + Juice Shop) ──────────────
|
||||
|
||||
# Phase 28 (Track R.1) mirror of Gate 6 for the JS/TS corpora. Same
|
||||
# wall-clock split (10 min dev reference / 15 min CI) and the same
|
||||
# report-only-by-default floor policy: NYX_JSTS_FLOOR_CAPS is empty, so the
|
||||
# per-cap confirmed-rate / precision / recall numbers are published but gate
|
||||
# nothing, while the per-(cap,lang) budget (unsupported_rate,
|
||||
# false_confirmed_rate) is hard-enforced. Promote a cap into the floor set
|
||||
# once it starts Confirming end to end.
|
||||
GATE7_WALLCLOCK_BUDGET="${NYX_JSTS_WALLCLOCK_BUDGET_SECONDS:-900}"
|
||||
GATE7_CONFIRMED_RATE_TARGET="${NYX_JSTS_CONFIRMED_RATE_TARGET:-0.40}"
|
||||
GATE7_PRECISION_TARGET="${NYX_JSTS_PRECISION_TARGET:-0.85}"
|
||||
GATE7_RECALL_TARGET="${NYX_JSTS_RECALL_TARGET:-0.40}"
|
||||
GATE7_FLOOR_CAPS="${NYX_JSTS_FLOOR_CAPS:-}"
|
||||
GATE7_BUDGET="${NYX_JSTS_BUDGET:-${REPO_ROOT}/tests/eval_corpus/budget.toml}"
|
||||
|
||||
gate_7_jsts_scale() {
|
||||
echo "── Gate 7: JS/TS real-corpus (NodeGoat + Juice Shop) verify acceptance ──"
|
||||
cargo build --release --quiet --features dynamic
|
||||
|
||||
# name : env var holding the corpus dir : committed ground-truth file
|
||||
local rows=(
|
||||
"nodegoat:NYX_NODEGOAT_CORPUS:nodegoat.json"
|
||||
"juiceshop:NYX_JUICESHOP_CORPUS:juiceshop.json"
|
||||
)
|
||||
local any_ran=0 any_failed=0
|
||||
for row in "${rows[@]}"; do
|
||||
local name envvar gtfile
|
||||
IFS=: read -r name envvar gtfile <<<"${row}"
|
||||
# When --sets names a single corpus, only run that row.
|
||||
if [[ -n "${SETS}" && "${SETS}" != "jsts" && "${SETS}" != "${name}" ]]; then
|
||||
continue
|
||||
fi
|
||||
local corpus="${!envvar:-}"
|
||||
if [[ -z "${corpus}" || ! -d "${corpus}" ]]; then
|
||||
echo " SKIP ${name}: set ${envvar} to a checkout to run this row."
|
||||
continue
|
||||
fi
|
||||
any_ran=1
|
||||
echo " ── ${name} (${corpus}) ──"
|
||||
# No --lang scope: NodeGoat/Juice Shop are single-language (js/ts), so
|
||||
# there is no cross-language asset noise to filter (unchanged Gate 7).
|
||||
if _run_corpus_acceptance "${name}" "${corpus}" \
|
||||
"${REPO_ROOT}/tests/eval_corpus/ground_truth/${gtfile}" \
|
||||
"${GATE7_WALLCLOCK_BUDGET}" "${GATE7_BUDGET}" "${GATE7_FLOOR_CAPS}" \
|
||||
"${GATE7_CONFIRMED_RATE_TARGET}" "${GATE7_PRECISION_TARGET}" \
|
||||
"${GATE7_RECALL_TARGET}" "NYX_JSTS_FLOOR_CAPS unset" ""; then
|
||||
echo " PASS ${name}"
|
||||
else
|
||||
any_failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${any_ran} -eq 0 ]]; then
|
||||
echo " SKIP: no JS/TS corpus configured (set NYX_NODEGOAT_CORPUS / NYX_JUICESHOP_CORPUS)."
|
||||
echo " (Gate 7 is Phase 28's headline acceptance for the JS/TS real corpora.)"
|
||||
return 0
|
||||
fi
|
||||
[[ ${any_failed} -eq 0 ]] || return 1
|
||||
echo " PASS"
|
||||
}
|
||||
|
||||
# ── Gate 8: Polyglot real-corpus acceptance (Track R.2 / Phase 29) ────────────
|
||||
|
||||
# RailsGoat (Rails, .rb) + DVWA (PHP) + DVPWA (aiohttp, .py) + gosec (Go) +
|
||||
# the RustSec advisory-db (Rust negative control). Same wall-clock split and
|
||||
# the same report-only-by-default floor policy as Gates 6/7: the per-(cap,lang)
|
||||
# budget in tests/eval_corpus/budget.toml is hard-enforced, while per-cap
|
||||
# confirmed-rate / precision / recall are published but gate nothing until
|
||||
# NYX_POLYGLOT_FLOOR_CAPS names a cap. Each row self-skips unless its
|
||||
# corpus env var points at a real checkout. The RustSec row is a NEGATIVE
|
||||
# CONTROL: advisory-db ships advisory metadata, not vulnerable source, so its
|
||||
# ground truth is empty by construction and the row asserts nyx Confirms
|
||||
# nothing there (false_confirmed_rate guard).
|
||||
GATE8_WALLCLOCK_BUDGET="${NYX_POLYGLOT_WALLCLOCK_BUDGET_SECONDS:-900}"
|
||||
GATE8_CONFIRMED_RATE_TARGET="${NYX_POLYGLOT_CONFIRMED_RATE_TARGET:-0.40}"
|
||||
GATE8_PRECISION_TARGET="${NYX_POLYGLOT_PRECISION_TARGET:-0.85}"
|
||||
GATE8_RECALL_TARGET="${NYX_POLYGLOT_RECALL_TARGET:-0.40}"
|
||||
GATE8_FLOOR_CAPS="${NYX_POLYGLOT_FLOOR_CAPS:-}"
|
||||
GATE8_BUDGET="${NYX_POLYGLOT_BUDGET:-${REPO_ROOT}/tests/eval_corpus/budget.toml}"
|
||||
|
||||
gate_8_polyglot_scale() {
|
||||
echo "── Gate 8: polyglot real-corpus (RailsGoat/DVWA/DVPWA/gosec/RustSec) verify acceptance ──"
|
||||
cargo build --release --quiet --features dynamic
|
||||
|
||||
# name : env var holding the corpus dir : committed ground-truth file :
|
||||
# target language (tabulation is scoped to it so incidental other-language
|
||||
# assets — e.g. vendored JS in the Rails / aiohttp apps — do not pollute
|
||||
# the corpus's per-cap metrics).
|
||||
local rows=(
|
||||
"railsgoat:NYX_RAILSGOAT_CORPUS:railsgoat.json:ruby"
|
||||
"dvwa:NYX_DVWA_CORPUS:dvwa.json:php"
|
||||
"dvpwa:NYX_DVPWA_CORPUS:dvpwa.json:python"
|
||||
"gosec:NYX_GOSEC_CORPUS:gosec.json:go"
|
||||
"rustsec:NYX_RUSTSEC_CORPUS:rustsec.json:rust"
|
||||
)
|
||||
local any_ran=0 any_failed=0
|
||||
for row in "${rows[@]}"; do
|
||||
local name envvar gtfile lang
|
||||
IFS=: read -r name envvar gtfile lang <<<"${row}"
|
||||
# When --sets names a single corpus, only run that row.
|
||||
if [[ -n "${SETS}" && "${SETS}" != "polyglot" && "${SETS}" != "${name}" ]]; then
|
||||
continue
|
||||
fi
|
||||
local corpus="${!envvar:-}"
|
||||
if [[ -z "${corpus}" || ! -d "${corpus}" ]]; then
|
||||
echo " SKIP ${name}: set ${envvar} to a checkout to run this row."
|
||||
continue
|
||||
fi
|
||||
any_ran=1
|
||||
echo " ── ${name} (${corpus}) ──"
|
||||
if _run_corpus_acceptance "${name}" "${corpus}" \
|
||||
"${REPO_ROOT}/tests/eval_corpus/ground_truth/${gtfile}" \
|
||||
"${GATE8_WALLCLOCK_BUDGET}" "${GATE8_BUDGET}" "${GATE8_FLOOR_CAPS}" \
|
||||
"${GATE8_CONFIRMED_RATE_TARGET}" "${GATE8_PRECISION_TARGET}" \
|
||||
"${GATE8_RECALL_TARGET}" "NYX_POLYGLOT_FLOOR_CAPS unset" "${lang}"; then
|
||||
echo " PASS ${name}"
|
||||
else
|
||||
any_failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${any_ran} -eq 0 ]]; then
|
||||
echo " SKIP: no polyglot corpus configured (set NYX_RAILSGOAT_CORPUS /"
|
||||
echo " NYX_DVWA_CORPUS / NYX_DVPWA_CORPUS / NYX_GOSEC_CORPUS / NYX_RUSTSEC_CORPUS)."
|
||||
echo " (Gate 8 is Phase 29's headline acceptance for the polyglot real corpora.)"
|
||||
return 0
|
||||
fi
|
||||
[[ ${any_failed} -eq 0 ]] || return 1
|
||||
echo " PASS"
|
||||
}
|
||||
|
||||
# ── Driver ────────────────────────────────────────────────────────────────────
|
||||
|
||||
declare -a FAILED=()
|
||||
run_gate() {
|
||||
local idx="$1" name="$2"
|
||||
if want_gate "${idx}"; then
|
||||
if ! "gate_${idx}_${name}"; then
|
||||
FAILED+=("${idx}")
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
run_gate 1 static_corpus
|
||||
run_gate 2 dynamic_tests
|
||||
run_gate 3 verify_ratio
|
||||
run_gate 4 sarif_schema
|
||||
run_gate 5 layering
|
||||
run_gate 6 owasp_scale
|
||||
run_gate 7 jsts_scale
|
||||
run_gate 8 polyglot_scale
|
||||
|
||||
if [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo
|
||||
echo "FAILED gates: ${FAILED[*]}"
|
||||
exit 1
|
||||
fi
|
||||
echo
|
||||
echo "All requested gates passed."
|
||||
48
scripts/update_dynamic_goldens.sh
Executable file
48
scripts/update_dynamic_goldens.sh
Executable file
|
|
@ -0,0 +1,48 @@
|
|||
#!/usr/bin/env bash
|
||||
# Regenerate dynamic-fixture golden verdicts.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/update_dynamic_goldens.sh [--test <name>]
|
||||
#
|
||||
# Re-runs the dynamic fixture suites under `NYX_UPDATE_GOLDENS=1` so each
|
||||
# fixture's harness overwrites its `.golden.json` file with the current
|
||||
# verdict. After this script completes, rerun without the env var to
|
||||
# confirm the goldens match.
|
||||
#
|
||||
# Default: refreshes both python_fixtures and rust_fixtures. Pass --test
|
||||
# to refresh only one suite (e.g. `--test python_fixtures`).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
SUITES=(python_fixtures rust_fixtures)
|
||||
if [[ $# -gt 0 ]]; then
|
||||
case "$1" in
|
||||
--test) SUITES=("$2"); shift 2 ;;
|
||||
-h|--help)
|
||||
sed -n '2,12p' "$0"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown arg: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
for suite in "${SUITES[@]}"; do
|
||||
echo "[update-goldens] refreshing $suite ..."
|
||||
NYX_UPDATE_GOLDENS=1 \
|
||||
cargo nextest run --features dynamic --test "$suite" --no-fail-fast
|
||||
done
|
||||
|
||||
echo "[update-goldens] re-running suites without NYX_UPDATE_GOLDENS=1 to verify ..."
|
||||
for suite in "${SUITES[@]}"; do
|
||||
cargo nextest run --features dynamic --test "$suite"
|
||||
done
|
||||
|
||||
echo "[update-goldens] done. Inspect git diff under tests/dynamic_fixtures/ before committing."
|
||||
Loading…
Add table
Add a link
Reference in a new issue