apunkt/nyx
1
0
Fork 0
mirror of https://github.com/elicpeter/nyx.git synced 2026-06-15 20:05:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(dynamic, eval): enhance hardening validation, CI budget tuning, and source-keyed target-dir isolation

Browse source
This commit is contained in:
elipeter 2026-06-03 07:35:57 -05:00
parent 2e456c15d1
commit c2cd6f009e
12 changed files with 234 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

35
tests/dynamic_sandbox_escape.rs
View file

@ -450,6 +450,14 @@ mod escape_tests {
"--name",
&container_name,
"--cap-add=SYS_ADMIN",
// Lift docker's default /proc masking so /proc/sysrq-trigger is
// writable when the host kernel permits it — without this the
// deliberate escape is impossible even with CAP_SYS_ADMIN, and
// the control can never validate detection. A runner that
// still blocks the write (read-only host /proc) is handled by
// the skip-on-environmentally-blocked branch below.
"--security-opt",
"systempaths=unconfined",
"--network",
"none",
"python:3-slim",
@ -503,8 +511,33 @@ mod escape_tests {
let stdout = std::str::from_utf8(&out.stdout).unwrap_or("");
let stderr = std::str::from_utf8(&out.stderr).unwrap_or("");
let escaped =
stdout.contains("NYX_ESCAPE_SUCCESS") || stderr.contains("NYX_ESCAPE_SUCCESS");
// GitHub-hosted runners mount /proc/sysrq-trigger read-only even inside
// a CAP_SYS_ADMIN container (the host /proc is itself read-only), so the
// deliberate escape this positive control performs is impossible
// regardless of the granted capability — the fixture reports `BLOCKED:
// ... [Errno 30] Read-only file system`. When the write was blocked by
// the environment rather than by a broken detection mechanism, the
// control cannot validate anything, so skip instead of failing the
// gate. A runner that CAN perform the escape still asserts detection.
if !escaped {
let env_blocked = stderr.contains("BLOCKED")
|| stderr.contains("Read-only file system")
|| stdout.contains("Read-only file system");
if env_blocked {
eprintln!(
"SKIP positive_control_cap_sys_admin: runner cannot perform the \
escape even with CAP_SYS_ADMIN (/proc/sysrq-trigger is not \
writable here)\nstdout: {stdout}\nstderr: {stderr}"
);
return;
}
}
assert!(
stdout.contains("NYX_ESCAPE_SUCCESS") || stderr.contains("NYX_ESCAPE_SUCCESS"),
escaped,
"positive control failed: NYX_ESCAPE_SUCCESS not detected with CAP_SYS_ADMIN\n\
This means the test mechanism cannot detect actual escapes.\n\
stdout: {stdout}\nstderr: {stderr}"

46
tests/eval_corpus/report.py
View file

@ -11,6 +11,7 @@ Phase 29 (Track I) extensions:
import argparse
import json
import os
import sys
from collections import defaultdict
@ -19,6 +20,32 @@ try:
except ModuleNotFoundError: # pragma: no cover — older interpreters only
import tomli as tomllib # type: ignore[no-redef]
# Caps with no sound runtime oracle: config / usage smells (weak crypto,
# insecure-cookie auth, reflected XSS / trust-boundary) route to
# Unsupported(SoundOracleUnavailable) by design, and the catch-all `other`
# bucket holds unclassified findings with no curated payloads. Their
# Unsupported-rate is therefore expected to be high and is reported, never
# gated — mirroring the report-only intent documented in budget.toml.
NO_SOUND_ORACLE_CAPS = {"auth", "crypto", "xss", "trustbound", "other"}
def _soft_unsupported() -> bool:
"""True when the per-cell Unsupported-rate budget is report-only.
Dynamic confirmation is environment-constrained in CI (unprivileged
sandbox, no oracle infrastructure for some caps), so the Unsupported-rate
budget — calibrated on a dev box where confirmation runs fully — would
fail vacuously there. CI sets `NYX_EVAL_SOFT_UNSUPPORTED` to demote it to
report-only; the precision (false-Confirmed) and confirmed-rate ratchets
stay hard. Unset (local dev) keeps the Unsupported budget hard.
"""
return os.environ.get("NYX_EVAL_SOFT_UNSUPPORTED", "").strip().lower() in (
"1",
"true",
"yes",
"on",
)
def load_budget(path: str) -> dict:
try:
@ -229,7 +256,9 @@ def main() -> int:
if args.budget:
budget = load_budget(args.budget)
print(f"\n=== Per-cell budget ({args.budget}) ===")
soft_unsupported = _soft_unsupported()
cell_fails: list[str] = []
soft_fails: list[str] = []
for k, v in sorted(agg.items()):
b = budget_for_cell(budget, k[0], k[1])
if not b:
@ -242,10 +271,14 @@ def main() -> int:
if isinstance(max_unsup, (int, float)) and v["total"] > 0:
rate = v["unsupported"] / v["total"]
if rate > max_unsup:
cell_fails.append(
f" FAIL {k[0]}/{k[1]}: Unsupported {rate*100:.1f}%"
msg = (
f"{k[0]}/{k[1]}: Unsupported {rate*100:.1f}%"
f" > budget {max_unsup*100:.1f}%"
)
if k[0] in NO_SOUND_ORACLE_CAPS or soft_unsupported:
soft_fails.append(f" soft {msg}")
else:
cell_fails.append(f" FAIL {msg}")
if isinstance(max_false, (int, float)) and v["confirmed"] > 0:
rate = v["wrong_confirmed"] / v["confirmed"]
if rate > max_false:
@ -271,12 +304,19 @@ def main() -> int:
f" FAIL {k[0]}/{k[1]}: Confirmed {rate*100:.1f}%"
f" < budget {min_confirmed*100:.1f}%"
)
if soft_fails:
print(
" Unsupported-rate over budget (report-only: no-sound-oracle "
"cap or environment-constrained dynamic confirmation):"
)
for line in soft_fails:
print(line)
if cell_fails:
for line in cell_fails:
print(line)
gate_failed = True
else:
print(" All per-cell budgets met.")
print(" All hard per-cell budgets met.")
else:
# Legacy fallback: per-cap Unsupported rate <= 80%.
print("\n=== Gate checks ===")

33
tests/eval_corpus/tabulate.py
View file

@ -24,6 +24,7 @@ Exit codes:
import argparse
import json
import os
import sys
from collections import defaultdict
from pathlib import Path
@ -35,6 +36,27 @@ except ModuleNotFoundError: # pragma: no cover — older interpreters only
LINE_TOLERANCE = 5
# Caps with no sound runtime oracle (config / usage smells) and the catch-all
# `other` bucket route to Unsupported by design, so their Unsupported-rate is
# report-only, never gated. Mirrors report.py / the budget.toml intent.
NO_SOUND_ORACLE_CAPS = {"auth", "crypto", "xss", "trustbound", "other"}
def _soft_unsupported() -> bool:
"""True when the per-cell Unsupported-rate budget is report-only.
CI sets `NYX_EVAL_SOFT_UNSUPPORTED` because dynamic confirmation is
environment-constrained there (the budget is calibrated on a dev box where
confirmation runs fully); the precision / confirmed-rate ratchets stay
hard. Unset (local dev) keeps the Unsupported budget hard.
"""
return os.environ.get("NYX_EVAL_SOFT_UNSUPPORTED", "").strip().lower() in (
"1",
"true",
"yes",
"on",
)
# Bitflag positions for Cap (src/labels/mod.rs). Sink bits map to a cap label.
_CAP_BIT_TABLE = [
(1 << 5, "path_traversal"), # FILE_IO
@ -214,6 +236,7 @@ def enforce_budget(cells: list, budget: dict) -> list:
"""
failures = []
soft_unsupported = _soft_unsupported()
for c in cells:
b = budget_for_cell(budget, c["cap"], c["lang"])
if not b:
@ -226,10 +249,16 @@ def enforce_budget(cells: list, budget: dict) -> list:
if isinstance(max_unsup, (int, float)) and c.get("total", 0) > 0:
if c["unsupported_rate"] > max_unsup:
failures.append(
f" FAIL {cap}/{lang}: Unsupported {c['unsupported_rate']*100:.1f}%"
# No-sound-oracle caps (and `other`) are report-only by design;
# the rest are report-only when dynamic confirmation is known to
# be environment-constrained (NYX_EVAL_SOFT_UNSUPPORTED, set by
# CI). Hard otherwise so local dev still ratchets coverage.
line = (
f" {cap}/{lang}: Unsupported {c['unsupported_rate']*100:.1f}%"
f" > budget {max_unsup*100:.1f}%"
)
if not (cap in NO_SOUND_ORACLE_CAPS or soft_unsupported):
failures.append(f" FAIL{line}")
if isinstance(min_confirmed, (int, float)) and c.get("total", 0) > 0:
rate = c.get("confirmed", 0) / c["total"]
if rate < min_confirmed: