mirror of
https://github.com/elicpeter/nyx.git
synced 2026-06-21 20:18:06 +02:00
updated CHANGELOG.md
This commit is contained in:
parent
25863d222a
commit
291fe5d7be
13 changed files with 320 additions and 73 deletions
|
|
@ -354,11 +354,17 @@ pub(crate) fn verify_findings_for_scan(
|
|||
config: &Config,
|
||||
verbose: bool,
|
||||
use_index_db: bool,
|
||||
progress: Option<&Arc<ScanProgress>>,
|
||||
) -> Option<crate::dynamic::verify::VerifyOptions> {
|
||||
if !config.scanner.verify {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(p) = progress {
|
||||
p.start_dynamic_verification(diags.len() as u64);
|
||||
}
|
||||
let verify_start = std::time::Instant::now();
|
||||
|
||||
let mut opts = crate::dynamic::verify::VerifyOptions::from_config(config);
|
||||
// Phase 30 (Track C observability): surface the per-finding
|
||||
// [`crate::dynamic::trace::VerifyTrace`] on stderr when the operator
|
||||
|
|
@ -405,14 +411,27 @@ pub(crate) fn verify_findings_for_scan(
|
|||
if let Some(trace) = &lane_trace {
|
||||
trace.print_to_stderr();
|
||||
}
|
||||
if let Some(p) = progress {
|
||||
p.inc_dynamic_completed(out.len() as u64);
|
||||
}
|
||||
out
|
||||
} else {
|
||||
diags
|
||||
.iter()
|
||||
.map(|d| crate::dynamic::verify::verify_finding(d, &opts))
|
||||
.map(|d| {
|
||||
let result = crate::dynamic::verify::verify_finding(d, &opts);
|
||||
if let Some(p) = progress {
|
||||
p.inc_dynamic_completed(1);
|
||||
}
|
||||
result
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
|
||||
if let Some(p) = progress {
|
||||
p.record_dynamic_verify_ms(verify_start.elapsed().as_millis() as u64);
|
||||
}
|
||||
|
||||
for (diag, mut result) in diags.iter_mut().zip(results) {
|
||||
if result.status == crate::dynamic::report::VerifyStatus::Confirmed
|
||||
&& let Some(ref log_path) = telemetry_log
|
||||
|
|
@ -808,6 +827,7 @@ pub fn handle(
|
|||
config,
|
||||
verbose,
|
||||
index_mode != IndexMode::Off,
|
||||
None,
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "dynamic"))]
|
||||
|
|
|
|||
|
|
@ -38,6 +38,9 @@ pub enum ServerEvent {
|
|||
files_skipped: u64,
|
||||
batches_total: u64,
|
||||
batches_completed: u64,
|
||||
dynamic_enabled: bool,
|
||||
dynamic_total: u64,
|
||||
dynamic_completed: u64,
|
||||
current_file: String,
|
||||
elapsed_ms: u64,
|
||||
timing: TimingBreakdown,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ use r2d2::Pool;
|
|||
use r2d2_sqlite::SqliteConnectionManager;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
use tokio::sync::broadcast;
|
||||
|
|
@ -107,6 +108,10 @@ impl JobManager {
|
|||
let progress = Arc::new(ScanProgress::new());
|
||||
let metrics = Arc::new(ScanMetrics::new());
|
||||
let log_collector = Arc::new(ScanLogCollector::default());
|
||||
#[cfg(feature = "dynamic")]
|
||||
if config.scanner.verify {
|
||||
progress.expect_dynamic_verification();
|
||||
}
|
||||
|
||||
let engine_version = env!("CARGO_PKG_VERSION").to_string();
|
||||
|
||||
|
|
@ -184,11 +189,12 @@ impl JobManager {
|
|||
let progress_for_sse = Arc::clone(&progress);
|
||||
let event_tx_sse = event_tx.clone();
|
||||
let jid_sse = job_id.clone();
|
||||
let progress_done = Arc::new(AtomicBool::new(false));
|
||||
let progress_done_sse = Arc::clone(&progress_done);
|
||||
std::thread::spawn(move || {
|
||||
loop {
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
let snap = progress_for_sse.snapshot();
|
||||
let is_complete = snap.stage == "complete";
|
||||
let _ = event_tx_sse.send(ServerEvent::ScanProgress {
|
||||
job_id: jid_sse.clone(),
|
||||
stage: snap.stage,
|
||||
|
|
@ -198,11 +204,14 @@ impl JobManager {
|
|||
files_skipped: snap.files_skipped,
|
||||
batches_total: snap.batches_total,
|
||||
batches_completed: snap.batches_completed,
|
||||
dynamic_enabled: snap.dynamic_enabled,
|
||||
dynamic_total: snap.dynamic_total,
|
||||
dynamic_completed: snap.dynamic_completed,
|
||||
current_file: snap.current_file,
|
||||
elapsed_ms: snap.elapsed_ms,
|
||||
timing: snap.timing,
|
||||
});
|
||||
if is_complete {
|
||||
if progress_done_sse.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -264,6 +273,7 @@ impl JobManager {
|
|||
&config,
|
||||
false,
|
||||
true,
|
||||
Some(&progress),
|
||||
);
|
||||
}
|
||||
Ok(diags)
|
||||
|
|
@ -271,6 +281,10 @@ impl JobManager {
|
|||
#[cfg(feature = "dynamic")]
|
||||
crate::dynamic::sandbox::cleanup_docker_containers();
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
if result.is_ok() {
|
||||
progress.finish_dynamic_verification();
|
||||
}
|
||||
progress_done.store(true, Ordering::Relaxed);
|
||||
|
||||
// Collect snapshots and do expensive work (post-processing,
|
||||
// JSON serialization) BEFORE acquiring the jobs mutex.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU8, AtomicU64, Ordering::Relaxed};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering::Relaxed};
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
|
|
@ -14,7 +14,8 @@ pub enum ScanStage {
|
|||
BuildingCallGraph = 4,
|
||||
Analyzing = 5,
|
||||
PostProcessing = 6,
|
||||
Complete = 7,
|
||||
DynamicVerification = 7,
|
||||
Complete = 8,
|
||||
}
|
||||
|
||||
impl ScanStage {
|
||||
|
|
@ -27,6 +28,7 @@ impl ScanStage {
|
|||
Self::BuildingCallGraph => "building_call_graph",
|
||||
Self::Analyzing => "analyzing",
|
||||
Self::PostProcessing => "post_processing",
|
||||
Self::DynamicVerification => "dynamic_verification",
|
||||
Self::Complete => "complete",
|
||||
}
|
||||
}
|
||||
|
|
@ -43,6 +45,10 @@ pub struct ScanProgress {
|
|||
files_skipped: AtomicU64,
|
||||
batches_total: AtomicU64,
|
||||
batches_completed: AtomicU64,
|
||||
dynamic_expected: AtomicBool,
|
||||
dynamic_finished: AtomicBool,
|
||||
dynamic_total: AtomicU64,
|
||||
dynamic_completed: AtomicU64,
|
||||
current_file: Mutex<String>,
|
||||
started_at: Instant,
|
||||
walk_ms: AtomicU64,
|
||||
|
|
@ -50,6 +56,7 @@ pub struct ScanProgress {
|
|||
call_graph_ms: AtomicU64,
|
||||
pass2_ms: AtomicU64,
|
||||
post_process_ms: AtomicU64,
|
||||
dynamic_verify_ms: AtomicU64,
|
||||
languages: Mutex<HashMap<String, u64>>,
|
||||
}
|
||||
|
||||
|
|
@ -69,6 +76,10 @@ impl ScanProgress {
|
|||
files_skipped: AtomicU64::new(0),
|
||||
batches_total: AtomicU64::new(0),
|
||||
batches_completed: AtomicU64::new(0),
|
||||
dynamic_expected: AtomicBool::new(false),
|
||||
dynamic_finished: AtomicBool::new(false),
|
||||
dynamic_total: AtomicU64::new(0),
|
||||
dynamic_completed: AtomicU64::new(0),
|
||||
current_file: Mutex::new(String::new()),
|
||||
started_at: Instant::now(),
|
||||
walk_ms: AtomicU64::new(0),
|
||||
|
|
@ -76,14 +87,52 @@ impl ScanProgress {
|
|||
call_graph_ms: AtomicU64::new(0),
|
||||
pass2_ms: AtomicU64::new(0),
|
||||
post_process_ms: AtomicU64::new(0),
|
||||
dynamic_verify_ms: AtomicU64::new(0),
|
||||
languages: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_stage(&self, stage: ScanStage) {
|
||||
let stage = if stage == ScanStage::Complete
|
||||
&& self.dynamic_expected.load(Relaxed)
|
||||
&& !self.dynamic_finished.load(Relaxed)
|
||||
{
|
||||
ScanStage::PostProcessing
|
||||
} else {
|
||||
stage
|
||||
};
|
||||
self.stage.store(stage as u8, Relaxed);
|
||||
}
|
||||
|
||||
pub fn expect_dynamic_verification(&self) {
|
||||
self.dynamic_expected.store(true, Relaxed);
|
||||
self.dynamic_finished.store(false, Relaxed);
|
||||
self.dynamic_total.store(0, Relaxed);
|
||||
self.dynamic_completed.store(0, Relaxed);
|
||||
}
|
||||
|
||||
pub fn start_dynamic_verification(&self, total: u64) {
|
||||
self.dynamic_expected.store(true, Relaxed);
|
||||
self.dynamic_finished.store(false, Relaxed);
|
||||
self.dynamic_total.store(total, Relaxed);
|
||||
self.dynamic_completed.store(0, Relaxed);
|
||||
self.stage
|
||||
.store(ScanStage::DynamicVerification as u8, Relaxed);
|
||||
}
|
||||
|
||||
pub fn inc_dynamic_completed(&self, n: u64) {
|
||||
self.dynamic_completed.fetch_add(n, Relaxed);
|
||||
}
|
||||
|
||||
pub fn finish_dynamic_verification(&self) {
|
||||
self.dynamic_finished.store(true, Relaxed);
|
||||
let total = self.dynamic_total.load(Relaxed);
|
||||
if total > 0 {
|
||||
self.dynamic_completed.store(total, Relaxed);
|
||||
}
|
||||
self.stage.store(ScanStage::Complete as u8, Relaxed);
|
||||
}
|
||||
|
||||
pub fn set_files_discovered(&self, count: u64) {
|
||||
self.files_discovered.store(count, Relaxed);
|
||||
}
|
||||
|
|
@ -143,6 +192,10 @@ impl ScanProgress {
|
|||
self.post_process_ms.fetch_add(ms, Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_dynamic_verify_ms(&self, ms: u64) {
|
||||
self.dynamic_verify_ms.fetch_add(ms, Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_language(&self, lang: &str) {
|
||||
if let Ok(mut langs) = self.languages.try_lock() {
|
||||
*langs.entry(lang.to_string()).or_insert(0) += 1;
|
||||
|
|
@ -158,6 +211,9 @@ impl ScanProgress {
|
|||
x if x == ScanStage::BuildingCallGraph as u8 => ScanStage::BuildingCallGraph.as_str(),
|
||||
x if x == ScanStage::Analyzing as u8 => ScanStage::Analyzing.as_str(),
|
||||
x if x == ScanStage::PostProcessing as u8 => ScanStage::PostProcessing.as_str(),
|
||||
x if x == ScanStage::DynamicVerification as u8 => {
|
||||
ScanStage::DynamicVerification.as_str()
|
||||
}
|
||||
x if x == ScanStage::Complete as u8 => ScanStage::Complete.as_str(),
|
||||
_ => "unknown",
|
||||
}
|
||||
|
|
@ -183,6 +239,9 @@ impl ScanProgress {
|
|||
files_skipped: self.files_skipped.load(Relaxed),
|
||||
batches_total: self.batches_total.load(Relaxed),
|
||||
batches_completed: self.batches_completed.load(Relaxed),
|
||||
dynamic_enabled: self.dynamic_expected.load(Relaxed),
|
||||
dynamic_total: self.dynamic_total.load(Relaxed),
|
||||
dynamic_completed: self.dynamic_completed.load(Relaxed),
|
||||
current_file,
|
||||
elapsed_ms: self.elapsed_ms(),
|
||||
timing: TimingBreakdown {
|
||||
|
|
@ -191,6 +250,7 @@ impl ScanProgress {
|
|||
call_graph_ms: self.call_graph_ms.load(Relaxed),
|
||||
pass2_ms: self.pass2_ms.load(Relaxed),
|
||||
post_process_ms: self.post_process_ms.load(Relaxed),
|
||||
dynamic_verify_ms: self.dynamic_verify_ms.load(Relaxed),
|
||||
},
|
||||
languages,
|
||||
}
|
||||
|
|
@ -207,6 +267,9 @@ pub struct ScanProgressSnapshot {
|
|||
pub files_skipped: u64,
|
||||
pub batches_total: u64,
|
||||
pub batches_completed: u64,
|
||||
pub dynamic_enabled: bool,
|
||||
pub dynamic_total: u64,
|
||||
pub dynamic_completed: u64,
|
||||
pub current_file: String,
|
||||
pub elapsed_ms: u64,
|
||||
pub timing: TimingBreakdown,
|
||||
|
|
@ -221,6 +284,8 @@ pub struct TimingBreakdown {
|
|||
pub call_graph_ms: u64,
|
||||
pub pass2_ms: u64,
|
||||
pub post_process_ms: u64,
|
||||
#[serde(default)]
|
||||
pub dynamic_verify_ms: u64,
|
||||
}
|
||||
|
||||
/// Engine-level metrics collected during a scan.
|
||||
|
|
@ -261,6 +326,39 @@ impl ScanMetrics {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn dynamic_verification_defers_static_complete_stage() {
|
||||
let progress = ScanProgress::new();
|
||||
|
||||
progress.expect_dynamic_verification();
|
||||
progress.set_stage(ScanStage::Complete);
|
||||
|
||||
let static_done = progress.snapshot();
|
||||
assert_eq!(static_done.stage, "post_processing");
|
||||
assert!(static_done.dynamic_enabled);
|
||||
assert_eq!(static_done.dynamic_total, 0);
|
||||
assert_eq!(static_done.dynamic_completed, 0);
|
||||
|
||||
progress.start_dynamic_verification(3);
|
||||
progress.inc_dynamic_completed(2);
|
||||
|
||||
let verifying = progress.snapshot();
|
||||
assert_eq!(verifying.stage, "dynamic_verification");
|
||||
assert_eq!(verifying.dynamic_total, 3);
|
||||
assert_eq!(verifying.dynamic_completed, 2);
|
||||
|
||||
progress.finish_dynamic_verification();
|
||||
|
||||
let complete = progress.snapshot();
|
||||
assert_eq!(complete.stage, "complete");
|
||||
assert_eq!(complete.dynamic_completed, 3);
|
||||
}
|
||||
}
|
||||
|
||||
/// Serializable snapshot of engine metrics.
|
||||
#[derive(Debug, Clone, Serialize, Default)]
|
||||
pub struct ScanMetricsSnapshot {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue