mirror of
https://github.com/elicpeter/nyx.git
synced 2026-06-06 19:35:13 +02:00
Add rayon for parallel file scanning in for no indexingscan.rs
- Integrated `rayon` for concurrent iteration in `scan_filesystem`, enhancing performance. - Updated `Cargo.toml` and `Cargo.lock` to include `rayon` and its dependencies. - Adjusted `scan.rs` to utilize `rayon`'s `par_bridge` for parallel file processing with error handling.
This commit is contained in:
parent
d7b8833ec6
commit
1933082b41
3 changed files with 47 additions and 8 deletions
27
Cargo.lock
generated
27
Cargo.lock
generated
|
|
@ -246,6 +246,12 @@ dependencies = [
|
|||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
||||
|
||||
[[package]]
|
||||
name = "encode_unicode"
|
||||
version = "1.0.0"
|
||||
|
|
@ -459,6 +465,7 @@ dependencies = [
|
|||
"ignore",
|
||||
"num_cpus",
|
||||
"once_cell",
|
||||
"rayon",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"toml",
|
||||
|
|
@ -536,6 +543,26 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
|
||||
dependencies = [
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.5.0"
|
||||
|
|
|
|||
|
|
@ -27,3 +27,4 @@ crossbeam-channel = "0.5.15"
|
|||
blake3 = "1.8.2"
|
||||
once_cell = "1.21.3"
|
||||
console = "0.15.11"
|
||||
rayon = "1.10.0"
|
||||
|
|
|
|||
|
|
@ -1,15 +1,18 @@
|
|||
use crate::utils::project::get_project_info;
|
||||
use console::style;
|
||||
use std::path::Path;
|
||||
|
||||
use std::sync::Mutex;
|
||||
use crate::database::index::{IssueRow, Indexer};
|
||||
use crate::patterns::Severity;
|
||||
use crate::utils::config::Config;
|
||||
use crate::utils::query_cache;
|
||||
use crate::walk::spawn_senders;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use tree_sitter::{Language, Parser, QueryCursor, StreamingIterator};
|
||||
|
||||
type DynError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Diag {
|
||||
pub(crate) path: String,
|
||||
|
|
@ -34,7 +37,7 @@ pub fn handle(
|
|||
let diags: Vec<Diag>;
|
||||
|
||||
if no_index {
|
||||
diags = scan_filesystem(&scan_path, config)?;
|
||||
diags = scan_filesystem(&scan_path, config).unwrap();
|
||||
} else {
|
||||
if rebuild_index || !db_path.exists() {
|
||||
tracing::debug!("Scanning filesystem index filesystem");
|
||||
|
|
@ -75,13 +78,21 @@ pub fn handle(
|
|||
fn scan_filesystem(
|
||||
root: &Path,
|
||||
cfg: &Config,
|
||||
) -> Result<Vec<Diag>, Box<dyn std::error::Error>> {
|
||||
) ->Result<Vec<Diag>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let rx = spawn_senders(root, cfg);
|
||||
let mut issues: Vec<Diag> = Vec::new();
|
||||
for batch in rx.iter().flatten() {
|
||||
issues.append(&mut run_rules_on_file(&batch, cfg)?);
|
||||
}
|
||||
Ok(issues)
|
||||
let acc = Mutex::new(Vec::new());
|
||||
|
||||
rx.into_iter()
|
||||
.flatten()
|
||||
.par_bridge() // rayon hand-off
|
||||
.try_for_each(|path| { // stable API
|
||||
let mut local = run_rules_on_file(&path, cfg).unwrap(); // <- same as before
|
||||
let mut guard = acc.lock().unwrap();
|
||||
guard.append(&mut local);
|
||||
Ok::<(), DynError>(()) // explicit error type
|
||||
})?; // propagate first error, if any
|
||||
|
||||
Ok(acc.into_inner().unwrap())
|
||||
}
|
||||
|
||||
fn scan_with_index(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue