Add rayon for parallel file scanning in for no indexingscan.rs

- Integrated `rayon` for concurrent iteration in `scan_filesystem`, enhancing performance.
- Updated `Cargo.toml` and `Cargo.lock` to include `rayon` and its dependencies.
- Adjusted `scan.rs` to utilize `rayon`'s `par_bridge` for parallel file processing with error handling.
This commit is contained in:
elipeter 2025-06-17 19:54:03 +02:00
parent d7b8833ec6
commit 1933082b41
3 changed files with 47 additions and 8 deletions

27
Cargo.lock generated
View file

@ -246,6 +246,12 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "either"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "encode_unicode"
version = "1.0.0"
@ -459,6 +465,7 @@ dependencies = [
"ignore",
"num_cpus",
"once_cell",
"rayon",
"rusqlite",
"serde",
"toml",
@ -536,6 +543,26 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rayon"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "redox_users"
version = "0.5.0"

View file

@ -27,3 +27,4 @@ crossbeam-channel = "0.5.15"
blake3 = "1.8.2"
once_cell = "1.21.3"
console = "0.15.11"
rayon = "1.10.0"

View file

@ -1,15 +1,18 @@
use crate::utils::project::get_project_info;
use console::style;
use std::path::Path;
use std::sync::Mutex;
use crate::database::index::{IssueRow, Indexer};
use crate::patterns::Severity;
use crate::utils::config::Config;
use crate::utils::query_cache;
use crate::walk::spawn_senders;
use rayon::prelude::*;
use tree_sitter::{Language, Parser, QueryCursor, StreamingIterator};
type DynError = Box<dyn std::error::Error + Send + Sync>;
#[derive(Debug)]
pub struct Diag {
pub(crate) path: String,
@ -34,7 +37,7 @@ pub fn handle(
let diags: Vec<Diag>;
if no_index {
diags = scan_filesystem(&scan_path, config)?;
diags = scan_filesystem(&scan_path, config).unwrap();
} else {
if rebuild_index || !db_path.exists() {
tracing::debug!("Scanning filesystem index filesystem");
@ -75,13 +78,21 @@ pub fn handle(
fn scan_filesystem(
root: &Path,
cfg: &Config,
) -> Result<Vec<Diag>, Box<dyn std::error::Error>> {
) ->Result<Vec<Diag>, Box<dyn std::error::Error + Send + Sync>> {
let rx = spawn_senders(root, cfg);
let mut issues: Vec<Diag> = Vec::new();
for batch in rx.iter().flatten() {
issues.append(&mut run_rules_on_file(&batch, cfg)?);
}
Ok(issues)
let acc = Mutex::new(Vec::new());
rx.into_iter()
.flatten()
.par_bridge() // rayon hand-off
.try_for_each(|path| { // stable API
let mut local = run_rules_on_file(&path, cfg).unwrap(); // <- same as before
let mut guard = acc.lock().unwrap();
guard.append(&mut local);
Ok::<(), DynError>(()) // explicit error type
})?; // propagate first error, if any
Ok(acc.into_inner().unwrap())
}
fn scan_with_index(