use crossbeam_channel::{bounded, Receiver}; use ignore::{WalkBuilder, WalkState}; use std::{path::{Path, PathBuf}, thread}; use ignore::overrides::OverrideBuilder; use crate::utils::Config; const BATCH_SIZE: usize = 5; type Batch = Vec; struct Batcher { tx: crossbeam_channel::Sender, batch: Batch, } impl Batcher { fn push(&mut self, p: PathBuf) { self.batch.push(p); if self.batch.len() == BATCH_SIZE { self.flush(); } } fn flush(&mut self) { if !self.batch.is_empty() { let _ = self.tx.send(std::mem::take(&mut self.batch)); } } } impl Drop for Batcher { fn drop(&mut self) { // guarantees the remainder is sent when the worker is dropped self.flush(); } } /// Walk `root`, send file paths to the returned receiver. pub fn spawn_senders( root: &Path, cfg: &Config ) -> Receiver { let mut ob = OverrideBuilder::new(root); for ext in &cfg.scanner.excluded_extensions { ob.add(&format!("!*.{ext}")).unwrap(); } for dir in &cfg.scanner.excluded_directories { ob.add(&format!("!**/{dir}/**")).unwrap(); } let overrides = ob.build().unwrap(); let worker_thrs = cfg.performance.worker_threads.unwrap_or(num_cpus::get()); let (tx, rx) = bounded::(worker_thrs * 2usize); let root = root.to_path_buf(); let scan_hidden = cfg.scanner.scan_hidden_files; let follow_links = cfg.scanner.follow_symlinks; thread::spawn(move || { let walker = WalkBuilder::new(root) .hidden(!scan_hidden) .follow_links(follow_links) .threads(worker_thrs) .overrides(overrides) .build_parallel(); walker.run(move || { let tx = tx.clone(); let mut batch = Vec::::with_capacity(256); Box::new(move |entry| { tracing::info!("walking: {:?}", entry); let mut b = Batcher { tx: tx.clone(), batch: Vec::with_capacity(BATCH_SIZE) }; match entry { Ok(e) if e.file_type().map_or(false, |ft| ft.is_file()) => { b.push(e.into_path()); if batch.len() == BATCH_SIZE { let _ = tx.send(std::mem::take(&mut batch)); } } Err(err) => eprintln!("walk error: {err}"), _ => {} } WalkState::Continue }) }); }); rx }