nyx/src/walk.rs

use crossbeam_channel::{bounded, Receiver};
use ignore::{WalkBuilder, WalkState};
use std::{path::{Path, PathBuf}, thread};
use ignore::overrides::OverrideBuilder;
use crate::utils::Config;

const BATCH_SIZE: usize = 5;

type Batch = Vec<PathBuf>;

struct Batcher {
    tx: crossbeam_channel::Sender<Batch>,
    batch: Batch,
}

impl Batcher {
    fn push(&mut self, p: PathBuf) {
        self.batch.push(p);
        if self.batch.len() == BATCH_SIZE {
            self.flush();
        }
    }
    fn flush(&mut self) {
        if !self.batch.is_empty() {
            let _ = self.tx.send(std::mem::take(&mut self.batch));
        }
    }
}

impl Drop for Batcher {
    fn drop(&mut self) {
        // guarantees the remainder is sent when the worker is dropped
        self.flush();
    }
}


/// Walk `root`, send file paths to the returned receiver.
pub fn spawn_senders(
    root: &Path, 
    cfg: &Config
) -> Receiver<Batch> {
    let mut ob = OverrideBuilder::new(root);

    for ext in &cfg.scanner.excluded_extensions {
        ob.add(&format!("!*.{ext}")).unwrap();           
    }

    for dir in &cfg.scanner.excluded_directories {
        ob.add(&format!("!**/{dir}/**")).unwrap();
    }
    
    let overrides = ob.build().unwrap();
    let worker_thrs  = cfg.performance.worker_threads.unwrap_or(num_cpus::get());
    
    let (tx, rx) = bounded::<Batch>(worker_thrs * 2usize);
    
    let root       = root.to_path_buf();
    let scan_hidden   = cfg.scanner.scan_hidden_files;
    let follow_links  = cfg.scanner.follow_symlinks;

    thread::spawn(move || {
        let walker = WalkBuilder::new(root)
          .hidden(!scan_hidden)
          .follow_links(follow_links)
          .threads(worker_thrs)
          .overrides(overrides)
          .build_parallel();

        walker.run(move || {
            let tx = tx.clone();
            let mut batch = Vec::<PathBuf>::with_capacity(256);

            Box::new(move |entry| {
                tracing::info!("walking: {:?}", entry);
                
                let mut b = Batcher { tx: tx.clone(), batch: Vec::with_capacity(BATCH_SIZE) };
                match entry {
                    Ok(e) if e.file_type().map_or(false, |ft| ft.is_file()) => {
                        b.push(e.into_path());
                        if batch.len() == BATCH_SIZE {
                            let _ = tx.send(std::mem::take(&mut batch));
                        }
                    }
                    Err(err) => eprintln!("walk error: {err}"),
                    _ => {}
                }
                WalkState::Continue
            })
        });

    });
    
    rx
}