feat: implement parallel document indexing in IndexingPipelineService

- Added `index_batch_parallel` method to enable concurrent indexing of documents with bounded concurrency, improving performance and efficiency.
- Refactored existing indexing logic to utilize `asyncio.to_thread` for non-blocking execution of embedding and chunking functions.
- Introduced unit tests to validate the functionality of the new parallel indexing method, ensuring robustness and error handling during document processing.
This commit is contained in:
Anish Sarkar 2026-03-26 19:33:49 +05:30
parent bbd5ee8a19
commit e5cb6bfacf
3 changed files with 76 additions and 4 deletions

View file

@ -5,10 +5,10 @@ import {
FileText,
Film,
Globe,
ImageIcon,
type LucideIcon,
Podcast,
ScanLine,
Sparkles,
Wrench,
} from "lucide-react";
@ -17,7 +17,7 @@ const TOOL_ICONS: Record<string, LucideIcon> = {
generate_podcast: Podcast,
generate_video_presentation: Film,
generate_report: FileText,
generate_image: Sparkles,
generate_image: ImageIcon,
scrape_webpage: ScanLine,
web_search: Globe,
search_surfsense_docs: BookOpen,