mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-18 23:11:12 +02:00
feat: implement parallel document indexing in IndexingPipelineService
- Added `index_batch_parallel` method to enable concurrent indexing of documents with bounded concurrency, improving performance and efficiency. - Refactored existing indexing logic to utilize `asyncio.to_thread` for non-blocking execution of embedding and chunking functions. - Introduced unit tests to validate the functionality of the new parallel indexing method, ensuring robustness and error handling during document processing.
This commit is contained in:
parent
bbd5ee8a19
commit
e5cb6bfacf
3 changed files with 76 additions and 4 deletions
|
|
@ -5,10 +5,10 @@ import {
|
|||
FileText,
|
||||
Film,
|
||||
Globe,
|
||||
ImageIcon,
|
||||
type LucideIcon,
|
||||
Podcast,
|
||||
ScanLine,
|
||||
Sparkles,
|
||||
Wrench,
|
||||
} from "lucide-react";
|
||||
|
||||
|
|
@ -17,7 +17,7 @@ const TOOL_ICONS: Record<string, LucideIcon> = {
|
|||
generate_podcast: Podcast,
|
||||
generate_video_presentation: Film,
|
||||
generate_report: FileText,
|
||||
generate_image: Sparkles,
|
||||
generate_image: ImageIcon,
|
||||
scrape_webpage: ScanLine,
|
||||
web_search: Globe,
|
||||
search_surfsense_docs: BookOpen,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue