mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-06 11:52:10 +02:00
feat: add document pipeline — PDF decoder, Ollama LLM, storage services
Add end-to-end document processing pipeline: - PDF decoder service (pdfjs-dist) extracts text per page from librarian docs - Ollama native LLM service for local model inference - FalkorDB triples store FlowProcessor consumer - Qdrant graph embeddings store FlowProcessor consumer - Fix spec name collisions in chunker/extractor (input→chunk-input, etc.) - Gateway /load endpoint to trigger document processing - Align flow manager blueprint and seed config with full pipeline topics - Add runner scripts and test coverage for document load Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
8f9de7604e
commit
8f7008822a
20 changed files with 894 additions and 37 deletions
|
|
@ -30,10 +30,10 @@ export class ChunkingService extends FlowProcessor {
|
|||
super(config);
|
||||
|
||||
this.registerSpecification(
|
||||
new ConsumerSpec<TextDocument>("input", this.onMessage.bind(this)),
|
||||
new ConsumerSpec<TextDocument>("chunk-input", this.onMessage.bind(this)),
|
||||
);
|
||||
this.registerSpecification(new ProducerSpec<Chunk>("output"));
|
||||
this.registerSpecification(new ProducerSpec<Triples>("triples"));
|
||||
this.registerSpecification(new ProducerSpec<Chunk>("chunk-output"));
|
||||
this.registerSpecification(new ProducerSpec<Triples>("chunk-triples"));
|
||||
this.registerSpecification(new ParameterSpec("chunk-size"));
|
||||
this.registerSpecification(new ParameterSpec("chunk-overlap"));
|
||||
|
||||
|
|
@ -75,7 +75,7 @@ export class ChunkingService extends FlowProcessor {
|
|||
`[ChunkingService] Split document ${msg.documentId} into ${chunks.length} chunks (size=${chunkSize}, overlap=${chunkOverlap})`,
|
||||
);
|
||||
|
||||
const outputProducer = flowCtx.flow.producer<Chunk>("output");
|
||||
const outputProducer = flowCtx.flow.producer<Chunk>("chunk-output");
|
||||
|
||||
for (const chunkText of chunks) {
|
||||
const chunk: Chunk = {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue