feat: add document pipeline — PDF decoder, Ollama LLM, storage services

Add end-to-end document processing pipeline:
- PDF decoder service (pdfjs-dist) extracts text per page from librarian docs
- Ollama native LLM service for local model inference
- FalkorDB triples store FlowProcessor consumer
- Qdrant graph embeddings store FlowProcessor consumer
- Fix spec name collisions in chunker/extractor (input→chunk-input, etc.)
- Gateway /load endpoint to trigger document processing
- Align flow manager blueprint and seed config with full pipeline topics
- Add runner scripts and test coverage for document load

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
elpresidank 2026-04-06 23:47:43 -05:00
parent 8f9de7604e
commit 8f7008822a
20 changed files with 894 additions and 37 deletions

View file

@ -0,0 +1,54 @@
/**
* Triples store service writes RDF triples to FalkorDB via FlowProcessor.
*
* A FlowProcessor that:
* 1. Consumes Triples messages
* 2. Writes each triple to FalkorDB using FalkorDBTriplesStore
*
* Python reference: trustgraph-flow/trustgraph/storage/triples/falkordb/service.py
*/
import {
FlowProcessor,
ConsumerSpec,
type ProcessorConfig,
type FlowContext,
type Triples,
} from "@trustgraph/base";
import { FalkorDBTriplesStore } from "./falkordb.js";
export class TriplesStoreService extends FlowProcessor {
private store: FalkorDBTriplesStore;
constructor(config: ProcessorConfig) {
super(config);
this.store = new FalkorDBTriplesStore();
this.registerSpecification(
new ConsumerSpec<Triples>("store-triples-input", this.onMessage.bind(this)),
);
console.log("[TriplesStore] Service initialized");
}
private async onMessage(
msg: Triples,
properties: Record<string, string>,
flowCtx: FlowContext,
): Promise<void> {
if (!msg.triples || msg.triples.length === 0) return;
const user = msg.metadata?.user ?? "default";
const collection = msg.metadata?.collection ?? "default";
await this.store.storeTriples(msg.triples, user, collection);
console.log(
`[TriplesStore] Stored ${msg.triples.length} triples for ${user}/${collection}`,
);
}
}
export async function run(): Promise<void> {
await TriplesStoreService.launch("triples-store");
}