apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-06 20:02:11 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

feat: add document pipeline — PDF decoder, Ollama LLM, storage services

Browse source 
Add end-to-end document processing pipeline:
- PDF decoder service (pdfjs-dist) extracts text per page from librarian docs
- Ollama native LLM service for local model inference
- FalkorDB triples store FlowProcessor consumer
- Qdrant graph embeddings store FlowProcessor consumer
- Fix spec name collisions in chunker/extractor (input→chunk-input, etc.)
- Gateway /load endpoint to trigger document processing
- Align flow manager blueprint and seed config with full pipeline topics
- Add runner scripts and test coverage for document load

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
elpresidank 2026-04-06 23:47:43 -05:00
parent 8f9de7604e
commit 8f7008822a
20 changed files with 894 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

49
ts/packages/flow/src/flow-manager/service.ts
View file

@ -48,22 +48,43 @@ interface Blueprint {
const DEFAULT_BLUEPRINT: Blueprint = {
description: "Default processing pipeline with all services",
topics: {
"request": "tg.flow.text-completion-request",
"response": "tg.flow.text-completion-response",
"prompt-request": "tg.flow.prompt-request",
"prompt-response": "tg.flow.prompt-response",
"graph-rag-request": "tg.flow.graph-rag-request",
"graph-rag-response": "tg.flow.graph-rag-response",
"document-rag-request": "tg.flow.document-rag-request",
"document-rag-response": "tg.flow.document-rag-response",
"triples-request": "tg.flow.triples-request",
"triples-response": "tg.flow.triples-response",
// Document processing pipeline
"decode-input": "tg.flow.document",
"decode-output": "tg.flow.text-document",
"decode-triples": "tg.flow.triples",
"chunk-input": "tg.flow.text-document",
"chunk-output": "tg.flow.chunk",
"chunk-triples": "tg.flow.triples",
"extract-input": "tg.flow.chunk",
"extract-triples": "tg.flow.triples",
"extract-entity-contexts": "tg.flow.entity-contexts",
// Storage consumers
"store-triples-input": "tg.flow.triples",
"store-graph-embeddings-input": "tg.flow.entity-contexts",
// LLM text completion
"text-completion-request": "tg.flow.text-completion-request",
"text-completion-response": "tg.flow.text-completion-response",
"input": "tg.flow.chunk",
"output": "tg.flow.chunk",
"triples": "tg.flow.triples",
"entity-contexts": "tg.flow.entity-contexts",
// Prompt service
"prompt-request": "tg.flow.prompt-request",
"prompt-response": "tg.flow.prompt-response",
// Graph RAG
"graph-rag-request": "tg.flow.graph-rag-request",
"graph-rag-response": "tg.flow.graph-rag-response",
// Document RAG
"document-rag-request": "tg.flow.document-rag-request",
"document-rag-response": "tg.flow.document-rag-response",
// Triples query
"triples-request": "tg.flow.triples-request",
"triples-response": "tg.flow.triples-response",
// Agent
"agent-request": "tg.flow.agent-request",
"agent-response": "tg.flow.agent-response",
// Embeddings
"embeddings-request": "tg.flow.embeddings-request",
"embeddings-response": "tg.flow.embeddings-response",
// Librarian RPC (for PDF decoder)
"librarian-request": "tg.flow.librarian-request",
"librarian-response": "tg.flow.librarian-response",
},
};