mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-06 11:52:10 +02:00
feat: add document pipeline — PDF decoder, Ollama LLM, storage services
Add end-to-end document processing pipeline: - PDF decoder service (pdfjs-dist) extracts text per page from librarian docs - Ollama native LLM service for local model inference - FalkorDB triples store FlowProcessor consumer - Qdrant graph embeddings store FlowProcessor consumer - Fix spec name collisions in chunker/extractor (input→chunk-input, etc.) - Gateway /load endpoint to trigger document processing - Align flow manager blueprint and seed config with full pipeline topics - Add runner scripts and test coverage for document load Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
8f9de7604e
commit
8f7008822a
20 changed files with 894 additions and 37 deletions
|
|
@ -95,6 +95,19 @@ async function main(): Promise<void> {
|
|||
await pushConfig(["flows"], {
|
||||
default: {
|
||||
topics: {
|
||||
// Document processing pipeline
|
||||
"decode-input": "tg.flow.document",
|
||||
"decode-output": "tg.flow.text-document",
|
||||
"decode-triples": "tg.flow.triples",
|
||||
"chunk-input": "tg.flow.text-document",
|
||||
"chunk-output": "tg.flow.chunk",
|
||||
"chunk-triples": "tg.flow.triples",
|
||||
"extract-input": "tg.flow.chunk",
|
||||
"extract-triples": "tg.flow.triples",
|
||||
"extract-entity-contexts": "tg.flow.entity-contexts",
|
||||
// Storage consumers
|
||||
"store-triples-input": "tg.flow.triples",
|
||||
"store-graph-embeddings-input": "tg.flow.entity-contexts",
|
||||
// LLM text completion
|
||||
"text-completion-request": "tg.flow.text-completion-request",
|
||||
"text-completion-response": "tg.flow.text-completion-response",
|
||||
|
|
@ -107,17 +120,18 @@ async function main(): Promise<void> {
|
|||
// Document RAG
|
||||
"document-rag-request": "tg.flow.document-rag-request",
|
||||
"document-rag-response": "tg.flow.document-rag-response",
|
||||
// Triples
|
||||
// Triples query
|
||||
"triples-request": "tg.flow.triples-request",
|
||||
"triples-response": "tg.flow.triples-response",
|
||||
// Agent
|
||||
"agent-request": "tg.flow.agent-request",
|
||||
"agent-response": "tg.flow.agent-response",
|
||||
// Chunking pipeline
|
||||
"input": "tg.flow.chunk",
|
||||
"output": "tg.flow.chunk",
|
||||
"triples": "tg.flow.triples",
|
||||
"entity-contexts": "tg.flow.entity-contexts",
|
||||
// Embeddings
|
||||
"embeddings-request": "tg.flow.embeddings-request",
|
||||
"embeddings-response": "tg.flow.embeddings-response",
|
||||
// Librarian RPC (for PDF decoder)
|
||||
"librarian-request": "tg.flow.librarian-request",
|
||||
"librarian-response": "tg.flow.librarian-response",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue