mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-01 09:29:38 +02:00
feat: add query/retrieval FlowProcessor services and missing runner scripts
Wire up the query and retrieval side of the pipeline so the agent can answer questions from stored knowledge: - Triples query service (FalkorDB) — all SPO pattern queries via NATS - Graph embeddings query service (Qdrant) — entity vector similarity - Document embeddings query service (Qdrant) — chunk vector similarity - Graph RAG service — full concept→entity→traverse→score→synthesize pipeline - Document RAG service — embed→find chunks→synthesize pipeline - Runner scripts for chunker, extractor, embeddings (missing from Phase 5) - Add DocumentEmbeddingsRequest/Response schema types - Add RAG prompt templates (extract-concepts, edge-scoring, synthesize) - Add graph/doc embeddings query topics to seed config + flow manager - Add all pipeline/query/retrieval services to docker-compose - 8 new runner scripts, 8 new pnpm script aliases Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
8f7008822a
commit
c545213224
19 changed files with 763 additions and 1 deletions
6
ts/scripts/run-chunker.ts
Normal file
6
ts/scripts/run-chunker.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import { run } from "../packages/flow/src/chunking/service.js";
|
||||
|
||||
run().catch((err) => {
|
||||
console.error("Chunking service failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
6
ts/scripts/run-doc-embeddings-query.ts
Normal file
6
ts/scripts/run-doc-embeddings-query.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import { run } from "../packages/flow/src/query/embeddings/qdrant-doc-service.js";
|
||||
|
||||
run().catch((err) => {
|
||||
console.error("Document embeddings query service failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
6
ts/scripts/run-document-rag.ts
Normal file
6
ts/scripts/run-document-rag.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import { run } from "../packages/flow/src/retrieval/document-rag-service.js";
|
||||
|
||||
run().catch((err) => {
|
||||
console.error("Document RAG service failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
6
ts/scripts/run-embeddings.ts
Normal file
6
ts/scripts/run-embeddings.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import { run } from "../packages/flow/src/embeddings/ollama.js";
|
||||
|
||||
run().catch((err) => {
|
||||
console.error("Embeddings service failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
6
ts/scripts/run-extractor.ts
Normal file
6
ts/scripts/run-extractor.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import { run } from "../packages/flow/src/extract/knowledge-extract.js";
|
||||
|
||||
run().catch((err) => {
|
||||
console.error("Knowledge extract service failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
6
ts/scripts/run-graph-embeddings-query.ts
Normal file
6
ts/scripts/run-graph-embeddings-query.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import { run } from "../packages/flow/src/query/embeddings/qdrant-graph-service.js";
|
||||
|
||||
run().catch((err) => {
|
||||
console.error("Graph embeddings query service failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
6
ts/scripts/run-graph-rag.ts
Normal file
6
ts/scripts/run-graph-rag.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import { run } from "../packages/flow/src/retrieval/graph-rag-service.js";
|
||||
|
||||
run().catch((err) => {
|
||||
console.error("Graph RAG service failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
6
ts/scripts/run-triples-query.ts
Normal file
6
ts/scripts/run-triples-query.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import { run } from "../packages/flow/src/query/triples/falkordb-service.js";
|
||||
|
||||
run().catch((err) => {
|
||||
console.error("Triples query service failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -88,6 +88,56 @@ async function main(): Promise<void> {
|
|||
"Question: {query}",
|
||||
].join("\n"),
|
||||
},
|
||||
"extract-concepts": {
|
||||
system: "You extract key concepts and entities from questions.",
|
||||
prompt: [
|
||||
"Extract the key concepts and entities from the following question.",
|
||||
"Return one concept per line, no numbering or bullets.",
|
||||
"",
|
||||
"Question: {query}",
|
||||
].join("\n"),
|
||||
},
|
||||
"kg-edge-scoring": {
|
||||
system: "You are a knowledge graph expert that scores the relevance of graph edges to a query.",
|
||||
prompt: [
|
||||
"Given the following question and a list of knowledge graph edges,",
|
||||
"score each edge for relevance to answering the question.",
|
||||
"Return a JSON array of objects with 'id' and 'score' (0.0 to 1.0).",
|
||||
"",
|
||||
"Question: {query}",
|
||||
"",
|
||||
"Edges:",
|
||||
"{knowledge}",
|
||||
"",
|
||||
"Requirements:",
|
||||
"- Respond only with a valid JSON array.",
|
||||
"- Example: [{\"id\": \"0\", \"score\": 0.9}, {\"id\": \"1\", \"score\": 0.2}]",
|
||||
].join("\n"),
|
||||
},
|
||||
"graph-rag-synthesize": {
|
||||
system: "You are a helpful assistant that answers questions using knowledge graph data. Only use the provided context.",
|
||||
prompt: [
|
||||
"Use the following knowledge graph relationships to answer the question.",
|
||||
"Do not speculate if the answer is not found in the context.",
|
||||
"",
|
||||
"Knowledge:",
|
||||
"{context}",
|
||||
"",
|
||||
"Question: {query}",
|
||||
].join("\n"),
|
||||
},
|
||||
"document-rag-synthesize": {
|
||||
system: "You are a helpful assistant. Use only the provided document context to answer questions.",
|
||||
prompt: [
|
||||
"Use the following document excerpts to answer the question.",
|
||||
"Do not speculate if the answer is not found in the context.",
|
||||
"",
|
||||
"Documents:",
|
||||
"{context}",
|
||||
"",
|
||||
"Question: {query}",
|
||||
].join("\n"),
|
||||
},
|
||||
});
|
||||
|
||||
// 2. Flow definitions (default flow with all topic mappings)
|
||||
|
|
@ -129,6 +179,12 @@ async function main(): Promise<void> {
|
|||
// Embeddings
|
||||
"embeddings-request": "tg.flow.embeddings-request",
|
||||
"embeddings-response": "tg.flow.embeddings-response",
|
||||
// Graph embeddings query
|
||||
"graph-embeddings-request": "tg.flow.graph-embeddings-request",
|
||||
"graph-embeddings-response": "tg.flow.graph-embeddings-response",
|
||||
// Document embeddings query
|
||||
"document-embeddings-request": "tg.flow.document-embeddings-request",
|
||||
"document-embeddings-response": "tg.flow.document-embeddings-response",
|
||||
// Librarian RPC (for PDF decoder)
|
||||
"librarian-request": "tg.flow.librarian-request",
|
||||
"librarian-response": "tg.flow.librarian-response",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue