/** * Flow manager service -- manages flow lifecycle (start/stop/list) and blueprints. * * An AsyncProcessor (NOT FlowProcessor) that: * 1. Listens on flow-request topic * 2. Handles operations: list-flows, get-flow, start-flow, stop-flow, * list-blueprints, get-blueprint, delete-blueprint * 3. Stores flows and blueprints in-memory * 4. On start/stop: pushes updated flow config to the config service * * Wire format uses kebab-case field names to match the client. * Access fields via bracket notation: request["flow-id"], response["flow-ids"]. * * Python reference: trustgraph-flow/trustgraph/flow/service.py */ import { AsyncProcessor, type ProcessorConfig, topics, RequestResponse, type ConfigRequest, type ConfigResponse, } from "@trustgraph/base"; import { makeProcessorProgram } from "@trustgraph/base"; import type { BackendProducer, BackendConsumer, Message, } from "@trustgraph/base"; // ---------- Internal state types ---------- interface FlowInstance { id: string; blueprintName: string; description: string; parameters: Record; status: "running" | "stopped"; } interface Blueprint { description: string; topics: Record; } // ---------- Default blueprint ---------- const DEFAULT_BLUEPRINT: Blueprint = { description: "Default processing pipeline with all services", topics: { // Document processing pipeline "decode-input": "tg.flow.document", "decode-output": "tg.flow.text-document", "decode-triples": "tg.flow.triples", "chunk-input": "tg.flow.text-document", "chunk-output": "tg.flow.chunk", "chunk-triples": "tg.flow.triples", "extract-input": "tg.flow.chunk", "extract-triples": "tg.flow.triples", "extract-entity-contexts": "tg.flow.entity-contexts", // Storage consumers "store-triples-input": "tg.flow.triples", "store-graph-embeddings-input": "tg.flow.entity-contexts", // LLM text completion "text-completion-request": "tg.flow.text-completion-request", "text-completion-response": "tg.flow.text-completion-response", // Prompt service "prompt-request": "tg.flow.prompt-request", "prompt-response": "tg.flow.prompt-response", // Graph RAG "graph-rag-request": "tg.flow.graph-rag-request", "graph-rag-response": "tg.flow.graph-rag-response", // Document RAG "document-rag-request": "tg.flow.document-rag-request", "document-rag-response": "tg.flow.document-rag-response", // Triples query "triples-request": "tg.flow.triples-request", "triples-response": "tg.flow.triples-response", // Agent "agent-request": "tg.flow.agent-request", "agent-response": "tg.flow.agent-response", // Embeddings "embeddings-request": "tg.flow.embeddings-request", "embeddings-response": "tg.flow.embeddings-response", // Graph embeddings query "graph-embeddings-request": "tg.flow.graph-embeddings-request", "graph-embeddings-response": "tg.flow.graph-embeddings-response", // Document embeddings query "document-embeddings-request": "tg.flow.document-embeddings-request", "document-embeddings-response": "tg.flow.document-embeddings-response", // Librarian RPC (for PDF decoder) "librarian-request": "tg.flow.librarian-request", "librarian-response": "tg.flow.librarian-response", }, }; // ---------- Service ---------- export class FlowManagerService extends AsyncProcessor { private flows = new Map(); private blueprints = new Map(); private consumer: BackendConsumer> | null = null; private responseProducer: BackendProducer> | null = null; private configClient: RequestResponse | null = null; constructor(config: ProcessorConfig) { super(config); this.blueprints.set("default", DEFAULT_BLUEPRINT); } protected override async run(): Promise { // Create config client for pushing flow configs to the config service this.configClient = new RequestResponse({ pubsub: this.pubsub, requestTopic: topics.configRequest, responseTopic: topics.configResponse, subscription: `${this.config.id}-config-client`, }); await this.configClient.start(); // Create producer for flow-response topic this.responseProducer = await this.pubsub.createProducer>({ topic: topics.flowResponse, }); // Create consumer for flow-request topic this.consumer = await this.pubsub.createConsumer>({ topic: topics.flowRequest, subscription: `${this.config.id}-flow-request`, }); console.log(`[FlowManager] Listening on ${topics.flowRequest}`); // Main consume loop (same pattern as ConfigService) while (this.running) { try { const msg = await this.consumer.receive(2000); if (msg === null) continue; await this.handleMessage(msg); await this.consumer.acknowledge(msg); } catch (err) { if (!this.running) break; console.error("[FlowManager] Error in consume loop:", err); await sleep(1000); } } } private async handleMessage( msg: Message>, ): Promise { const request = msg.value(); const props = msg.properties(); const requestId = props.id; if (requestId === undefined || requestId.length === 0) { console.warn("[FlowManager] Received request without id, ignoring"); return; } try { const response = await this.handleOperation(request); await this.responseProducer!.send(response, { id: requestId }); } catch (err) { const message = err instanceof Error ? err.message : String(err); await this.responseProducer!.send( { error: { type: "flow-error", message }, }, { id: requestId }, ); } } private async handleOperation( request: Record, ): Promise> { const op = request.operation as string; switch (op) { case "list-blueprints": return this.handleListBlueprints(); case "get-blueprint": return this.handleGetBlueprint(request); case "delete-blueprint": return this.handleDeleteBlueprint(request); case "list-flows": return this.handleListFlows(); case "get-flow": return this.handleGetFlow(request); case "start-flow": return await this.handleStartFlow(request); case "stop-flow": return await this.handleStopFlow(request); default: throw new Error(`Unknown flow operation: ${op}`); } } // ---------- Blueprint operations ---------- private handleListBlueprints(): Record { return { "blueprint-names": [...this.blueprints.keys()], }; } private handleGetBlueprint( request: Record, ): Record { const name = request["blueprint-name"] as string | undefined; if (name === undefined || name.length === 0) { throw new Error("Missing blueprint-name"); } const blueprint = this.blueprints.get(name); if (blueprint === undefined) { throw new Error(`Blueprint not found: ${name}`); } return { "blueprint-definition": JSON.stringify(blueprint), }; } private handleDeleteBlueprint( request: Record, ): Record { const name = request["blueprint-name"] as string | undefined; if (name === undefined || name.length === 0) { throw new Error("Missing blueprint-name"); } if (name === "default") { throw new Error("Cannot delete the default blueprint"); } const existed = this.blueprints.delete(name); if (!existed) { throw new Error(`Blueprint not found: ${name}`); } return {}; } // ---------- Flow operations ---------- private handleListFlows(): Record { return { "flow-ids": [...this.flows.keys()], }; } private handleGetFlow( request: Record, ): Record { const id = request["flow-id"] as string | undefined; if (id === undefined || id.length === 0) { throw new Error("Missing flow-id"); } const inst = this.flows.get(id); if (inst === undefined) { throw new Error(`Flow not found: ${id}`); } return { flow: JSON.stringify({ "blueprint-name": inst.blueprintName, description: inst.description, parameters: inst.parameters, }), }; } private async handleStartFlow( request: Record, ): Promise> { const id = request["flow-id"] as string | undefined; const blueprintName = (request["blueprint-name"] as string) ?? "default"; const description = (request["description"] as string) ?? ""; const parameters = (request["parameters"] as Record) ?? {}; if (id === undefined || id.length === 0) { throw new Error("Missing flow-id"); } if (this.flows.has(id)) { throw new Error(`Flow already exists: ${id}`); } const blueprint = this.blueprints.get(blueprintName); if (blueprint === undefined) { throw new Error(`Blueprint not found: ${blueprintName}`); } // Create the flow instance const inst: FlowInstance = { id, blueprintName, description, parameters, status: "running", }; this.flows.set(id, inst); console.log( `[FlowManager] Started flow "${id}" with blueprint "${blueprintName}"`, ); // Push updated flows config to the config service await this.pushFlowsConfig(); return {}; } private async handleStopFlow( request: Record, ): Promise> { const id = request["flow-id"] as string | undefined; if (id === undefined || id.length === 0) { throw new Error("Missing flow-id"); } const inst = this.flows.get(id); if (inst === undefined) { throw new Error(`Flow not found: ${id}`); } this.flows.delete(id); console.log(`[FlowManager] Stopped flow "${id}"`); // Push updated flows config (without the removed flow) await this.pushFlowsConfig(); return {}; } // ---------- Config push ---------- /** * Build the flows config object from all running flows and push it * to the config service via a PUT operation. */ private async pushFlowsConfig(): Promise { if (this.configClient === null) return; const flowsConfig: Record }> = {}; for (const [id, inst] of this.flows) { const blueprint = this.blueprints.get(inst.blueprintName); if (blueprint !== undefined) { flowsConfig[id] = { topics: blueprint.topics }; } } try { await this.configClient.request({ operation: "put", keys: ["flows"], values: flowsConfig, }); console.log( `[FlowManager] Pushed flows config (${this.flows.size} active flows)`, ); } catch (err) { console.error("[FlowManager] Failed to push flows config:", err); } } // ---------- Lifecycle ---------- override async stop(): Promise { if (this.consumer !== null) { await this.consumer.close(); this.consumer = null; } if (this.responseProducer !== null) { await this.responseProducer.close(); this.responseProducer = null; } if (this.configClient !== null) { await this.configClient.stop(); this.configClient = null; } await super.stop(); } } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } export const program = makeProcessorProgram({ id: "flow-manager", make: (config) => new FlowManagerService(config), }); export async function run(): Promise { await FlowManagerService.launch("flow-manager"); }