fix: NATS pipeline bugs, add integration tests and service runners

Fix three critical bugs preventing the NATS message pipeline from working:

- FlowProcessor now subscribes to config-push topic (was missing entirely),
  using DeliverPolicy.All to replay config on service restart
- NATS streams use wildcard subjects (tg.flow.>) instead of per-topic
  narrow filters that caused 503 errors on publish
- Subscriber dispatch loop has exponential backoff on errors to prevent
  tight error loops

Add service runner scripts (gateway, config, LLM) and a 7-test
integration suite that verifies config CRUD, WebSocket round-trip,
and full LLM text-completion through the NATS pipeline.

Fix Docker Compose infra: pin Tempo to v2.6.1, remove deprecated Loki
config fields, add user:0 for volume permissions, remap conflicting
ports (FalkorDB 6380, OTLP 4327/4328).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

ts/packages/base/src/backend/nats.ts

@@ -98,38 +98,28 @@ class NatsConsumer<T> implements BackendConsumer<T> {
     private readonly subject: string,
     private readonly subscription: string,
     private readonly initialPosition: "latest" | "earliest",
+    private readonly streamName: string,
   ) {}
 
   async init(): Promise<void> {
-    // Ensure stream exists
-    const streamName = this.streamNameFromSubject(this.subject);
-    try {
-      await this.jsm.streams.info(streamName);
-    } catch {
-      await this.jsm.streams.add({
-        name: streamName,
-        subjects: [this.subject],
-      });
-    }
-
+    // Stream is already ensured by NatsBackend.ensureStream().
     // Create or bind to durable consumer.
-    // Try to get an existing durable consumer first; if it doesn't exist, create it.
     try {
-      this.consumer = await this.js.consumers.get(streamName, this.subscription);
+      this.consumer = await this.js.consumers.get(this.streamName, this.subscription);
     } catch {
       const deliverPolicy =
         this.initialPosition === "earliest"
           ? DeliverPolicy.All
           : DeliverPolicy.New;
 
-      await this.jsm.consumers.add(streamName, {
+      await this.jsm.consumers.add(this.streamName, {
         durable_name: this.subscription,
         ack_policy: AckPolicy.Explicit,
         deliver_policy: deliverPolicy,
         filter_subject: this.subject,
       });
 
-      this.consumer = await this.js.consumers.get(streamName, this.subscription);
+      this.consumer = await this.js.consumers.get(this.streamName, this.subscription);
     }
   }
 
@@ -164,18 +154,13 @@ class NatsConsumer<T> implements BackendConsumer<T> {
   async close(): Promise<void> {
     this.consumer = null;
   }
-
-  private streamNameFromSubject(subject: string): string {
-    // Convert topic like "tg.flow.text-completion" to stream name "tg_flow"
-    const parts = subject.split(".");
-    return parts.slice(0, 2).join("_");
-  }
 }
 
 export class NatsBackend implements PubSubBackend {
   private connection: NatsConnection | null = null;
   private js: JetStreamClient | null = null;
   private jsm: JetStreamManager | null = null;
+  private initializedStreams = new Set<string>();
 
   constructor(private readonly url: string = "nats://localhost:4222") {}
 
@@ -187,19 +172,46 @@ export class NatsBackend implements PubSubBackend {
     }
   }
 
+  /**
+   * Ensure the stream for a given subject exists with a wildcard filter.
+   * E.g. subject "tg.flow.config-request" → stream "tg_flow" with subjects ["tg.flow.>"]
+   */
+  private async ensureStream(subject: string): Promise<string> {
+    const parts = subject.split(".");
+    const streamName = parts.slice(0, 2).join("_");
+
+    if (this.initializedStreams.has(streamName)) return streamName;
+
+    const wildcardSubject = `${parts.slice(0, 2).join(".")}.>`;
+
+    try {
+      await this.jsm!.streams.info(streamName);
+    } catch {
+      await this.jsm!.streams.add({
+        name: streamName,
+        subjects: [wildcardSubject],
+      });
+    }
+    this.initializedStreams.add(streamName);
+    return streamName;
+  }
+
   async createProducer<T>(options: CreateProducerOptions): Promise<BackendProducer<T>> {
     await this.ensureConnected();
+    await this.ensureStream(options.topic);
     return new NatsProducer<T>(this.js!, options.topic);
   }
 
   async createConsumer<T>(options: CreateConsumerOptions): Promise<BackendConsumer<T>> {
     await this.ensureConnected();
+    const streamName = await this.ensureStream(options.topic);
     const consumer = new NatsConsumer<T>(
       this.js!,
       this.jsm!,
       options.topic,
       options.subscription,
       options.initialPosition ?? "latest",
+      streamName,
     );
     await consumer.init();
     return consumer;

ts/packages/base/src/messaging/subscriber.ts

@@ -111,11 +111,14 @@ export class Subscriber<T> {
   }
 
   private async dispatchLoop(): Promise<void> {
+    let consecutiveErrors = 0;
     while (this.running) {
       try {
         const msg = await this.backend!.receive(2000);
         if (!msg) continue;
 
+        consecutiveErrors = 0;
+
         const props = msg.properties();
         const id = props.id;
         const value = msg.value();
@@ -136,7 +139,15 @@ export class Subscriber<T> {
         await this.backend!.acknowledge(msg);
       } catch (err) {
         if (!this.running) break;
-        console.error("[Subscriber] Error:", err);
+        consecutiveErrors++;
+        if (consecutiveErrors <= 3) {
+          console.error("[Subscriber] Error:", err);
+        } else if (consecutiveErrors === 4) {
+          console.error("[Subscriber] Suppressing further errors (will retry with backoff)");
+        }
+        // Exponential backoff: 1s, 2s, 4s, max 10s
+        const delay = Math.min(1000 * Math.pow(2, consecutiveErrors - 1), 10_000);
+        await new Promise((r) => setTimeout(r, delay));
       }
     }
   }

ts/packages/base/src/processor/async-processor.ts

@@ -24,7 +24,7 @@ export type ConfigHandler = (
 export abstract class AsyncProcessor {
   protected pubsub: PubSubBackend;
   protected running = false;
-  private configHandlers: ConfigHandler[] = [];
+  protected configHandlers: ConfigHandler[] = [];
   private shutdownCallbacks: Array<() => Promise<void>> = [];
 
   constructor(protected readonly config: ProcessorConfig) {

ts/packages/base/src/processor/flow-processor.ts

@@ -1,20 +1,30 @@
 /**
  * Flow-aware processor that manages dynamic flow instances.
  *
+ * Subscribes to config-push topic and dynamically creates/destroys
+ * flow instances based on the configuration received.
+ *
  * Python reference: trustgraph-base/trustgraph/base/flow_processor.py
  */
 
 import { AsyncProcessor, type ProcessorConfig } from "./async-processor.js";
 import type { Spec } from "../spec/types.js";
+import type { BackendConsumer } from "../backend/types.js";
 import { Flow, type FlowDefinition } from "./flow.js";
+import { topics } from "../schema/topics.js";
+
+interface ConfigPush {
+  version: number;
+  config: Record<string, unknown>;
+}
 
 export abstract class FlowProcessor extends AsyncProcessor {
   private specifications: Spec[] = [];
   private flows = new Map<string, Flow>();
+  private configConsumer: BackendConsumer<ConfigPush> | null = null;
 
   constructor(config: ProcessorConfig) {
     super(config);
-    this.registerConfigHandler(this.onConfigureFlows.bind(this));
   }
 
   registerSpecification(spec: Spec): void {
@@ -22,16 +32,38 @@ export abstract class FlowProcessor extends AsyncProcessor {
   }
 
   protected async run(): Promise<void> {
-    // The processor sits idle waiting for flow configurations
-    // to arrive via the config push topic. In the meantime,
-    // the consumer loop runs in the background.
-    await new Promise<void>((resolve) => {
-      const check = () => {
-        if (!this.running) resolve();
-        else setTimeout(check, 1000);
-      };
-      check();
+    // Subscribe to config-push topic to receive flow definitions.
+    // Use "earliest" to replay any config pushes that arrived before this service started.
+    this.configConsumer = await this.pubsub.createConsumer<ConfigPush>({
+      topic: topics.configPush,
+      subscription: `${this.config.id}-config-push`,
+      initialPosition: "earliest",
     });
+
+    console.log(`[${this.config.id}] Listening for config pushes on ${topics.configPush}`);
+
+    while (this.running) {
+      try {
+        const msg = await this.configConsumer.receive(2000);
+        if (!msg) continue;
+
+        const push = msg.value();
+        console.log(`[${this.config.id}] Received config push version=${push.version}`);
+
+        await this.onConfigureFlows(push.config, push.version);
+
+        // Also call any registered config handlers
+        for (const handler of this.configHandlers) {
+          await handler(push.config, push.version);
+        }
+
+        await this.configConsumer.acknowledge(msg);
+      } catch (err) {
+        if (!this.running) break;
+        console.error(`[${this.config.id}] Config consumer error:`, err);
+        await sleep(1000);
+      }
+    }
   }
 
   private async onConfigureFlows(
@@ -39,27 +71,43 @@ export abstract class FlowProcessor extends AsyncProcessor {
     version: number,
   ): Promise<void> {
     const flowDefs = config.flows as Record<string, FlowDefinition> | undefined;
-    if (!flowDefs) return;
+    if (!flowDefs) {
+      console.log(`[${this.config.id}] No flows in config push, skipping`);
+      return;
+    }
 
     // Stop removed flows
     for (const [name, flow] of this.flows) {
       if (!(name in flowDefs)) {
+        console.log(`[${this.config.id}] Stopping removed flow: ${name}`);
         await flow.stop();
         this.flows.delete(name);
       }
     }
 
-    // Start new flows
+    // Start or update flows
     for (const [name, defn] of Object.entries(flowDefs)) {
+      // Skip invalid definitions (e.g., stringified JSON)
+      if (typeof defn !== "object" || defn === null) {
+        console.warn(`[${this.config.id}] Skipping flow "${name}": definition is not an object`);
+        continue;
+      }
+
       if (!this.flows.has(name)) {
+        console.log(`[${this.config.id}] Starting flow "${name}" with topics:`, defn.topics);
         const flow = new Flow(name, this.config.id, this.pubsub, defn, this.specifications);
         await flow.start();
         this.flows.set(name, flow);
+        console.log(`[${this.config.id}] Flow "${name}" started`);
       }
     }
   }
 
   override async stop(): Promise<void> {
+    if (this.configConsumer) {
+      await this.configConsumer.close();
+      this.configConsumer = null;
+    }
     for (const flow of this.flows.values()) {
       await flow.stop();
     }
@@ -67,3 +115,7 @@ export abstract class FlowProcessor extends AsyncProcessor {
     await super.stop();
   }
 }
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}