saving

ts/packages/base/src/backend/index.ts

@@ -0,0 +1,12 @@
+export type {
+  Message,
+  BackendProducer,
+  BackendConsumer,
+  PubSubBackend,
+  CreateProducerOptions,
+  CreateConsumerOptions,
+  ConsumerType,
+  InitialPosition,
+} from "./types.js";
+
+export { NatsBackend } from "./nats.js";

ts/packages/base/src/backend/nats.ts

@@ -0,0 +1,196 @@
+/**
+ * NATS JetStream backend implementation.
+ *
+ * Replaces Pulsar as the message broker. NATS JetStream provides
+ * at-least-once delivery, consumer groups, and replay — matching
+ * the QoS levels used by the Python Pulsar backend.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/pulsar_backend.py
+ */
+
+import {
+  connect,
+  type NatsConnection,
+  type JetStreamClient,
+  type JetStreamManager,
+  type ConsumerMessages,
+  type JsMsg,
+  StringCodec,
+  AckPolicy,
+} from "nats";
+
+import type {
+  PubSubBackend,
+  BackendProducer,
+  BackendConsumer,
+  CreateProducerOptions,
+  CreateConsumerOptions,
+  Message,
+} from "./types.js";
+
+const sc = StringCodec();
+
+class NatsMessage<T> implements Message<T> {
+  constructor(
+    private readonly msg: JsMsg,
+    private readonly decoded: T,
+  ) {}
+
+  value(): T {
+    return this.decoded;
+  }
+
+  properties(): Record<string, string> {
+    const headers = this.msg.headers;
+    const props: Record<string, string> = {};
+    if (headers) {
+      for (const [key, values] of headers) {
+        props[key] = values[0];
+      }
+    }
+    return props;
+  }
+}
+
+class NatsProducer<T> implements BackendProducer<T> {
+  constructor(
+    private readonly js: JetStreamClient,
+    private readonly subject: string,
+  ) {}
+
+  async send(message: T, properties?: Record<string, string>): Promise<void> {
+    const data = sc.encode(JSON.stringify(message));
+    const opts: Record<string, unknown> = {};
+
+    if (properties && Object.keys(properties).length > 0) {
+      const { headers } = await import("nats");
+      const hdrs = headers();
+      for (const [key, val] of Object.entries(properties)) {
+        hdrs.append(key, val);
+      }
+      opts.headers = hdrs;
+    }
+
+    await this.js.publish(this.subject, data, opts);
+  }
+
+  async flush(): Promise<void> {
+    // NATS publishes are flushed on the connection level
+  }
+
+  async close(): Promise<void> {
+    // No per-producer cleanup needed for NATS
+  }
+}
+
+class NatsConsumer<T> implements BackendConsumer<T> {
+  private messages: ConsumerMessages | null = null;
+
+  constructor(
+    private readonly js: JetStreamClient,
+    private readonly jsm: JetStreamManager,
+    private readonly subject: string,
+    private readonly subscription: string,
+    private readonly initialPosition: "latest" | "earliest",
+  ) {}
+
+  async init(): Promise<void> {
+    // Ensure stream exists
+    const streamName = this.streamNameFromSubject(this.subject);
+    try {
+      await this.jsm.streams.info(streamName);
+    } catch {
+      await this.jsm.streams.add({
+        name: streamName,
+        subjects: [this.subject],
+      });
+    }
+
+    // Create or bind to durable consumer
+    const consumer = await this.js.consumers.get(streamName, this.subscription);
+    this.messages = await consumer.consume();
+  }
+
+  async receive(timeoutMs = 2000): Promise<Message<T> | null> {
+    if (!this.messages) throw new Error("Consumer not initialized");
+
+    const deadline = Date.now() + timeoutMs;
+    for await (const msg of this.messages) {
+      const decoded = JSON.parse(sc.decode(msg.data)) as T;
+      return new NatsMessage(msg, decoded);
+    }
+
+    if (Date.now() >= deadline) return null;
+    return null;
+  }
+
+  async acknowledge(message: Message<T>): Promise<void> {
+    const natsMsg = message as NatsMessage<T>;
+    // Access internal JsMsg for ack — in practice we'd store the ref
+    // This is a simplified version; real impl tracks msg refs
+    void natsMsg;
+  }
+
+  async negativeAcknowledge(message: Message<T>): Promise<void> {
+    void message;
+  }
+
+  async unsubscribe(): Promise<void> {
+    // Drain and close consumer
+  }
+
+  async close(): Promise<void> {
+    if (this.messages) {
+      this.messages.stop();
+    }
+  }
+
+  private streamNameFromSubject(subject: string): string {
+    // Convert topic like "tg.flow.text-completion" to stream name "tg_flow"
+    const parts = subject.split(".");
+    return parts.slice(0, 2).join("_");
+  }
+}
+
+export class NatsBackend implements PubSubBackend {
+  private connection: NatsConnection | null = null;
+  private js: JetStreamClient | null = null;
+  private jsm: JetStreamManager | null = null;
+
+  constructor(private readonly url: string = "nats://localhost:4222") {}
+
+  private async ensureConnected(): Promise<void> {
+    if (!this.connection) {
+      this.connection = await connect({ servers: this.url });
+      this.js = this.connection.jetstream();
+      this.jsm = await this.connection.jetstreamManager();
+    }
+  }
+
+  async createProducer<T>(options: CreateProducerOptions): Promise<BackendProducer<T>> {
+    await this.ensureConnected();
+    return new NatsProducer<T>(this.js!, options.topic);
+  }
+
+  async createConsumer<T>(options: CreateConsumerOptions): Promise<BackendConsumer<T>> {
+    await this.ensureConnected();
+    const consumer = new NatsConsumer<T>(
+      this.js!,
+      this.jsm!,
+      options.topic,
+      options.subscription,
+      options.initialPosition ?? "latest",
+    );
+    await consumer.init();
+    return consumer;
+  }
+
+  async close(): Promise<void> {
+    if (this.connection) {
+      await this.connection.drain();
+      this.connection = null;
+      this.js = null;
+      this.jsm = null;
+    }
+  }
+}

ts/packages/base/src/backend/types.ts

@@ -0,0 +1,45 @@
+/**
+ * Core pub/sub backend abstraction.
+ *
+ * Mirrors Python's backend.py Protocol classes. Any message broker
+ * (NATS, Pulsar, Redis Streams) implements these interfaces.
+ */
+
+export interface Message<T = unknown> {
+  value(): T;
+  properties(): Record<string, string>;
+}
+
+export interface BackendProducer<T = unknown> {
+  send(message: T, properties?: Record<string, string>): Promise<void>;
+  flush(): Promise<void>;
+  close(): Promise<void>;
+}
+
+export interface BackendConsumer<T = unknown> {
+  receive(timeoutMs?: number): Promise<Message<T> | null>;
+  acknowledge(message: Message<T>): Promise<void>;
+  negativeAcknowledge(message: Message<T>): Promise<void>;
+  unsubscribe(): Promise<void>;
+  close(): Promise<void>;
+}
+
+export type ConsumerType = "shared" | "exclusive" | "failover";
+export type InitialPosition = "latest" | "earliest";
+
+export interface CreateProducerOptions {
+  topic: string;
+}
+
+export interface CreateConsumerOptions {
+  topic: string;
+  subscription: string;
+  initialPosition?: InitialPosition;
+  consumerType?: ConsumerType;
+}
+
+export interface PubSubBackend {
+  createProducer<T>(options: CreateProducerOptions): Promise<BackendProducer<T>>;
+  createConsumer<T>(options: CreateConsumerOptions): Promise<BackendConsumer<T>>;
+  close(): Promise<void>;
+}

ts/packages/base/src/errors.ts

@@ -0,0 +1,29 @@
+/**
+ * Custom error types.
+ *
+ * Python reference: trustgraph-base/trustgraph/exceptions.py
+ */
+
+export class TooManyRequestsError extends Error {
+  constructor(message = "Rate limit exceeded") {
+    super(message);
+    this.name = "TooManyRequestsError";
+  }
+}
+
+export class LlmError extends Error {
+  constructor(
+    message: string,
+    public readonly errorType: string = "llm-error",
+  ) {
+    super(message);
+    this.name = "LlmError";
+  }
+}
+
+export class ParseError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ParseError";
+  }
+}

ts/packages/base/src/index.ts

@@ -0,0 +1,10 @@
+// @trustgraph/base — core abstractions for the TrustGraph TypeScript port
+
+export * from "./backend/index.js";
+export * from "./messaging/index.js";
+export * from "./processor/index.js";
+export * from "./schema/index.js";
+export * from "./spec/index.js";
+export * from "./services/index.js";
+export * from "./metrics/index.js";
+export * from "./errors.js";

ts/packages/base/src/messaging/consumer.ts

@@ -0,0 +1,105 @@
+/**
+ * High-level consumer with concurrency, retry, and rate-limit handling.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/consumer.py
+ */
+
+import type { PubSubBackend, BackendConsumer, Message } from "../backend/types.js";
+import { TooManyRequestsError } from "../errors.js";
+
+export type MessageHandler<T> = (
+  message: T,
+  properties: Record<string, string>,
+  flow: FlowContext,
+) => Promise<void>;
+
+export interface FlowContext {
+  id: string;
+  name: string;
+}
+
+export interface ConsumerOptions<T> {
+  pubsub: PubSubBackend;
+  topic: string;
+  subscription: string;
+  handler: MessageHandler<T>;
+  concurrency?: number;
+  initialPosition?: "latest" | "earliest";
+  rateLimitRetryMs?: number;
+  rateLimitTimeoutMs?: number;
+}
+
+export class Consumer<T> {
+  private backend: BackendConsumer<T> | null = null;
+  private running = false;
+  private abortController = new AbortController();
+
+  private readonly concurrency: number;
+  private readonly rateLimitRetryMs: number;
+
+  constructor(private readonly options: ConsumerOptions<T>) {
+    this.concurrency = options.concurrency ?? 1;
+    this.rateLimitRetryMs = options.rateLimitRetryMs ?? 10_000;
+  }
+
+  async start(flow: FlowContext): Promise<void> {
+    this.backend = await this.options.pubsub.createConsumer<T>({
+      topic: this.options.topic,
+      subscription: this.options.subscription,
+      initialPosition: this.options.initialPosition ?? "latest",
+    });
+
+    this.running = true;
+
+    // Spawn concurrent consumer tasks
+    const tasks = Array.from({ length: this.concurrency }, () =>
+      this.consumeLoop(flow),
+    );
+    // Run all concurrently — first rejection stops all
+    await Promise.all(tasks);
+  }
+
+  async stop(): Promise<void> {
+    this.running = false;
+    this.abortController.abort();
+    if (this.backend) {
+      await this.backend.close();
+      this.backend = null;
+    }
+  }
+
+  private async consumeLoop(flow: FlowContext): Promise<void> {
+    while (this.running) {
+      try {
+        const msg = await this.backend!.receive(2000);
+        if (!msg) continue;
+
+        await this.handleWithRetry(msg, flow);
+        await this.backend!.acknowledge(msg);
+      } catch (err) {
+        if (!this.running) break;
+        console.error("[Consumer] Error in consume loop:", err);
+        // Brief pause before retry
+        await sleep(1000);
+      }
+    }
+  }
+
+  private async handleWithRetry(msg: Message<T>, flow: FlowContext): Promise<void> {
+    try {
+      await this.options.handler(msg.value(), msg.properties(), flow);
+    } catch (err) {
+      if (err instanceof TooManyRequestsError) {
+        console.warn(`[Consumer] Rate limited, retrying in ${this.rateLimitRetryMs}ms`);
+        await sleep(this.rateLimitRetryMs);
+        await this.options.handler(msg.value(), msg.properties(), flow);
+      } else {
+        throw err;
+      }
+    }
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

ts/packages/base/src/messaging/index.ts

@@ -0,0 +1,4 @@
+export { Producer } from "./producer.js";
+export { Consumer, type MessageHandler, type FlowContext, type ConsumerOptions } from "./consumer.js";
+export { Subscriber, AsyncQueue } from "./subscriber.js";
+export { RequestResponse, type RequestResponseOptions } from "./request-response.js";

ts/packages/base/src/messaging/producer.ts

@@ -0,0 +1,40 @@
+/**
+ * High-level async producer with queue buffering and retry.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/producer.py
+ */
+
+import type { PubSubBackend, BackendProducer } from "../backend/types.js";
+import type { ProducerMetrics } from "../metrics/prometheus.js";
+
+export class Producer<T> {
+  private backend: BackendProducer<T> | null = null;
+  private running = false;
+
+  constructor(
+    private readonly pubsub: PubSubBackend,
+    private readonly topic: string,
+    private readonly metrics?: ProducerMetrics,
+  ) {}
+
+  async start(): Promise<void> {
+    this.backend = await this.pubsub.createProducer<T>({ topic: this.topic });
+    this.running = true;
+  }
+
+  async send(id: string, message: T): Promise<void> {
+    if (!this.backend) throw new Error("Producer not started");
+
+    await this.backend.send(message, { id });
+    this.metrics?.inc();
+  }
+
+  async stop(): Promise<void> {
+    this.running = false;
+    if (this.backend) {
+      await this.backend.flush();
+      await this.backend.close();
+      this.backend = null;
+    }
+  }
+}

ts/packages/base/src/messaging/request-response.ts

@@ -0,0 +1,91 @@
+/**
+ * Request/response pattern over pub/sub.
+ *
+ * Sends a request with a unique ID, subscribes for matching responses.
+ * Supports streaming (multiple responses per request) via a recipient callback.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/request_response_spec.py
+ */
+
+import { randomUUID } from "node:crypto";
+import { Producer } from "./producer.js";
+import { Subscriber } from "./subscriber.js";
+import type { PubSubBackend } from "../backend/types.js";
+
+export interface RequestResponseOptions {
+  pubsub: PubSubBackend;
+  requestTopic: string;
+  responseTopic: string;
+  subscription: string;
+}
+
+export class RequestResponse<TReq, TRes> {
+  private producer: Producer<TReq>;
+  private subscriber: Subscriber<TRes>;
+
+  constructor(private readonly options: RequestResponseOptions) {
+    this.producer = new Producer<TReq>(options.pubsub, options.requestTopic);
+    this.subscriber = new Subscriber<TRes>(
+      options.pubsub,
+      options.responseTopic,
+      options.subscription,
+    );
+  }
+
+  async start(): Promise<void> {
+    await this.producer.start();
+    await this.subscriber.start();
+  }
+
+  async stop(): Promise<void> {
+    await this.producer.stop();
+    await this.subscriber.stop();
+  }
+
+  /**
+   * Send a request and wait for responses.
+   *
+   * @param request - The request payload
+   * @param options.timeoutMs - Total timeout in milliseconds (default: 300s)
+   * @param options.recipient - Optional callback for streaming responses.
+   *   Return `true` to indicate the final response has been received.
+   *   If omitted, returns the first response.
+   */
+  async request(
+    request: TReq,
+    options?: {
+      timeoutMs?: number;
+      recipient?: (response: TRes) => Promise<boolean>;
+    },
+  ): Promise<TRes> {
+    const id = randomUUID();
+    const timeoutMs = options?.timeoutMs ?? 300_000;
+    const recipient = options?.recipient;
+
+    const queue = this.subscriber.subscribe(id);
+
+    try {
+      await this.producer.send(id, request);
+
+      const deadline = Date.now() + timeoutMs;
+
+      while (true) {
+        const remaining = deadline - Date.now();
+        if (remaining <= 0) {
+          throw new Error(`Request timed out after ${timeoutMs}ms`);
+        }
+
+        const response = await queue.pop(remaining);
+
+        if (recipient) {
+          const isFinal = await recipient(response);
+          if (isFinal) return response;
+        } else {
+          return response;
+        }
+      }
+    } finally {
+      this.subscriber.unsubscribe(id);
+    }
+  }
+}

ts/packages/base/src/messaging/subscriber.ts

@@ -0,0 +1,143 @@
+/**
+ * Fan-out subscriber: routes responses to waiting callers by request ID.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/subscriber.py
+ */
+
+import type { PubSubBackend, BackendConsumer } from "../backend/types.js";
+
+type Resolver<T> = {
+  queue: AsyncQueue<T>;
+};
+
+/**
+ * Simple async queue for inter-task communication (replaces asyncio.Queue).
+ */
+export class AsyncQueue<T> {
+  private buffer: T[] = [];
+  private waiters: Array<(value: T) => void> = [];
+
+  push(item: T): void {
+    const waiter = this.waiters.shift();
+    if (waiter) {
+      waiter(item);
+    } else {
+      this.buffer.push(item);
+    }
+  }
+
+  async pop(timeoutMs?: number): Promise<T> {
+    const buffered = this.buffer.shift();
+    if (buffered !== undefined) return buffered;
+
+    return new Promise<T>((resolve, reject) => {
+      let timer: ReturnType<typeof setTimeout> | undefined;
+
+      const waiter = (value: T) => {
+        if (timer) clearTimeout(timer);
+        resolve(value);
+      };
+
+      this.waiters.push(waiter);
+
+      if (timeoutMs !== undefined) {
+        timer = setTimeout(() => {
+          const idx = this.waiters.indexOf(waiter);
+          if (idx !== -1) this.waiters.splice(idx, 1);
+          reject(new Error(`Queue.pop timed out after ${timeoutMs}ms`));
+        }, timeoutMs);
+      }
+    });
+  }
+
+  get length(): number {
+    return this.buffer.length;
+  }
+}
+
+export class Subscriber<T> {
+  private backend: BackendConsumer<T> | null = null;
+  private running = false;
+
+  // ID-specific subscriptions (request/response correlation)
+  private idSubscribers = new Map<string, Resolver<T>>();
+  // Wildcard subscribers (receive all messages)
+  private allSubscribers = new Map<string, Resolver<T>>();
+
+  constructor(
+    private readonly pubsub: PubSubBackend,
+    private readonly topic: string,
+    private readonly subscription: string,
+  ) {}
+
+  async start(): Promise<void> {
+    this.backend = await this.pubsub.createConsumer<T>({
+      topic: this.topic,
+      subscription: this.subscription,
+    });
+    this.running = true;
+    // Start the dispatch loop (fire and forget — runs until stop)
+    this.dispatchLoop().catch((err) => {
+      if (this.running) console.error("[Subscriber] dispatch loop error:", err);
+    });
+  }
+
+  async stop(): Promise<void> {
+    this.running = false;
+    if (this.backend) {
+      await this.backend.close();
+      this.backend = null;
+    }
+  }
+
+  subscribe(id: string): AsyncQueue<T> {
+    const queue = new AsyncQueue<T>();
+    this.idSubscribers.set(id, { queue });
+    return queue;
+  }
+
+  subscribeAll(id: string): AsyncQueue<T> {
+    const queue = new AsyncQueue<T>();
+    this.allSubscribers.set(id, { queue });
+    return queue;
+  }
+
+  unsubscribe(id: string): void {
+    this.idSubscribers.delete(id);
+  }
+
+  unsubscribeAll(id: string): void {
+    this.allSubscribers.delete(id);
+  }
+
+  private async dispatchLoop(): Promise<void> {
+    while (this.running) {
+      try {
+        const msg = await this.backend!.receive(2000);
+        if (!msg) continue;
+
+        const props = msg.properties();
+        const id = props.id;
+        const value = msg.value();
+
+        // Route to ID-specific subscriber
+        if (id) {
+          const sub = this.idSubscribers.get(id);
+          if (sub) {
+            sub.queue.push(value);
+          }
+        }
+
+        // Broadcast to all-subscribers
+        for (const sub of this.allSubscribers.values()) {
+          sub.queue.push(value);
+        }
+
+        await this.backend!.acknowledge(msg);
+      } catch (err) {
+        if (!this.running) break;
+        console.error("[Subscriber] Error:", err);
+      }
+    }
+  }
+}

ts/packages/base/src/metrics/index.ts

@@ -0,0 +1 @@
+export { ConsumerMetrics, ProducerMetrics, registry } from "./prometheus.js";

ts/packages/base/src/metrics/prometheus.ts

@@ -0,0 +1,68 @@
+/**
+ * Prometheus metrics wrappers.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/metrics.py
+ */
+
+import { Counter, Histogram, Registry, collectDefaultMetrics } from "prom-client";
+
+export const registry = new Registry();
+collectDefaultMetrics({ register: registry });
+
+export class ConsumerMetrics {
+  private requestHistogram: Histogram;
+  private processingCounter: Counter;
+  private rateLimitCounter: Counter;
+
+  constructor(processor: string, flow: string, name: string) {
+    this.requestHistogram = new Histogram({
+      name: "tg_consumer_request_duration_seconds",
+      help: "Consumer request processing time",
+      labelNames: ["processor", "flow", "name"],
+      registers: [registry],
+    });
+
+    this.processingCounter = new Counter({
+      name: "tg_consumer_processing_total",
+      help: "Consumer processing outcomes",
+      labelNames: ["processor", "flow", "name", "status"],
+      registers: [registry],
+    });
+
+    this.rateLimitCounter = new Counter({
+      name: "tg_consumer_rate_limit_total",
+      help: "Consumer rate limit events",
+      labelNames: ["processor", "flow", "name"],
+      registers: [registry],
+    });
+  }
+
+  recordTime(seconds: number): void {
+    this.requestHistogram.observe(seconds);
+  }
+
+  process(status: "success" | "error"): void {
+    this.processingCounter.inc({ status });
+  }
+
+  rateLimit(): void {
+    this.rateLimitCounter.inc();
+  }
+}
+
+export class ProducerMetrics {
+  private counter: Counter;
+
+  constructor(processor: string, flow: string, name: string) {
+    this.counter = new Counter({
+      name: "tg_producer_items_total",
+      help: "Producer items sent",
+      labelNames: ["processor", "flow", "name"],
+      registers: [registry],
+    });
+  }
+
+  inc(): void {
+    this.counter.inc();
+  }
+}

ts/packages/base/src/processor/async-processor.ts

@@ -0,0 +1,83 @@
+/**
+ * Base async processor — foundation for all TrustGraph services.
+ *
+ * Handles pub/sub lifecycle, configuration subscription, and graceful shutdown.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/async_processor.py
+ */
+
+import type { PubSubBackend } from "../backend/types.js";
+import { NatsBackend } from "../backend/nats.js";
+import { topics } from "../schema/topics.js";
+
+export interface ProcessorConfig {
+  id: string;
+  pubsubUrl?: string;
+  metricsPort?: number;
+}
+
+export type ConfigHandler = (
+  config: Record<string, unknown>,
+  version: number,
+) => Promise<void>;
+
+export abstract class AsyncProcessor {
+  protected pubsub: PubSubBackend;
+  protected running = false;
+  private configHandlers: ConfigHandler[] = [];
+  private shutdownCallbacks: Array<() => Promise<void>> = [];
+
+  constructor(protected readonly config: ProcessorConfig) {
+    this.pubsub = new NatsBackend(config.pubsubUrl ?? "nats://localhost:4222");
+  }
+
+  registerConfigHandler(handler: ConfigHandler): void {
+    this.configHandlers.push(handler);
+  }
+
+  async start(): Promise<void> {
+    this.running = true;
+    // Set up graceful shutdown
+    const shutdown = async () => {
+      console.log(`[${this.config.id}] Shutting down...`);
+      await this.stop();
+      process.exit(0);
+    };
+    process.on("SIGINT", shutdown);
+    process.on("SIGTERM", shutdown);
+
+    await this.run();
+  }
+
+  async stop(): Promise<void> {
+    this.running = false;
+    for (const cb of this.shutdownCallbacks) {
+      await cb();
+    }
+    await this.pubsub.close();
+  }
+
+  protected onShutdown(callback: () => Promise<void>): void {
+    this.shutdownCallbacks.push(callback);
+  }
+
+  protected abstract run(): Promise<void>;
+
+  /**
+   * Static launch helper — parses env/args and starts the processor.
+   * Subclasses call: `MyProcessor.launch("my-service")`
+   */
+  static async launch<T extends AsyncProcessor>(
+    this: new (config: ProcessorConfig) => T,
+    id: string,
+  ): Promise<void> {
+    const config: ProcessorConfig = {
+      id,
+      pubsubUrl: process.env.NATS_URL ?? process.env.PULSAR_HOST,
+      metricsPort: parseInt(process.env.METRICS_PORT ?? "8000", 10),
+    };
+
+    const processor = new this(config);
+    await processor.start();
+  }
+}

ts/packages/base/src/processor/flow-processor.ts

@@ -0,0 +1,69 @@
+/**
+ * Flow-aware processor that manages dynamic flow instances.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/flow_processor.py
+ */
+
+import { AsyncProcessor, type ProcessorConfig } from "./async-processor.js";
+import type { Spec } from "../spec/types.js";
+import { Flow, type FlowDefinition } from "./flow.js";
+
+export abstract class FlowProcessor extends AsyncProcessor {
+  private specifications: Spec[] = [];
+  private flows = new Map<string, Flow>();
+
+  constructor(config: ProcessorConfig) {
+    super(config);
+    this.registerConfigHandler(this.onConfigureFlows.bind(this));
+  }
+
+  registerSpecification(spec: Spec): void {
+    this.specifications.push(spec);
+  }
+
+  protected async run(): Promise<void> {
+    // The processor sits idle waiting for flow configurations
+    // to arrive via the config push topic. In the meantime,
+    // the consumer loop runs in the background.
+    await new Promise<void>((resolve) => {
+      const check = () => {
+        if (!this.running) resolve();
+        else setTimeout(check, 1000);
+      };
+      check();
+    });
+  }
+
+  private async onConfigureFlows(
+    config: Record<string, unknown>,
+    version: number,
+  ): Promise<void> {
+    const flowDefs = config.flows as Record<string, FlowDefinition> | undefined;
+    if (!flowDefs) return;
+
+    // Stop removed flows
+    for (const [name, flow] of this.flows) {
+      if (!(name in flowDefs)) {
+        await flow.stop();
+        this.flows.delete(name);
+      }
+    }
+
+    // Start new flows
+    for (const [name, defn] of Object.entries(flowDefs)) {
+      if (!this.flows.has(name)) {
+        const flow = new Flow(name, this.config.id, this.pubsub, defn, this.specifications);
+        await flow.start();
+        this.flows.set(name, flow);
+      }
+    }
+  }
+
+  override async stop(): Promise<void> {
+    for (const flow of this.flows.values()) {
+      await flow.stop();
+    }
+    this.flows.clear();
+    await super.stop();
+  }
+}

ts/packages/base/src/processor/flow.ts

@@ -0,0 +1,83 @@
+/**
+ * Runtime flow instance — created by FlowProcessor for each configured flow.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/flow.py
+ */
+
+import type { PubSubBackend } from "../backend/types.js";
+import type { Spec } from "../spec/types.js";
+import type { Producer } from "../messaging/producer.js";
+import type { Consumer } from "../messaging/consumer.js";
+
+export interface FlowDefinition {
+  /** Topic overrides keyed by spec name */
+  topics?: Record<string, string>;
+  /** Parameter values keyed by spec name */
+  parameters?: Record<string, unknown>;
+}
+
+export class Flow {
+  private producers = new Map<string, Producer<unknown>>();
+  private consumers = new Map<string, Consumer<unknown>>();
+  private parameters = new Map<string, unknown>();
+
+  constructor(
+    public readonly name: string,
+    public readonly processorId: string,
+    private readonly pubsub: PubSubBackend,
+    private readonly definition: FlowDefinition,
+    private readonly specifications: Spec[],
+  ) {}
+
+  async start(): Promise<void> {
+    for (const spec of this.specifications) {
+      await spec.add(this, this.pubsub, this.definition);
+    }
+
+    // Start all consumers
+    for (const consumer of this.consumers.values()) {
+      consumer.start({ id: this.processorId, name: this.name }).catch((err) => {
+        console.error(`[Flow:${this.name}] Consumer error:`, err);
+      });
+    }
+  }
+
+  async stop(): Promise<void> {
+    for (const consumer of this.consumers.values()) {
+      await consumer.stop();
+    }
+    for (const producer of this.producers.values()) {
+      await producer.stop();
+    }
+  }
+
+  registerProducer(name: string, producer: Producer<unknown>): void {
+    this.producers.set(name, producer);
+  }
+
+  registerConsumer(name: string, consumer: Consumer<unknown>): void {
+    this.consumers.set(name, consumer);
+  }
+
+  setParameter(name: string, value: unknown): void {
+    this.parameters.set(name, value);
+  }
+
+  producer<T>(name: string): Producer<T> {
+    const p = this.producers.get(name);
+    if (!p) throw new Error(`Producer "${name}" not found in flow "${this.name}"`);
+    return p as Producer<T>;
+  }
+
+  consumer<T>(name: string): Consumer<T> {
+    const c = this.consumers.get(name);
+    if (!c) throw new Error(`Consumer "${name}" not found in flow "${this.name}"`);
+    return c as Consumer<T>;
+  }
+
+  parameter<T>(name: string): T {
+    const v = this.parameters.get(name);
+    if (v === undefined) throw new Error(`Parameter "${name}" not found in flow "${this.name}"`);
+    return v as T;
+  }
+}

ts/packages/base/src/processor/index.ts

@@ -0,0 +1,3 @@
+export { AsyncProcessor, type ProcessorConfig, type ConfigHandler } from "./async-processor.js";
+export { FlowProcessor } from "./flow-processor.js";
+export { Flow, type FlowDefinition } from "./flow.js";

ts/packages/base/src/schema/index.ts

@@ -0,0 +1,3 @@
+export * from "./primitives.js";
+export * from "./topics.js";
+export * from "./messages.js";

ts/packages/base/src/schema/messages.ts

@@ -0,0 +1,135 @@
+/**
+ * Message types for service communication.
+ *
+ * Python reference: trustgraph-base/trustgraph/schema/services/
+ */
+
+import type { TgError, Triple, Term, RowSchema } from "./primitives.js";
+
+// Text completion
+export interface TextCompletionRequest {
+  system: string;
+  prompt: string;
+  model?: string;
+  temperature?: number;
+  streaming?: boolean;
+}
+
+export interface TextCompletionResponse {
+  response: string;
+  model?: string;
+  inToken?: number;
+  outToken?: number;
+  error?: TgError;
+  endOfStream?: boolean;
+}
+
+// Embeddings
+export interface EmbeddingsRequest {
+  text: string[];
+  model?: string;
+}
+
+export interface EmbeddingsResponse {
+  vectors: number[][];
+  error?: TgError;
+}
+
+// Graph RAG
+export interface GraphRagRequest {
+  query: string;
+  collection?: string;
+  entityLimit?: number;
+  tripleLimit?: number;
+  maxSubgraphSize?: number;
+  maxPathLength?: number;
+  streaming?: boolean;
+}
+
+export interface GraphRagResponse {
+  response: string;
+  error?: TgError;
+  endOfStream?: boolean;
+}
+
+// Document RAG
+export interface DocumentRagRequest {
+  query: string;
+  collection?: string;
+  streaming?: boolean;
+}
+
+export interface DocumentRagResponse {
+  response: string;
+  error?: TgError;
+  endOfStream?: boolean;
+}
+
+// Agent
+export interface AgentRequest {
+  question: string;
+  collection?: string;
+  streaming?: boolean;
+}
+
+export interface AgentResponse {
+  answer: string;
+  error?: TgError;
+  endOfStream?: boolean;
+  endOfSession?: boolean;
+}
+
+// Triples query
+export interface TriplesQueryRequest {
+  s?: Term;
+  p?: Term;
+  o?: Term;
+  collection?: string;
+  limit?: number;
+}
+
+export interface TriplesQueryResponse {
+  triples: Triple[];
+  error?: TgError;
+}
+
+// Graph embeddings query
+export interface GraphEmbeddingsRequest {
+  vectors: number[][];
+  limit?: number;
+  collection?: string;
+}
+
+export interface GraphEmbeddingsResponse {
+  entities: Term[];
+  error?: TgError;
+}
+
+// Config
+export type ConfigOperation = "get" | "list" | "delete" | "put" | "config";
+
+export interface ConfigRequest {
+  operation: ConfigOperation;
+  keys?: string[];
+  values?: Record<string, unknown>;
+}
+
+export interface ConfigResponse {
+  version?: number;
+  values?: Record<string, unknown>;
+  directory?: string[];
+  config?: Record<string, unknown>;
+  error?: TgError;
+}
+
+// Prompt
+export interface PromptRequest {
+  name: string;
+  variables?: Record<string, string>;
+}
+
+export interface PromptResponse {
+  system: string;
+  prompt: string;
+  error?: TgError;
+}

ts/packages/base/src/schema/primitives.ts

@@ -0,0 +1,72 @@
+/**
+ * Core data types mirroring the Python schema primitives.
+ *
+ * Python reference: trustgraph-base/trustgraph/schema/core/primitives.py
+ */
+
+export interface TgError {
+  type: string;
+  message: string;
+}
+
+// RDF Term types — discriminated union
+export type TermType = "IRI" | "BLANK" | "LITERAL" | "TRIPLE";
+
+export interface IriTerm {
+  type: "IRI";
+  iri: string;
+}
+
+export interface BlankTerm {
+  type: "BLANK";
+  id: string;
+}
+
+export interface LiteralTerm {
+  type: "LITERAL";
+  value: string;
+  datatype?: string;
+  language?: string;
+}
+
+export interface TripleTerm {
+  type: "TRIPLE";
+  triple: Triple;
+}
+
+export type Term = IriTerm | BlankTerm | LiteralTerm | TripleTerm;
+
+export interface Triple {
+  s: Term;
+  p: Term;
+  o: Term;
+  g?: Term; // Named graph (optional quad)
+}
+
+export interface Field {
+  name: string;
+  type: string;
+  description?: string;
+}
+
+export interface RowSchema {
+  name: string;
+  description?: string;
+  fields: Field[];
+}
+
+// LLM-related types
+export interface LlmResult {
+  text: string;
+  inToken: number;
+  outToken: number;
+  model: string;
+}
+
+export interface LlmChunk {
+  text: string;
+  inToken: number | null;
+  outToken: number | null;
+  model: string;
+  isFinal: boolean;
+}

ts/packages/base/src/schema/topics.ts

@@ -0,0 +1,62 @@
+/**
+ * Topic naming conventions.
+ *
+ * Python reference: trustgraph-base/trustgraph/schema/core/topic.py
+ *
+ * The Python version uses Pulsar URI format: "q1/tg/flow/queue-name"
+ * We use NATS subject format: "tg.flow.queue-name"
+ */
+
+export type QoS = "q0" | "q1" | "q2";
+
+export function topic(
+  name: string,
+  tenant = "tg",
+  namespace = "flow",
+): string {
+  return `${tenant}.${namespace}.${name}`;
+}
+
+// Well-known topics from the Python schema
+export const topics = {
+  // Config
+  configRequest: topic("config-request"),
+  configResponse: topic("config-response"),
+  configPush: topic("config-push"),
+
+  // Text completion
+  textCompletionRequest: topic("text-completion-request"),
+  textCompletionResponse: topic("text-completion-response"),
+
+  // Embeddings
+  embeddingsRequest: topic("embeddings-request"),
+  embeddingsResponse: topic("embeddings-response"),
+
+  // Graph RAG
+  graphRagRequest: topic("graph-rag-request"),
+  graphRagResponse: topic("graph-rag-response"),
+
+  // Document RAG
+  documentRagRequest: topic("document-rag-request"),
+  documentRagResponse: topic("document-rag-response"),
+
+  // Agent
+  agentRequest: topic("agent-request"),
+  agentResponse: topic("agent-response"),
+
+  // Triples
+  triplesRequest: topic("triples-request"),
+  triplesResponse: topic("triples-response"),
+
+  // Graph embeddings
+  graphEmbeddingsRequest: topic("graph-embeddings-request"),
+  graphEmbeddingsResponse: topic("graph-embeddings-response"),
+
+  // Document embeddings
+  docEmbeddingsRequest: topic("doc-embeddings-request"),
+  docEmbeddingsResponse: topic("doc-embeddings-response"),
+
+  // Prompt
+  promptRequest: topic("prompt-request"),
+  promptResponse: topic("prompt-response"),
+} as const;

ts/packages/base/src/services/embeddings-service.ts

@@ -0,0 +1,46 @@
+/**
+ * Base embeddings service.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/embeddings_service.py
+ */
+
+import { FlowProcessor } from "../processor/flow-processor.js";
+import { ConsumerSpec } from "../spec/consumer-spec.js";
+import { ProducerSpec } from "../spec/producer-spec.js";
+import { ParameterSpec } from "../spec/parameter-spec.js";
+import type { ProcessorConfig } from "../processor/async-processor.js";
+import type { FlowContext } from "../messaging/consumer.js";
+import type { EmbeddingsRequest, EmbeddingsResponse } from "../schema/messages.js";
+
+export abstract class EmbeddingsService extends FlowProcessor {
+  constructor(config: ProcessorConfig) {
+    super(config);
+
+    this.registerSpecification(
+      new ConsumerSpec<EmbeddingsRequest>(
+        "request",
+        this.onRequest.bind(this),
+      ),
+    );
+    this.registerSpecification(new ProducerSpec<EmbeddingsResponse>("response"));
+    this.registerSpecification(new ParameterSpec("model"));
+  }
+
+  private async onRequest(
+    msg: EmbeddingsRequest,
+    properties: Record<string, string>,
+    _flowCtx: FlowContext,
+  ): Promise<void> {
+    const requestId = properties.id;
+    if (!requestId) return;
+
+    try {
+      const vectors = await this.onEmbeddings(msg.text, msg.model);
+      void vectors; // Producer send would go here
+    } catch (err) {
+      console.error(`[EmbeddingsService] Error processing request:`, err);
+    }
+  }
+
+  abstract onEmbeddings(texts: string[], model?: string): Promise<number[][]>;
+}

ts/packages/base/src/services/index.ts

@@ -0,0 +1,2 @@
+export { LlmService } from "./llm-service.js";
+export { EmbeddingsService } from "./embeddings-service.js";

ts/packages/base/src/services/llm-service.ts

@@ -0,0 +1,88 @@
+/**
+ * Base LLM service — handles message plumbing, subclasses implement the LLM call.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/llm_service.py
+ */
+
+import { FlowProcessor } from "../processor/flow-processor.js";
+import { ConsumerSpec } from "../spec/consumer-spec.js";
+import { ProducerSpec } from "../spec/producer-spec.js";
+import { ParameterSpec } from "../spec/parameter-spec.js";
+import type { ProcessorConfig } from "../processor/async-processor.js";
+import type { FlowContext } from "../messaging/consumer.js";
+import type { Flow } from "../processor/flow.js";
+import type {
+  TextCompletionRequest,
+  TextCompletionResponse,
+} from "../schema/messages.js";
+import type { LlmResult, LlmChunk } from "../schema/primitives.js";
+
+export abstract class LlmService extends FlowProcessor {
+  constructor(config: ProcessorConfig) {
+    super(config);
+
+    this.registerSpecification(
+      new ConsumerSpec<TextCompletionRequest>(
+        "request",
+        this.onRequest.bind(this),
+      ),
+    );
+    this.registerSpecification(new ProducerSpec<TextCompletionResponse>("response"));
+    this.registerSpecification(new ParameterSpec("model"));
+    this.registerSpecification(new ParameterSpec("temperature"));
+  }
+
+  private async onRequest(
+    msg: TextCompletionRequest,
+    properties: Record<string, string>,
+    flowCtx: FlowContext,
+  ): Promise<void> {
+    // We need the actual flow instance to access producers/parameters.
+    // In the full implementation, FlowContext would carry a flow reference.
+    // For now this shows the pattern.
+    const requestId = properties.id;
+    if (!requestId) return;
+
+    try {
+      if (msg.streaming && this.supportsStreaming()) {
+        for await (const chunk of this.generateContentStream(
+          msg.system,
+          msg.prompt,
+          msg.model,
+          msg.temperature,
+        )) {
+          // Send each chunk as a response with the same request ID
+          void chunk; // Producer send would go here
+        }
+      } else {
+        const result = await this.generateContent(
+          msg.system,
+          msg.prompt,
+          msg.model,
+          msg.temperature,
+        );
+        void result; // Producer send would go here
+      }
+    } catch (err) {
+      console.error(`[LlmService] Error processing request:`, err);
+    }
+  }
+
+  abstract generateContent(
+    system: string,
+    prompt: string,
+    model?: string,
+    temperature?: number,
+  ): Promise<LlmResult>;
+
+  abstract generateContentStream(
+    system: string,
+    prompt: string,
+    model?: string,
+    temperature?: number,
+  ): AsyncGenerator<LlmChunk>;
+
+  supportsStreaming(): boolean {
+    return false;
+  }
+}

ts/packages/base/src/spec/consumer-spec.ts

@@ -0,0 +1,32 @@
+/**
+ * Consumer specification — declares a message consumer for a flow.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/consumer_spec.py
+ */
+
+import type { Spec } from "./types.js";
+import type { PubSubBackend } from "../backend/types.js";
+import type { Flow, FlowDefinition } from "../processor/flow.js";
+import { Consumer, type MessageHandler } from "../messaging/consumer.js";
+
+export class ConsumerSpec<T> implements Spec {
+  constructor(
+    public readonly name: string,
+    private readonly handler: MessageHandler<T>,
+    private readonly concurrency = 1,
+  ) {}
+
+  async add(flow: Flow, pubsub: PubSubBackend, definition: FlowDefinition): Promise<void> {
+    const topic = definition.topics?.[this.name] ?? this.name;
+
+    const consumer = new Consumer<T>({
+      pubsub,
+      topic,
+      subscription: `${flow.processorId}-${flow.name}-${this.name}`,
+      handler: this.handler,
+      concurrency: this.concurrency,
+    });
+
+    flow.registerConsumer(this.name, consumer as Consumer<unknown>);
+  }
+}

ts/packages/base/src/spec/index.ts

@@ -0,0 +1,4 @@
+export type { Spec } from "./types.js";
+export { ConsumerSpec } from "./consumer-spec.js";
+export { ProducerSpec } from "./producer-spec.js";
+export { ParameterSpec } from "./parameter-spec.js";

ts/packages/base/src/spec/parameter-spec.ts

@@ -0,0 +1,18 @@
+/**
+ * Parameter specification — declares a configuration parameter for a flow.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/parameter_spec.py
+ */
+
+import type { Spec } from "./types.js";
+import type { PubSubBackend } from "../backend/types.js";
+import type { Flow, FlowDefinition } from "../processor/flow.js";
+
+export class ParameterSpec implements Spec {
+  constructor(public readonly name: string) {}
+
+  async add(flow: Flow, _pubsub: PubSubBackend, definition: FlowDefinition): Promise<void> {
+    const value = definition.parameters?.[this.name];
+    flow.setParameter(this.name, value);
+  }
+}

ts/packages/base/src/spec/producer-spec.ts

@@ -0,0 +1,21 @@
+/**
+ * Producer specification — declares a message producer for a flow.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/producer_spec.py
+ */
+
+import type { Spec } from "./types.js";
+import type { PubSubBackend } from "../backend/types.js";
+import type { Flow, FlowDefinition } from "../processor/flow.js";
+import { Producer } from "../messaging/producer.js";
+
+export class ProducerSpec<T> implements Spec {
+  constructor(public readonly name: string) {}
+
+  async add(flow: Flow, pubsub: PubSubBackend, definition: FlowDefinition): Promise<void> {
+    const topic = definition.topics?.[this.name] ?? this.name;
+    const producer = new Producer<T>(pubsub, topic);
+    await producer.start();
+    flow.registerProducer(this.name, producer as Producer<unknown>);
+  }
+}

ts/packages/base/src/spec/types.ts

@@ -0,0 +1,13 @@
+/**
+ * Specification types for declarative flow configuration.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/spec.py and siblings
+ */
+
+import type { PubSubBackend } from "../backend/types.js";
+import type { Flow, FlowDefinition } from "../processor/flow.js";
+
+export interface Spec {
+  name: string;
+  add(flow: Flow, pubsub: PubSubBackend, definition: FlowDefinition): Promise<void>;
+}