saving

ts/packages/base/src/messaging/consumer.ts

@@ -0,0 +1,105 @@
+/**
+ * High-level consumer with concurrency, retry, and rate-limit handling.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/consumer.py
+ */
+
+import type { PubSubBackend, BackendConsumer, Message } from "../backend/types.js";
+import { TooManyRequestsError } from "../errors.js";
+
+export type MessageHandler<T> = (
+  message: T,
+  properties: Record<string, string>,
+  flow: FlowContext,
+) => Promise<void>;
+
+export interface FlowContext {
+  id: string;
+  name: string;
+}
+
+export interface ConsumerOptions<T> {
+  pubsub: PubSubBackend;
+  topic: string;
+  subscription: string;
+  handler: MessageHandler<T>;
+  concurrency?: number;
+  initialPosition?: "latest" | "earliest";
+  rateLimitRetryMs?: number;
+  rateLimitTimeoutMs?: number;
+}
+
+export class Consumer<T> {
+  private backend: BackendConsumer<T> | null = null;
+  private running = false;
+  private abortController = new AbortController();
+
+  private readonly concurrency: number;
+  private readonly rateLimitRetryMs: number;
+
+  constructor(private readonly options: ConsumerOptions<T>) {
+    this.concurrency = options.concurrency ?? 1;
+    this.rateLimitRetryMs = options.rateLimitRetryMs ?? 10_000;
+  }
+
+  async start(flow: FlowContext): Promise<void> {
+    this.backend = await this.options.pubsub.createConsumer<T>({
+      topic: this.options.topic,
+      subscription: this.options.subscription,
+      initialPosition: this.options.initialPosition ?? "latest",
+    });
+
+    this.running = true;
+
+    // Spawn concurrent consumer tasks
+    const tasks = Array.from({ length: this.concurrency }, () =>
+      this.consumeLoop(flow),
+    );
+    // Run all concurrently — first rejection stops all
+    await Promise.all(tasks);
+  }
+
+  async stop(): Promise<void> {
+    this.running = false;
+    this.abortController.abort();
+    if (this.backend) {
+      await this.backend.close();
+      this.backend = null;
+    }
+  }
+
+  private async consumeLoop(flow: FlowContext): Promise<void> {
+    while (this.running) {
+      try {
+        const msg = await this.backend!.receive(2000);
+        if (!msg) continue;
+
+        await this.handleWithRetry(msg, flow);
+        await this.backend!.acknowledge(msg);
+      } catch (err) {
+        if (!this.running) break;
+        console.error("[Consumer] Error in consume loop:", err);
+        // Brief pause before retry
+        await sleep(1000);
+      }
+    }
+  }
+
+  private async handleWithRetry(msg: Message<T>, flow: FlowContext): Promise<void> {
+    try {
+      await this.options.handler(msg.value(), msg.properties(), flow);
+    } catch (err) {
+      if (err instanceof TooManyRequestsError) {
+        console.warn(`[Consumer] Rate limited, retrying in ${this.rateLimitRetryMs}ms`);
+        await sleep(this.rateLimitRetryMs);
+        await this.options.handler(msg.value(), msg.properties(), flow);
+      } else {
+        throw err;
+      }
+    }
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

ts/packages/base/src/messaging/index.ts

@@ -0,0 +1,4 @@
+export { Producer } from "./producer.js";
+export { Consumer, type MessageHandler, type FlowContext, type ConsumerOptions } from "./consumer.js";
+export { Subscriber, AsyncQueue } from "./subscriber.js";
+export { RequestResponse, type RequestResponseOptions } from "./request-response.js";

ts/packages/base/src/messaging/producer.ts

@@ -0,0 +1,40 @@
+/**
+ * High-level async producer with queue buffering and retry.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/producer.py
+ */
+
+import type { PubSubBackend, BackendProducer } from "../backend/types.js";
+import type { ProducerMetrics } from "../metrics/prometheus.js";
+
+export class Producer<T> {
+  private backend: BackendProducer<T> | null = null;
+  private running = false;
+
+  constructor(
+    private readonly pubsub: PubSubBackend,
+    private readonly topic: string,
+    private readonly metrics?: ProducerMetrics,
+  ) {}
+
+  async start(): Promise<void> {
+    this.backend = await this.pubsub.createProducer<T>({ topic: this.topic });
+    this.running = true;
+  }
+
+  async send(id: string, message: T): Promise<void> {
+    if (!this.backend) throw new Error("Producer not started");
+
+    await this.backend.send(message, { id });
+    this.metrics?.inc();
+  }
+
+  async stop(): Promise<void> {
+    this.running = false;
+    if (this.backend) {
+      await this.backend.flush();
+      await this.backend.close();
+      this.backend = null;
+    }
+  }
+}

ts/packages/base/src/messaging/request-response.ts

@@ -0,0 +1,91 @@
+/**
+ * Request/response pattern over pub/sub.
+ *
+ * Sends a request with a unique ID, subscribes for matching responses.
+ * Supports streaming (multiple responses per request) via a recipient callback.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/request_response_spec.py
+ */
+
+import { randomUUID } from "node:crypto";
+import { Producer } from "./producer.js";
+import { Subscriber } from "./subscriber.js";
+import type { PubSubBackend } from "../backend/types.js";
+
+export interface RequestResponseOptions {
+  pubsub: PubSubBackend;
+  requestTopic: string;
+  responseTopic: string;
+  subscription: string;
+}
+
+export class RequestResponse<TReq, TRes> {
+  private producer: Producer<TReq>;
+  private subscriber: Subscriber<TRes>;
+
+  constructor(private readonly options: RequestResponseOptions) {
+    this.producer = new Producer<TReq>(options.pubsub, options.requestTopic);
+    this.subscriber = new Subscriber<TRes>(
+      options.pubsub,
+      options.responseTopic,
+      options.subscription,
+    );
+  }
+
+  async start(): Promise<void> {
+    await this.producer.start();
+    await this.subscriber.start();
+  }
+
+  async stop(): Promise<void> {
+    await this.producer.stop();
+    await this.subscriber.stop();
+  }
+
+  /**
+   * Send a request and wait for responses.
+   *
+   * @param request - The request payload
+   * @param options.timeoutMs - Total timeout in milliseconds (default: 300s)
+   * @param options.recipient - Optional callback for streaming responses.
+   *   Return `true` to indicate the final response has been received.
+   *   If omitted, returns the first response.
+   */
+  async request(
+    request: TReq,
+    options?: {
+      timeoutMs?: number;
+      recipient?: (response: TRes) => Promise<boolean>;
+    },
+  ): Promise<TRes> {
+    const id = randomUUID();
+    const timeoutMs = options?.timeoutMs ?? 300_000;
+    const recipient = options?.recipient;
+
+    const queue = this.subscriber.subscribe(id);
+
+    try {
+      await this.producer.send(id, request);
+
+      const deadline = Date.now() + timeoutMs;
+
+      while (true) {
+        const remaining = deadline - Date.now();
+        if (remaining <= 0) {
+          throw new Error(`Request timed out after ${timeoutMs}ms`);
+        }
+
+        const response = await queue.pop(remaining);
+
+        if (recipient) {
+          const isFinal = await recipient(response);
+          if (isFinal) return response;
+        } else {
+          return response;
+        }
+      }
+    } finally {
+      this.subscriber.unsubscribe(id);
+    }
+  }
+}

ts/packages/base/src/messaging/subscriber.ts

@@ -0,0 +1,143 @@
+/**
+ * Fan-out subscriber: routes responses to waiting callers by request ID.
+ *
+ * Python reference: trustgraph-base/trustgraph/base/subscriber.py
+ */
+
+import type { PubSubBackend, BackendConsumer } from "../backend/types.js";
+
+type Resolver<T> = {
+  queue: AsyncQueue<T>;
+};
+
+/**
+ * Simple async queue for inter-task communication (replaces asyncio.Queue).
+ */
+export class AsyncQueue<T> {
+  private buffer: T[] = [];
+  private waiters: Array<(value: T) => void> = [];
+
+  push(item: T): void {
+    const waiter = this.waiters.shift();
+    if (waiter) {
+      waiter(item);
+    } else {
+      this.buffer.push(item);
+    }
+  }
+
+  async pop(timeoutMs?: number): Promise<T> {
+    const buffered = this.buffer.shift();
+    if (buffered !== undefined) return buffered;
+
+    return new Promise<T>((resolve, reject) => {
+      let timer: ReturnType<typeof setTimeout> | undefined;
+
+      const waiter = (value: T) => {
+        if (timer) clearTimeout(timer);
+        resolve(value);
+      };
+
+      this.waiters.push(waiter);
+
+      if (timeoutMs !== undefined) {
+        timer = setTimeout(() => {
+          const idx = this.waiters.indexOf(waiter);
+          if (idx !== -1) this.waiters.splice(idx, 1);
+          reject(new Error(`Queue.pop timed out after ${timeoutMs}ms`));
+        }, timeoutMs);
+      }
+    });
+  }
+
+  get length(): number {
+    return this.buffer.length;
+  }
+}
+
+export class Subscriber<T> {
+  private backend: BackendConsumer<T> | null = null;
+  private running = false;
+
+  // ID-specific subscriptions (request/response correlation)
+  private idSubscribers = new Map<string, Resolver<T>>();
+  // Wildcard subscribers (receive all messages)
+  private allSubscribers = new Map<string, Resolver<T>>();
+
+  constructor(
+    private readonly pubsub: PubSubBackend,
+    private readonly topic: string,
+    private readonly subscription: string,
+  ) {}
+
+  async start(): Promise<void> {
+    this.backend = await this.pubsub.createConsumer<T>({
+      topic: this.topic,
+      subscription: this.subscription,
+    });
+    this.running = true;
+    // Start the dispatch loop (fire and forget — runs until stop)
+    this.dispatchLoop().catch((err) => {
+      if (this.running) console.error("[Subscriber] dispatch loop error:", err);
+    });
+  }
+
+  async stop(): Promise<void> {
+    this.running = false;
+    if (this.backend) {
+      await this.backend.close();
+      this.backend = null;
+    }
+  }
+
+  subscribe(id: string): AsyncQueue<T> {
+    const queue = new AsyncQueue<T>();
+    this.idSubscribers.set(id, { queue });
+    return queue;
+  }
+
+  subscribeAll(id: string): AsyncQueue<T> {
+    const queue = new AsyncQueue<T>();
+    this.allSubscribers.set(id, { queue });
+    return queue;
+  }
+
+  unsubscribe(id: string): void {
+    this.idSubscribers.delete(id);
+  }
+
+  unsubscribeAll(id: string): void {
+    this.allSubscribers.delete(id);
+  }
+
+  private async dispatchLoop(): Promise<void> {
+    while (this.running) {
+      try {
+        const msg = await this.backend!.receive(2000);
+        if (!msg) continue;
+
+        const props = msg.properties();
+        const id = props.id;
+        const value = msg.value();
+
+        // Route to ID-specific subscriber
+        if (id) {
+          const sub = this.idSubscribers.get(id);
+          if (sub) {
+            sub.queue.push(value);
+          }
+        }
+
+        // Broadcast to all-subscribers
+        for (const sub of this.allSubscribers.values()) {
+          sub.queue.push(value);
+        }
+
+        await this.backend!.acknowledge(msg);
+      } catch (err) {
+        if (!this.running) break;
+        console.error("[Subscriber] Error:", err);
+      }
+    }
+  }
+}