/** * NATS JetStream backend implementation. * * Replaces Pulsar as the message broker. NATS JetStream provides * at-least-once delivery, consumer groups, and replay — matching * the QoS levels used by the Python Pulsar backend. * * Python reference: trustgraph-base/trustgraph/base/pulsar_backend.py */ import { connect, type NatsConnection, type JetStreamClient, type JetStreamManager, type Consumer as NatsJsConsumer, type JsMsg, StringCodec, AckPolicy, DeliverPolicy, } from "nats"; import * as S from "effect/Schema"; import type { PubSubBackend, BackendProducer, BackendConsumer, CreateProducerOptions, CreateConsumerOptions, Message, } from "./types.js"; const sc = StringCodec(); interface NatsMessage extends Message { /** Exposed so acknowledge/negativeAcknowledge can access the raw JsMsg */ readonly _jsMsg: JsMsg; } function makeNatsMessage(msg: JsMsg, decoded: T): NatsMessage { return { _jsMsg: msg, value: () => decoded, properties: () => { const headers = msg.headers; const props: Record = {}; if (headers !== undefined) { for (const [key, values] of headers) { const value = values[0]; if (value !== undefined) { props[key] = value; } } } return props; }, }; } function makeNatsProducer( js: JetStreamClient, subject: string, schema?: S.Top, ): BackendProducer { return { send: async (message, properties) => { const encoded = schema !== undefined ? S.encodeUnknownSync(schema as S.Codec)(message) : message; const data = sc.encode(JSON.stringify(encoded)); const opts: Record = {}; if (properties !== undefined && Object.keys(properties).length > 0) { const { headers } = await import("nats"); const hdrs = headers(); for (const [key, val] of Object.entries(properties)) { hdrs.append(key, val); } opts.headers = hdrs; } await js.publish(subject, data, opts); }, flush: async () => { // NATS publishes are flushed on the connection level. }, close: async () => { // No per-producer cleanup needed for NATS. }, }; } interface InitializableBackendConsumer extends BackendConsumer { readonly init: () => Promise; } function makeNatsConsumer( js: JetStreamClient, jsm: JetStreamManager, subject: string, subscription: string, initialPosition: "latest" | "earliest", streamName: string, schema?: S.Top, ): InitializableBackendConsumer { let consumer: NatsJsConsumer | null = null; return { init: async () => { // Stream is already ensured by makeNatsBackend(). Create or bind to a durable consumer. try { consumer = await js.consumers.get(streamName, subscription); } catch { const deliverPolicy = initialPosition === "earliest" ? DeliverPolicy.All : DeliverPolicy.New; await jsm.consumers.add(streamName, { durable_name: subscription, ack_policy: AckPolicy.Explicit, deliver_policy: deliverPolicy, filter_subject: subject, }); consumer = await js.consumers.get(streamName, subscription); } }, receive: async (timeoutMs = 2000) => { if (consumer === null) throw new Error("Consumer not initialized"); // Pull a single message with a timeout using the pull-based API. // consumer.next() returns a JsMsg or null when the timeout expires. const msg = await consumer.next({ expires: timeoutMs }); if (msg === null) return null; const parsed = JSON.parse(sc.decode(msg.data)); const decoded = schema !== undefined ? S.decodeUnknownSync(schema as S.Codec)(parsed) as T : parsed as T; return makeNatsMessage(msg, decoded); }, acknowledge: async (message) => { const natsMsg = message as NatsMessage; natsMsg._jsMsg.ack(); }, negativeAcknowledge: async (message) => { const natsMsg = message as NatsMessage; natsMsg._jsMsg.nak(); }, unsubscribe: async () => { // The pull-based consumer does not have a persistent subscription to drain. // Clearing the reference is sufficient; the durable consumer persists server-side. consumer = null; }, close: async () => { consumer = null; }, }; } export function makeNatsBackend(url = "nats://localhost:4222"): PubSubBackend { let connection: NatsConnection | null = null; let js: JetStreamClient | null = null; let jsm: JetStreamManager | null = null; const initializedStreams = new Set(); const ensureConnected = async (): Promise => { if (connection === null) { connection = await connect({ servers: url }); js = connection.jetstream(); jsm = await connection.jetstreamManager(); } }; /** * Ensure the stream for a given subject exists with a wildcard filter. * E.g. subject "tg.flow.config-request" → stream "tg_flow" with subjects ["tg.flow.>"] */ const ensureStream = async (subject: string): Promise => { const parts = subject.split("."); const streamName = parts.slice(0, 2).join("_"); if (initializedStreams.has(streamName)) return streamName; const wildcardSubject = `${parts.slice(0, 2).join(".")}.>`; const manager = jsm; if (manager === null) throw new Error("NATS backend not connected"); try { await manager.streams.info(streamName); } catch { await manager.streams.add({ name: streamName, subjects: [wildcardSubject], }); } initializedStreams.add(streamName); return streamName; }; return { createProducer: async (options: CreateProducerOptions) => { await ensureConnected(); await ensureStream(options.topic); const client = js; if (client === null) throw new Error("NATS backend not connected"); return makeNatsProducer(client, options.topic, options.schema); }, createConsumer: async (options: CreateConsumerOptions) => { await ensureConnected(); const streamName = await ensureStream(options.topic); const client = js; const manager = jsm; if (client === null || manager === null) throw new Error("NATS backend not connected"); const consumer = makeNatsConsumer( client, manager, options.topic, options.subscription, options.initialPosition ?? "latest", streamName, options.schema, ); await consumer.init(); return consumer; }, close: async () => { if (connection !== null) { await connection.drain(); connection = null; js = null; jsm = null; } }, }; }