saving

ts/packages/base/src/spec/consumer-spec.ts

@@ -4,29 +4,84 @@
  * Python reference: trustgraph-base/trustgraph/base/consumer_spec.py
  */
 
+import { Effect } from "effect";
+import * as S from "effect/Schema";
 import type { Spec } from "./types.js";
+import type { SpecRuntimeRequirements } from "./types.js";
 import type { PubSubBackend } from "../backend/types.js";
 import type { Flow, FlowDefinition } from "../processor/flow.js";
-import { Consumer, type MessageHandler } from "../messaging/consumer.js";
+import { type MessageHandler } from "../messaging/consumer.js";
+import {
+  ConsumerFactory,
+  type EffectMessageHandler,
+} from "../messaging/runtime.js";
+import {
+  messagingHandlerError,
+  TooManyRequestsError,
+  type MessagingHandlerError,
+  type PubSubError,
+} from "../errors.js";
+
+const isTooManyRequestsError = S.is(TooManyRequestsError);
+
+export class ConsumerSpec<T, E = never, R = never> implements Spec<R> {
+  public readonly name: string;
+  private readonly handler: EffectMessageHandler<T, E, R>;
+  private readonly concurrency: number;
 
-export class ConsumerSpec<T> implements Spec {
   constructor(
-    public readonly name: string,
-    private readonly handler: MessageHandler<T>,
-    private readonly concurrency = 1,
-  ) {}
+    name: string,
+    handler: EffectMessageHandler<T, E, R>,
+    concurrency = 1,
+  ) {
+    this.name = name;
+    this.handler = handler;
+    this.concurrency = concurrency;
+  }
+
+  static fromPromise<T>(
+    name: string,
+    handler: MessageHandler<T>,
+    concurrency = 1,
+  ): ConsumerSpec<T, TooManyRequestsError | MessagingHandlerError> {
+    return new ConsumerSpec<T, TooManyRequestsError | MessagingHandlerError>(
+      name,
+      (message, properties, flow) =>
+        Effect.tryPromise({
+          try: () => handler(message, properties, flow),
+          catch: (error) =>
+            isTooManyRequestsError(error)
+              ? error
+              : messagingHandlerError(name, `${flow.id}-${flow.name}-${name}`, error),
+        }),
+      concurrency,
+    );
+  }
+
+  addEffect(flow: Flow<R>, definition: FlowDefinition) {
+    const spec = this;
+    return Effect.gen(function* () {
+      const topic = definition.topics?.[spec.name] ?? spec.name;
+      const factory = yield* ConsumerFactory;
+      const consumer = yield* factory.run<T, E, R>(
+        {
+          topic,
+          subscription: `${flow.processorId}-${flow.name}-${spec.name}`,
+          handler: spec.handler,
+          concurrency: spec.concurrency,
+        },
+        { id: flow.processorId, name: flow.name, flow },
+      );
+      flow.registerConsumer(spec.name, consumer);
+    });
+  }
 
   async add(flow: Flow, pubsub: PubSubBackend, definition: FlowDefinition): Promise<void> {
-    const topic = definition.topics?.[this.name] ?? this.name;
-
-    const consumer = new Consumer<T>({
-      pubsub,
-      topic,
-      subscription: `${flow.processorId}-${flow.name}-${this.name}`,
-      handler: this.handler,
-      concurrency: this.concurrency,
-    });
-
-    flow.registerConsumer(this.name, consumer as Consumer<unknown>);
+    const effect = this.addEffect(flow, definition) as Effect.Effect<
+      void,
+      PubSubError,
+      SpecRuntimeRequirements
+    >;
+    await flow.runInCompatibilityScope(effect, pubsub);
   }
 }

ts/packages/base/src/spec/index.ts

@@ -1,4 +1,4 @@
-export type { Spec } from "./types.js";
+export type { Spec, SpecRuntimeError, SpecRuntimeRequirements } from "./types.js";
 export { ConsumerSpec } from "./consumer-spec.js";
 export { ProducerSpec } from "./producer-spec.js";
 export { ParameterSpec } from "./parameter-spec.js";

ts/packages/base/src/spec/parameter-spec.ts

@@ -4,15 +4,27 @@
  * Python reference: trustgraph-base/trustgraph/base/parameter_spec.py
  */
 
+import { Effect } from "effect";
 import type { Spec } from "./types.js";
 import type { PubSubBackend } from "../backend/types.js";
 import type { Flow, FlowDefinition } from "../processor/flow.js";
 
 export class ParameterSpec implements Spec {
-  constructor(public readonly name: string) {}
+  public readonly name: string;
+
+  constructor(name: string) {
+    this.name = name;
+  }
+
+  addEffect(flow: Flow, definition: FlowDefinition) {
+    const spec = this;
+    return Effect.sync(() => {
+      const value = definition.parameters?.[spec.name];
+      flow.setParameter(spec.name, value);
+    });
+  }
 
   async add(flow: Flow, _pubsub: PubSubBackend, definition: FlowDefinition): Promise<void> {
-    const value = definition.parameters?.[this.name];
-    flow.setParameter(this.name, value);
+    await Effect.runPromise(this.addEffect(flow, definition));
   }
 }

ts/packages/base/src/spec/producer-spec.ts

@@ -4,18 +4,33 @@
  * Python reference: trustgraph-base/trustgraph/base/producer_spec.py
  */
 
+import { Effect } from "effect";
 import type { Spec } from "./types.js";
 import type { PubSubBackend } from "../backend/types.js";
 import type { Flow, FlowDefinition } from "../processor/flow.js";
-import { Producer } from "../messaging/producer.js";
+import {
+  ProducerFactory,
+  type EffectProducer,
+} from "../messaging/runtime.js";
 
 export class ProducerSpec<T> implements Spec {
-  constructor(public readonly name: string) {}
+  public readonly name: string;
+
+  constructor(name: string) {
+    this.name = name;
+  }
+
+  addEffect(flow: Flow, definition: FlowDefinition) {
+    const spec = this;
+    return Effect.gen(function* () {
+      const topic = definition.topics?.[spec.name] ?? spec.name;
+      const factory = yield* ProducerFactory;
+      const producer = yield* factory.make<T>({ topic });
+      flow.registerProducer(spec.name, producer as EffectProducer<unknown>);
+    });
+  }
 
   async add(flow: Flow, pubsub: PubSubBackend, definition: FlowDefinition): Promise<void> {
-    const topic = definition.topics?.[this.name] ?? this.name;
-    const producer = new Producer<T>(pubsub, topic);
-    await producer.start();
-    flow.registerProducer(this.name, producer as Producer<unknown>);
+    await flow.runInCompatibilityScope(this.addEffect(flow, definition), pubsub);
   }
 }

ts/packages/base/src/spec/request-response-spec.ts

@@ -7,30 +7,46 @@
  * Python reference: trustgraph-base/trustgraph/base/prompt_client_spec.py
  */
 
+import { Effect } from "effect";
 import type { Spec } from "./types.js";
 import type { PubSubBackend } from "../backend/types.js";
 import type { Flow, FlowDefinition } from "../processor/flow.js";
-import { RequestResponse } from "../messaging/request-response.js";
+import {
+  RequestResponseFactory,
+  type EffectRequestResponse,
+} from "../messaging/runtime.js";
 
 export class RequestResponseSpec<TReq, TRes> implements Spec {
+  public readonly name: string;
+  private readonly requestTopicName: string;
+  private readonly responseTopicName: string;
+
   constructor(
-    public readonly name: string,
-    private readonly requestTopicName: string,
-    private readonly responseTopicName: string,
-  ) {}
+    name: string,
+    requestTopicName: string,
+    responseTopicName: string,
+  ) {
+    this.name = name;
+    this.requestTopicName = requestTopicName;
+    this.responseTopicName = responseTopicName;
+  }
+
+  addEffect(flow: Flow, definition: FlowDefinition) {
+    const spec = this;
+    return Effect.gen(function* () {
+      const requestTopic = definition.topics?.[spec.requestTopicName] ?? spec.requestTopicName;
+      const responseTopic = definition.topics?.[spec.responseTopicName] ?? spec.responseTopicName;
+      const factory = yield* RequestResponseFactory;
+      const requestor = yield* factory.make<TReq, TRes>({
+        requestTopic,
+        responseTopic,
+        subscription: `${flow.processorId}-${flow.name}-${spec.name}`,
+      });
+      flow.registerRequestor(spec.name, requestor as EffectRequestResponse<unknown, unknown>);
+    });
+  }
 
   async add(flow: Flow, pubsub: PubSubBackend, definition: FlowDefinition): Promise<void> {
-    const requestTopic = definition.topics?.[this.requestTopicName] ?? this.requestTopicName;
-    const responseTopic = definition.topics?.[this.responseTopicName] ?? this.responseTopicName;
-
-    const rr = new RequestResponse<TReq, TRes>({
-      pubsub,
-      requestTopic,
-      responseTopic,
-      subscription: `${flow.processorId}-${flow.name}-${this.name}`,
-    });
-    await rr.start();
-
-    flow.registerRequestor(this.name, rr as RequestResponse<unknown, unknown>);
+    await flow.runInCompatibilityScope(this.addEffect(flow, definition), pubsub);
   }
 }

ts/packages/base/src/spec/types.ts

@@ -4,10 +4,29 @@
  * Python reference: trustgraph-base/trustgraph/base/spec.py and siblings
  */
 
+import type { Effect, Scope } from "effect";
 import type { PubSubBackend } from "../backend/types.js";
+import type {
+  ConsumerFactory,
+  ProducerFactory,
+  RequestResponseFactory,
+} from "../messaging/runtime.js";
 import type { Flow, FlowDefinition } from "../processor/flow.js";
+import type { PubSubError } from "../errors.js";
 
-export interface Spec {
+export type SpecRuntimeRequirements =
+  | Scope.Scope
+  | ProducerFactory
+  | ConsumerFactory
+  | RequestResponseFactory;
+
+export type SpecRuntimeError = PubSubError;
+
+export interface Spec<Requirements = never> {
   name: string;
+  addEffect(
+    flow: Flow<Requirements>,
+    definition: FlowDefinition,
+  ): Effect.Effect<void, SpecRuntimeError, SpecRuntimeRequirements | Requirements>;
   add(flow: Flow, pubsub: PubSubBackend, definition: FlowDefinition): Promise<void>;
 }