Migrate strict Effect runtime surfaces

This commit is contained in:
elpresidank 2026-06-02 00:22:04 -05:00
parent f6878d4dd7
commit b4ee2b691f
35 changed files with 1717 additions and 1410 deletions

View file

@ -4,9 +4,16 @@
* Python reference: trustgraph-base/trustgraph/base/consumer.py
*/
import type { PubSubBackend, BackendConsumer, Message } from "../backend/types.js";
import type { BackendConsumer, Message, PubSubBackend } from "../backend/types.js";
import type { Flow } from "../processor/flow.js";
import { TooManyRequestsError } from "../errors.js";
import {
MessagingHandlerError,
TooManyRequestsError,
messagingDeliveryError,
messagingHandlerError,
messagingLifecycleError,
} from "../errors.js";
import { Duration, Effect } from "effect";
import * as S from "effect/Schema";
export type MessageHandler<T> = (
@ -44,83 +51,140 @@ export interface Consumer<T> {
export function makeConsumer<T>(options: ConsumerOptions<T>): Consumer<T> {
let backend: BackendConsumer<T> | null = null;
let running = false;
let abortController = new AbortController();
const isTooManyRequestsError = S.is(TooManyRequestsError);
const concurrency = options.concurrency ?? 1;
const rateLimitRetryMs = options.rateLimitRetryMs ?? 10_000;
const handleWithRetry = async (msg: Message<T>, flow: FlowContext): Promise<void> => {
try {
await options.handler(msg.value(), msg.properties(), flow);
} catch (err) {
if (S.is(TooManyRequestsError)(err)) {
console.warn(`[Consumer] Rate limited, retrying in ${rateLimitRetryMs}ms`);
await sleep(rateLimitRetryMs);
await options.handler(msg.value(), msg.properties(), flow);
} else {
throw err;
}
const runHandler = (
message: T,
properties: Record<string, string>,
flow: FlowContext,
): Effect.Effect<void, TooManyRequestsError | MessagingHandlerError> =>
Effect.tryPromise({
try: () => options.handler(message, properties, flow),
catch: (error) =>
isTooManyRequestsError(error)
? error
: messagingHandlerError(options.topic, options.subscription, error),
});
const handleWithRetry = Effect.fn("Consumer.handleWithRetry")(function* (
message: Message<T>,
flow: FlowContext,
) {
const callHandler = runHandler(message.value(), message.properties(), flow);
yield* callHandler.pipe(
Effect.catchTag("TooManyRequestsError", () =>
Effect.logWarning("[Consumer] Rate limited, retrying", {
topic: options.topic,
subscription: options.subscription,
retryMs: rateLimitRetryMs,
}).pipe(
Effect.flatMap(() => Effect.sleep(Duration.millis(rateLimitRetryMs))),
Effect.flatMap(() => callHandler),
),
),
);
});
const consumeOnce = Effect.fn("Consumer.consumeOnce")(function* (flow: FlowContext) {
const currentBackend = backend;
if (currentBackend === null) {
return yield* messagingLifecycleError(
`${options.topic}:${options.subscription}`,
"receive",
"Consumer backend not started",
);
}
};
const consumeLoop = async (flow: FlowContext): Promise<void> => {
while (running) {
let msg: Message<T> | null = null;
try {
const currentBackend = backend;
if (currentBackend === null) throw new Error("Consumer backend not started");
const message = yield* Effect.tryPromise({
try: () => currentBackend.receive(2000),
catch: (error) => messagingDeliveryError(options.topic, "receive", error),
});
if (message === null) return;
msg = await currentBackend.receive(2000);
if (msg === null) continue;
yield* handleWithRetry(message, flow).pipe(
Effect.flatMap(() =>
Effect.tryPromise({
try: () => currentBackend.acknowledge(message),
catch: (error) => messagingDeliveryError(options.topic, "acknowledge", error),
}),
),
Effect.catch((error) =>
Effect.tryPromise({
try: () => currentBackend.negativeAcknowledge(message),
catch: (nakError) => messagingDeliveryError(options.topic, "negative-acknowledge", nakError),
}).pipe(
Effect.catch((nakError) =>
Effect.logError("[Consumer] Failed to negative-acknowledge message", {
error: nakError.message,
topic: nakError.topic,
}),
),
Effect.flatMap(() => Effect.fail(error)),
),
),
);
});
await handleWithRetry(msg, flow);
await currentBackend.acknowledge(msg);
} catch (err) {
if (!running) break;
console.error("[Consumer] Error in consume loop:", err);
if (msg !== null) {
try {
const currentBackend = backend;
if (currentBackend !== null) {
await currentBackend.negativeAcknowledge(msg);
}
} catch (nakErr) {
console.error("[Consumer] Failed to nak message:", nakErr);
}
}
await sleep(1000);
}
}
};
const consumeLoop = Effect.fn("Consumer.consumeLoop")(function* (flow: FlowContext) {
yield* Effect.whileLoop({
while: () => running,
body: () =>
consumeOnce(flow).pipe(
Effect.catch((error) => {
if (!running) return Effect.void;
return Effect.logError("[Consumer] Error in consume loop", {
error: error.message,
topic: options.topic,
subscription: options.subscription,
}).pipe(
Effect.flatMap(() => Effect.sleep(Duration.millis(1000))),
);
}),
),
step: () => undefined,
});
});
return {
start: async (flow) => {
backend = await options.pubsub.createConsumer<T>({
topic: options.topic,
subscription: options.subscription,
initialPosition: options.initialPosition ?? "latest",
});
start: (flow) =>
Effect.runPromise(
Effect.gen(function* () {
backend = yield* Effect.tryPromise({
try: () =>
options.pubsub.createConsumer<T>({
topic: options.topic,
subscription: options.subscription,
initialPosition: options.initialPosition ?? "latest",
}),
catch: (error) =>
messagingLifecycleError(`${options.topic}:${options.subscription}`, "create-consumer", error),
});
running = true;
running = true;
// Spawn concurrent consumer tasks.
const tasks = Array.from({ length: concurrency }, () =>
consumeLoop(flow),
);
// Run all concurrently: first rejection stops all.
await Promise.all(tasks);
},
stop: async () => {
running = false;
abortController.abort();
abortController = new AbortController();
if (backend !== null) {
await backend.close();
backend = null;
}
},
const workerIndexes = Array.from({ length: concurrency }, (_value, index) => index);
yield* Effect.forEach(workerIndexes, () => consumeLoop(flow), {
concurrency: "unbounded",
discard: true,
});
}),
),
stop: () =>
Effect.runPromise(
Effect.gen(function* () {
running = false;
const currentBackend = backend;
backend = null;
if (currentBackend !== null) {
yield* Effect.tryPromise({
try: () => currentBackend.close(),
catch: (error) =>
messagingLifecycleError(`${options.topic}:${options.subscription}`, "close-consumer", error),
});
}
}),
),
};
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

View file

@ -8,6 +8,7 @@ import type { PubSubBackend } from "../backend/types.js";
import type { ProducerMetrics } from "../metrics/prometheus.js";
import { Effect } from "effect";
import { makeEffectProducerHandle, type EffectProducer } from "./runtime.js";
import { messagingLifecycleError } from "../errors.js";
export interface Producer<T> {
readonly start: () => Promise<void>;
@ -23,28 +24,38 @@ export function makeProducer<T>(
let effectProducer: EffectProducer<T> | null = null;
return {
start: async () => {
const backend = await pubsub.createProducer<T>({ topic });
effectProducer = makeEffectProducerHandle(backend, {
topic,
...(metrics === undefined ? {} : { metrics }),
});
},
send: async (id, message) => {
if (effectProducer === null) throw new Error("Producer not started");
await Effect.runPromise(effectProducer.send(id, message));
},
stop: async () => {
if (effectProducer !== null) {
const producer = effectProducer;
await Effect.runPromise(
producer.flush.pipe(
Effect.flatMap(() => producer.close),
),
);
effectProducer = null;
}
},
start: () =>
Effect.runPromise(
Effect.gen(function* () {
const backend = yield* Effect.tryPromise({
try: () => pubsub.createProducer<T>({ topic }),
catch: (error) => messagingLifecycleError(topic, "create-producer", error),
});
effectProducer = makeEffectProducerHandle(backend, {
topic,
...(metrics === undefined ? {} : { metrics }),
});
}),
),
send: (id, message) =>
effectProducer === null
? Effect.runPromise(Effect.fail(messagingLifecycleError(topic, "send", "Producer not started")))
: Effect.runPromise(effectProducer.send(id, message)),
stop: () =>
Effect.runPromise(
Effect.gen(function* () {
if (effectProducer !== null) {
const producer = effectProducer;
yield* producer.flush.pipe(
Effect.flatMap(() => producer.close),
Effect.ensuring(
Effect.sync(() => {
effectProducer = null;
}),
),
);
}
}),
),
};
}

View file

@ -44,37 +44,42 @@ export function makeRequestResponse<TReq, TRes>(
let runtime: RequestResponseRuntime<TReq, TRes> | null = null;
return {
start: async () => {
if (runtime !== null) return;
start: () =>
runtime !== null
? Promise.resolve()
: Effect.runPromise(
Effect.gen(function* () {
const scope = yield* Scope.make();
const startRuntime = Effect.gen(function* () {
const config = yield* loadMessagingRuntimeConfig();
const requestor = yield* makeEffectRequestResponseFromPubSub<TReq, TRes>(
PubSub.fromBackend(options.pubsub),
config,
{
requestTopic: options.requestTopic,
responseTopic: options.responseTopic,
subscription: options.subscription,
},
).pipe(Scope.provide(scope));
const scope = await Effect.runPromise(Scope.make());
runtime = { scope, requestor };
});
try {
const config = await Effect.runPromise(loadMessagingRuntimeConfig());
const requestor = await Effect.runPromise(
makeEffectRequestResponseFromPubSub<TReq, TRes>(
PubSub.fromBackend(options.pubsub),
config,
{
requestTopic: options.requestTopic,
responseTopic: options.responseTopic,
subscription: options.subscription,
},
).pipe(Scope.provide(scope)),
);
runtime = { scope, requestor };
} catch (error) {
await Effect.runPromise(Scope.close(scope, Exit.fail(error))).catch(() => undefined);
throw error;
}
},
stop: async () => {
yield* startRuntime.pipe(
Effect.catch((error) =>
Scope.close(scope, Exit.fail(error)).pipe(
Effect.flatMap(() => Effect.fail(error)),
),
),
);
}),
),
stop: () => {
const current = runtime;
runtime = null;
if (current === null) return;
await Effect.runPromise(Scope.close(current.scope, Exit.void));
return current === null
? Promise.resolve()
: Effect.runPromise(Scope.close(current.scope, Exit.void));
},
/**
* Send a request and wait for responses.
@ -85,20 +90,24 @@ export function makeRequestResponse<TReq, TRes>(
* Return `true` to indicate the final response has been received.
* If omitted, returns the first response.
*/
request: async (request, requestOptions) => {
request: (request, requestOptions) => {
const current = runtime;
if (current === null) {
throw messagingLifecycleError(
`${options.requestTopic}:${options.responseTopic}`,
"request",
"RequestResponse not started",
return Effect.runPromise(
Effect.fail(
messagingLifecycleError(
`${options.requestTopic}:${options.responseTopic}`,
"request",
"RequestResponse not started",
),
),
);
}
const timeoutMs = requestOptions?.timeoutMs ?? 300_000;
const recipient = requestOptions?.recipient;
return await Effect.runPromise(
return Effect.runPromise(
current.requestor.request(request, {
timeoutMs,
...(recipient === undefined

View file

@ -44,9 +44,9 @@ export type EffectMessageHandler<T, E = never, R = never> = (
flow: FlowContext<R>,
) => Effect.Effect<void, E, R>;
export interface EffectProducerOptions {
export interface EffectProducerOptions<T = unknown> {
readonly topic: string;
readonly schema?: S.Top;
readonly schema?: S.Codec<T, unknown>;
readonly metrics?: ProducerMetrics;
}
@ -62,7 +62,7 @@ export interface EffectConsumerOptions<T, E = never, R = never> {
readonly handler: EffectMessageHandler<T, E, R>;
readonly concurrency?: number;
readonly initialPosition?: "latest" | "earliest";
readonly schema?: S.Top;
readonly schema?: S.Codec<T, unknown>;
readonly receiveTimeoutMs?: number;
readonly errorBackoffMs?: number;
readonly rateLimitRetryMs?: number;
@ -73,12 +73,12 @@ export interface EffectConsumer {
readonly fibers: ReadonlyArray<Fiber.Fiber<void, never>>;
}
export interface EffectRequestResponseOptions {
export interface EffectRequestResponseOptions<TReq = unknown, TRes = unknown> {
readonly requestTopic: string;
readonly responseTopic: string;
readonly subscription: string;
readonly requestSchema?: S.Top;
readonly responseSchema?: S.Top;
readonly requestSchema?: S.Codec<TReq, unknown>;
readonly responseSchema?: S.Codec<TRes, unknown>;
}
export interface EffectRequestOptions<TRes, E = never, R = never> {
@ -96,7 +96,7 @@ export interface EffectRequestResponse<TReq, TRes> {
export interface ProducerFactoryService {
readonly make: <T>(
options: EffectProducerOptions,
options: EffectProducerOptions<T>,
) => Effect.Effect<EffectProducer<T>, PubSubError, Scope.Scope>;
}
@ -109,7 +109,7 @@ export interface ConsumerFactoryService {
export interface RequestResponseFactoryService {
readonly make: <TReq, TRes>(
options: EffectRequestResponseOptions,
options: EffectRequestResponseOptions<TReq, TRes>,
) => Effect.Effect<EffectRequestResponse<TReq, TRes>, PubSubError, Scope.Scope>;
}
@ -138,7 +138,7 @@ export class FlowRuntime extends Context.Service<FlowRuntime, FlowRuntimeService
export function makeEffectProducerHandle<T>(
backend: BackendProducer<T>,
options: EffectProducerOptions,
options: EffectProducerOptions<T>,
): EffectProducer<T> {
return {
send: Effect.fn(`Producer.send:${options.topic}`)((id: string, message: T) =>
@ -168,9 +168,9 @@ export function makeEffectProducerHandle<T>(
export const makeEffectProducerFromPubSub = Effect.fn("makeEffectProducerFromPubSub")(function* <T>(
pubsub: PubSubService,
options: EffectProducerOptions,
options: EffectProducerOptions<T>,
) {
const createOptions: CreateProducerOptions = options.schema === undefined
const createOptions: CreateProducerOptions<T> = options.schema === undefined
? { topic: options.topic }
: { topic: options.topic, schema: options.schema };
const backend = yield* pubsub.createProducer<T>(createOptions);
@ -326,7 +326,7 @@ export const makeEffectConsumerFromPubSub = Effect.fn("makeEffectConsumerFromPub
options: EffectConsumerOptions<T, E, R>,
flow: FlowContext<R>,
) {
const createOptions: CreateConsumerOptions = {
const createOptions: CreateConsumerOptions<T> = {
topic: options.topic,
subscription: options.subscription,
...(options.initialPosition === undefined ? {} : { initialPosition: options.initialPosition }),
@ -422,9 +422,9 @@ export const makeEffectRequestResponseFromPubSub = Effect.fn("makeEffectRequestR
>(
pubsub: PubSubService,
config: MessagingRuntimeConfig,
options: EffectRequestResponseOptions,
options: EffectRequestResponseOptions<TReq, TRes>,
) {
const producerOptions: CreateProducerOptions = options.requestSchema === undefined
const producerOptions: CreateProducerOptions<TReq> = options.requestSchema === undefined
? { topic: options.requestTopic }
: { topic: options.requestTopic, schema: options.requestSchema };
const producerBackend = yield* pubsub.createProducer<TReq>(producerOptions);
@ -432,7 +432,7 @@ export const makeEffectRequestResponseFromPubSub = Effect.fn("makeEffectRequestR
topic: options.requestTopic,
...(options.requestSchema === undefined ? {} : { schema: options.requestSchema }),
});
const createOptions: CreateConsumerOptions = {
const createOptions: CreateConsumerOptions<TRes> = {
topic: options.responseTopic,
subscription: options.subscription,
...(options.responseSchema === undefined ? {} : { schema: options.responseSchema }),
@ -502,7 +502,7 @@ export const makeEffectRequestResponseFromPubSub = Effect.fn("makeEffectRequestR
export function makeProducerFactoryService(pubsub: PubSubService): ProducerFactoryService {
return {
make: Effect.fn("ProducerFactory.make")(<T>(options: EffectProducerOptions) =>
make: Effect.fn("ProducerFactory.make")(<T>(options: EffectProducerOptions<T>) =>
makeEffectProducerFromPubSub<T>(pubsub, options),
),
};
@ -526,13 +526,11 @@ export function makeRequestResponseFactoryService(
pubsub: PubSubService,
config: MessagingRuntimeConfig,
): RequestResponseFactoryService {
const make = Effect.fn("RequestResponseFactory.make")(function* <TReq, TRes>(
options: EffectRequestResponseOptions,
) {
return yield* makeEffectRequestResponseFromPubSub<TReq, TRes>(pubsub, config, options);
}) as RequestResponseFactoryService["make"];
return { make };
return {
make: Effect.fn("RequestResponseFactory.make")(<TReq, TRes>(
options: EffectRequestResponseOptions<TReq, TRes>,
) => makeEffectRequestResponseFromPubSub<TReq, TRes>(pubsub, config, options)),
};
}
export const ProducerFactoryLive = Layer.effect(
@ -589,7 +587,7 @@ export const MessagingRuntimeLive = Layer.mergeAll(
);
export const runEffectProducerScoped = Effect.fn("runEffectProducerScoped")(function* <T>(
options: EffectProducerOptions,
options: EffectProducerOptions<T>,
) {
const pubsub = yield* PubSub;
return yield* makeEffectProducerFromPubSub<T>(pubsub, options);
@ -605,7 +603,7 @@ export const runEffectConsumerScoped = Effect.fn("runEffectConsumerScoped")(func
});
export const runEffectRequestResponseScoped = Effect.fn("runEffectRequestResponseScoped")(function* <TReq, TRes>(
options: EffectRequestResponseOptions,
options: EffectRequestResponseOptions<TReq, TRes>,
) {
const pubsub = yield* PubSub;
const config = yield* loadMessagingRuntimeConfig();

View file

@ -5,6 +5,8 @@
*/
import type { PubSubBackend, BackendConsumer } from "../backend/types.js";
import { Duration, Effect, Fiber } from "effect";
import { messagingDeliveryError, messagingLifecycleError, messagingTimeoutError } from "../errors.js";
type Resolver<T> = {
queue: AsyncQueue<T>;
@ -32,28 +34,33 @@ export function makeAsyncQueue<T>(): AsyncQueue<T> {
buffer.push(item);
}
},
pop: async (timeoutMs) => {
pop: (timeoutMs) => {
const buffered = buffer.shift();
if (buffered !== undefined) return buffered;
return new Promise<T>((resolve, reject) => {
let timer: ReturnType<typeof setTimeout> | undefined;
if (buffered !== undefined) return Promise.resolve(buffered);
const take = Effect.callback<T>((resume) => {
const waiter = (value: T) => {
if (timer !== undefined) clearTimeout(timer);
resolve(value);
resume(Effect.succeed(value));
};
waiters.push(waiter);
if (timeoutMs !== undefined) {
timer = setTimeout(() => {
const idx = waiters.indexOf(waiter);
if (idx !== -1) waiters.splice(idx, 1);
reject(new Error(`Queue.pop timed out after ${timeoutMs}ms`));
}, timeoutMs);
}
return Effect.sync(() => {
const idx = waiters.indexOf(waiter);
if (idx !== -1) waiters.splice(idx, 1);
});
});
return Effect.runPromise(
timeoutMs === undefined
? take
: take.pipe(
Effect.timeout(Duration.millis(timeoutMs)),
Effect.catchTag("TimeoutError", () =>
Effect.fail(messagingTimeoutError("queue.pop", timeoutMs)),
),
),
);
},
get length() {
return buffer.length;
@ -77,76 +84,113 @@ export function makeSubscriber<T>(
): Subscriber<T> {
let backend: BackendConsumer<T> | null = null;
let running = false;
let fiber: Fiber.Fiber<void, never> | null = null;
// ID-specific subscriptions (request/response correlation)
const idSubscribers = new Map<string, Resolver<T>>();
// Wildcard subscribers (receive all messages)
const allSubscribers = new Map<string, Resolver<T>>();
const dispatchLoop = async (): Promise<void> => {
const dispatchLoop = Effect.fn("Subscriber.dispatchLoop")(function* () {
let consecutiveErrors = 0;
while (running) {
try {
const currentBackend = backend;
if (currentBackend === null) throw new Error("Subscriber backend not started");
const dispatchOnce = Effect.fn("Subscriber.dispatchOnce")(function* () {
const currentBackend = backend;
if (currentBackend === null) {
return yield* messagingLifecycleError(
`${topic}:${subscription}`,
"dispatch",
"Subscriber backend not started",
);
}
const msg = await currentBackend.receive(2000);
if (msg === null) continue;
const msg = yield* Effect.tryPromise({
try: () => currentBackend.receive(2000),
catch: (error) => messagingDeliveryError(topic, "receive", error),
});
if (msg === null) return;
consecutiveErrors = 0;
consecutiveErrors = 0;
const props = msg.properties();
const id = props.id;
const value = msg.value();
const props = msg.properties();
const id = props.id;
const value = msg.value();
// Route to ID-specific subscriber
if (id !== undefined && id.length > 0) {
const sub = idSubscribers.get(id);
if (sub !== undefined) {
// Route to ID-specific subscriber
if (id !== undefined && id.length > 0) {
const sub = idSubscribers.get(id);
if (sub !== undefined) {
sub.queue.push(value);
}
}
// Broadcast to all-subscribers
for (const sub of allSubscribers.values()) {
sub.queue.push(value);
}
}
// Broadcast to all-subscribers
for (const sub of allSubscribers.values()) {
sub.queue.push(value);
}
yield* Effect.tryPromise({
try: () => currentBackend.acknowledge(msg),
catch: (error) => messagingDeliveryError(topic, "acknowledge", error),
});
});
await currentBackend.acknowledge(msg);
} catch (err) {
if (!running) break;
consecutiveErrors++;
if (consecutiveErrors <= 3) {
console.error("[Subscriber] Error:", err);
} else if (consecutiveErrors === 4) {
console.error("[Subscriber] Suppressing further errors (will retry with backoff)");
}
// Exponential backoff: 1s, 2s, 4s, max 10s
const delay = Math.min(1000 * Math.pow(2, consecutiveErrors - 1), 10_000);
await new Promise((r) => setTimeout(r, delay));
}
}
};
yield* Effect.whileLoop({
while: () => running,
body: () =>
dispatchOnce().pipe(
Effect.catch((error) => {
if (!running) return Effect.void;
consecutiveErrors++;
const logEffect = consecutiveErrors <= 3
? Effect.logError("[Subscriber] Error", { error })
: consecutiveErrors === 4
? Effect.logError("[Subscriber] Suppressing further errors (will retry with backoff)", { error })
: Effect.void;
const delay = Math.min(1000 * 2 ** (consecutiveErrors - 1), 10_000);
return logEffect.pipe(Effect.flatMap(() => Effect.sleep(Duration.millis(delay))));
}),
),
step: () => undefined,
});
});
return {
start: async () => {
backend = await pubsub.createConsumer<T>({
topic,
subscription,
});
running = true;
// Start the dispatch loop (fire and forget; runs until stop).
dispatchLoop().catch((err) => {
if (running === true) console.error("[Subscriber] dispatch loop error:", err);
});
},
stop: async () => {
running = false;
if (backend !== null) {
await backend.close();
backend = null;
}
},
start: () =>
Effect.runPromise(
Effect.gen(function* () {
backend = yield* Effect.tryPromise({
try: () =>
pubsub.createConsumer<T>({
topic,
subscription,
}),
catch: (error) =>
messagingLifecycleError(`${topic}:${subscription}`, "create-consumer", error),
});
running = true;
fiber = yield* dispatchLoop().pipe(Effect.forkDetach);
}),
),
stop: () =>
Effect.runPromise(
Effect.gen(function* () {
running = false;
const activeFiber = fiber;
fiber = null;
if (activeFiber !== null) {
yield* Fiber.interrupt(activeFiber);
}
const currentBackend = backend;
if (currentBackend !== null) {
backend = null;
yield* Effect.tryPromise({
try: () => currentBackend.close(),
catch: (error) =>
messagingLifecycleError(`${topic}:${subscription}`, "close-consumer", error),
});
}
}),
),
subscribe: (id) => {
const queue = makeAsyncQueue<T>();
idSubscribers.set(id, { queue });