apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-01 09:29:38 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Isolate concurrent Effect consumers

Browse source
This commit is contained in:
elpresidank 2026-06-02 06:08:49 -05:00
parent eaa7921314
commit 0fb943c0ef
3 changed files with 120 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

34
ts/packages/base/src/messaging/runtime.ts
View file

@ -3,7 +3,7 @@
*/
import { randomUUID } from "node:crypto";
import { Context, Duration, Effect, Fiber, Layer, Queue, Result, Schedule, Scope, Stream } from "effect";
import { Context, Duration, Effect, Fiber, Layer, Queue, Ref, Result, Schedule, Scope, Stream } from "effect";
import * as O from "effect/Option";
import * as S from "effect/Schema";
import type {
@ -346,20 +346,32 @@ export const makeEffectConsumerFromPubSub = Effect.fn("makeEffectConsumerFromPub
...(options.initialPosition === undefined ? {} : { initialPosition: options.initialPosition }),
...(options.schema === undefined ? {} : { schema: options.schema }),
};
const backend = yield* pubsub.createConsumer<T>(createOptions);
const concurrency = Math.max(1, options.concurrency ?? 1);
const workerIndexes = Array.from({ length: concurrency }, (_value, index) => index);
const fibers = yield* Effect.forEach(workerIndexes, () =>
consumerLoop(backend, options, flow, {
...config,
rateLimitRetryMs: options.rateLimitRetryMs ?? config.rateLimitRetryMs,
rateLimitTimeoutMs: options.rateLimitTimeoutMs ?? config.rateLimitTimeoutMs,
}).pipe(Effect.forkChild),
const workerConfig = {
...config,
rateLimitRetryMs: options.rateLimitRetryMs ?? config.rateLimitRetryMs,
rateLimitTimeoutMs: options.rateLimitTimeoutMs ?? config.rateLimitTimeoutMs,
};
const workers = yield* Effect.forEach(workerIndexes, () =>
Effect.gen(function* () {
const backend = yield* pubsub.createConsumer<T>(createOptions);
const fiber = yield* consumerLoop(backend, options, flow, workerConfig).pipe(Effect.forkChild);
return { backend, fiber };
}),
);
const stopped = yield* Ref.make(false);
const stop = Effect.fn(`Consumer.stop:${options.topic}`)(function* () {
yield* Effect.forEach(fibers, Fiber.interrupt, { discard: true });
yield* closeConsumerBackend(backend, options.topic, options.subscription);
const alreadyStopped = yield* Ref.getAndSet(stopped, true);
if (alreadyStopped) return;
yield* Effect.forEach(workers, (worker) => Fiber.interrupt(worker.fiber), { discard: true });
yield* Effect.forEach(
workers,
(worker) => closeConsumerBackend(worker.backend, options.topic, options.subscription),
{ discard: true },
);
});
yield* Effect.addFinalizer(() =>
@ -375,7 +387,7 @@ export const makeEffectConsumerFromPubSub = Effect.fn("makeEffectConsumerFromPub
);
return {
fibers,
fibers: workers.map((worker) => worker.fiber),
stop: stop(),
} satisfies EffectConsumer;
});