mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-01 17:39:39 +02:00
saving
This commit is contained in:
elpresidank
parent
e8c7a4f6e0
commit
ffd97375a8
160 changed files with 6704 additions and 1895 deletions
|
|
@ -4,11 +4,28 @@
|
|||
* Python reference: trustgraph-base/trustgraph/base/flow.py
|
||||
*/
|
||||
|
||||
import { Effect, Exit, Scope } from "effect";
|
||||
import type { PubSubBackend } from "../backend/types.js";
|
||||
import type { Spec } from "../spec/types.js";
|
||||
import type { Producer } from "../messaging/producer.js";
|
||||
import type { Consumer } from "../messaging/consumer.js";
|
||||
import type { RequestResponse } from "../messaging/request-response.js";
|
||||
import { makePubSubService } from "../backend/pubsub.js";
|
||||
import {
|
||||
flowResourceNotFoundError,
|
||||
type FlowResourceNotFoundError,
|
||||
type PubSubError,
|
||||
} from "../errors.js";
|
||||
import {
|
||||
ConsumerFactory,
|
||||
ProducerFactory,
|
||||
RequestResponseFactory,
|
||||
type EffectConsumer,
|
||||
type EffectProducer,
|
||||
type EffectRequestOptions,
|
||||
type EffectRequestResponse,
|
||||
makeConsumerFactoryService,
|
||||
makeProducerFactoryService,
|
||||
makeRequestResponseFactoryService,
|
||||
} from "../messaging/runtime.js";
|
||||
import { loadMessagingRuntimeConfig } from "../runtime/messaging-config.js";
|
||||
import type { Spec, SpecRuntimeRequirements } from "../spec/types.js";
|
||||
|
||||
export interface FlowDefinition {
|
||||
/** Topic overrides keyed by spec name */
|
||||
|
|
@ -17,54 +34,119 @@ export interface FlowDefinition {
|
|||
parameters?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export class Flow {
|
||||
private producers = new Map<string, Producer<unknown>>();
|
||||
private consumers = new Map<string, Consumer<unknown>>();
|
||||
private requestors = new Map<string, RequestResponse<unknown, unknown>>();
|
||||
export interface FlowProducer<T> {
|
||||
readonly send: (id: string, message: T) => Promise<void>;
|
||||
readonly flush: () => Promise<void>;
|
||||
readonly stop: () => Promise<void>;
|
||||
}
|
||||
|
||||
export interface FlowConsumer {
|
||||
readonly stop: () => Promise<void>;
|
||||
}
|
||||
|
||||
export interface FlowRequestOptions<TRes> {
|
||||
readonly timeoutMs?: number;
|
||||
readonly recipient?: (response: TRes) => Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface FlowRequestor<TReq, TRes> {
|
||||
readonly request: (
|
||||
request: TReq,
|
||||
options?: FlowRequestOptions<TRes>,
|
||||
) => Promise<TRes>;
|
||||
readonly stop: () => Promise<void>;
|
||||
}
|
||||
|
||||
export class Flow<Requirements = never> {
|
||||
private producers = new Map<string, EffectProducer<unknown>>();
|
||||
private consumers = new Map<string, EffectConsumer>();
|
||||
private requestors = new Map<string, EffectRequestResponse<unknown, unknown>>();
|
||||
private parameters = new Map<string, unknown>();
|
||||
private compatibilityScope: Scope.Closeable | null = null;
|
||||
public readonly name: string;
|
||||
public readonly processorId: string;
|
||||
private readonly pubsub: PubSubBackend;
|
||||
private readonly definition: FlowDefinition;
|
||||
private readonly specifications: ReadonlyArray<Spec<Requirements>>;
|
||||
|
||||
constructor(
|
||||
public readonly name: string,
|
||||
public readonly processorId: string,
|
||||
private readonly pubsub: PubSubBackend,
|
||||
private readonly definition: FlowDefinition,
|
||||
private readonly specifications: Spec[],
|
||||
) {}
|
||||
name: string,
|
||||
processorId: string,
|
||||
pubsub: PubSubBackend,
|
||||
definition: FlowDefinition,
|
||||
specifications: ReadonlyArray<Spec<Requirements>>,
|
||||
) {
|
||||
this.name = name;
|
||||
this.processorId = processorId;
|
||||
this.pubsub = pubsub;
|
||||
this.definition = definition;
|
||||
this.specifications = specifications;
|
||||
}
|
||||
|
||||
startEffect(): Effect.Effect<void, PubSubError, SpecRuntimeRequirements | Requirements> {
|
||||
const flow = this;
|
||||
return Effect.gen(function* () {
|
||||
for (const spec of flow.specifications) {
|
||||
yield* spec.addEffect(flow, flow.definition);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
for (const spec of this.specifications) {
|
||||
await spec.add(this, this.pubsub, this.definition);
|
||||
}
|
||||
|
||||
// Start all consumers, passing this Flow instance via FlowContext
|
||||
for (const consumer of this.consumers.values()) {
|
||||
consumer.start({ id: this.processorId, name: this.name, flow: this }).catch((err) => {
|
||||
console.error(`[Flow:${this.name}] Consumer error:`, err);
|
||||
});
|
||||
if (this.compatibilityScope !== null) {
|
||||
await this.stop();
|
||||
}
|
||||
await this.runInCompatibilityScope(
|
||||
this.startEffect() as Effect.Effect<void, PubSubError, SpecRuntimeRequirements>,
|
||||
this.pubsub,
|
||||
);
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
for (const consumer of this.consumers.values()) {
|
||||
await consumer.stop();
|
||||
}
|
||||
for (const producer of this.producers.values()) {
|
||||
await producer.stop();
|
||||
}
|
||||
for (const rr of this.requestors.values()) {
|
||||
await rr.stop();
|
||||
const scope = this.compatibilityScope;
|
||||
this.compatibilityScope = null;
|
||||
if (scope !== null) {
|
||||
await Effect.runPromise(Scope.close(scope, Exit.void));
|
||||
}
|
||||
this.clearResources();
|
||||
}
|
||||
|
||||
registerProducer(name: string, producer: Producer<unknown>): void {
|
||||
async runInCompatibilityScope<A, E>(
|
||||
effect: Effect.Effect<A, E, SpecRuntimeRequirements>,
|
||||
pubsub: PubSubBackend,
|
||||
): Promise<A> {
|
||||
const scope = await this.ensureCompatibilityScope();
|
||||
const pubsubService = makePubSubService(pubsub);
|
||||
const messagingConfig = await Effect.runPromise(loadMessagingRuntimeConfig());
|
||||
return await Effect.runPromise(
|
||||
effect.pipe(
|
||||
Effect.provideService(ProducerFactory, ProducerFactory.of(makeProducerFactoryService(pubsubService))),
|
||||
Effect.provideService(ConsumerFactory, ConsumerFactory.of(makeConsumerFactoryService(pubsubService, messagingConfig))),
|
||||
Effect.provideService(
|
||||
RequestResponseFactory,
|
||||
RequestResponseFactory.of(makeRequestResponseFactoryService(pubsubService, messagingConfig)),
|
||||
),
|
||||
Scope.provide(scope),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
clearResources(): void {
|
||||
this.producers.clear();
|
||||
this.consumers.clear();
|
||||
this.requestors.clear();
|
||||
this.parameters.clear();
|
||||
}
|
||||
|
||||
registerProducer(name: string, producer: EffectProducer<unknown>): void {
|
||||
this.producers.set(name, producer);
|
||||
}
|
||||
|
||||
registerConsumer(name: string, consumer: Consumer<unknown>): void {
|
||||
registerConsumer(name: string, consumer: EffectConsumer): void {
|
||||
this.consumers.set(name, consumer);
|
||||
}
|
||||
|
||||
registerRequestor(name: string, rr: RequestResponse<unknown, unknown>): void {
|
||||
registerRequestor(name: string, rr: EffectRequestResponse<unknown, unknown>): void {
|
||||
this.requestors.set(name, rr);
|
||||
}
|
||||
|
||||
|
|
@ -72,27 +154,97 @@ export class Flow {
|
|||
this.parameters.set(name, value);
|
||||
}
|
||||
|
||||
producer<T>(name: string): Producer<T> {
|
||||
producerEffect<T>(name: string): Effect.Effect<EffectProducer<T>, FlowResourceNotFoundError> {
|
||||
const p = this.producers.get(name);
|
||||
if (!p) throw new Error(`Producer "${name}" not found in flow "${this.name}"`);
|
||||
return p as Producer<T>;
|
||||
return p === undefined
|
||||
? Effect.fail(flowResourceNotFoundError(this.name, "producer", name))
|
||||
: Effect.succeed(p as EffectProducer<T>);
|
||||
}
|
||||
|
||||
consumer<T>(name: string): Consumer<T> {
|
||||
consumerEffect(name: string): Effect.Effect<EffectConsumer, FlowResourceNotFoundError> {
|
||||
const c = this.consumers.get(name);
|
||||
if (!c) throw new Error(`Consumer "${name}" not found in flow "${this.name}"`);
|
||||
return c as Consumer<T>;
|
||||
return c === undefined
|
||||
? Effect.fail(flowResourceNotFoundError(this.name, "consumer", name))
|
||||
: Effect.succeed(c);
|
||||
}
|
||||
|
||||
requestor<TReq, TRes>(name: string): RequestResponse<TReq, TRes> {
|
||||
requestorEffect<TReq, TRes>(
|
||||
name: string,
|
||||
): Effect.Effect<EffectRequestResponse<TReq, TRes>, FlowResourceNotFoundError> {
|
||||
const rr = this.requestors.get(name);
|
||||
if (!rr) throw new Error(`Requestor "${name}" not found in flow "${this.name}"`);
|
||||
return rr as RequestResponse<TReq, TRes>;
|
||||
return rr === undefined
|
||||
? Effect.fail(flowResourceNotFoundError(this.name, "requestor", name))
|
||||
: Effect.succeed(rr as EffectRequestResponse<TReq, TRes>);
|
||||
}
|
||||
|
||||
parameterEffect<T>(name: string): Effect.Effect<T, FlowResourceNotFoundError> {
|
||||
const v = this.parameters.get(name);
|
||||
return v === undefined
|
||||
? Effect.fail(flowResourceNotFoundError(this.name, "parameter", name))
|
||||
: Effect.succeed(v as T);
|
||||
}
|
||||
|
||||
producer<T>(name: string): FlowProducer<T> {
|
||||
const p = this.producers.get(name);
|
||||
if (p === undefined) throw flowResourceNotFoundError(this.name, "producer", name);
|
||||
return {
|
||||
send: (id, message) => Effect.runPromise((p as EffectProducer<T>).send(id, message)),
|
||||
flush: () => Effect.runPromise(p.flush),
|
||||
stop: () => Effect.runPromise(p.flush.pipe(Effect.flatMap(() => p.close))),
|
||||
};
|
||||
}
|
||||
|
||||
consumer(name: string): FlowConsumer {
|
||||
const c = this.consumers.get(name);
|
||||
if (c === undefined) throw flowResourceNotFoundError(this.name, "consumer", name);
|
||||
return {
|
||||
stop: () => Effect.runPromise(c.stop),
|
||||
};
|
||||
}
|
||||
|
||||
requestor<TReq, TRes>(name: string): FlowRequestor<TReq, TRes> {
|
||||
const rr = this.requestors.get(name);
|
||||
if (rr === undefined) throw flowResourceNotFoundError(this.name, "requestor", name);
|
||||
return {
|
||||
request: (request, options) =>
|
||||
Effect.runPromise(
|
||||
(rr as EffectRequestResponse<TReq, TRes>).request(
|
||||
request,
|
||||
this.toEffectRequestOptions(options),
|
||||
),
|
||||
),
|
||||
stop: () => Effect.runPromise(rr.stop),
|
||||
};
|
||||
}
|
||||
|
||||
parameter<T>(name: string): T {
|
||||
const v = this.parameters.get(name);
|
||||
if (v === undefined) throw new Error(`Parameter "${name}" not found in flow "${this.name}"`);
|
||||
if (v === undefined) throw flowResourceNotFoundError(this.name, "parameter", name);
|
||||
return v as T;
|
||||
}
|
||||
|
||||
private async ensureCompatibilityScope(): Promise<Scope.Closeable> {
|
||||
if (this.compatibilityScope !== null) {
|
||||
return this.compatibilityScope;
|
||||
}
|
||||
this.compatibilityScope = await Effect.runPromise(Scope.make());
|
||||
return this.compatibilityScope;
|
||||
}
|
||||
|
||||
private toEffectRequestOptions<TRes>(
|
||||
options: FlowRequestOptions<TRes> | undefined,
|
||||
): EffectRequestOptions<TRes> | undefined {
|
||||
if (options === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const recipient = options.recipient;
|
||||
return {
|
||||
...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
|
||||
...(recipient === undefined
|
||||
? {}
|
||||
: {
|
||||
recipient: (response: TRes) => Effect.promise(() => recipient(response)),
|
||||
}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue