mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-06 03:42:11 +02:00
Use MutableHashMap for base flow registries
This commit is contained in:
parent
fba0f97723
commit
4ffa84dbe7
3 changed files with 54 additions and 34 deletions
|
|
@ -2290,6 +2290,24 @@ Notes:
|
||||||
- `cd ts && bun run lint`
|
- `cd ts && bun run lint`
|
||||||
- `git diff --check`
|
- `git diff --check`
|
||||||
|
|
||||||
|
### 2026-06-04: Base Processor Registry MutableHashMap Slice
|
||||||
|
|
||||||
|
- Status: migrated and package-verified.
|
||||||
|
- Completed:
|
||||||
|
- `ts/packages/base/src/processor/flow.ts` now stores producers, consumers,
|
||||||
|
requestors, and parameters in `MutableHashMap` instead of native `Map`.
|
||||||
|
- `ts/packages/base/src/processor/flow-processor.ts` now tracks active flow
|
||||||
|
scopes in `MutableHashMap`, including `Option`-based lookups and
|
||||||
|
`MutableHashMap.clear` / `remove` / `size` / `set` for lifecycle changes.
|
||||||
|
- Verification:
|
||||||
|
- `cd ts && bun run --cwd packages/base build`
|
||||||
|
- `cd ts/packages/base && bunx --bun vitest run src/__tests__/flow-processor-runtime.test.ts src/__tests__/flow-spec-runtime.test.ts`
|
||||||
|
- `cd ts && bun run check:tsgo`
|
||||||
|
- `cd ts && bun run build`
|
||||||
|
- `cd ts && bun run test`
|
||||||
|
- `cd ts && bun run lint`
|
||||||
|
- `git diff --check`
|
||||||
|
|
||||||
## Subagent Findings To Preserve
|
## Subagent Findings To Preserve
|
||||||
|
|
||||||
- MCP/workbench:
|
- MCP/workbench:
|
||||||
|
|
@ -2475,8 +2493,8 @@ Notes:
|
||||||
compatibility facades, gateway/librarian helpers, and CLI command actions.
|
compatibility facades, gateway/librarian helpers, and CLI command actions.
|
||||||
The workbench random id helper is complete; the remaining workbench
|
The workbench random id helper is complete; the remaining workbench
|
||||||
`Effect.gen` match is a local one-shot command effect value.
|
`Effect.gen` match is a local one-shot command effect value.
|
||||||
- Remaining real long-lived native collection targets include base processor
|
- Remaining real long-lived native collection target is Librarian service
|
||||||
registries and Librarian service state. The standalone Librarian collection
|
state. Base processor registries, the standalone Librarian collection
|
||||||
manager, prompt template cache, and workbench explain triples module cache
|
manager, prompt template cache, and workbench explain triples module cache
|
||||||
are complete. Local traversal sets and test fakes remain no-op boundaries.
|
are complete. Local traversal sets and test fakes remain no-op boundaries.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,7 @@ import {
|
||||||
import { makePubSubService, PubSub } from "../backend/pubsub.js";
|
import { makePubSubService, PubSub } from "../backend/pubsub.js";
|
||||||
import { loadMessagingRuntimeConfig } from "../runtime/index.ts";
|
import { loadMessagingRuntimeConfig } from "../runtime/index.ts";
|
||||||
import { Context, Duration, Effect, Exit, Layer, ManagedRuntime, Scope } from "effect";
|
import { Context, Duration, Effect, Exit, Layer, ManagedRuntime, Scope } from "effect";
|
||||||
|
import * as MutableHashMap from "effect/MutableHashMap";
|
||||||
import * as O from "effect/Option";
|
import * as O from "effect/Option";
|
||||||
import * as S from "effect/Schema";
|
import * as S from "effect/Schema";
|
||||||
|
|
||||||
|
|
@ -136,7 +137,7 @@ export function runFlowProcessorDefinitionScoped<
|
||||||
| FlowRequirements
|
| FlowRequirements
|
||||||
| ConfigHandlerRequirements
|
| ConfigHandlerRequirements
|
||||||
> {
|
> {
|
||||||
const flows = new Map<string, ActiveFlow>();
|
const flows = MutableHashMap.empty<string, ActiveFlow>();
|
||||||
let configConsumer: BackendConsumer<ConfigPush> | null = null;
|
let configConsumer: BackendConsumer<ConfigPush> | null = null;
|
||||||
let lastFlowsJson = "";
|
let lastFlowsJson = "";
|
||||||
const isRunning = options.isRunning ?? (() => true);
|
const isRunning = options.isRunning ?? (() => true);
|
||||||
|
|
@ -147,11 +148,11 @@ export function runFlowProcessorDefinitionScoped<
|
||||||
);
|
);
|
||||||
|
|
||||||
const closeAllFlowsEffect = Effect.gen(function* () {
|
const closeAllFlowsEffect = Effect.gen(function* () {
|
||||||
const activeFlows = Array.from(flows.entries());
|
const activeFlows = Array.from(flows);
|
||||||
for (const [name, activeFlow] of activeFlows) {
|
for (const [name, activeFlow] of activeFlows) {
|
||||||
yield* closeFlowEffect(name, activeFlow);
|
yield* closeFlowEffect(name, activeFlow);
|
||||||
}
|
}
|
||||||
flows.clear();
|
MutableHashMap.clear(flows);
|
||||||
});
|
});
|
||||||
|
|
||||||
const closeConfigConsumerEffect = (): Effect.Effect<void> => {
|
const closeConfigConsumerEffect = (): Effect.Effect<void> => {
|
||||||
|
|
@ -215,7 +216,7 @@ export function runFlowProcessorDefinitionScoped<
|
||||||
const flowsJson = yield* S.encodeUnknownEffect(S.UnknownFromJsonString)(flowDefinitions).pipe(
|
const flowsJson = yield* S.encodeUnknownEffect(S.UnknownFromJsonString)(flowDefinitions).pipe(
|
||||||
Effect.catch((error) => Effect.succeed(String(error))),
|
Effect.catch((error) => Effect.succeed(String(error))),
|
||||||
);
|
);
|
||||||
if (lastFlowsJson.length > 0 && flowsJson === lastFlowsJson && flows.size > 0) {
|
if (lastFlowsJson.length > 0 && flowsJson === lastFlowsJson && MutableHashMap.size(flows) > 0) {
|
||||||
yield* Effect.log(`[${options.id}] Flow definitions unchanged, skipping restart`);
|
yield* Effect.log(`[${options.id}] Flow definitions unchanged, skipping restart`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -225,21 +226,21 @@ export function runFlowProcessorDefinitionScoped<
|
||||||
if (!(name in flowDefinitions)) {
|
if (!(name in flowDefinitions)) {
|
||||||
yield* Effect.log(`[${options.id}] Stopping removed flow: ${name}`);
|
yield* Effect.log(`[${options.id}] Stopping removed flow: ${name}`);
|
||||||
yield* closeFlowEffect(name, activeFlow);
|
yield* closeFlowEffect(name, activeFlow);
|
||||||
flows.delete(name);
|
MutableHashMap.remove(flows, name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const [name, defn] of Object.entries(flowDefinitions)) {
|
for (const [name, defn] of Object.entries(flowDefinitions)) {
|
||||||
const existing = flows.get(name);
|
const existing = O.getOrUndefined(MutableHashMap.get(flows, name));
|
||||||
if (existing !== undefined) {
|
if (existing !== undefined) {
|
||||||
yield* Effect.log(`[${options.id}] Restarting flow "${name}" with updated config`);
|
yield* Effect.log(`[${options.id}] Restarting flow "${name}" with updated config`);
|
||||||
yield* closeFlowEffect(name, existing);
|
yield* closeFlowEffect(name, existing);
|
||||||
flows.delete(name);
|
MutableHashMap.remove(flows, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
yield* Effect.log(`[${options.id}] Starting flow "${name}"`);
|
yield* Effect.log(`[${options.id}] Starting flow "${name}"`);
|
||||||
const activeFlow = yield* startFlowEffect(name, defn);
|
const activeFlow = yield* startFlowEffect(name, defn);
|
||||||
flows.set(name, activeFlow);
|
MutableHashMap.set(flows, name, activeFlow);
|
||||||
yield* Effect.log(`[${options.id}] Flow "${name}" started`);
|
yield* Effect.log(`[${options.id}] Flow "${name}" started`);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { Config as EffectConfig, Context, Effect, Exit, Layer, ManagedRuntime, Scope } from "effect";
|
import { Config as EffectConfig, Context, Effect, Exit, Layer, ManagedRuntime, Scope } from "effect";
|
||||||
|
import * as MutableHashMap from "effect/MutableHashMap";
|
||||||
import * as O from "effect/Option";
|
import * as O from "effect/Option";
|
||||||
import * as S from "effect/Schema";
|
import * as S from "effect/Schema";
|
||||||
import type { PubSubBackend } from "../backend/types.js";
|
import type { PubSubBackend } from "../backend/types.js";
|
||||||
|
|
@ -128,10 +129,10 @@ export function makeFlow<Requirements = never>(
|
||||||
definition: FlowDefinition,
|
definition: FlowDefinition,
|
||||||
specifications: ReadonlyArray<Spec<Requirements>>,
|
specifications: ReadonlyArray<Spec<Requirements>>,
|
||||||
): Flow<Requirements> {
|
): Flow<Requirements> {
|
||||||
const producers = new Map<string, EffectProducer<never>>();
|
const producers = MutableHashMap.empty<string, EffectProducer<never>>();
|
||||||
const consumers = new Map<string, EffectConsumer>();
|
const consumers = MutableHashMap.empty<string, EffectConsumer>();
|
||||||
const requestors = new Map<string, EffectRequestResponse<never, unknown>>();
|
const requestors = MutableHashMap.empty<string, EffectRequestResponse<never, unknown>>();
|
||||||
const parameters = new Map<string, unknown>();
|
const parameters = MutableHashMap.empty<string, unknown>();
|
||||||
let compatibilityScope: Scope.Closeable | null = null;
|
let compatibilityScope: Scope.Closeable | null = null;
|
||||||
const compatibilityRuntime = ManagedRuntime.make(Layer.empty);
|
const compatibilityRuntime = ManagedRuntime.make(Layer.empty);
|
||||||
|
|
||||||
|
|
@ -162,14 +163,14 @@ export function makeFlow<Requirements = never>(
|
||||||
};
|
};
|
||||||
|
|
||||||
const getParameterEffect = (parameterName: string): Effect.Effect<unknown, FlowResourceNotFoundError> => {
|
const getParameterEffect = (parameterName: string): Effect.Effect<unknown, FlowResourceNotFoundError> => {
|
||||||
const value = parameters.get(parameterName);
|
const value = O.getOrUndefined(MutableHashMap.get(parameters, parameterName));
|
||||||
return value === undefined
|
return value === undefined
|
||||||
? Effect.fail(flowResourceNotFoundError(name, "parameter", parameterName))
|
? Effect.fail(flowResourceNotFoundError(name, "parameter", parameterName))
|
||||||
: Effect.succeed(value);
|
: Effect.succeed(value);
|
||||||
};
|
};
|
||||||
|
|
||||||
const getParameter = (parameterName: string): unknown => {
|
const getParameter = (parameterName: string): unknown => {
|
||||||
const value = parameters.get(parameterName);
|
const value = O.getOrUndefined(MutableHashMap.get(parameters, parameterName));
|
||||||
if (value === undefined) throw flowResourceNotFoundError(name, "parameter", parameterName);
|
if (value === undefined) throw flowResourceNotFoundError(name, "parameter", parameterName);
|
||||||
return value;
|
return value;
|
||||||
};
|
};
|
||||||
|
|
@ -191,14 +192,14 @@ export function makeFlow<Requirements = never>(
|
||||||
const getProducerEffect = (
|
const getProducerEffect = (
|
||||||
producerName: string,
|
producerName: string,
|
||||||
): Effect.Effect<EffectProducer<never>, FlowResourceNotFoundError> => {
|
): Effect.Effect<EffectProducer<never>, FlowResourceNotFoundError> => {
|
||||||
const producer = producers.get(producerName);
|
const producer = O.getOrUndefined(MutableHashMap.get(producers, producerName));
|
||||||
return producer === undefined
|
return producer === undefined
|
||||||
? Effect.fail(flowResourceNotFoundError(name, "producer", producerName))
|
? Effect.fail(flowResourceNotFoundError(name, "producer", producerName))
|
||||||
: Effect.succeed(producer);
|
: Effect.succeed(producer);
|
||||||
};
|
};
|
||||||
|
|
||||||
const getProducer = (producerName: string): EffectProducer<never> => {
|
const getProducer = (producerName: string): EffectProducer<never> => {
|
||||||
const producer = producers.get(producerName);
|
const producer = O.getOrUndefined(MutableHashMap.get(producers, producerName));
|
||||||
if (producer === undefined) throw flowResourceNotFoundError(name, "producer", producerName);
|
if (producer === undefined) throw flowResourceNotFoundError(name, "producer", producerName);
|
||||||
return producer;
|
return producer;
|
||||||
};
|
};
|
||||||
|
|
@ -206,7 +207,7 @@ export function makeFlow<Requirements = never>(
|
||||||
const getRequestorEffect = (
|
const getRequestorEffect = (
|
||||||
requestorName: string,
|
requestorName: string,
|
||||||
): Effect.Effect<EffectRequestResponse<never, unknown>, FlowResourceNotFoundError> => {
|
): Effect.Effect<EffectRequestResponse<never, unknown>, FlowResourceNotFoundError> => {
|
||||||
const requestor = requestors.get(requestorName);
|
const requestor = O.getOrUndefined(MutableHashMap.get(requestors, requestorName));
|
||||||
return requestor === undefined
|
return requestor === undefined
|
||||||
? Effect.fail(flowResourceNotFoundError(name, "requestor", requestorName))
|
? Effect.fail(flowResourceNotFoundError(name, "requestor", requestorName))
|
||||||
: Effect.succeed(requestor);
|
: Effect.succeed(requestor);
|
||||||
|
|
@ -215,7 +216,7 @@ export function makeFlow<Requirements = never>(
|
||||||
const getRequestor = (
|
const getRequestor = (
|
||||||
requestorName: string,
|
requestorName: string,
|
||||||
): EffectRequestResponse<never, unknown> => {
|
): EffectRequestResponse<never, unknown> => {
|
||||||
const requestor = requestors.get(requestorName);
|
const requestor = O.getOrUndefined(MutableHashMap.get(requestors, requestorName));
|
||||||
if (requestor === undefined) throw flowResourceNotFoundError(name, "requestor", requestorName);
|
if (requestor === undefined) throw flowResourceNotFoundError(name, "requestor", requestorName);
|
||||||
return requestor;
|
return requestor;
|
||||||
};
|
};
|
||||||
|
|
@ -251,7 +252,7 @@ export function makeFlow<Requirements = never>(
|
||||||
if (typeof producer === "string") {
|
if (typeof producer === "string") {
|
||||||
return getProducerEffect(producer);
|
return getProducerEffect(producer);
|
||||||
}
|
}
|
||||||
if (!producers.has(producer.name)) {
|
if (!MutableHashMap.has(producers, producer.name)) {
|
||||||
return Effect.fail(flowResourceNotFoundError(name, "producer", producer.name));
|
return Effect.fail(flowResourceNotFoundError(name, "producer", producer.name));
|
||||||
}
|
}
|
||||||
return producer.producerEffect(flow);
|
return producer.producerEffect(flow);
|
||||||
|
|
@ -269,7 +270,7 @@ export function makeFlow<Requirements = never>(
|
||||||
if (typeof requestor === "string") {
|
if (typeof requestor === "string") {
|
||||||
return getRequestorEffect(requestor);
|
return getRequestorEffect(requestor);
|
||||||
}
|
}
|
||||||
if (!requestors.has(requestor.name)) {
|
if (!MutableHashMap.has(requestors, requestor.name)) {
|
||||||
return Effect.fail(flowResourceNotFoundError(name, "requestor", requestor.name));
|
return Effect.fail(flowResourceNotFoundError(name, "requestor", requestor.name));
|
||||||
}
|
}
|
||||||
return requestor.requestorEffect(flow);
|
return requestor.requestorEffect(flow);
|
||||||
|
|
@ -308,7 +309,7 @@ export function makeFlow<Requirements = never>(
|
||||||
if (typeof producer === "string") {
|
if (typeof producer === "string") {
|
||||||
return toFlowProducer(getProducer(producer));
|
return toFlowProducer(getProducer(producer));
|
||||||
}
|
}
|
||||||
if (!producers.has(producer.name)) {
|
if (!MutableHashMap.has(producers, producer.name)) {
|
||||||
throw flowResourceNotFoundError(name, "producer", producer.name);
|
throw flowResourceNotFoundError(name, "producer", producer.name);
|
||||||
}
|
}
|
||||||
return toFlowProducer(compatibilityRuntime.runSync(producer.producerEffect(flow)));
|
return toFlowProducer(compatibilityRuntime.runSync(producer.producerEffect(flow)));
|
||||||
|
|
@ -324,7 +325,7 @@ export function makeFlow<Requirements = never>(
|
||||||
if (typeof requestor === "string") {
|
if (typeof requestor === "string") {
|
||||||
return toFlowRequestor(getRequestor(requestor));
|
return toFlowRequestor(getRequestor(requestor));
|
||||||
}
|
}
|
||||||
if (!requestors.has(requestor.name)) {
|
if (!MutableHashMap.has(requestors, requestor.name)) {
|
||||||
throw flowResourceNotFoundError(name, "requestor", requestor.name);
|
throw flowResourceNotFoundError(name, "requestor", requestor.name);
|
||||||
}
|
}
|
||||||
return toFlowRequestor(compatibilityRuntime.runSync(requestor.requestorEffect(flow)));
|
return toFlowRequestor(compatibilityRuntime.runSync(requestor.requestorEffect(flow)));
|
||||||
|
|
@ -388,26 +389,26 @@ export function makeFlow<Requirements = never>(
|
||||||
return compatibilityRuntime.runPromise(flow.runInCompatibilityScopeEffect(effect, runtimePubsub, context));
|
return compatibilityRuntime.runPromise(flow.runInCompatibilityScopeEffect(effect, runtimePubsub, context));
|
||||||
},
|
},
|
||||||
clearResources(): void {
|
clearResources(): void {
|
||||||
producers.clear();
|
MutableHashMap.clear(producers);
|
||||||
consumers.clear();
|
MutableHashMap.clear(consumers);
|
||||||
requestors.clear();
|
MutableHashMap.clear(requestors);
|
||||||
parameters.clear();
|
MutableHashMap.clear(parameters);
|
||||||
},
|
},
|
||||||
registerProducer<T>(registerName: string, producer: EffectProducer<T>): void {
|
registerProducer<T>(registerName: string, producer: EffectProducer<T>): void {
|
||||||
producers.set(registerName, producer);
|
MutableHashMap.set(producers, registerName, producer);
|
||||||
},
|
},
|
||||||
registerConsumer(registerName: string, consumer: EffectConsumer): void {
|
registerConsumer(registerName: string, consumer: EffectConsumer): void {
|
||||||
consumers.set(registerName, consumer);
|
MutableHashMap.set(consumers, registerName, consumer);
|
||||||
},
|
},
|
||||||
registerRequestor<TReq, TRes>(registerName: string, rr: EffectRequestResponse<TReq, TRes>): void {
|
registerRequestor<TReq, TRes>(registerName: string, rr: EffectRequestResponse<TReq, TRes>): void {
|
||||||
requestors.set(registerName, rr);
|
MutableHashMap.set(requestors, registerName, rr);
|
||||||
},
|
},
|
||||||
setParameter(parameterName: string, value: unknown): void {
|
setParameter(parameterName: string, value: unknown): void {
|
||||||
parameters.set(parameterName, value);
|
MutableHashMap.set(parameters, parameterName, value);
|
||||||
},
|
},
|
||||||
producerEffect,
|
producerEffect,
|
||||||
consumerEffect(consumerName: string): Effect.Effect<EffectConsumer, FlowResourceNotFoundError> {
|
consumerEffect(consumerName: string): Effect.Effect<EffectConsumer, FlowResourceNotFoundError> {
|
||||||
const c = consumers.get(consumerName);
|
const c = O.getOrUndefined(MutableHashMap.get(consumers, consumerName));
|
||||||
return c === undefined
|
return c === undefined
|
||||||
? Effect.fail(flowResourceNotFoundError(name, "consumer", consumerName))
|
? Effect.fail(flowResourceNotFoundError(name, "consumer", consumerName))
|
||||||
: Effect.succeed(c);
|
: Effect.succeed(c);
|
||||||
|
|
@ -416,7 +417,7 @@ export function makeFlow<Requirements = never>(
|
||||||
parameterEffect,
|
parameterEffect,
|
||||||
producer,
|
producer,
|
||||||
consumer(consumerName: string): FlowConsumer {
|
consumer(consumerName: string): FlowConsumer {
|
||||||
const c = consumers.get(consumerName);
|
const c = O.getOrUndefined(MutableHashMap.get(consumers, consumerName));
|
||||||
if (c === undefined) throw flowResourceNotFoundError(name, "consumer", consumerName);
|
if (c === undefined) throw flowResourceNotFoundError(name, "consumer", consumerName);
|
||||||
return {
|
return {
|
||||||
stop: () => compatibilityRuntime.runPromise(c.stop),
|
stop: () => compatibilityRuntime.runPromise(c.stop),
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue