Use MutableHashMap for base flow registries

This commit is contained in:
elpresidank 2026-06-04 07:53:49 -05:00
parent fba0f97723
commit 4ffa84dbe7
3 changed files with 54 additions and 34 deletions

View file

@ -2290,6 +2290,24 @@ Notes:
- `cd ts && bun run lint` - `cd ts && bun run lint`
- `git diff --check` - `git diff --check`
### 2026-06-04: Base Processor Registry MutableHashMap Slice
- Status: migrated and package-verified.
- Completed:
- `ts/packages/base/src/processor/flow.ts` now stores producers, consumers,
requestors, and parameters in `MutableHashMap` instead of native `Map`.
- `ts/packages/base/src/processor/flow-processor.ts` now tracks active flow
scopes in `MutableHashMap`, including `Option`-based lookups and
`MutableHashMap.clear` / `remove` / `size` / `set` for lifecycle changes.
- Verification:
- `cd ts && bun run --cwd packages/base build`
- `cd ts/packages/base && bunx --bun vitest run src/__tests__/flow-processor-runtime.test.ts src/__tests__/flow-spec-runtime.test.ts`
- `cd ts && bun run check:tsgo`
- `cd ts && bun run build`
- `cd ts && bun run test`
- `cd ts && bun run lint`
- `git diff --check`
## Subagent Findings To Preserve ## Subagent Findings To Preserve
- MCP/workbench: - MCP/workbench:
@ -2475,8 +2493,8 @@ Notes:
compatibility facades, gateway/librarian helpers, and CLI command actions. compatibility facades, gateway/librarian helpers, and CLI command actions.
The workbench random id helper is complete; the remaining workbench The workbench random id helper is complete; the remaining workbench
`Effect.gen` match is a local one-shot command effect value. `Effect.gen` match is a local one-shot command effect value.
- Remaining real long-lived native collection targets include base processor - Remaining real long-lived native collection target is Librarian service
registries and Librarian service state. The standalone Librarian collection state. Base processor registries, the standalone Librarian collection
manager, prompt template cache, and workbench explain triples module cache manager, prompt template cache, and workbench explain triples module cache
are complete. Local traversal sets and test fakes remain no-op boundaries. are complete. Local traversal sets and test fakes remain no-op boundaries.

View file

@ -38,6 +38,7 @@ import {
import { makePubSubService, PubSub } from "../backend/pubsub.js"; import { makePubSubService, PubSub } from "../backend/pubsub.js";
import { loadMessagingRuntimeConfig } from "../runtime/index.ts"; import { loadMessagingRuntimeConfig } from "../runtime/index.ts";
import { Context, Duration, Effect, Exit, Layer, ManagedRuntime, Scope } from "effect"; import { Context, Duration, Effect, Exit, Layer, ManagedRuntime, Scope } from "effect";
import * as MutableHashMap from "effect/MutableHashMap";
import * as O from "effect/Option"; import * as O from "effect/Option";
import * as S from "effect/Schema"; import * as S from "effect/Schema";
@ -136,7 +137,7 @@ export function runFlowProcessorDefinitionScoped<
| FlowRequirements | FlowRequirements
| ConfigHandlerRequirements | ConfigHandlerRequirements
> { > {
const flows = new Map<string, ActiveFlow>(); const flows = MutableHashMap.empty<string, ActiveFlow>();
let configConsumer: BackendConsumer<ConfigPush> | null = null; let configConsumer: BackendConsumer<ConfigPush> | null = null;
let lastFlowsJson = ""; let lastFlowsJson = "";
const isRunning = options.isRunning ?? (() => true); const isRunning = options.isRunning ?? (() => true);
@ -147,11 +148,11 @@ export function runFlowProcessorDefinitionScoped<
); );
const closeAllFlowsEffect = Effect.gen(function* () { const closeAllFlowsEffect = Effect.gen(function* () {
const activeFlows = Array.from(flows.entries()); const activeFlows = Array.from(flows);
for (const [name, activeFlow] of activeFlows) { for (const [name, activeFlow] of activeFlows) {
yield* closeFlowEffect(name, activeFlow); yield* closeFlowEffect(name, activeFlow);
} }
flows.clear(); MutableHashMap.clear(flows);
}); });
const closeConfigConsumerEffect = (): Effect.Effect<void> => { const closeConfigConsumerEffect = (): Effect.Effect<void> => {
@ -215,7 +216,7 @@ export function runFlowProcessorDefinitionScoped<
const flowsJson = yield* S.encodeUnknownEffect(S.UnknownFromJsonString)(flowDefinitions).pipe( const flowsJson = yield* S.encodeUnknownEffect(S.UnknownFromJsonString)(flowDefinitions).pipe(
Effect.catch((error) => Effect.succeed(String(error))), Effect.catch((error) => Effect.succeed(String(error))),
); );
if (lastFlowsJson.length > 0 && flowsJson === lastFlowsJson && flows.size > 0) { if (lastFlowsJson.length > 0 && flowsJson === lastFlowsJson && MutableHashMap.size(flows) > 0) {
yield* Effect.log(`[${options.id}] Flow definitions unchanged, skipping restart`); yield* Effect.log(`[${options.id}] Flow definitions unchanged, skipping restart`);
return; return;
} }
@ -225,21 +226,21 @@ export function runFlowProcessorDefinitionScoped<
if (!(name in flowDefinitions)) { if (!(name in flowDefinitions)) {
yield* Effect.log(`[${options.id}] Stopping removed flow: ${name}`); yield* Effect.log(`[${options.id}] Stopping removed flow: ${name}`);
yield* closeFlowEffect(name, activeFlow); yield* closeFlowEffect(name, activeFlow);
flows.delete(name); MutableHashMap.remove(flows, name);
} }
} }
for (const [name, defn] of Object.entries(flowDefinitions)) { for (const [name, defn] of Object.entries(flowDefinitions)) {
const existing = flows.get(name); const existing = O.getOrUndefined(MutableHashMap.get(flows, name));
if (existing !== undefined) { if (existing !== undefined) {
yield* Effect.log(`[${options.id}] Restarting flow "${name}" with updated config`); yield* Effect.log(`[${options.id}] Restarting flow "${name}" with updated config`);
yield* closeFlowEffect(name, existing); yield* closeFlowEffect(name, existing);
flows.delete(name); MutableHashMap.remove(flows, name);
} }
yield* Effect.log(`[${options.id}] Starting flow "${name}"`); yield* Effect.log(`[${options.id}] Starting flow "${name}"`);
const activeFlow = yield* startFlowEffect(name, defn); const activeFlow = yield* startFlowEffect(name, defn);
flows.set(name, activeFlow); MutableHashMap.set(flows, name, activeFlow);
yield* Effect.log(`[${options.id}] Flow "${name}" started`); yield* Effect.log(`[${options.id}] Flow "${name}" started`);
} }
}); });

View file

@ -5,6 +5,7 @@
*/ */
import { Config as EffectConfig, Context, Effect, Exit, Layer, ManagedRuntime, Scope } from "effect"; import { Config as EffectConfig, Context, Effect, Exit, Layer, ManagedRuntime, Scope } from "effect";
import * as MutableHashMap from "effect/MutableHashMap";
import * as O from "effect/Option"; import * as O from "effect/Option";
import * as S from "effect/Schema"; import * as S from "effect/Schema";
import type { PubSubBackend } from "../backend/types.js"; import type { PubSubBackend } from "../backend/types.js";
@ -128,10 +129,10 @@ export function makeFlow<Requirements = never>(
definition: FlowDefinition, definition: FlowDefinition,
specifications: ReadonlyArray<Spec<Requirements>>, specifications: ReadonlyArray<Spec<Requirements>>,
): Flow<Requirements> { ): Flow<Requirements> {
const producers = new Map<string, EffectProducer<never>>(); const producers = MutableHashMap.empty<string, EffectProducer<never>>();
const consumers = new Map<string, EffectConsumer>(); const consumers = MutableHashMap.empty<string, EffectConsumer>();
const requestors = new Map<string, EffectRequestResponse<never, unknown>>(); const requestors = MutableHashMap.empty<string, EffectRequestResponse<never, unknown>>();
const parameters = new Map<string, unknown>(); const parameters = MutableHashMap.empty<string, unknown>();
let compatibilityScope: Scope.Closeable | null = null; let compatibilityScope: Scope.Closeable | null = null;
const compatibilityRuntime = ManagedRuntime.make(Layer.empty); const compatibilityRuntime = ManagedRuntime.make(Layer.empty);
@ -162,14 +163,14 @@ export function makeFlow<Requirements = never>(
}; };
const getParameterEffect = (parameterName: string): Effect.Effect<unknown, FlowResourceNotFoundError> => { const getParameterEffect = (parameterName: string): Effect.Effect<unknown, FlowResourceNotFoundError> => {
const value = parameters.get(parameterName); const value = O.getOrUndefined(MutableHashMap.get(parameters, parameterName));
return value === undefined return value === undefined
? Effect.fail(flowResourceNotFoundError(name, "parameter", parameterName)) ? Effect.fail(flowResourceNotFoundError(name, "parameter", parameterName))
: Effect.succeed(value); : Effect.succeed(value);
}; };
const getParameter = (parameterName: string): unknown => { const getParameter = (parameterName: string): unknown => {
const value = parameters.get(parameterName); const value = O.getOrUndefined(MutableHashMap.get(parameters, parameterName));
if (value === undefined) throw flowResourceNotFoundError(name, "parameter", parameterName); if (value === undefined) throw flowResourceNotFoundError(name, "parameter", parameterName);
return value; return value;
}; };
@ -191,14 +192,14 @@ export function makeFlow<Requirements = never>(
const getProducerEffect = ( const getProducerEffect = (
producerName: string, producerName: string,
): Effect.Effect<EffectProducer<never>, FlowResourceNotFoundError> => { ): Effect.Effect<EffectProducer<never>, FlowResourceNotFoundError> => {
const producer = producers.get(producerName); const producer = O.getOrUndefined(MutableHashMap.get(producers, producerName));
return producer === undefined return producer === undefined
? Effect.fail(flowResourceNotFoundError(name, "producer", producerName)) ? Effect.fail(flowResourceNotFoundError(name, "producer", producerName))
: Effect.succeed(producer); : Effect.succeed(producer);
}; };
const getProducer = (producerName: string): EffectProducer<never> => { const getProducer = (producerName: string): EffectProducer<never> => {
const producer = producers.get(producerName); const producer = O.getOrUndefined(MutableHashMap.get(producers, producerName));
if (producer === undefined) throw flowResourceNotFoundError(name, "producer", producerName); if (producer === undefined) throw flowResourceNotFoundError(name, "producer", producerName);
return producer; return producer;
}; };
@ -206,7 +207,7 @@ export function makeFlow<Requirements = never>(
const getRequestorEffect = ( const getRequestorEffect = (
requestorName: string, requestorName: string,
): Effect.Effect<EffectRequestResponse<never, unknown>, FlowResourceNotFoundError> => { ): Effect.Effect<EffectRequestResponse<never, unknown>, FlowResourceNotFoundError> => {
const requestor = requestors.get(requestorName); const requestor = O.getOrUndefined(MutableHashMap.get(requestors, requestorName));
return requestor === undefined return requestor === undefined
? Effect.fail(flowResourceNotFoundError(name, "requestor", requestorName)) ? Effect.fail(flowResourceNotFoundError(name, "requestor", requestorName))
: Effect.succeed(requestor); : Effect.succeed(requestor);
@ -215,7 +216,7 @@ export function makeFlow<Requirements = never>(
const getRequestor = ( const getRequestor = (
requestorName: string, requestorName: string,
): EffectRequestResponse<never, unknown> => { ): EffectRequestResponse<never, unknown> => {
const requestor = requestors.get(requestorName); const requestor = O.getOrUndefined(MutableHashMap.get(requestors, requestorName));
if (requestor === undefined) throw flowResourceNotFoundError(name, "requestor", requestorName); if (requestor === undefined) throw flowResourceNotFoundError(name, "requestor", requestorName);
return requestor; return requestor;
}; };
@ -251,7 +252,7 @@ export function makeFlow<Requirements = never>(
if (typeof producer === "string") { if (typeof producer === "string") {
return getProducerEffect(producer); return getProducerEffect(producer);
} }
if (!producers.has(producer.name)) { if (!MutableHashMap.has(producers, producer.name)) {
return Effect.fail(flowResourceNotFoundError(name, "producer", producer.name)); return Effect.fail(flowResourceNotFoundError(name, "producer", producer.name));
} }
return producer.producerEffect(flow); return producer.producerEffect(flow);
@ -269,7 +270,7 @@ export function makeFlow<Requirements = never>(
if (typeof requestor === "string") { if (typeof requestor === "string") {
return getRequestorEffect(requestor); return getRequestorEffect(requestor);
} }
if (!requestors.has(requestor.name)) { if (!MutableHashMap.has(requestors, requestor.name)) {
return Effect.fail(flowResourceNotFoundError(name, "requestor", requestor.name)); return Effect.fail(flowResourceNotFoundError(name, "requestor", requestor.name));
} }
return requestor.requestorEffect(flow); return requestor.requestorEffect(flow);
@ -308,7 +309,7 @@ export function makeFlow<Requirements = never>(
if (typeof producer === "string") { if (typeof producer === "string") {
return toFlowProducer(getProducer(producer)); return toFlowProducer(getProducer(producer));
} }
if (!producers.has(producer.name)) { if (!MutableHashMap.has(producers, producer.name)) {
throw flowResourceNotFoundError(name, "producer", producer.name); throw flowResourceNotFoundError(name, "producer", producer.name);
} }
return toFlowProducer(compatibilityRuntime.runSync(producer.producerEffect(flow))); return toFlowProducer(compatibilityRuntime.runSync(producer.producerEffect(flow)));
@ -324,7 +325,7 @@ export function makeFlow<Requirements = never>(
if (typeof requestor === "string") { if (typeof requestor === "string") {
return toFlowRequestor(getRequestor(requestor)); return toFlowRequestor(getRequestor(requestor));
} }
if (!requestors.has(requestor.name)) { if (!MutableHashMap.has(requestors, requestor.name)) {
throw flowResourceNotFoundError(name, "requestor", requestor.name); throw flowResourceNotFoundError(name, "requestor", requestor.name);
} }
return toFlowRequestor(compatibilityRuntime.runSync(requestor.requestorEffect(flow))); return toFlowRequestor(compatibilityRuntime.runSync(requestor.requestorEffect(flow)));
@ -388,26 +389,26 @@ export function makeFlow<Requirements = never>(
return compatibilityRuntime.runPromise(flow.runInCompatibilityScopeEffect(effect, runtimePubsub, context)); return compatibilityRuntime.runPromise(flow.runInCompatibilityScopeEffect(effect, runtimePubsub, context));
}, },
clearResources(): void { clearResources(): void {
producers.clear(); MutableHashMap.clear(producers);
consumers.clear(); MutableHashMap.clear(consumers);
requestors.clear(); MutableHashMap.clear(requestors);
parameters.clear(); MutableHashMap.clear(parameters);
}, },
registerProducer<T>(registerName: string, producer: EffectProducer<T>): void { registerProducer<T>(registerName: string, producer: EffectProducer<T>): void {
producers.set(registerName, producer); MutableHashMap.set(producers, registerName, producer);
}, },
registerConsumer(registerName: string, consumer: EffectConsumer): void { registerConsumer(registerName: string, consumer: EffectConsumer): void {
consumers.set(registerName, consumer); MutableHashMap.set(consumers, registerName, consumer);
}, },
registerRequestor<TReq, TRes>(registerName: string, rr: EffectRequestResponse<TReq, TRes>): void { registerRequestor<TReq, TRes>(registerName: string, rr: EffectRequestResponse<TReq, TRes>): void {
requestors.set(registerName, rr); MutableHashMap.set(requestors, registerName, rr);
}, },
setParameter(parameterName: string, value: unknown): void { setParameter(parameterName: string, value: unknown): void {
parameters.set(parameterName, value); MutableHashMap.set(parameters, parameterName, value);
}, },
producerEffect, producerEffect,
consumerEffect(consumerName: string): Effect.Effect<EffectConsumer, FlowResourceNotFoundError> { consumerEffect(consumerName: string): Effect.Effect<EffectConsumer, FlowResourceNotFoundError> {
const c = consumers.get(consumerName); const c = O.getOrUndefined(MutableHashMap.get(consumers, consumerName));
return c === undefined return c === undefined
? Effect.fail(flowResourceNotFoundError(name, "consumer", consumerName)) ? Effect.fail(flowResourceNotFoundError(name, "consumer", consumerName))
: Effect.succeed(c); : Effect.succeed(c);
@ -416,7 +417,7 @@ export function makeFlow<Requirements = never>(
parameterEffect, parameterEffect,
producer, producer,
consumer(consumerName: string): FlowConsumer { consumer(consumerName: string): FlowConsumer {
const c = consumers.get(consumerName); const c = O.getOrUndefined(MutableHashMap.get(consumers, consumerName));
if (c === undefined) throw flowResourceNotFoundError(name, "consumer", consumerName); if (c === undefined) throw flowResourceNotFoundError(name, "consumer", consumerName);
return { return {
stop: () => compatibilityRuntime.runPromise(c.stop), stop: () => compatibilityRuntime.runPromise(c.stop),