mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-01 09:29:38 +02:00
Decode flow definitions with schema
This commit is contained in:
parent
4ec7e72532
commit
3070ce2b47
3 changed files with 84 additions and 28 deletions
|
|
@ -12,8 +12,8 @@ Verified source roots:
|
||||||
- Effect v4 subtree: `/home/elpresidank/YeeBois/projects/beep-effect2/.repos/effect-v4`
|
- Effect v4 subtree: `/home/elpresidank/YeeBois/projects/beep-effect2/.repos/effect-v4`
|
||||||
- Installed Effect beta used by this workspace: `ts/node_modules/effect`
|
- Installed Effect beta used by this workspace: `ts/node_modules/effect`
|
||||||
|
|
||||||
Current signal counts from `ts/packages` after the 2026-06-02 Base processor
|
Current signal counts from `ts/packages` after the 2026-06-02 Base flow
|
||||||
compatibility runtime slice:
|
definition schema slice:
|
||||||
|
|
||||||
| Signal | Count |
|
| Signal | Count |
|
||||||
| --- | ---: |
|
| --- | ---: |
|
||||||
|
|
@ -65,6 +65,9 @@ Notes:
|
||||||
- The base processor compatibility runtime slice dropped the
|
- The base processor compatibility runtime slice dropped the
|
||||||
`Effect.runPromise` count again by moving `AsyncProcessor`, `Flow`, and
|
`Effect.runPromise` count again by moving `AsyncProcessor`, `Flow`, and
|
||||||
`FlowProcessor` Promise compatibility facades onto `ManagedRuntime`.
|
`FlowProcessor` Promise compatibility facades onto `ManagedRuntime`.
|
||||||
|
- The base flow definition schema slice removed hand-rolled
|
||||||
|
`Predicate`/object narrowing from `flow-processor.ts`; signal counts are
|
||||||
|
unchanged because this was a validation-quality migration.
|
||||||
- `Record<string, any>` and `throwLibrarianServiceError` are now clean in
|
- `Record<string, any>` and `throwLibrarianServiceError` are now clean in
|
||||||
`ts/packages`.
|
`ts/packages`.
|
||||||
|
|
||||||
|
|
@ -520,6 +523,26 @@ Notes:
|
||||||
- `cd ts && bun run build`
|
- `cd ts && bun run build`
|
||||||
- `cd ts && bun run test`
|
- `cd ts && bun run test`
|
||||||
|
|
||||||
|
### 2026-06-02: Base Flow Definition Schema Slice
|
||||||
|
|
||||||
|
- Status: migrated and root-verified.
|
||||||
|
- Completed:
|
||||||
|
- `ts/packages/base/src/processor/flow-processor.ts` now validates
|
||||||
|
`config.flows` with Effect Schema instead of local
|
||||||
|
`Predicate`/object/string-record guards.
|
||||||
|
- Invalid flow definition payloads still log/skip and preserve the existing
|
||||||
|
config-handler and acknowledgement behavior.
|
||||||
|
- `ts/packages/base/src/__tests__/flow-processor-runtime.test.ts` now covers
|
||||||
|
an invalid nested flow definition that is acknowledged without starting
|
||||||
|
resources.
|
||||||
|
- Verification:
|
||||||
|
- `bun run --cwd ts/packages/base test -- src/__tests__/flow-processor-runtime.test.ts`
|
||||||
|
- `bun run --cwd ts/packages/base build`
|
||||||
|
- `cd ts && bun run check`
|
||||||
|
- `cd ts && bun run build`
|
||||||
|
- `cd ts && bun run test`
|
||||||
|
- `git diff --check`
|
||||||
|
|
||||||
## Subagent Findings To Preserve
|
## Subagent Findings To Preserve
|
||||||
|
|
||||||
- MCP/workbench:
|
- MCP/workbench:
|
||||||
|
|
@ -568,11 +591,10 @@ Notes:
|
||||||
|
|
||||||
## Ranked Findings
|
## Ranked Findings
|
||||||
|
|
||||||
### P1: Base Flow Definition Schemas And Typed Spec Accessors
|
### P1: Base Typed Spec Accessors
|
||||||
|
|
||||||
- TrustGraph evidence:
|
- TrustGraph evidence:
|
||||||
- `ts/packages/base/src/processor/flow.ts`
|
- `ts/packages/base/src/processor/flow.ts`
|
||||||
- `ts/packages/base/src/processor/flow-processor.ts`
|
|
||||||
- `ts/packages/base/src/spec/parameter-spec.ts`
|
- `ts/packages/base/src/spec/parameter-spec.ts`
|
||||||
- `ts/packages/base/src/spec/producer-spec.ts`
|
- `ts/packages/base/src/spec/producer-spec.ts`
|
||||||
- `ts/packages/base/src/spec/request-response-spec.ts`
|
- `ts/packages/base/src/spec/request-response-spec.ts`
|
||||||
|
|
@ -580,8 +602,6 @@ Notes:
|
||||||
- Schema-backed registries, `Context`, `Layer`, `Effect.fn`, `Option`,
|
- Schema-backed registries, `Context`, `Layer`, `Effect.fn`, `Option`,
|
||||||
`Predicate`, `HashMap`/`MutableHashMap`.
|
`Predicate`, `HashMap`/`MutableHashMap`.
|
||||||
- Rewrite shape:
|
- Rewrite shape:
|
||||||
- Replace hand-rolled `isStringRecord` / `isFlowDefinition` narrowing with
|
|
||||||
Schema decoding plus `Option`/`Match`-style branches.
|
|
||||||
- Add schema-backed generic parameter specs and spec-object accessors such as
|
- Add schema-backed generic parameter specs and spec-object accessors such as
|
||||||
`flow.parameterEffect(spec)`, then keep string accessors as compatibility
|
`flow.parameterEffect(spec)`, then keep string accessors as compatibility
|
||||||
escapes.
|
escapes.
|
||||||
|
|
@ -645,7 +665,7 @@ Notes:
|
||||||
|
|
||||||
## Recommended PR Order
|
## Recommended PR Order
|
||||||
|
|
||||||
1. Base flow definition schema decoding and typed spec accessors.
|
1. Base typed spec accessors.
|
||||||
2. Gateway RPC callback and client streaming completion cleanup.
|
2. Gateway RPC callback and client streaming completion cleanup.
|
||||||
3. Storage/provider managed resource cleanup.
|
3. Storage/provider managed resource cleanup.
|
||||||
4. MCP parity/deletion decision and workbench platform polish.
|
4. MCP parity/deletion decision and workbench platform polish.
|
||||||
|
|
|
||||||
|
|
@ -136,6 +136,10 @@ class FlowProcessorBackend implements PubSubBackend {
|
||||||
}
|
}
|
||||||
|
|
||||||
pushConfig(version: number, flows: Record<string, unknown>): void {
|
pushConfig(version: number, flows: Record<string, unknown>): void {
|
||||||
|
this.pushFlowConfig(version, flows);
|
||||||
|
}
|
||||||
|
|
||||||
|
pushFlowConfig(version: number, flows: unknown): void {
|
||||||
this.configConsumer.push(createMessage({ version, config: { flows } }));
|
this.configConsumer.push(createMessage({ version, config: { flows } }));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -254,4 +258,44 @@ describe("Effect-native FlowProcessor runtime", () => {
|
||||||
expect(backend.closeCount).toBe(1);
|
expect(backend.closeCount).toBe(1);
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
it.effect(
|
||||||
|
"schema-decodes flow definitions before starting resources",
|
||||||
|
Effect.fnUntraced(function* () {
|
||||||
|
const backend = new FlowProcessorBackend();
|
||||||
|
const events: Array<string> = [];
|
||||||
|
|
||||||
|
yield* Effect.scoped(
|
||||||
|
Effect.gen(function* () {
|
||||||
|
const fiber = yield* runFlowProcessorDefinitionScoped({
|
||||||
|
id: "schema-flow-processor-test",
|
||||||
|
pubsub: backend,
|
||||||
|
specifications: [makeProducerSpec<string>("output")],
|
||||||
|
configHandlers: [
|
||||||
|
(_config, version) => Effect.sync(() => {
|
||||||
|
events.push(`handler:${version}`);
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
}).pipe(
|
||||||
|
Effect.provide(MessagingRuntimeLive),
|
||||||
|
Effect.provide(PubSub.layer(backend)),
|
||||||
|
Effect.provide(fastMessagingConfig),
|
||||||
|
Effect.forkChild,
|
||||||
|
);
|
||||||
|
|
||||||
|
yield* waitFor(() => backend.consumerOptions.length === 1, "schema config subscription");
|
||||||
|
|
||||||
|
backend.pushFlowConfig(1, { default: { topics: { output: 42 } } });
|
||||||
|
yield* waitFor(() => backend.configConsumer.acknowledged.length === 1, "schema config ack");
|
||||||
|
|
||||||
|
yield* Fiber.interrupt(fiber);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(backend.producers).toHaveLength(0);
|
||||||
|
expect(events).toEqual(["handler:1"]);
|
||||||
|
expect(backend.configConsumer.closeCount).toBeGreaterThanOrEqual(1);
|
||||||
|
expect(backend.closeCount).toBe(1);
|
||||||
|
}),
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@ import {
|
||||||
import { makePubSubService, PubSub } from "../backend/pubsub.js";
|
import { makePubSubService, PubSub } from "../backend/pubsub.js";
|
||||||
import { loadMessagingRuntimeConfig } from "../runtime/messaging-config.js";
|
import { loadMessagingRuntimeConfig } from "../runtime/messaging-config.js";
|
||||||
import { Duration, Effect, Exit, Layer, ManagedRuntime, Scope } from "effect";
|
import { Duration, Effect, Exit, Layer, ManagedRuntime, Scope } from "effect";
|
||||||
import * as Predicate from "effect/Predicate";
|
import * as O from "effect/Option";
|
||||||
import * as S from "effect/Schema";
|
import * as S from "effect/Schema";
|
||||||
|
|
||||||
interface ConfigPush {
|
interface ConfigPush {
|
||||||
|
|
@ -105,19 +105,14 @@ const ConfigPushSchema = S.Struct({
|
||||||
config: S.Record(S.String, S.Unknown),
|
config: S.Record(S.String, S.Unknown),
|
||||||
});
|
});
|
||||||
|
|
||||||
const isStringRecord = (value: unknown): value is Record<string, unknown> =>
|
const FlowDefinitionSchema = S.Struct({
|
||||||
Predicate.isObject(value) && !Array.isArray(value);
|
topics: S.optionalKey(S.Record(S.String, S.String)),
|
||||||
|
parameters: S.optionalKey(S.Record(S.String, S.Unknown)),
|
||||||
|
});
|
||||||
|
|
||||||
const isTopicsRecord = (value: unknown): value is Record<string, string> =>
|
const FlowDefinitionsSchema = S.Record(S.String, FlowDefinitionSchema);
|
||||||
isStringRecord(value) && Object.values(value).every((item) => typeof item === "string");
|
|
||||||
|
|
||||||
const isFlowDefinition = (value: unknown): value is FlowDefinition => {
|
const decodeFlowDefinitions = S.decodeUnknownOption(FlowDefinitionsSchema);
|
||||||
if (!isStringRecord(value)) return false;
|
|
||||||
const topics = value.topics;
|
|
||||||
const parameters = value.parameters;
|
|
||||||
return (topics === undefined || isTopicsRecord(topics)) &&
|
|
||||||
(parameters === undefined || isStringRecord(parameters));
|
|
||||||
};
|
|
||||||
|
|
||||||
export function runFlowProcessorDefinitionScoped<
|
export function runFlowProcessorDefinitionScoped<
|
||||||
FlowRequirements = never,
|
FlowRequirements = never,
|
||||||
|
|
@ -220,12 +215,14 @@ export function runFlowProcessorDefinitionScoped<
|
||||||
yield* Effect.log(`[${options.id}] No flows in config push, skipping`);
|
yield* Effect.log(`[${options.id}] No flows in config push, skipping`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!isStringRecord(flowDefs)) {
|
const decodedFlowDefs = decodeFlowDefinitions(flowDefs);
|
||||||
|
if (O.isNone(decodedFlowDefs)) {
|
||||||
yield* Effect.logWarning(`[${options.id}] Skipping config push: flows is not an object`);
|
yield* Effect.logWarning(`[${options.id}] Skipping config push: flows is not an object`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const flowDefinitions = decodedFlowDefs.value;
|
||||||
|
|
||||||
const flowsJson = yield* S.encodeUnknownEffect(S.UnknownFromJsonString)(flowDefs).pipe(
|
const flowsJson = yield* S.encodeUnknownEffect(S.UnknownFromJsonString)(flowDefinitions).pipe(
|
||||||
Effect.catch((error) => Effect.succeed(String(error))),
|
Effect.catch((error) => Effect.succeed(String(error))),
|
||||||
);
|
);
|
||||||
if (lastFlowsJson.length > 0 && flowsJson === lastFlowsJson && flows.size > 0) {
|
if (lastFlowsJson.length > 0 && flowsJson === lastFlowsJson && flows.size > 0) {
|
||||||
|
|
@ -235,19 +232,14 @@ export function runFlowProcessorDefinitionScoped<
|
||||||
lastFlowsJson = flowsJson;
|
lastFlowsJson = flowsJson;
|
||||||
|
|
||||||
for (const [name, activeFlow] of flows) {
|
for (const [name, activeFlow] of flows) {
|
||||||
if (!(name in flowDefs)) {
|
if (!(name in flowDefinitions)) {
|
||||||
yield* Effect.log(`[${options.id}] Stopping removed flow: ${name}`);
|
yield* Effect.log(`[${options.id}] Stopping removed flow: ${name}`);
|
||||||
yield* closeFlowEffect(name, activeFlow);
|
yield* closeFlowEffect(name, activeFlow);
|
||||||
flows.delete(name);
|
flows.delete(name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const [name, defn] of Object.entries(flowDefs)) {
|
for (const [name, defn] of Object.entries(flowDefinitions)) {
|
||||||
if (!isFlowDefinition(defn)) {
|
|
||||||
yield* Effect.logWarning(`[${options.id}] Skipping flow "${name}": definition is not an object`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const existing = flows.get(name);
|
const existing = flows.get(name);
|
||||||
if (existing !== undefined) {
|
if (existing !== undefined) {
|
||||||
yield* Effect.log(`[${options.id}] Restarting flow "${name}" with updated config`);
|
yield* Effect.log(`[${options.id}] Restarting flow "${name}" with updated config`);
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue