From fab718dce8eabd9a19120e4ee897632070281f43 Mon Sep 17 00:00:00 2001 From: elpresidank Date: Thu, 11 Jun 2026 08:34:59 -0500 Subject: [PATCH] chore(ts): consolidate effect native closeout --- ts/CLASS_EFFECT_GOAL.md | 151 - ts/EFFECT_NATIVE_REWRITE_AUDIT.md | 2924 ----------------- ts/EFFECT_NATIVE_REWRITE_PLAN.md | 127 - ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md | 225 -- ts/EFFECT_NATIVE_STATUS.md | 80 + ts/packages/base/src/schema/primitives.ts | 4 +- ts/packages/client/src/index.ts | 9 +- .../client/src/socket/trustgraph-socket.ts | 10 +- ts/packages/flow/src/config/service.ts | 19 +- ts/packages/flow/src/cores/service.ts | 9 +- ts/packages/flow/src/flow-manager/service.ts | 9 +- ts/packages/flow/src/librarian/service.ts | 4 +- ts/packages/flow/src/retrieval/graph-rag.ts | 8 +- ts/packages/mcp/src/server-effect.ts | 6 +- ts/packages/workbench/src/atoms/workbench.ts | 15 +- .../src/components/chat/explain-graph.tsx | 3 +- ts/packages/workbench/src/pages/graph.tsx | 3 +- .../workbench/src/qa/initial-values.ts | 4 +- ts/packages/workbench/src/qa/mock-api.ts | 65 +- ts/scripts/effect-laws.allowlist.json | 2 +- ts/scripts/test-pipeline.ts | 55 +- 21 files changed, 199 insertions(+), 3533 deletions(-) delete mode 100644 ts/CLASS_EFFECT_GOAL.md delete mode 100644 ts/EFFECT_NATIVE_REWRITE_AUDIT.md delete mode 100644 ts/EFFECT_NATIVE_REWRITE_PLAN.md delete mode 100644 ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md create mode 100644 ts/EFFECT_NATIVE_STATUS.md diff --git a/ts/CLASS_EFFECT_GOAL.md b/ts/CLASS_EFFECT_GOAL.md deleted file mode 100644 index 1505725c..00000000 --- a/ts/CLASS_EFFECT_GOAL.md +++ /dev/null @@ -1,151 +0,0 @@ -# TrustGraph TS Native Class To Functional Effect Services Goal - -Use the `grill-me` skill from: - -`/home/elpresidank/YeeBois/projects/beep-effect/.agents/skills/grill-me/SKILL.md` - -## Objective - -Refactor the TrustGraph TypeScript port so production runtime code no longer -uses native TypeScript or JavaScript class syntax except where an Effect module -API truly requires class syntax and no practical functional form exists. - -The preferred style is functional and pure: - -- Functions, factories, closures, and object-literal service implementations. -- Functions returning functions or service objects instead of constructors. -- `Context.Service` tags plus `Layer` builders for dependency provision. -- `Effect.gen`, `Effect.fn`, `Effect.acquireRelease`, `Effect.addFinalizer`, - and scoped Layers for lifecycle. -- Top-level `Effect.runPromise` only at CLI/bootstrap boundaries. - -Terminal condition: repeated inventory agents find zero blocking native classes -in production runtime source like these current smells: - -- `ts/packages/base/src/processor/async-processor.ts#L38-162` -- `ts/packages/base/src/processor/flow-processor.ts#L284-358` - -## Class Policy - -Blocking scope is production runtime source under `ts/packages/**/src`, -excluding `__tests__`, `*.test.ts`, and `*.spec.ts`. - -Tests and scripts must still be inventoried separately, but they do not block -completion unless they are required to verify migrated production behavior. - -Allowed class syntax is intentionally narrow and proof-based. Do not preserve -class syntax just because it is Effect-adjacent. For every candidate exemption, -prove that the specific Effect API requires class syntax and that a functional -alternative is not better. - -Candidate exemptions to investigate, not blindly preserve: - -- `S.Class`, `S.TaggedClass`, `S.TaggedErrorClass`, `S.ErrorClass` -- `Data.TaggedError` -- `Context.Service` -- Effect RPC / HTTP API class forms such as `Rpc.make` and `HttpApi.make` - -If a functional Effect equivalent exists, use the functional form. The target -style is functions returning functions or service objects, not class-shaped -Effect code. - -Blocking class forms include: - -- Abstract base classes and inheritance-based processors. -- Service classes extending `AsyncProcessor`, `FlowProcessor`, or `LlmService`. -- Client API wrapper classes. -- Backend adapter classes. -- Spec/resource classes. -- Query/store/engine classes. -- Metrics/parser/lifecycle classes. -- React class components such as error boundaries. - -## Required Workflow - -1. Read and follow `grill-me`. -2. Before asking the user questions, use parallel read-only sub-agents to - inventory class declarations. -3. Use AST-based inspection where possible; regex alone is not enough. -4. Split inventory agents by subsystem: - - Base processor/messaging/runtime/specs/backend. - - Flow service processors and LLM/embedding/RAG services. - - Query/store engines and adapters. - - Client socket/API wrappers. - - Workbench production source. -5. Consolidate findings into: - - Blocking production native classes. - - Effect-native class forms with proof. - - Functional replacements available for prior exemptions. - - Non-blocking test/script classes. -6. Use `grill-me` to resolve any remaining API or rollout tradeoffs. -7. Produce a decision-complete implementation plan. -8. After approval, implement in phases and rerun inventory until blocking count - is zero. - -## Refactor Direction - -Replace inheritance and mutable class instances with: - -- Plain TypeScript service interfaces. -- `Context.Service` tags only where needed for Effect dependency injection. -- `Layer.succeed`, `Layer.effect`, and `Layer.scoped` for construction. -- Closure-held state only where necessary, preferably scoped and finalized. -- Object-literal services instead of class instances. -- Factory functions like `makeXService`, `makeXLayer`, and `runX`. -- `Effect.tryPromise` for external Promise APIs. -- `Effect.fn` for reusable effectful service methods. - -Preserve behavior and in-repo call semantics, but do not preserve native class -constructors as public API. Replace exported classes with interfaces, service -tags, factory functions, or layer builders, and update all in-repo callers. - -## Verification Requirements - -Required gates: - -- `cd ts && bun run check:tsgo` -- `cd ts && bun run build` -- `cd ts && bun run test` -- `cd ts && bun run workbench:qa` -- Add or provide an inventory command/script that fails on blocking production - classes. -- Run the inventory command after every migration phase. -- Final inventory must report zero blocking production native classes. - -Live smoke if service surfaces changed: - -- `cd ts/deploy && docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build` -- `cd ts && bun run seed` -- `cd ts && bun run seed:demo` -- `cd ts && bun run seed:flows` -- `cd ts && SKIP_LLM=1 bun run test:pipeline` - -## New Session Prompt - -Paste this into a new Codex session: - -```text -Use the grill-me skill from /home/elpresidank/YeeBois/projects/beep-effect/.agents/skills/grill-me/SKILL.md. - -The repo is /home/elpresidank/YeeBois/dev/trustgraph and the TypeScript port is in ./ts. I want to remove native TypeScript/JavaScript class syntax from production runtime code and convert the port to functional, pure Effect services. - -Before asking me questions, use parallel read-only sub-agents to inventory every class declaration under ts/packages/**/src. Production runtime source is the blocking scope; tests and scripts should be inventoried separately but are non-blocking. Use AST-based inspection where possible, not regex alone. - -Allowed class syntax is extremely narrow and proof-based. Even Effect-native-looking classes such as S.Class, S.TaggedClass, S.TaggedErrorClass, S.ErrorClass, Data.TaggedError, Context.Service, Rpc.make, and HttpApi.make must be treated as candidate exemptions, not automatic exemptions. If a functional Effect equivalent exists, use the functional form. I prefer functions, factories, closures, object-literal service implementations, Context.Service tags, and Layers over all class-shaped code. - -Use these files as representative smells: -- ts/packages/base/src/processor/async-processor.ts#L38-162 -- ts/packages/base/src/processor/flow-processor.ts#L284-358 - -Inventory by subsystem: base processor/messaging/runtime/specs/backend; flow service processors and LLM/embedding/RAG services; query/store engines and adapters; client socket/API wrappers; workbench production source. - -After inventory, use grill-me to resolve any remaining API or rollout tradeoffs, then produce a decision-complete implementation plan. After I approve the plan, implement in phases and repeat inventory until zero blocking production native classes remain. - -Required verification: cd ts && bun run check:tsgo; cd ts && bun run build; cd ts && bun run test; cd ts && bun run workbench:qa; add or provide an inventory command/script that fails on blocking production classes. If service surfaces changed, also run the live docker/seed/pipeline smoke from this file. -``` - -## Goal Command - -```text -/goal Use ts/CLASS_EFFECT_GOAL.md to run the TrustGraph TS native-class-to-functional-Effect-services migration. First use grill-me and parallel read-only sub-agents to inventory every native class declaration. Then produce and execute a decision-complete refactor plan that replaces classes with pure functions, factories, closures, object-literal services, Context.Service tags, and Layers. Terminal condition: repeated inventory agents and the repo inventory command report zero blocking native classes in production runtime source under ts/packages/**/src. Even Effect-native class forms such as S.Class, TaggedError classes, Context.Service, Rpc.make, and HttpApi.make must be treated as narrow proof-based exemptions; if a functional Effect equivalent exists, use it. Verify with check:tsgo, build, test, workbench:qa, and live smoke when service surfaces change. Preserve unrelated local files. -``` diff --git a/ts/EFFECT_NATIVE_REWRITE_AUDIT.md b/ts/EFFECT_NATIVE_REWRITE_AUDIT.md deleted file mode 100644 index dd17ce7f..00000000 --- a/ts/EFFECT_NATIVE_REWRITE_AUDIT.md +++ /dev/null @@ -1,2924 +0,0 @@ -# TrustGraph Effect-Native Rewrite Opportunity Audit - -This is the current backlog snapshot for the playbook in -`ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md`. The branch is `ts-port-effect-v4`. -The unrelated local file `.idea/effect.intellij.xml` must stay uncommitted. - -## Inputs - -Verified source roots: - -- TrustGraph TS port: `/home/elpresidank/YeeBois/dev/trustgraph/ts` -- Effect v4 subtree: `/home/elpresidank/YeeBois/projects/beep-effect2/.repos/effect-v4` -- Installed Effect beta used by this workspace: `ts/node_modules/effect` - -Current signal counts from `ts/packages` after the 2026-06-02 dispatcher -Effect collections slice: - -| Signal | Count | -| --- | ---: | -| `Effect.runPromise` | 175 | -| `Effect.runPromiseWith` | 0 | -| `Effect.cached` | 0 | -| `Layer.succeed` | 13 | -| `Map<` | 86 | -| `WebSocket` | 72 | -| `new Map` | 56 | -| `new Set` | 15 | -| `Set<` | 9 | -| `toPromiseRequestor` | 0 | -| `makeAsyncProcessor` | 19 | -| `receive(` | 17 | -| `while (` | 2 | -| `new Error` | 7 | -| `new Promise` | 9 | -| `JSON.parse` | 4 | -| `localStorage` | 9 | -| `JSON.stringify` | 8 | -| `setTimeout` | 3 | -| `process.env` | 3 | - -Notes: - -- The remaining `process.env` hits are in `packages/workbench/playwright.config.ts`. -- In production `packages/base`, `packages/cli`, and `packages/mcp` sources, - the strict scans for `new Error`, `new Promise`, `setTimeout`, - `JSON.parse`, `JSON.stringify`, and direct `process.env` reads are clean. -- `Effect.runPromise` is expected at external Promise compatibility - boundaries, but each match should still be audited for avoidable internal - runtime ownership. -- The dispatcher Effect collections slice removed native `Map`/`Set` from the - gateway service registries, streaming membership set, and scoped requestor - cache. Remaining broad `Map`/`Set` matches include tests/fakes, WeakMap - compatibility caches, short-lived pure traversal collections, and larger - ref-backed service state that still needs focused `HashMap`/`MutableHashMap` - cleanup. -- The `Effect.runPromise` and `WebSocket` counts dropped in this snapshot - because `EffectRpcClient` now owns its RPC/socket layer with - `ManagedRuntime` and uses Effect's WebSocket constructor layer. -- The raw `WebSocket` count increased in this snapshot because the adapter - slice added focused tests and typed adapter names; production - `websocket-adapter.ts` is now clean of `try`/`catch`, normal `Error`, and - the previous constructor assertions. -- The `new Error` count dropped because `websocket-adapter.ts` now throws - `S.TaggedErrorClass` adapter errors. -- The latest client socket slice removed the remaining production - `trustgraph-socket.ts` normal `Error`, raw `JSON.parse`, and listener - `try`/`catch` matches. The remaining client socket modernization signal is - the shared `newableFactory` constructor assertion pattern. -- The service entrypoint runtime slice dropped the `Effect.runPromise` count by - replacing remaining flow service `run()` program facades with - `ManagedRuntime` and routing local `ts/scripts/run-*` launchers through - `runMain()`/`NodeRuntime.runMain`. -- The base processor compatibility runtime slice dropped the - `Effect.runPromise` count again by moving `AsyncProcessor`, `Flow`, and - `FlowProcessor` Promise compatibility facades onto `ManagedRuntime`. -- The base flow definition schema slice removed hand-rolled - `Predicate`/object narrowing from `flow-processor.ts`; signal counts are - unchanged because this was a validation-quality migration. -- The text completion stream sentinel slice removed the duplicated - `Effect.void as Effect.Effect` assertions from provider stream - unfold branches. Counts are unchanged because this was an Effect diagnostic - and type-channel cleanup. -- The text completion generator boundary slice removed the - `Effect.runPromise(Effect.fail(...))` fallback and the related - `AsyncGenerator`/`IteratorResult` assertions from - `model/text-completion/common.ts`. -- The text completion provider status slice replaced manual status/statusCode - record assertions with `effect/Predicate` narrowing. -- The base parameter spec accessor slice added Schema-backed - `ParameterSpec` values plus `flow.parameterEffect(spec)` and - `flow.parameter(spec)`. Bare string parameter lookup remains available as an - `unknown` compatibility escape, while typed parameter access now decodes - through Schema and fails with a tagged `FlowParameterDecodeError`. -- The base producer/requestor spec accessor slice added typed spec-object - accessors for `ProducerSpec` and `RequestResponseSpec`, then - migrated flow service producer/requestor lookups off caller-chosen generic - string calls. Spec object handles are scoped per `Flow` through WeakMaps and - finalizers delete only the handle they registered. -- The native PubSub boundary slice removed the unused legacy - `messaging/subscriber.ts` async queue/fanout implementation. Effect's native - `PubSub` is an in-process hub and does not replace the broker-backed - `PubSubBackend`/NATS boundary, but it should be preferred for future - in-process broadcast/fanout needs. -- The base producer scoped runtime slice moved the legacy `makeProducer` - Promise facade onto the existing `makeEffectProducerFromPubSub` scoped - factory. Public `start`/`send`/`stop` remain Promise compatibility - boundaries, while producer allocation, flush, and finalizer close now go - through the Effect runtime path. -- The NATS typed boundary slice removed the dynamic `import("nats")` header - path and maps header construction plus `ack()`/`nak()` failures into tagged - `PubSubError`s with `Effect.try`. The `receive(` and `JSON.stringify` count - increases are from the new mocked NATS backend test, not production code. -- The NATS selective 404 slice replaced catch-all stream/consumer create - fallbacks with an internal `S.TaggedErrorClass` lookup wrapper plus - `Effect.catchIf` recovery only for NATS JetStream missing-resource errors. - Non-missing lookup failures now stay on the typed failure path without - attempting to create streams or durable consumers. -- The consumer rate-limit retry slice wired the previously unused - `rateLimitTimeoutMs` option and `TG_RATE_LIMIT_TIMEOUT_MS` config into both - legacy and Effect-native consumers. Repeated `TooManyRequestsError` failures - now retry with `Schedule.spaced` until success or a tagged rate-limit timeout. - The `new Error` count dropped by one because a touched consumer test fixture - no longer uses a normal `Error`. -- The consumer concurrency ownership slice changed the Effect-native consumer - runtime so `concurrency > 1` allocates one backend consumer per worker instead - of sharing a single `BackendConsumer.receive()` handle. `stop` is now - idempotent through `Ref`, so explicit stop and scoped finalizers do not close - workers twice. -- The request-response stop signal slice added a `Deferred` shutdown signal to - `makeEffectRequestResponseFromPubSub`. Pending requests now race response - waiting against runtime stop and fail promptly with a tagged - `MessagingLifecycleError` instead of waiting for timeout. -- The legacy consumer facade slice moved `makeConsumer` onto - `makeEffectConsumerFromPubSub` with a `ManagedRuntime` Promise boundary and a - closeable `Scope`. Consumer workers now use `Effect.forkScoped` so their - lifetime is owned by the caller scope rather than the parent fiber. The - `Effect.runPromise`, `receive(`, `new Promise`, and `setTimeout` counts - dropped because the old blocking facade loop and its test timer shim were - removed. -- The workbench theme storage slice stopped mirroring `themeAtom` into the - legacy `tg-theme` localStorage key. The canonical - `trustgraph-workbench-theme-v1` value remains owned by `Atom.kvs` over - `BrowserKeyValueStore.layerLocalStorage`; the first-paint host script reads - that JSON-encoded key before React mounts and falls back to `tg-theme` only - for legacy installs. -- The Effect AI `LanguageModel` adapter slice added a reusable - `makeLanguageModelProvider` bridge in text-completion common code. It maps - `generateText` responses to `LlmResult`, maps streaming `text-delta` and - final `finish.usage` parts to TrustGraph chunks, and converts Effect AI rate - and quota failures into `TooManyRequestsError`. No concrete provider has - been flipped yet. -- The native request/response PubSub slice removed the local - `Map` response subscriber fanout in - `makeEffectRequestResponseFromPubSub`. Response dispatch now publishes - `{ id, value }` envelopes through native `effect/PubSub`, and each request - uses a scoped `PubSub.Subscription` plus `Stream.fromSubscription` to wait - for its matching response. -- The Claude Effect AI slice moved the Claude provider off the direct - `@anthropic-ai/sdk` wrapper and onto `@effect/ai-anthropic` - `AnthropicLanguageModel` through `makeLanguageModelProvider`. The direct SDK - dependency was removed from `@trustgraph/flow`. -- A focused broker-backend scout found no remaining P0 broker runtime rewrite - after the producer, NATS, consumer concurrency, rate-limit, and - request-response stop slices. `PubSubBackend` remains an intentional - Promise-returning adapter boundary wrapped by `PubSub`/Effect services. -- The gateway streaming callback slice added Effect-returning dispatcher - streaming methods, switched the RPC stream server off nested - `Effect.runPromiseWith(context)` queue offers, and replaced the client - `StopStreaming` sentinel error with `Stream.runForEachWhile`. -- The FalkorDB scoped client lifecycle slice removed the remaining - `Effect.cached` matches from `ts/packages`. FalkorDB triples store/query - Live layers and direct compatibility factories now acquire clients through - `Effect.acquireRelease` and disconnect them on scope close. The - `Effect.runPromise` count increased by two because the new lifecycle tests - run scoped programs at the test boundary. -- The Qdrant config/schema/fakeability slice removed direct production - `new QdrantClient`, sync config loading, payload casts, and Qdrant - `Layer.succeed` service construction from graph/doc store/query modules. - The installed Qdrant client exposes no public close/disconnect method, so - this remains a fakeable construction and Schema decode slice rather than a - scoped finalizer slice. `Effect.runPromise` increased because the new tests - and legacy service initialization logs run Effects at compatibility - boundaries. -- The client streaming facade slice did not change signal counts. It - centralized the legacy streaming `{ response, complete, error }` envelope - decode in `trustgraph-socket.ts`, uses Schema plus `effect/Predicate` - property narrowing for streaming payload reads, and leaves service-specific - legacy completion markers only where they preserve public callback behavior. -- The Ollama embeddings effectful layer slice dropped one `Layer.succeed` - match by making `OllamaEmbeddingsLive` effectful and mapping config/load - failures to `EmbeddingsError`. The `JSON.stringify` count increased by one - because the new layer test uses a JSON response fixture. -- The text completion provider stream helper slice removed all provider-local - `Stream.unfold` pull loops, dropped the `while (` count from 9 to 3, and - removed the Mistral `content as string` assertion. The only remaining - text-completion `iterator.next` match is the `toAsyncGenerator` - compatibility adapter that exposes Effect streams through the public - `AsyncGenerator` provider contract. -- The request-response queue stream slice replaced the Effectful - `waitForResponse` generator loop with `Stream.fromQueue`, - `Stream.filterMapEffect`, `Result`, and `Stream.runHead`, dropping the - remaining `while (` count from 3 to 2. The two remaining production `while` - hits are synchronous parsing/CLI traversal loops, not async polling loops. -- The gateway RPC WebSocket cause-handling slice removed the Promise `.catch` - around the socket program by sandboxing the Effect and handling the resulting - `Cause` in the Effect pipeline before the Fastify fire-and-forget - `runPromise` boundary. -- The client RPC acquisition cause tap slice removed the Promise `.catch` used - only to update connection state on runtime/client acquisition failure. - `effect-rpc-client.ts` now uses `Effect.tapCause` and `Cause.pretty` before - the public Promise boundary. -- The client socket close Effect boundary slice removed the Promise `.catch` - from `BaseApi.close()`. The void public facade now runs `rpc.close()` through - `Effect.tryPromise` and logs the tagged socket close error through - `Effect.catch`. -- The client streaming callback Effect boundary slice removed the remaining - production Promise `.catch` matches from `trustgraph-socket.ts` by - centralizing legacy callback request failures in `runLegacyStreamingRequest`. - The public callback facades still return/ignore Promises where required, but - failure mapping now uses `Effect.tryPromise` and `Effect.catch`. -- The text completion provider effectful layer slice dropped six - `Layer.succeed` matches by moving OpenAI, OpenAI-compatible, Azure OpenAI, - Claude, Mistral, and Ollama processor layers onto - `makeTextCompletionLayer(makeXProviderEffect(config))`. SDK construction and - config lookup now live in Effect; sync `makeXProvider` exports remain - compatibility facades. -- The gateway dispatcher ownership and serialization slice did not change broad - signal counts. It stopped closing injected pubsub backends, brackets - one-shot publish producers with `Effect.acquireUseRelease`, and routes - gateway request/response translation through `Effect.try` wrappers returning - tagged `DispatchSerializationError` failures. -- `Record` and `throwLibrarianServiceError` are now clean in - `ts/packages`. - -## Loop Passes - -### 2026-06-02: Base Request/Response Facade - -- Status: migrated and verified. -- Completed: - - Request/response startup now owns a scoped Effect runtime handle and maps - failures to TrustGraph tagged messaging errors. - - Runtime shutdown is idempotent and uses scoped fibers. - - Tests cover Promise compatibility, tagged timeout errors, and tagged - lifecycle errors. -- Verification: - - `bun run --cwd ts/packages/base test` - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts check:tsgo` - - `bun run --cwd ts build` - - `bun run --cwd ts test` - -### 2026-06-02: Gateway Dispatcher Requestor Cache - -- Status: migrated and package-verified. -- Completed: - - Gateway dispatcher caches scoped `EffectRequestResponse` handles instead - of `Promise` values. - - Lazy requestor creation is serialized with `SynchronizedRef.modifyEffect`. - - Streaming final-marker detection is centralized. - - Dispatcher cleanup uses Effect scope/error handling instead of manual - `try`/`catch`. -- Verification: - - `bun run --cwd ts/packages/flow test` - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts check:tsgo` - -### 2026-06-02: Gateway Dispatcher Effect Collections Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/gateway/dispatch/manager.ts` now stores the - flow/global service registries in `effect/HashMap` instead of native - `ReadonlyMap`, while explicit entry arrays preserve the public service-name - ordering. - - Streaming service membership now uses `effect/HashSet` instead of native - `Set`. - - The scoped requestor cache now stores - `HashMap>` in the existing - `SynchronizedRef`, replacing `new Map` cloning with immutable - `HashMap.set`. - - Cache hits and service topic lookups now use `HashMap.get` plus - `effect/Option`, and ref update tuples use `effect/Tuple.make` instead of - `as const` assertions. - - Gateway dispatcher tests now cover concurrent same-key dispatches so the - cache still creates exactly one scoped producer/consumer pair. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/gateway-dispatcher.test.ts` - - `cd ts && bun run check:tsgo` - -### 2026-06-02: Strict Base, CLI, MCP, And tsgo Slice - -- Status: migrated, root-verified, committed, and pushed. -- Completed: - - Base messaging, NATS backend, producer, consumer, subscriber, - request/response, runtime factories, processor programs, flow specs, and - LLM service now use Effect-native boundaries, schema codecs, scoped - cleanup, and `S.TaggedErrorClass.make(...)` errors. - - CLI commands now run Effect programs at the command boundary, wrap socket - lifecycle with `Effect.acquireUseRelease`, encode JSON through Effect - Schema, and write output without `console.log`. - - MCP Effect server now loads env/config through `Config`, wraps gateway - calls with `Effect.tryPromise`, constructs schema classes with `.make`, and - uses tagged errors. - - MCP stdio compatibility server keeps `createMcpServer` and `run`, but uses - Effect callbacks/tryPromise/schema encoding internally. `run()` uses - `ManagedRuntime`; `runMain()` uses `NodeRuntime.runMain`. - - Flow stateful service launch sites now pass an explicit `Context.Context` - into the base processor runtime instead of hiding requirements behind - assertions. -- Verification: - - `cd ts && bun run check` - - `cd ts && bun run test` - - `cd ts && bun run build` - - `git diff --check` - -### 2026-06-02: ConfigService Ref-Backed State Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/config/service.ts` now models runtime state as a - `SynchronizedRef` instead of adding mutable - `store`, `version`, consumer, and producer fields onto the processor - object. - - Config operations have Effect-returning handlers with Promise facades only - on the exported compatibility methods. - - Request narrowing now uses `effect/Predicate` rather than request-record - type assertions. - - Persistence remains schema-backed and now reads/writes snapshots from the - ref-backed state. - - The consume loop now uses `Effect.whileLoop`; the remaining - `consumer.receive(2000)` call is a pubsub boundary for this service. - - Service startup now exposes `runMain()` through `NodeRuntime.runMain`. - The legacy `run()` Promise facade uses `ManagedRuntime`, and - `ts/scripts/run-config.ts` delegates directly to `runMain()` instead of - owning its own catch/process-exit wrapper. - - Config-service tests cover tagged invalid mutation errors, workspace - persistence, legacy load, concurrent ref-backed mutations, and push - publishing from the stored producer handle. -- Verification: - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - - `git diff --check` - -### 2026-06-02: ConfigService Operation Match Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/config/service.ts` now dispatches - `ConfigOperation` with `effect/Match` instead of a native `switch`. - - The dispatcher is a named `Effect.fn` and uses `Match.exhaustive` against - the schema-derived `ConfigOperation` union. - - The per-message response sender now uses `Effect.fnUntraced` instead of an - arrow function returning `Effect.gen(...)`. - - Config-service tests now cover all seven operations through - `handleOperation`, including tagged invalid mutation failures. - - Existing explicit `never` annotations on `persistEffect`, - `loadFromDiskEffect`, `persistStateEffect`, and - `readPersistedConfigEffect` were removed so Effect can infer the channel. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/config-service.test.ts` - - `cd ts && bun run check:tsgo` - -### 2026-06-02: Librarian Operation Match Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/librarian/service.ts` now dispatches librarian and - collection-management operations with `effect/Match` instead of native - `switch` statements. - - Both dispatchers intentionally use `Match.orElse` rather than - `Match.exhaustive` because raw broker message values can still contain - unknown runtime operations before a schema boundary rejects them. - - Existing tagged `LibrarianServiceError` operation labels are preserved for - promise, sync, Effect-helper, stream-only, and unknown-operation branches. - - Librarian-service tests now cover representative Match-backed librarian - dispatch paths, collection list/update/delete dispatch, and runtime - fallback errors without type assertions. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/librarian-service.test.ts` - - `cd ts && bun run check:tsgo` - -### 2026-06-04: FlowManager Operation Match Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/flow-manager/service.ts` now dispatches flow - operations with `effect/Match` instead of a native `switch`. - - The dispatcher keeps the existing config refresh behavior before routing - and uses `Match.orElse` because `FlowRequest.operation` is a public - wire-level `string`, not a closed schema literal union. - - Existing tagged `FlowManagerError` behavior is preserved for unknown - operations and branch-specific failures. - - Flow-manager tests now cover all eight flow operations through - `handleOperation`, including config-client blueprint mutations and the - runtime unknown-operation fallback. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/flow-manager-service.test.ts` - - `cd ts && bun run check:tsgo` - -### 2026-06-04: Client Connection State SubscriptionRef Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/client/src/socket/effect-rpc-client.ts` now owns RPC - connection state in `effect/SubscriptionRef` instead of a mutable state - variable plus manual listener `Set`. - - `ts/packages/client/src/socket/trustgraph-socket.ts` now bridges UI - connection-state listeners through `SubscriptionRef.changes` instead of a - hand-rolled listener array. - - Both public `subscribe` APIs preserve synchronous immediate replay and - unsubscribe compatibility while using Effect fibers for later updates. - - Client tests now drive a fake RPC state source to prove immediate replay, - connection updates, and unsubscribe behavior. -- Verification: - - `bun run --cwd ts/packages/client test -- src/__tests__/rpc-timeout.test.ts` - - `cd ts && bun run check:tsgo` - -### 2026-06-04: Client Callback Match Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/client/src/socket/trustgraph-socket.ts` now maps RPC - connection status with `effect/Match` instead of a native `switch`. - - Flow agent streaming chunk callbacks now use `effect/Match` for - `thought`, `observation`, `answer`, `final-answer`, and `action`, while - preserving ignored behavior for unknown chunk types with `Match.orElse`. - - Client RPC tests now drive agent stream chunks through the fake RPC stream - to prove callback dispatch, ignored fallback behavior, completion signals, - and metadata forwarding. -- Verification: - - `bun run --cwd ts/packages/client test -- src/__tests__/rpc-timeout.test.ts` - - `cd ts && bun run check:tsgo` - -### 2026-06-04: Gateway Term Service HashSet Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/gateway/dispatch/serialize.ts` now uses - `effect/HashSet` for static term-bearing request/response service - membership instead of native `Set`. - - Request and response translators preserve the same deep client/internal - term conversion behavior via `HashSet.has` membership checks. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/gateway-dispatcher.test.ts` - - `cd ts && bun run check:tsgo` - -### 2026-06-04: Effect AI Stream Part Match Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/model/text-completion/common.ts` now maps - `Response.StreamPart` values with `effect/Match` instead of a native - `switch`. - - The matcher handles `text-delta`, `finish`, and `error` explicitly, while - preserving ignored behavior for other valid stream parts with - `Match.orElse`. - - Text-completion common tests now include an ignored `text-start` stream - part before text deltas to prove the fallback path remains silent. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/text-completion-common.test.ts` - - `cd ts && bun run check:tsgo` - -### 2026-06-04: MCP Tool Transport Adapter Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/agent/mcp-tool/service.ts` no longer uses an - `as unknown as Parameters[0]` assertion when connecting - the MCP SDK client. - - The concrete `StreamableHTTPClientTransport` is wrapped in a small - `Transport` adapter that forwards callbacks, send, close, and protocol - version updates while avoiding the SDK's exact-optional `sessionId` type - mismatch. - - The transport release path still closes the concrete SDK transport. -- Verification: - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check:tsgo` - -### 2026-06-04: Agent Service Match Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/agent/react/service.ts` no longer uses native - `switch` for configured tool construction, live tool wiring, or ReAct - continuation parsing. - - Configured tool construction now uses `Effect.fn` plus `effect/Match` and - preserves unknown-tool logging/fallback behavior. - - The ReAct parser is exported for focused tests and uses an exhaustive - `Match` over continuation sections. - - Agent service tests cover configured-tool loading, unknown configured-tool - fallback, default descriptions, MCP tool args, continuation parsing, and - final-answer parsing. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/agent-service.test.ts` - - `cd ts && bun run check:tsgo` - -### 2026-06-04: CLI Library MIME Match Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/cli/src/commands/library.ts` now uses `effect/Match` for - extension-to-MIME mapping instead of a native `switch`. - - `guessMimeType` is exported for focused CLI helper coverage. - - CLI tests now cover known extensions, case normalization, `html`/`htm` - aliases, and fallback behavior for unknown or extensionless paths. - - `ts/packages/cli/package.json` excludes `dist/**` from Vitest so built - test output is not executed a second time after root build lanes. -- Verification: - - `bun run --cwd ts/packages/cli test -- src/__tests__/library.test.ts` - - `cd ts && bun run check:tsgo` - -### 2026-06-04: Streaming ReAct Parser Match Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/agent/react/parser.ts` now uses `effect/Match` for - ReAct state emission instead of a native `switch`. - - Marker states now use an inferred `MarkerState` type instead of per-entry - `as ReActState` assertions. - - Flush now processes an unterminated final buffer line through marker - detection, fixing `Final Answer:` chunks that arrive without a trailing - newline. - - Parser tests cover split markers, pre-marker thought fallback, - continuations, action input, and final-answer routing. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/agent-parser.test.ts` - - `cd ts && bun run check:tsgo` - -### 2026-06-02: RAG And Agent Requestor Bridge Slice - -- Status: migrated, root-verified, committed, and pushed. -- Completed: - - `ts/packages/flow/src/retrieval/graph-rag.ts` and - `ts/packages/flow/src/retrieval/document-rag.ts` now accept - `EffectRequestResponse` clients directly. The engines no longer adapt - Effect requestors back to Promise requestors and then wrap those calls in - `Effect.tryPromise`. - - `ts/packages/flow/src/retrieval/graph-rag-service.ts` and - `ts/packages/flow/src/retrieval/document-rag-service.ts` now pass native - flow requestors directly into the engines. - - `ts/packages/flow/src/agent/react/tools.ts` now accepts - `EffectRequestResponse` clients directly for graph RAG, document RAG, - triples, and MCP tool calls. Tool input narrowing uses Schema and - `effect/Predicate` rather than local request/response type assertions. - - `ts/packages/flow/src/agent/react/service.ts` wires default and configured - tools with native Effect requestors instead of `toPromiseRequestor`. - - Graph RAG, document RAG, and agent service startup now expose `runMain()` - through `NodeRuntime.runMain`; their legacy `run()` Promise facades use - `ManagedRuntime`. - - `ts/scripts/run-graph-rag.ts`, `ts/scripts/run-document-rag.ts`, and - `ts/scripts/run-agent.ts` now delegate to `runMain()`. -- Verification: - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - -### 2026-06-02: KnowledgeCore Ref-Backed State Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/cores/service.ts` now exposes a typed - `KnowledgeCoreService` instead of `AsyncProcessorRuntime & Record`. - - Runtime state now lives in - `SynchronizedRef` with `kgCores`, `deCores`, - the request consumer, and response producer. - - Knowledge operations now have Effect-returning handlers with Promise - facades only on exported compatibility methods. - - Persistence now decodes legacy and current snapshot shapes with Effect - Schema and encodes JSON through Schema rather than raw - `JSON.parse`/`JSON.stringify` plus assertions. - - The consume loop now uses `Effect.whileLoop`; the remaining - `consumer.receive(2000)` call is a pubsub boundary for this service. - - The service exposes `runMain()` through `NodeRuntime.runMain`; legacy - `run()` uses `ManagedRuntime`, and `ts/scripts/run-knowledge.ts` delegates - to `runMain()`. - - `ts/packages/base/src/schema/messages.ts` now models legacy hyphenated - knowledge request/response aliases so the service can preserve the wire - shape without response type assertions. - - New knowledge-core tests cover ref-backed mutation, graph embedding alias - responses, concurrent state updates, and legacy persistence loading. -- Verification: - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - -### 2026-06-02: KnowledgeCore Operation Match Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/cores/service.ts` now dispatches - `KnowledgeOperation` with `effect/Match` instead of a native `switch`. - - The dispatcher is a named `Effect.fn` and uses `Match.exhaustive` against - the schema-derived `KnowledgeOperation` union, so newly modeled operations - should surface as type/check failures until handled. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/knowledge-core-service.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-02: Flow Manager And Librarian Runtime Normalization - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/flow-manager/service.ts` and - `ts/packages/flow/src/librarian/service.ts` now expose `runMain()` through - `NodeRuntime.runMain`. - - Their legacy `run()` Promise facades now use `ManagedRuntime` instead of - directly owning `Effect.runPromise`. - - `ts/scripts/run-flow-manager.ts` and `ts/scripts/run-librarian.ts` now - delegate to `runMain()` instead of wrapping startup with local - `.catch(console.error/process.exit)` handlers. -- Verification: - - `bun run --cwd ts/packages/flow build` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: FlowManager Ref-Backed State Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/flow-manager/service.ts` now exposes a typed - `FlowManagerService` instead of `AsyncProcessorRuntime & Record`. - - Runtime state now lives in - `SynchronizedRef` with `flows`, `blueprints`, the - request consumer, response producer, and config request client. - - Flow operations now have Effect-returning handlers with Promise facades - only on exported compatibility methods. - - Blueprint config loading now narrows runtime values before constructing - `Blueprint` records, replacing the prior `parsed as Blueprint` shortcut. - - `start-flow` and `stop-flow` mutate the flow map through - `SynchronizedRef.modifyEffect`, making duplicate checks and map updates - atomic. - - The consume loop now uses `Effect.whileLoop`; the remaining - `consumer.receive(2000)` call is a pubsub boundary for this service. - - New flow-manager tests cover tagged errors, ref-backed flow mutation, - config push/delete requests, blueprint narrowing, duplicate concurrent - starts, and message-level flow-error responses. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/flow-manager-service.test.ts` - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: FlowManager Effect.fn Normalization Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/flow-manager/service.ts` no longer defines - reusable helpers as arrow functions that immediately return - `Effect.gen(...)`. - - Config request, blueprint refresh, flow refresh, blueprint handlers, flow - handlers, config push/delete, resource close, consume, run, and local - operation handling now use named `Effect.fn` providers. - - Hot local helpers for one-message consumption and response sending use - `Effect.fnUntraced`. - - `pushFlowsConfigEffect` keeps its best-effort logging/swallowing contract - through the `Effect.fn` pipeable form instead of a wrapper generator. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/flow-manager-service.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-02: Librarian Schema And Assertion Cleanup Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/base/src/schema/messages.ts` now models librarian upload and - stream request/response fields directly, instead of requiring service-side - `as LibrarianResponse` casts for the existing wire protocol. - - `ts/packages/flow/src/librarian/service.ts` now decodes persisted - librarian state through a concrete `S.fromJsonString` schema instead of a - generic JSON decode plus `as A`. - - Document metadata `metadata` triples now narrow through Schema decoding - with `Option` before being included in normalized metadata. - - Upload, stream, and complete-upload request/response constructors now rely - on the schema-modeled fields instead of local type assertions. - - New librarian tests cover modeled upload fields, concrete persisted-state - loading, and schema-backed metadata triple normalization. -- Remaining: - - Resolved by the typed runtime loop and ref-backed state slices below. -- Verification: - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/flow test -- src/__tests__/librarian-service.test.ts` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Librarian Tagged Operation Helper Slice - -- Status: migrated and root-verified. -- Completed: - - Removed the librarian `throwLibrarianServiceError` helper. - - `get-document-metadata`, `list-children`, `upload-chunk`, - `get-upload-status`, and `abort-upload` now dispatch through local - Effect-returning helpers that fail with `LibrarianServiceError`. - - Compatibility methods for those operations now return Promise facades - backed by `Effect.runPromise`. - - The librarian tests now await the Promise compatibility facade for upload - status. -- Remaining: - - Resolved by the typed runtime loop and ref-backed state slices below. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/librarian-service.test.ts` - - `bun run --cwd ts/packages/flow build` - - `cd ts && bun run check` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Librarian Typed Runtime Loop Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/librarian/service.ts` now exposes a typed - `LibrarianService` interface instead of `AsyncProcessorRuntime & - Record`. - - Service construction now uses `makeAsyncProcessor` - with `runEffect`; the old method-bag `run` override and - `as LibrarianService` cast are gone. - - The librarian startup poller now uses `Effect.whileLoop`. - - The local operation helpers retrieve the initialized service through an - Effect gate rather than closing over an unsafe partially built value. -- Remaining: - - Resolved by the ref-backed state slice below. -- Verification: - - `bun run --cwd ts/packages/flow build` - - `cd ts && bun run check` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Librarian Ref-Backed State Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/librarian/service.ts` now stores documents, - processing records, upload sessions, collection manager, and pubsub - handles in `SynchronizedRef`. - - Document, processing, upload, collection, persistence, load, and stop paths - now read snapshots or mutate cloned maps/managers through the ref instead - of writing fields on the service object. - - Upload chunk updates clone nested `UploadSession.chunks` before replacing - the upload map entry, avoiding mutable nested state hidden behind the ref. - - Librarian response producers and consumers are read/nullified through - ref-backed handles. -- Verification: - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/flow test -- src/__tests__/librarian-service.test.ts` - - `cd ts && bun run check` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Client RPC Managed Runtime Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/client/src/socket/effect-rpc-client.ts` now builds one - `ManagedRuntime` from the RPC client layer instead of manually creating a - `Scope`, building the layer, and calling `Effect.runPromise` for every - operation. - - RPC dispatch and stream dispatch continue to expose the existing - Promise-returning `EffectRpcClient` facade, but they run through the managed - runtime and close with `runtime.dispose()`. - - The Effect RPC socket path now consumes `Socket.layerWebSocketConstructorGlobal` - instead of a duplicate local WebSocket constructor layer. - - Dispatch payload construction now uses `DispatchPayload.make(...)` so - schema classes are not instantiated with `new`. - - Client socket logging and timestamp creation now use Effect `Logger` and - `Clock` instead of direct console and `Date.now()` calls in the touched - surface. -- Verification: - - `bun run --cwd ts/packages/client build` - - `cd ts && bun run check` - - `bun run --cwd ts/packages/client test` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Client WebSocket Adapter Error Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/client/src/socket/websocket-adapter.ts` now models host - fallback failures with `WebSocketAdapterError` via - `S.TaggedErrorClass`. - - Synchronous `getWebSocketConstructor()` and `getRandomValues()` facades - keep their public signatures while using `Result.try` instead of local - `try`/`catch` blocks. - - Runtime predicates now narrow WebSocket constructor modules and crypto - modules without the previous constructor/result type assertions. - - New adapter tests cover global WebSocket selection, optional `ws` - fallback, global crypto, typed crypto failure, and typed adapter errors. -- Verification: - - `bun run --cwd ts/packages/client build` - - `bun run --cwd ts/packages/client test -- src/__tests__/websocket-adapter.test.ts` - - `bun run --cwd ts/packages/client test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - - `git diff --check` - -### 2026-06-02: Client Socket Tagged Error And JSON Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/client/src/socket/trustgraph-socket.ts` now models socket API - failures with `TrustGraphSocketError` via `S.TaggedErrorClass`. - - Flow/blueprint JSON response parsing now uses Schema decoding through - `S.UnknownFromJsonString` instead of raw `JSON.parse`. - - Token-cost config JSON keeps the previous invalid-string fallback behavior - while decoding through Schema/Option. - - Connection-state listener isolation now uses `Result.try` and typed socket - errors instead of a local `try`/`catch`. - - Flow start, row embeddings, collection update, and response-error - failures now reject with tagged socket errors instead of normal `Error`. - - Flow API tests cover invalid JSON and response-error rejections. -- Verification: - - `bun run --cwd ts/packages/client build` - - `bun run --cwd ts/packages/client test -- src/__tests__/flows-api.test.ts` - - `cd ts && bun run check` - - `bun run --cwd ts/packages/client test` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Client Newable Factory Compatibility Decision - -- Status: documented no-op for the current loop. -- Evidence: - - `ts/packages/workbench/src/atoms/workbench.ts` constructs - `new BaseApi(...)`. - - `ts/packages/client/src/__tests__/flows-api.test.ts` constructs - `new FlowsApi(...)`, and sibling API facades expose the same constructor - shape. - - `EffectRpcClient` and `BaseApi` also preserve callable factory exports for - compatibility with the vendored TrustGraph client shape. -- Decision: - - The remaining `newableFactory(... ) as unknown as NewableFactory<...>` - assertions in client socket files are TypeScript compatibility boundaries, - not Effect error/requirement channel assertions and not replacements for an - Effect primitive. - - Removing them safely requires a deliberate public API redesign or explicit - class implementations for every API facade, not a local Effect-native - rewrite. -- Verification: - - Current client/root verification from the tagged error slice covers this - no-op decision. - -### 2026-06-02: Service Entrypoint Runtime Slice - -- Status: migrated and root-verified. -- Completed: - - Remaining flow service `run(): Promise` program facades now use - `ManagedRuntime.make(Layer.empty)` instead of direct - `Effect.runPromise(program)`. - - Remaining flow service modules now expose `runMain()` through - `NodeRuntime.runMain(program)`. - - Local `ts/scripts/run-*` launchers for gateway, prompt, chunker, - extractor, PDF decoder, embeddings, triples, graph/document embeddings, - text-completion providers, and MCP tool service now delegate directly to - `runMain()`. - - Direct `Effect.runPromise(program)` matches in `ts/packages/flow/src` are - clean. Remaining `Effect.runPromise` matches are callback/Promise - compatibility boundaries for later slices. -- Verification: - - `bun run --cwd ts/packages/flow build` - - `git diff --check` - - `cd ts && bun run check` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run build` - - `cd ts && bun run test` - -### 2026-06-02: Base Processor Compatibility Runtime Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/base/src/processor/async-processor.ts` now uses a - `ManagedRuntime` for Promise compatibility methods, signal-shutdown - execution, and legacy `AsyncProcessor.launch`. - - `ts/packages/base/src/processor/flow.ts` now owns a per-flow - `ManagedRuntime` for `start`, `stop`, `runInCompatibilityScope`, and - Promise resource facades. - - `ts/packages/base/src/processor/flow-processor.ts` now uses a - `ManagedRuntime` for the public `start(context)` facade instead of a local - `Effect.runPromiseWith` runner. - - `ts/packages/base/src/spec/parameter-spec.ts` now routes legacy `add` - through `flow.runInCompatibilityScope(...)`, matching the other specs. - - Subagent checks confirmed `NodeRuntime` is process-entrypoint-only here; - `@trustgraph/base` should not add an `@effect/platform-node` dependency - for these compatibility facades. -- Remaining: - - Constructor `as unknown as` shims in base processors preserve - callable-plus-newable public exports and are compatibility boundaries for - this loop. - - Typed string lookup casts in `Flow` need a real typed-spec/key redesign; - `HashMap`/`MutableHashMap` alone cannot infer `T` from a bare string. -- Verification: - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/base test` - - `cd ts && bun run check` - - `git diff --check` - - `cd ts && bun run build` - - `cd ts && bun run test` - -### 2026-06-02: Base Flow Definition Schema Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/base/src/processor/flow-processor.ts` now validates - `config.flows` with Effect Schema instead of local - `Predicate`/object/string-record guards. - - Invalid flow definition payloads still log/skip and preserve the existing - config-handler and acknowledgement behavior. - - `ts/packages/base/src/__tests__/flow-processor-runtime.test.ts` now covers - an invalid nested flow definition that is acknowledged without starting - resources. -- Verification: - - `bun run --cwd ts/packages/base test -- src/__tests__/flow-processor-runtime.test.ts` - - `bun run --cwd ts/packages/base build` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Text Completion Stream Sentinel Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/model/text-completion/{ollama,openai,mistral,azure-openai,claude,openai-compatible}.ts` - now return the `Stream.unfold` end sentinel with - `Effect.as(Effect.void, undefined)`. - - Removed six `Effect.void as Effect.Effect` assertions without - replacing them with `Effect.succeed(undefined)`, which `@effect/tsgo` - flags as a diagnostic. -- Verification: - - `bun run --cwd ts/packages/flow build` - - `cd ts && bun run check` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Text Completion Generator Boundary Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/model/text-completion/common.ts` now rejects - fallback `AsyncGenerator.throw(...)` calls with the mapped tagged provider - error directly instead of running `Effect.fail(...)` through - `Effect.runPromise`. - - The custom generator object no longer uses `as AsyncGenerator`, - `as Promise>`, or `as LlmChunk` assertions. - - Added a focused unit test for fallback throw mapping. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/text-completion-common.test.ts` - - `bun run --cwd ts/packages/flow build` - - `cd ts && bun run check` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Text Completion Provider Status Narrowing Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/model/text-completion/common.ts` now uses - `effect/Predicate` narrowing for provider `status` / `statusCode` - inspection instead of local record assertions. - - `ts/packages/flow/src/__tests__/text-completion-common.test.ts` covers - both rate-limit status fields. -- Verification: - - `bun run --cwd ts/packages/flow test -- src/__tests__/text-completion-common.test.ts` - - `bun run --cwd ts/packages/flow build` - - `cd ts && bun run check` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Base Parameter Spec Accessor Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/base/src/spec/parameter-spec.ts` now models - `ParameterSpec` with an Effect Schema codec. Legacy parameter specs - default to `S.Unknown`, preserving name-based registration while making - typed access schema-backed. - - `ts/packages/base/src/processor/flow.ts` now exposes - `flow.parameterEffect(spec)` and `flow.parameter(spec)` for inferred, - Schema-decoded parameter values. String lookup remains available as an - `unknown` compatibility escape instead of a caller-chosen generic type. - - Parameter schema failures now fail with the tagged - `FlowParameterDecodeError` rather than a normal `Error`. - - `ts/packages/flow/src/chunking/service.ts` now declares numeric chunk - parameters once and retrieves them through the typed spec-object accessor. - - `ts/packages/base/src/__tests__/flow-spec-runtime.test.ts` covers typed - parameter decoding, legacy string lookup, missing parameter errors, sync - accessor decoding, and schema mismatch errors. -- Remaining: - - Add typed spec-object accessors for producers and requestors so call sites - can stop spelling generic string lookups for those registries too. -- Verification: - - `bun run --cwd ts/packages/base test -- src/__tests__/flow-spec-runtime.test.ts` - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/flow build` - - `cd ts && bun run check` - - `bun run --cwd ts/packages/base test` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Base Producer And Requestor Spec Accessor Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/base/src/spec/producer-spec.ts` now exposes - `ProducerSpec.producerEffect(flow)` and stores typed producer handles in - a per-spec WeakMap keyed by `Flow`. - - `ts/packages/base/src/spec/request-response-spec.ts` now exposes - `RequestResponseSpec.requestorEffect(flow)` and stores typed - requestor handles in a per-spec WeakMap keyed by `Flow`. - - Spec finalizers remove only the exact handle they registered, avoiding - stale finalizers deleting newer registrations for the same flow/spec pair. - - `ts/packages/base/src/processor/flow.ts` now supports - `flow.producerEffect(spec)`, `flow.requestorEffect(spec)`, - `flow.producer(spec)`, and `flow.requestor(spec)` while keeping string - accessors as untyped compatibility escapes. - - Base service adapters and flow service handlers now reuse the same hoisted - producer/requestor spec object in their spec arrays and handler lookups. - - `ts/packages/base/src/__tests__/flow-spec-runtime.test.ts` covers typed - spec-object lookups, duplicate spec identity failures, and scoped - finalizer cleanup for producer and requestor handles. -- Remaining: - - Bare string `Flow` producer/requestor accessors remain compatibility - escapes for external/legacy callers, but new Effect service code should use - spec objects. -- Verification: - - `bun run --cwd ts/packages/base test -- src/__tests__/flow-spec-runtime.test.ts` - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/base test` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Native PubSub Boundary Slice - -- Status: migrated and package-verified. -- Completed: - - Confirmed Effect's native `PubSub` module is an in-process asynchronous hub - with scoped subscriptions, not a NATS/Pulsar-compatible broker boundary. - - Kept TrustGraph's `PubSubBackend` and `PubSub` service as the broker - adapter layer because it owns topics, broker producers/consumers, - acknowledgement, schema codecs, and backend lifecycle. - - Removed the unused legacy `ts/packages/base/src/messaging/subscriber.ts` - implementation, which duplicated in-process async queue/fanout behavior. - - Removed the corresponding `makeAsyncQueue`, `makeSubscriber`, - `Subscriber`, and `AsyncQueue` barrel exports from - `ts/packages/base/src/messaging/index.ts`. -- Remaining: - - Future in-process fanout or request-streaming code should use - `effect/PubSub`, `Queue`, `Stream.fromPubSub`, or `Channel.fromPubSub` - rather than adding another local async queue implementation. - - Do not replace `PubSubBackend` with `effect/PubSub` unless the code path is - explicitly local-only and does not need broker semantics. -- Verification: - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/base test` - - `cd ts && bun run check` - -### 2026-06-02: Gateway Streaming Callback Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/gateway/dispatch/manager.ts` now exposes - `dispatchGlobalServiceStreamingEffect` and - `dispatchFlowServiceStreamingEffect` so Effect callers can handle stream - chunks without Promise callback re-entry. - - The existing Promise-returning streaming methods remain as compatibility - facades and wrap responders with `Effect.tryPromise`. - - `ts/packages/flow/src/gateway/rpc-server.ts` now writes stream chunks into - the RPC queue through the dispatcher Effect path, removing the prior - `Effect.context` plus `Effect.runPromiseWith(context)` bridge. - - `ts/packages/client/src/socket/effect-rpc-client.ts` now uses - `Stream.runForEachWhile` for early stream termination instead of throwing a - synthetic `StopStreaming` tagged error. - - Gateway dispatcher tests now exercise both the Promise compatibility - streaming path and the Effect-native responder path. -- Remaining: - - `ts/packages/flow/src/gateway/rpc-protocol.ts` remains a Fastify socket - compatibility bridge, not a direct replacement target for Effect RPC - server layers yet. Local parser validation and collection cleanup inside - that bridge remain valid migration targets. -- Verification: - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/client build` - - `bunx --bun vitest run src/__tests__/gateway-dispatcher.test.ts` - - `bunx --bun vitest run src/__tests__/rpc-timeout.test.ts` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Client Streaming Facade Normalization Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/client/src/socket/trustgraph-socket.ts` now decodes the - legacy streaming envelope with Schema before service-specific callback - handling. - - Streaming payload reads now use `effect/Predicate` property narrowing - helpers instead of repeated response-wrapper assertions. - - Graph RAG, document RAG, text completion, prompt, agent, and document - stream callbacks now use a shared `streamComplete(...)` helper. The RPC - `DispatchStreamChunk.complete` bit is the default transport completion - source, with legacy service markers preserved for public compatibility. - - Explainability triples are decoded through a recursive Schema instead of - `as Triple[]`. - - The focused client test now proves normalized `DispatchStreamChunk` - completion flows through `graphRagStreaming` and final metadata. -- Verification: - - `bunx --bun vitest run src/__tests__/rpc-timeout.test.ts` - - `bun run --cwd ts/packages/client build` - - `cd ts && bun run check:tsgo` - - `bun run --cwd ts/packages/client test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Ollama Embeddings Effectful Layer Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/embeddings/ollama.ts` now exposes - `makeOllamaEmbeddingsEffect` for effectful config loading and service - construction. - - `OllamaEmbeddingsLive` now uses `Layer.effect` and maps config/load - failures into `EmbeddingsError` instead of preconstructing the service with - `Layer.succeed`. - - The direct `makeOllamaEmbeddings(config)` factory remains as a - compatibility facade, while the canonical `program` entrypoint preserves - the provider tagged error channel. - - Ollama response JSON parsing no longer uses a Promise type assertion. - - The focused embeddings tests now cover both direct factory use and the - effectful `OllamaEmbeddingsLive` layer. -- Verification: - - `bunx --bun vitest run src/__tests__/ollama-embeddings.test.ts` - - `bun run --cwd ts/packages/flow build` - - `cd ts && bun run check:tsgo` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - -### 2026-06-02: FalkorDB Scoped Client Lifecycle Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/storage/triples/falkordb.ts` and - `ts/packages/flow/src/query/triples/falkordb.ts` now model FalkorDB client - acquisition with `Effect.acquireRelease`. - - FalkorDB Live layers now use `Layer.effect` and own Redis client - disconnect finalizers through the layer scope. - - Direct Promise compatibility factories and direct service factories now - bracket each operation with scoped acquisition instead of hiding mutable - `Effect.cached` connection slots. - - Legacy `makeTriplesStoreService` and `makeTriplesQueryService` provider - hooks now acquire scoped FalkorDB services and map acquisition failures to - `ProcessorLifecycleError`; modern `program` entrypoints preserve the - FalkorDB tagged layer error type. - - FalkorDB query row field extraction now uses `effect/Predicate` narrowing - instead of record/string type assertions. - - New lifecycle tests use fake clients/graphs to prove connect on acquire - and disconnect on scope close for both triples store and triples query. -- Verification: - - `bunx --bun vitest run src/__tests__/falkordb-lifecycle.test.ts` - - `bun run --cwd ts/packages/flow build` - - `cd ts && bun run check` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Qdrant Config, Schema, And Fakeable Construction Slice - -- Status: migrated and root-verified. -- Completed: - - Added `ts/packages/flow/src/qdrant/client.ts` as the narrow fakeable - Qdrant surface used by graph/doc embedding store/query modules. - - Graph and document Qdrant store/query constructors now create clients - through `Effect.try`, load Qdrant config in Effect, and map config/client - failures into their existing `S.TaggedErrorClass` errors. - - Graph and document query payload extraction now uses - `Schema.decodeUnknownEffect(...).pipe(Effect.option)` and skips malformed - Qdrant payloads without type assertions. - - Qdrant graph/doc query Live layers and graph store Live layer now use - `Layer.effect` instead of preconstructing services with `Layer.succeed`. - - Legacy graph store/query/doc query processor providers now acquire Qdrant - services with named `Effect.fn` providers and map startup failures to - `ProcessorLifecycleError`. - - The installed Qdrant client still has no public close/disconnect method, - so no `Effect.acquireRelease` finalizer was added for Qdrant. -- Verification: - - `bunx --bun vitest run src/__tests__/qdrant-embeddings.test.ts` - - `bun run --cwd ts/packages/flow build` - - `cd ts && bun run check:tsgo` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Text Completion Provider Stream Helper Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/model/text-completion/common.ts` now exposes - `streamTextCompletionChunks`, an Effect-native helper built on - `Stream.fromAsyncIterable`, `Ref`, `Stream.filterMap`/`Result`, and a final - token chunk append. - - OpenAI, Azure OpenAI, OpenAI-compatible, Mistral, Ollama, and Claude - streaming providers now share the helper instead of each hand-rolling - `Stream.unfold` plus `iterator.next` loops. - - Mistral non-streaming and streaming content normalization now uses - `effect/Predicate` and `Option` narrowing through `textFromContent`, removing - the prior `content as string` assertion. - - The helper uses the installed Effect beta's `Option.fromNullishOr` and - Result-shaped `Stream.filterMap` API, verified by `check:tsgo`. - - `ts/packages/flow/src/__tests__/text-completion-common.test.ts` covers - token accumulation/final chunk emission and non-string content narrowing. -- Remaining: - - Full Effect AI provider swaps still need parity tests first; current OpenAI - and Azure behavior is Chat Completions based, no installed - Azure/Mistral/Ollama Effect AI provider exists, and Anthropic needs explicit - text/token/streaming/rate-limit parity coverage before replacement. -- Verification: - - `bunx --bun vitest run src/__tests__/text-completion-common.test.ts` - - `cd ts && bun run check:tsgo` - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Request-Response Queue Stream Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/base/src/messaging/runtime.ts` now waits for accepted - request-response replies by converting the response `Queue` to a - `Stream.fromQueue`. - - Recipient filtering now uses `Stream.filterMapEffect` with `Result` to skip - partial responses until the recipient returns `true`. - - `Stream.runHead` replaces the prior `while (true)`/`Queue.take` loop and - preserves the existing timeout behavior around request-response calls. - - `ts/packages/base/src/__tests__/messaging-runtime.test.ts` now covers - recipient filtering across partial and final responses. -- Remaining: - - The two remaining production `while (` matches are `agent/react/parser.ts` - line-buffer parsing and `cli/src/commands/util.ts` Commander parent - traversal; neither is async polling or resource ownership. -- Verification: - - `bunx --bun vitest run src/__tests__/messaging-runtime.test.ts` - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/base test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Gateway RPC WebSocket Cause Handling Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/flow/src/gateway/server.ts` now handles RPC WebSocket program - defects and interruptions inside the Effect pipeline with `Effect.sandbox`, - `Effect.catch`, and `Cause.pretty`. - - The previous Promise `.catch(...)` around `Effect.runPromise(...)`, plus the - nested `Effect.runPromise` used only for logging and socket close, is removed. - - The outer `Effect.runPromise` remains as the Fastify WebSocket host boundary. -- Verification: - - `cd ts && bun run check:tsgo` - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Client RPC Acquisition Cause Tap Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/client/src/socket/effect-rpc-client.ts` now observes - runtime/client acquisition failures with `Effect.tapCause` and - `Cause.pretty`. - - Removed the Promise `.catch(...)` that only updated local connection state - after `runtime.runPromise(TrustGraphRpcClientService)`. - - Removed the local `errorMessage` helper and its message-field assertion. - - Public `dispatch`, `dispatchStream`, and `close` Promise facades remain - compatibility boundaries. -- Verification: - - `cd ts && bun run check:tsgo` - - `bun run --cwd ts/packages/client build` - - `bun run --cwd ts/packages/client test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Client Socket Close Effect Boundary Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/client/src/socket/trustgraph-socket.ts` now wraps - `rpc.close()` with `Effect.tryPromise` inside the public `close(): void` - facade. - - Close failures are mapped to the existing tagged `TrustGraphSocketError` - shape and logged through `Effect.catch` instead of a Promise `.catch`. - - The remaining client socket Promise `.catch` matches were streaming - callback compatibility bridges and are now handled by the follow-up slice. -- Verification: - - `cd ts && bun run check:tsgo` - - `bun run --cwd ts/packages/client build` - - `bun run --cwd ts/packages/client test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Client Streaming Callback Effect Boundary Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/client/src/socket/trustgraph-socket.ts` now routes the - legacy `agent`, `graphRagStreaming`, and `documentRagStreaming` callback - request failures through `runLegacyStreamingRequest`. - - `runLegacyStreamingRequest` uses `Effect.tryPromise` to map failures into - tagged `TrustGraphSocketError` values, then uses `Effect.catch` to invoke - the public legacy error callback. - - Production `trustgraph-socket.ts` no longer has Promise `.catch` matches; - remaining matches in that file are `Effect.catch` only. - - Rechecked the PubSub replacement question against Effect v4 source: - Effect's native `PubSub` is an in-process async hub over Effect queues. - TrustGraph's `PubSubBackend` remains the broker adapter boundary for - NATS/Pulsar-style topics, subscriptions, acknowledgement, schema codecs, - and backend lifecycle. -- Verification: - - `cd ts && bun run check:tsgo` - - `bun run --cwd ts/packages/client build` - - `bun run --cwd ts/packages/client test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Text Completion Provider Effectful Layer Slice - -- Status: migrated and root-verified. -- Completed: - - Added shared `makeTextCompletionLayer` for constructing `Llm` from an - effectful `LlmProvider`. - - Added `makeOpenAIProviderEffect`, - `makeOpenAICompatibleProviderEffect`, `makeAzureOpenAIProviderEffect`, - `makeClaudeProviderEffect`, `makeMistralProviderEffect`, and - `makeOllamaProviderEffect`. - - Processor `program.layer` definitions now use `Layer.effect` via the - shared helper instead of constructing providers inside `Layer.succeed`. - - Provider object assembly is split into pure `makeXProviderFromClient` - helpers so Promise-returning provider methods remain external - compatibility facades and do not trigger `effect(runEffectInsideEffect)`. - - Added tests for explicit provider config, shared `Llm` layer provisioning, - and tagged missing-config errors. -- Verification: - - `bunx --bun vitest run src/__tests__/text-completion-providers.test.ts src/__tests__/text-completion-common.test.ts` - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Gateway Dispatcher Ownership And Serialization Slice - -- Status: migrated and root-verified. -- Completed: - - `makeDispatcherManager` now tracks whether it owns the pubsub backend and - no longer closes injected `PubSubBackend` instances on `stop()`. - - `publishToTopic` now uses `Effect.acquireUseRelease` so the one-shot - producer is closed even when `send` fails. - - Gateway dispatch paths now call `translateRequestEffect` and - `translateResponseEffect`, which wrap serialization with `Effect.try` and - return tagged `DispatchSerializationError` failures. - - Streaming dispatch recipients are named `Effect.fn` callbacks, satisfying - strict Effect diagnostics while preserving responder behavior. - - Tests cover injected backend ownership, typed serialization failure before - requestor startup, and producer close on send failure. -- Verification: - - `bunx --bun vitest run src/__tests__/gateway-dispatcher.test.ts` - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Base Producer Scoped Runtime Slice - -- Status: migrated and root-verified. -- Completed: - - Kept `PubSubBackend` as the broker adapter boundary; Effect native - `PubSub` remains an in-process primitive and is not a replacement for - broker-backed topics, subscriptions, acknowledgements, codecs, or backend - lifecycle. - - Reworked `makeProducer` so the legacy Promise facade allocates producers - through `makeEffectProducerFromPubSub` inside a closeable `Scope`. - - `stop()` now flushes the Effect producer and closes the scope with the - registered producer finalizer, including the flush-failure path. - - Added focused producer facade coverage for send routing, idempotent stop, - tagged not-started lifecycle errors, and close-on-flush-failure behavior. -- Verification: - - `cd ts && bun run check:tsgo` - - `bun run --cwd ts/packages/base build` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/producer.test.ts` - - `bun run --cwd ts/packages/base test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - -### 2026-06-02: NATS Typed Boundary Slice - -- Status: migrated and root-verified. -- Completed: - - Replaced the dynamic header import inside `makeNatsProducer` with the - static NATS `headers` export. - - Wrapped publish header construction in `Effect.try`, so invalid header - names/values fail as tagged `PubSubError` values instead of defects. - - Wrapped NATS `ack()` and `nak()` calls in `Effect.try`, preserving the - existing wrong-message guard and mapping thrown acknowledgement failures - into tagged `PubSubError`s. - - Added a mocked NATS backend test covering invalid publish headers and - thrown ack/nak failures through the public `makeNatsBackend` path. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/nats-backend.test.ts` - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/base test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - -### 2026-06-02: NATS Selective 404 Slice - -- Status: migrated and root-verified. -- Completed: - - Added an internal `NatsLookupError` tagged error to preserve lookup causes - without leaving `unknown` in Effect error channels. - - Stream creation now happens only when `manager.streams.info()` fails with - a NATS JetStream 404/missing-resource error. - - Durable consumer creation now happens only when `js.consumers.get()` fails - with a NATS JetStream 404/missing-resource error. - - Added mocked NATS tests proving 404 lookups create resources while - permission-style lookup failures do not. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/nats-backend.test.ts` - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/base test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - -### 2026-06-02: Consumer Rate-Limit Retry Slice - -- Status: migrated and root-verified. -- Completed: - - Added `rateLimitTimeoutMs` to the Effect-native messaging runtime config, - backed by `TG_RATE_LIMIT_TIMEOUT_MS` and the Python-compatible default of - `7_200_000ms`. - - Reworked legacy `makeConsumer` retry handling to use `Schedule.spaced`, - retry repeated `TooManyRequestsError`s, and fail with a tagged - `MessagingTimeoutError` when the rate-limit timeout elapses. - - Reworked `makeEffectConsumerFromPubSub` handler retry handling with the - same schedule/timeout behavior while keeping handler failures in typed - Effect error channels. - - Added legacy and Effect-native tests for repeated rate-limit retry until - success and negative acknowledgement after retry timeout. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/consumer.test.ts src/__tests__/messaging-runtime.test.ts` - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/base test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - -### 2026-06-02: Consumer Concurrency Ownership Slice - -- Status: migrated and root-verified. -- Completed: - - `makeEffectConsumerFromPubSub` now creates one backend consumer per - concurrency worker rather than sharing a single backend consumer across - parallel receive loops. - - Consumer runtime `stop` is idempotent via `Ref.getAndSet`, so explicit - `consumer.stop` and scope finalization do not double-close worker handles. - - Added Effect-native runtime coverage proving `concurrency: 3` creates and - closes three independent backend consumers exactly once. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/messaging-runtime.test.ts` - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/base test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - -### 2026-06-02: Request-Response Stop Signal Slice - -- Status: migrated and root-verified. -- Completed: - - `makeEffectRequestResponseFromPubSub` now owns a `Deferred` stop signal for - the lifetime of the scoped request-response runtime. - - `request()` races response waiting against that stop signal before applying - the request timeout, so pending calls fail promptly when the runtime stops. - - `stop()` fails the stop signal with a tagged `MessagingLifecycleError` - before interrupting the dispatch loop and closing the producer/consumer - resources. - - Flow PDF decoder and graph embeddings service error unions now include - `MessagingLifecycleError` because requestor failures can surface shutdown. - - Added Effect-native runtime coverage proving a pending request fails with - the tagged lifecycle error when the request-response runtime stops. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/messaging-runtime.test.ts` - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/base test` - - `bun run --cwd ts/packages/flow build` - - `bun run --cwd ts/packages/flow test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - -### 2026-06-02: Legacy Consumer Facade Slice - -- Status: migrated and root-verified. -- Completed: - - `makeConsumer` is now a Promise compatibility facade over - `makeEffectConsumerFromPubSub` instead of owning a separate mutable - backend, `running` flag, retry loop, and direct `BackendConsumer.receive`. - - The facade uses a module `ManagedRuntime`, `Scope.make`, `Scope.provide`, - and `Scope.close` to keep public `start()`/`stop()` Promises at the - boundary while the actual consumer lifetime stays scoped. - - Legacy Promise handlers are adapted with `Effect.tryPromise` and preserve - `TooManyRequestsError` as a typed retry signal. - - `makeEffectConsumerFromPubSub` now forks workers with `Effect.forkScoped`, - so a caller-owned scope keeps workers alive until stop/finalization. - - `consumer.test.ts` no longer encodes `start()` as the blocking consume-loop - join; it waits for observable handler/ack/nak effects and then stops the - scoped consumer. -- Verification: - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/consumer.test.ts` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/messaging-runtime.test.ts src/__tests__/flow-spec-runtime.test.ts` - - `cd ts && bun run check:tsgo` - - `bun run --cwd ts/packages/base build` - - `bun run --cwd ts/packages/base test` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `git diff --check` - -### 2026-06-02: Workbench Theme KeyValueStore Slice - -- Status: migrated and root-verified. -- Completed: - - `themeClassAtom` no longer writes the legacy `tg-theme` localStorage - mirror. Theme persistence stays in the canonical - `trustgraph-workbench-theme-v1` `Atom.kvs` entry backed by - `BrowserKeyValueStore.layerLocalStorage`. - - The pre-paint host script in `index.html` now restores the canonical - JSON-encoded theme key before React mounts, with `tg-theme` retained only - as a legacy fallback. - - Workbench QA now asserts that changing theme writes the canonical key and - does not recreate `tg-theme`. -- Verification: - - `cd ts && bun run check` - - `cd ts && bun run workbench:qa` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - -### 2026-06-02: Effect AI LanguageModel Adapter Slice - -- Status: migrated and package-verified. -- Completed: - - Added `makeLanguageModelProvider`, a bridge from - `effect/unstable/ai/LanguageModel` into the existing TrustGraph - `LlmProvider` contract. - - Covered non-streaming text/token mapping, streaming text/final-token - mapping, and Effect AI rate/quota failure mapping with fake - `LanguageModel` tests. - - Kept concrete provider swaps deferred until provider-specific parity is - proven. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/text-completion-common.test.ts src/__tests__/text-completion-providers.test.ts` - -### 2026-06-02: Native Request/Response PubSub Fanout Slice - -- Status: migrated and package-verified. -- Completed: - - Replaced the request/response runtime's hand-managed - `Map` response fanout with native `effect/PubSub`. - - Each request subscribes before sending, consumes through - `Stream.fromSubscription`, filters by response id, and releases the - subscription at scope exit. - - Kept `PubSubBackend` as the broker boundary because Effect native PubSub is - in-process only and does not provide NATS topics, ack/nack, durable - subscriptions, schema codecs, or backend lifecycle. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/messaging-runtime.test.ts src/__tests__/request-response.test.ts src/__tests__/flow-spec-runtime.test.ts` - -### 2026-06-02: Claude Effect AI Provider Slice - -- Status: migrated and package-verified. -- Completed: - - Replaced the direct `@anthropic-ai/sdk` provider implementation with - `@effect/ai-anthropic` `AnthropicLanguageModel` plus the shared - `makeLanguageModelProvider` adapter. - - Preserved `CLAUDE_KEY`, default model, temperature, max output, processor, - and public `LlmProvider` compatibility behavior. - - Added Claude provider config coverage for `CLAUDE_KEY` env fallback and the - missing-key tagged config error. - - Removed the direct `@anthropic-ai/sdk` dependency and its lockfile entries - from `@trustgraph/flow`. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/text-completion-providers.test.ts src/__tests__/text-completion-common.test.ts` - - `cd ts/packages/flow && bun run build` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-02: RPC Dispatch Tagged Error Slice - -- Status: migrated and package-verified. -- Completed: - - Replaced the remaining production `S.ErrorClass` usage in the Flow gateway - RPC contract with `S.TaggedErrorClass`. - - Normalized the client-side RPC `DispatchError` counterpart to the same - tagged-error schema shape so both wire contract copies stay aligned. - - Closed the scratch-note `S.ErrorClass` finding for production code. The - remaining plain `new Error` matches are test-only helpers or external - host-boundary simulations. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/gateway-dispatcher.test.ts` - - `cd ts/packages/client && bunx --bun vitest run src/__tests__/rpc-timeout.test.ts` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-02: Effect Metrics Prometheus Slice - -- Status: migrated and root-verified. -- Completed: - - Replaced the `@trustgraph/base` `prom-client` metric wrappers with - Effect-native `Metric.counter`, `Metric.histogram`, and - `effect/unstable/observability` `PrometheusMetrics.format`. - - Kept the existing Prometheus metric names and gateway - `/api/v1/metrics` scrape boundary while removing the direct `prom-client` - dependency and lockfile entries. - - Changed producer metric recording from a sync callback to an Effect value - that runs inside the producer send pipeline. - - Added isolated metric-registry tests for producer and consumer Prometheus - formatting. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/metrics-effect.test.ts src/__tests__/producer.test.ts src/__tests__/messaging-runtime.test.ts` - - `cd ts/packages/base && bun run build` - - `cd ts/packages/base && bun run test` - - `cd ts/packages/flow && bun run build` - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/gateway-dispatcher.test.ts` - - `cd ts && bun run check` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-02: MCP Effect Stdio Entrypoint Slice - -- Status: migrated and root-verified. -- Completed: - - Added an Effect-native stdio layer and process entrypoint with - `McpServer.layerStdio`, `NodeStdio.layer`, and `NodeRuntime.runMain`. - - Reused the same `TrustGraphMcpToolkitLive` path for HTTP and stdio through - a shared toolkit-layer helper. - - Kept the legacy SDK/Zod stdio export as a compatibility surface until - protocol-level `tools/list` and `tools/call` parity tests prove it can be - flipped or removed. - - Added focused coverage that the Effect toolkit names remain stable and - the stdio layer/entrypoint are exported. - - Updated the MCP test script to ignore compiled `dist/**` output so root - builds do not cause duplicate Vitest runs from generated tests. -- Scratch-note triage: - - Metrics, in-process PubSub fanout, Claude Effect AI, RPC - `S.TaggedErrorClass`, and `@effect/tsgo` setup are already migrated. - - Scratch targets migrated or parked since this triage include MCP tool-name - parity, Duration config cleanup, Term/ClientTerm tagged-union matching, - service `Effect.fn` normalization, chunking `Chunk`, and long-lived - `Map`/`Set` state. Remaining follow-ups are future API design or - compatibility decisions: MCP tool-call parity before deleting legacy stdio, - `effect/unstable/cli`, stream/RPC public facades, and cores Promise APIs. -- Verification: - - `cd ts/packages/mcp && bun run test` - - `cd ts/packages/mcp && bun run build` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-02: Term And ClientTerm Match Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/base/src/schema/primitives.ts` now exposes `Term` as a - recursive `S.toTaggedUnion("type")` schema and aligns `Triple.g` with the - Python/client wire contract as an optional graph string. - - `ts/packages/flow/src/gateway/dispatch/serialize.ts` now defines compact - client-term schemas with `S.tag`, decodes unknown term-shaped values with - Schema/Option, and translates terms with - `Match.discriminatorsExhaustive`. - - Removed the gateway serializer's native term switches and unsafe - pass-through casts. Malformed known-tag objects now stay ordinary payload - objects during deep translation instead of being cast into invalid terms. - - Replaced pure term helper switches in FalkorDB triples store/query, - Qdrant graph embeddings store, Graph RAG, agent tools, and workbench graph - utilities with exhaustive `Match` discriminators. - - Added tests for named graph string decoding, nested compact/internal term - round trips, and malformed known-tag payload preservation. -- Remaining: - - The client socket streaming term schema is still a local recursive union - and can be centralized later if drift appears. It has no native term - switch in the current scan. - - Operation dispatch switches in config, cores, librarian, and flow-manager - are separate service-command refactors, not part of this term wire slice. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/schema-effect.test.ts` - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/gateway-dispatcher.test.ts src/__tests__/falkordb-lifecycle.test.ts src/__tests__/qdrant-embeddings.test.ts src/__tests__/retrieval-rag.test.ts` - - `cd ts/packages/flow && bun run test` - - `cd ts/packages/base && bun run build` - - `cd ts/packages/flow && bun run build` - - `cd ts/packages/workbench && bun run build` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-02: Messaging Runtime Duration Config Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/base/src/runtime/messaging-config.ts` now stores internal - runtime timing fields as `Duration.Duration` instead of number-shaped - millisecond fields. - - `loadMessagingRuntimeConfig()` reads `TG_*` timing env vars with - `Config.duration` while preserving legacy bare-number millisecond env - values through a `Config.number(...).map(Duration.millis)` fallback. - - `ts/packages/base/src/messaging/runtime.ts` now uses `Duration` values - directly for sleeps, retry schedules, and timeout options. It converts back - to milliseconds only at the broker `receive(timeoutMs)` boundary and for - existing timeout error payloads. - - Public compatibility options such as `receiveTimeoutMs`, - `rateLimitRetryMs`, `rateLimitTimeoutMs`, and request `timeoutMs` remain - numeric millisecond inputs. - - Tests now cover both legacy numeric env values and native Effect duration - strings. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/schema-effect.test.ts src/__tests__/messaging-runtime.test.ts src/__tests__/consumer.test.ts` - - `cd ts/packages/base && bun run build` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-02: Qdrant MutableHashSet Cache Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/storage/embeddings/qdrant-graph.ts` now uses - `MutableHashSet` for its known-collection cache. - - `ts/packages/flow/src/storage/embeddings/qdrant-doc.ts` now uses - `MutableHashSet` inside its store effect shape instead of a native - `Set`. - - Collection cache membership, insertion, and invalidation now use - `MutableHashSet.has`, `MutableHashSet.add`, and - `MutableHashSet.remove`. - - Qdrant tests now prove the graph store service avoids repeated collection - existence checks while the collection is cached and checks again after - delete invalidates the cache. -- Remaining: - - The document store's public direct facade still constructs a fresh store - effect per operation, so its cache lifetime is per constructed effect - shape. Changing that facade to own a shared runtime/service is a separate - lifecycle design slice, not part of this collection primitive migration. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/qdrant-embeddings.test.ts` - - `cd ts/packages/flow && bun run build` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Workbench Match Dispatch Slice - -- Status: migrated and root-verified. -- Completed: - - `ts/packages/workbench/src/qa/mock-api.ts` now dispatches mock RPC - service requests with `effect/Match` and keeps the existing unknown-service - `{}` fallback. - - `ts/packages/workbench/src/atoms/workbench.ts` now dispatches chat submit - behavior for `graph-rag`, `document-rag`, and `agent` through - `Match.exhaustive` over the `ChatMode` union. - - The strict scan for native `switch` statements in `ts/packages` is clean: - `rg -n "\bswitch\s*\(" ts/packages --glob '*.ts' --glob '*.tsx'` returns - no matches. -- Verification: - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Request-Response Stop Ref Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/base/src/messaging/runtime.ts` now tracks - `makeEffectRequestResponseFromPubSub` stop idempotence with `Ref` instead - of local mutable state. - - The request-response runtime now matches the consumer runtime stop pattern: - `Ref.getAndSet` gates shutdown, signal failure, PubSub shutdown, fiber - interruption, producer close, and backend consumer close. - - `ts/packages/base/src/__tests__/messaging-runtime.test.ts` now asserts that - explicit request-response stop plus scoped finalization closes the producer - and response consumer exactly once. -- Verification: - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/messaging-runtime.test.ts src/__tests__/request-response.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Config Service HashMap State Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/config/service.ts` now stores workspace and - namespace config state in nested `HashMap` values inside the existing - `SynchronizedRef`. - - Put/delete operations now update immutable `HashMap` snapshots with - `HashMap.set` and `HashMap.remove` instead of cloning and mutating native - `Map` instances. - - Persistence, config dump, list, and get-values handlers keep plain - `Record`/array shapes only at API and JSON boundaries, with deterministic - ordering applied while converting out of `HashMap`. - - The focused scan for native map state in `config/service.ts` is clean; the - remaining map matches in that file are `HashMap` and `SynchronizedRef` - operations. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/config-service.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: KnowledgeCore Effect.fn Helper Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/cores/service.ts` now defines its reusable - generator helpers with `Effect.fn` or `Effect.fnUntraced` instead of - arrow functions returning `Effect.gen`. - - `sendResponse` and `consumeOnceEffect` use `Effect.fnUntraced` because - they are small hot-path helper functions. - - `readPersistedKnowledgeEffect`, `persistStateEffect`, - `closeKnowledgeResourcesEffect`, and `runKnowledgeCoreServiceEffect` use - named `Effect.fn` wrappers while preserving their existing catch/logging - behavior. - - The focused scan for `=> Effect.gen` in `cores/service.ts` is clean. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/knowledge-core-service.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Config Service Effect.fn Helper Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/config/service.ts` now defines its reusable - generator helpers with `Effect.fn` or `Effect.fnUntraced` instead of - arrow functions returning `Effect.gen`. - - `consumeOnceEffect` uses `Effect.fnUntraced`; persistence, push, read, - put/delete, close, and run helpers use named `Effect.fn` wrappers. - - Existing persistence read/write catch/logging behavior is preserved through - `Effect.fn` post-processing functions. - - The focused scan for config-service helper `=> Effect.gen` patterns is - clean. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/config-service.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: NATS MutableHashSet Stream Cache Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/base/src/backend/nats.ts` now tracks initialized stream names - in `MutableHashSet` instead of a native `Set`. - - `ensureStream` uses `MutableHashSet.has` and `MutableHashSet.add` while - preserving the NATS broker adapter boundary and the selective 404 - create-on-missing behavior. - - `ts/packages/base/src/__tests__/nats-backend.test.ts` now proves repeated - producer/consumer creation for subjects in the same stream only performs - one stream lookup. -- Verification: - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/nats-backend.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: KnowledgeCore HashMap State Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/cores/service.ts` now stores knowledge-core and - document-core state in `HashMap` inside the existing `SynchronizedRef`. - - Put/delete/load/get operations now read and update immutable `HashMap` - snapshots with `HashMap.get`, `HashMap.set`, `HashMap.remove`, and - `HashMap.has`. - - Persistence and list-response helpers convert `HashMap` state to - deterministic sorted records/arrays only at the API/JSON boundaries. - - `ts/packages/flow/src/__tests__/knowledge-core-service.test.ts` now reads - service state through `HashMap.get` and `Option`. - - The focused scan for native map state in `cores/service.ts` is clean. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/knowledge-core-service.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Librarian Effect.fn Helper Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/librarian/service.ts` now defines its reusable - librarian generator helpers with `Effect.fn` or `Effect.fnUntraced` - instead of arrow functions returning `Effect.gen`. - - `consumeOnceEffect` and both per-message response senders use - `Effect.fnUntraced` for small hot-path callbacks. - - `runLibrarianServiceEffect`, `getDocumentMetadataEffect`, - `listChildrenEffect`, `uploadChunkEffect`, `getUploadStatusEffect`, and - `abortUploadEffect` use named `Effect.fn` wrappers while preserving their - tagged-error behavior and Promise compatibility method surfaces. - - The focused scan for librarian-service helper `=> Effect.gen` patterns is - clean. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/librarian-service.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: FalkorDB Effect.fn Helper Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/query/triples/falkordb.ts` now defines reusable - connection, pattern-match, and query helper programs with `Effect.fn` or - `Effect.fnUntraced` instead of arrow functions returning `Effect.gen`. - - `ts/packages/flow/src/storage/triples/falkordb.ts` now uses a named - `Effect.fn` for its connection helper, matching the existing scoped - acquisition/finalizer boundary. - - The public Promise facades and scoped layer acquisition behavior are - unchanged; this slice only removes repeated helper generator construction. - - The focused scan for FalkorDB helper `=> Effect.gen` patterns is clean. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/falkordb-lifecycle.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: FlowManager HashMap State Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/flow-manager/service.ts` now stores long-lived - flow and blueprint state in immutable `HashMap` snapshots inside the - existing `SynchronizedRef`. - - Refresh, start, stop, delete, list, get, and config-push paths now use - `HashMap.get`, `HashMap.has`, `HashMap.set`, `HashMap.remove`, and - `HashMap.size` instead of cloned native `Map` state. - - List responses and config-push iteration convert `HashMap` state through - sorted entries only at the API/config boundary for deterministic output. - - `ts/packages/flow/src/__tests__/flow-manager-service.test.ts` now reads - service state through `HashMap.get` plus `Option` and keeps the test fake - pubsub `Map` as a compatibility fixture. - - The focused scan for native flow-manager map state is clean. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/flow-manager-service.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Gateway RPC Protocol Decode Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/gateway/rpc-protocol.ts` no longer casts - `parser.decode(data)` to `ReadonlyArray`. - - The socket protocol now validates parser output through a local Schema for - client-to-server `Request`, `Ack`, `Interrupt`, `Ping`, and `Eof` - envelopes before calling the server protocol write hook. - - Invalid client frames and parser failures now map to a tagged - `RpcProtocolDecodeError` and are written back as encoded RPC defects. - - The existing server-response defect encoding no longer uses a non-null - assertion on `parser.encode`. - - `ts/packages/flow/src/__tests__/gateway-rpc-protocol.test.ts` covers valid - request header prepending, control-frame forwarding, and rejection of - server response envelopes received from a client socket. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/gateway-rpc-protocol.test.ts` - - `cd ts/packages/flow && bunx --bun vitest run` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Gateway RPC Protocol Mutable Collections Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/gateway/rpc-protocol.ts` now tracks active socket - clients with `MutableHashMap` and `MutableHashSet` instead of native - `Map` / `Set` closure state. - - The protocol still returns a native `ReadonlySet` snapshot at the - `RpcServer.Protocol.clientIds` API boundary because that is the Effect RPC - protocol contract. - - `ts/packages/flow/src/__tests__/gateway-rpc-protocol.test.ts` now covers - server response sends through the registered client and the public - `clientIds` snapshot. - - The focused scan for native map/set state in `gateway/rpc-protocol.ts` is - clean. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/gateway-rpc-protocol.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Retrieval RAG Effect.fn Helper Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/retrieval/document-rag.ts` now defines its reusable - document RAG query program with `Effect.fn` instead of a function returning - `Effect.gen`. - - `ts/packages/flow/src/retrieval/graph-rag.ts` now defines the graph RAG - query, concept extraction, vector lookup, entity lookup, edge traversal, - edge scoring, and synthesis helpers with named `Effect.fn` providers. - - Public Promise facades and requestor compatibility surfaces are unchanged. - - The focused scan for retrieval `return Effect.gen(...)` helper patterns is - clean. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/retrieval-rag.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Ollama Embeddings Effect.fn Helper Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/embeddings/ollama.ts` now defines the `embed` - service method as a named `Effect.fn` generator instead of returning a - nested `Effect.gen` from an `Effect.fn` callback. - - Empty embedding requests now return through the same generator path without - allocating a separate `Effect.succeed` helper. - - The public sync factory, effectful layer, `ManagedRuntime` facade, and - `NodeRuntime.runMain` entrypoint are unchanged. - - The focused scan for Ollama `return Effect.gen(...)` helper patterns is - clean. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/ollama-embeddings.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Base Processor Effect.fn Helper Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/base/src/processor/flow.ts` now exposes `startEffect` and - `stopEffect` as Effect values with spans instead of lazy zero-argument - Effect methods. - - `Flow.runInCompatibilityScopeEffect` is now a named `Effect.fn` helper - instead of a method returning nested `Effect.gen`. - - The public `Flow` shape is explicit so Effect error and requirement - channels do not widen through recursive object inference. - - Producer, request/response, and parameter specs are generic over the - receiving flow requirements, so reusable specs remain mountable in flows - with additional service dependencies without type assertions. - - `ts/packages/base/src/processor/flow-processor.ts` now defines - `startFlow`, `configureFlows`, `processNextConfigPush`, the safe consume - wrapper, and the Promise `start` compatibility helper with named - `Effect.fn` functions. - - The focused scan for callable `return Effect.gen(...)` helpers in - `flow.ts` is clean. Remaining `flow-processor.ts` matches are one-shot - program values / exported program factories and should be scoped - separately from reusable helper normalization. -- Verification: - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/flow-spec-runtime.test.ts src/__tests__/flow-processor-runtime.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Chunking Chunk Collection Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/chunking/recursive-splitter.ts` now returns - `Chunk.Chunk` and uses `effect/Chunk` for splitter result, merge, - recursive append, and overlap collections. - - The chunking service behavior is unchanged; `Chunk` is iterable and still - provides `length` for logging and output counting. - - Splitter and service tests convert `Chunk` results to readonly arrays only - at assertion boundaries. - - The focused chunking scan no longer has array-backed splitter result state. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/recursive-splitter.test.ts src/__tests__/chunking-service.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Workbench Random ID Effect.fn Helper Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/workbench/src/atoms/workbench.ts` now defines the reusable - random id helper with named `Effect.fn` instead of a function returning - `Effect.gen`. - - Existing notification, upload, chat request, and message id call sites are - unchanged and still yield the helper directly. - - The remaining workbench `Effect.gen` scan hit is the local - `submitUploadDocument` one-shot effect value inside an existing - `Effect.fn` command, not a reusable helper factory. -- Verification: - - `cd ts/packages/workbench && bun run build` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Librarian Collection Manager MutableHashMap Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/librarian/collection-manager.ts` now stores - in-memory collection entries in `MutableHashMap` instead of native `Map`. - - Public `listCollections`, `getCollection`, `updateCollection`, - `deleteCollection`, `ensureCollectionExists`, `toJSON`, and `loadFromJSON` - behavior stays array/boolean/undefined based at the API and persistence - boundaries. - - New focused coverage verifies create/update/list/get/delete and JSON - restore behavior through the migrated collection state. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/collection-manager.test.ts src/__tests__/librarian-service.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Prompt Template MutableHashMap Cache Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/prompt/template.ts` now stores loaded prompt - templates in `MutableHashMap` instead of a native `Map`. - - Config reload clears and repopulates the Effect collection, request lookup - narrows through `Option`, and logging uses `MutableHashMap.size` / `keys`. - - New focused coverage verifies a config-loaded prompt template renders - through the service request/response flow. -- Verification: - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/prompt-template.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Workbench Explain Triples MutableHashMap Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/workbench/src/atoms/workbench.ts` now stores the - `explainTriplesAtom` input cache in `MutableHashMap` instead of a native - `Map`. - - Lookup narrows through `Option`, writes use `MutableHashMap.set`, and the - adjacent `Triple[]` assertions were replaced with callback return types and - inferred array copies. -- Verification: - - `cd ts && bun run --cwd packages/workbench build` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Base Processor Registry MutableHashMap Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/base/src/processor/flow.ts` now stores producers, consumers, - requestors, and parameters in `MutableHashMap` instead of native `Map`. - - `ts/packages/base/src/processor/flow-processor.ts` now tracks active flow - scopes in `MutableHashMap`, including `Option`-based lookups and - `MutableHashMap.clear` / `remove` / `size` / `set` for lifecycle changes. -- Verification: - - `cd ts && bun run --cwd packages/base build` - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/flow-processor-runtime.test.ts src/__tests__/flow-spec-runtime.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Librarian Service State MutableHashMap Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/librarian/service.ts` now stores live documents, - processing records, uploads, and per-upload chunks in `MutableHashMap` - instead of native `Map`. - - State updates continue to clone the current collection before swapping it - into `SynchronizedRef`, but now use `MutableHashMap.fromIterable`, - `Option`-based lookups, and `MutableHashMap.set` / `remove` / `size` / - `keys` / `values` for service-state behavior. - - Persistence stays at the JSON boundary through `Object.fromEntries` over - the Effect collection iterable; persisted load repopulates - `MutableHashMap` state. -- Verification: - - `cd ts && bun run --cwd packages/flow build` - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/librarian-service.test.ts src/__tests__/collection-manager.test.ts` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Tagged Test Error Cleanup Slice - -- Status: migrated and package-verified. -- Completed: - - Removed the remaining native `new Error` / `extends Error` matches from - tests and client wire models. - - Polling helpers in embeddings, flow processor, and chunking tests now fail - with `S.TaggedErrorClass` timeout values. - - Runtime service fake failures and the embeddings provider failure fixture - now use tagged test errors. - - The NATS SDK error mock preserves `code` and `jsError()` behavior through a - `TaggedErrorClass`, and the client model no longer exports a wire type - alias named `Error`. -- Verification: - - `cd ts/packages/base && bunx --bun vitest run src/__tests__/nats-backend.test.ts src/__tests__/runtime-services.test.ts src/__tests__/embeddings-service.test.ts src/__tests__/flow-processor-runtime.test.ts` - - `cd ts/packages/flow && bunx --bun vitest run src/__tests__/chunking-service.test.ts` - - `cd ts && bun run --cwd packages/client build && bun run --cwd packages/base build && bun run --cwd packages/flow build` - - `rg -n "new Error\\(|extends Error|export type Error|error\\?: Error|S\\.ErrorClass" ts/packages --glob '*.ts' --glob '*.tsx'` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: MCP Legacy Tool Name Parity Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/mcp/src/__tests__/server-effect.test.ts` now verifies the - legacy SDK stdio server's registered tool names stay aligned with the - canonical Effect `TrustGraphMcpToolkit`. - - The test mocks the TrustGraph client constructor so parity inspection does - not open a gateway socket. - - `ts/packages/mcp/src/server.ts` uses a namespace `zod` import so the legacy - compatibility path works under the Bun/Vitest runtime used by the parity - test. -- Verification: - - `cd ts/packages/mcp && bunx --bun vitest run src/__tests__/server-effect.test.ts` - - `cd ts && bun run --cwd packages/mcp build` - - `cd ts && bun run check:tsgo` - - `cd ts && bun run build` - - `cd ts && bun run test` - - `cd ts && bun run lint` - - `git diff --check` - -### 2026-06-04: Predicate Error Message Cleanup Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/base/src/errors.ts` now narrows unknown message-bearing - failures through `effect/Predicate` instead of a structural type - assertion. - - `ts/packages/base/src/processor/flow-processor.ts` now logs config - consumer failures through the shared `errorMessage` helper instead of an - `instanceof Error` branch. - - `ts/packages/client/src/socket/trustgraph-socket.ts` now uses - `effect/Predicate` for legacy callback error-message extraction instead of - native `Error` checks or structural assertions. - - `ts/packages/workbench/src/atoms/workbench.ts` now uses - `effect/Predicate` for workbench promise-boundary error-message - extraction. -- Verification: - - `cd ts && bun run --cwd packages/base test` - - `cd ts && bun run --cwd packages/base build` - - `cd ts && bun run --cwd packages/client test` - - `cd ts && bun run --cwd packages/client build` - - `cd ts && bun run --cwd packages/workbench build` - - `cd ts && bun run check:tsgo` - - `rg -n "instanceof Error|\\bas \\{ message\\?: unknown \\}|\\bas \\{ message\\?:" ts/packages --glob '*.ts' --glob '*.tsx'` - -### 2026-06-04: Flow Persistence FileSystem Slice - -- Status: migrated and package-verified. -- Completed: - - `ts/packages/flow/src/runtime/effect-files.ts` now provides - Effect-native `ensureDirectoryEffect`, `readTextFileEffect`, - `readBinaryFileEffect`, `writeTextFileEffect`, `writeBinaryFileEffect`, - and `removePathEffect` helpers backed by `effect/FileSystem`. - - The existing Promise helpers remain compatibility wrappers over a - `ManagedRuntime` provided with `@effect/platform-bun/BunFileSystem.layer`, - preserving current config/core/librarian service call sites while removing - direct `Bun.$`, `Bun.file`, and `Bun.write` production IO from the helper. - - File operation error channels are typed as `effect/PlatformError`. -- Verification: - - `cd ts && bun run --cwd packages/flow build` - - `cd ts && bunx --bun vitest run packages/flow/src/__tests__/config-service.test.ts packages/flow/src/__tests__/knowledge-core-service.test.ts packages/flow/src/__tests__/librarian-service.test.ts` - - `rg -n "Bun\\.\\$|Bun\\.file|Bun\\.write" ts/packages/flow/src/runtime/effect-files.ts ts/packages/flow/src --glob '*.ts'` - -## Subagent Findings To Preserve - -- MCP/workbench: - - Make the Effect MCP server the canonical implementation. An Effect - `McpServer.layerStdio` entrypoint now exists; the old stdio server should - remain only as compatibility until protocol-level `tools/call` parity is - proved. Tool-name parity with the Effect toolkit is now covered. Do not - delete `server.ts` until call coverage exists, with special attention to - `text_completion` behavior. - - Workbench BaseApi atoms can move toward `AtomRpc` or `AtomHttpApi` after - the client API is less Promise-first. - - MCP env is now Config-backed; continue that policy for future MCP settings. - - Workbench persistent theme storage is now canonicalized through - `Atom.kvs`/`BrowserKeyValueStore`. Remaining workbench `localStorage` - matches are legacy migration fallbacks, QA assertions, or the pre-paint - host script. -- Flow stateful services: - - Config service operation dispatch, schema persistence, and nested - workspace state are complete: the long-lived config store now uses - `HashMap` inside `SynchronizedRef`, and plain records remain only at - persistence/API boundaries. - - KnowledgeCore service operation dispatch, helper functions, and ref-backed - core state are complete: `kgCores` and `deCores` now use `HashMap` inside - `SynchronizedRef`, and plain records remain only at persistence/API - boundaries. - - FlowManager operation dispatch, helper functions, and ref-backed flow / - blueprint state are complete: `flows` and `blueprints` now use `HashMap` - inside `SynchronizedRef`, and plain records remain only at config/API - boundaries. Librarian ref-backed state remains a larger collection target. - Follow-up service work should focus on scoped layers, schedules where - polling semantics allow, and managed persistence providers rather than - direct mutable service fields. - - Flow service startup facades now consistently use `ManagedRuntime`, and - local scripts should delegate to `runMain()` instead of adding local - `.catch(console.error/process.exit)` wrappers. - - Base `Flow.startEffect` / `stopEffect` are now Effect values, not - zero-argument lazy Effect methods. New runtime APIs should expose Effect - values where no arguments are needed, and reserve `Effect.fn` for reusable - helpers that take arguments. - - Persistence IO should move toward `FileSystem` or `KeyValueStore` where - the installed beta has the needed provider surface. -- Base messaging/processors: - - Processor/flow Promise compatibility now uses `ManagedRuntime`; keep - `NodeRuntime` only for process `runMain()` entrypoints. - - Subscriber queues/maps and dynamic flow state should continue moving - toward `Queue`, `Deferred`, `SynchronizedRef`, `Schedule`, and scoped - layers. - - The legacy `messaging/subscriber.ts` async queue/fanout implementation is - removed. Use native `effect/PubSub` for future in-process fanout, while - keeping `PubSubBackend` for broker-backed messaging. - - The legacy producer and consumer facades now delegate to scoped Effect - runtime factories. Public `start()`/`send()`/`stop()` Promises remain - compatibility boundaries. - - NATS header construction, ack/nak operations, and lookup create-on-missing - behavior now stay typed. `PubSubBackend` remains an intentional broker - adapter contract rather than a direct `effect/PubSub` replacement target. - - NATS initialized stream cache now uses Effect `MutableHashSet`; keep the - rest of the NATS SDK connection/JetStream manager state as an external - broker adapter boundary unless moving the whole backend lifecycle. - - Consumer rate-limit retry timeout behavior is now wired in both legacy and - Effect-native consumer paths. Effect-native consumer concurrency now owns - one backend consumer per worker, and request-response pending shutdown now - fails through a tagged lifecycle error. The legacy consumer facade blocking - shape is complete; do not reopen it unless a public API compatibility - issue appears. - - Existing constructor shims preserve callable-plus-newable public exports; - removing them needs a public API split or real class redesign. - - Typed string registries in `Flow` now have Schema-backed parameter specs - and typed producer/requestor spec-object accessors. New service handlers - should hoist spec objects and use those accessors; bare string accessors - remain compatibility escapes. - - Base metrics are now Effect-native and Prometheus-formatted through - `PrometheusMetrics.format`; do not reopen `prom-client` unless a future - scrape requirement cannot be represented by Effect metrics. - - The scratch note's `prom-client` claim is stale: `ts/packages` has no - `prom-client` matches, and `metrics/prometheus.ts` already uses - `effect/Metric` plus `PrometheusMetrics.format`. Remaining observability - work is OTLP/span coverage and wiring existing consumer metrics helpers. - - Numeric public timeout fields such as `timeoutMs` remain compatibility - surfaces. Internal runtime config with `Config.number(...Ms)` is still a - valid `Config.duration` / `Duration` cleanup target. -- Gateway/client: - - `EffectRpcClient` now owns its socket/RPC layer with `ManagedRuntime`. - Socket errors/JSON parsing now use tagged errors and Schema decoding. - The remaining client `newableFactory` assertions are documented as public - API compatibility boundaries for this loop. - - `EffectRpcClient` remains Promise-shaped at the public facade. A future - client API slice can add an Effect/Stream-native surface beside the - Promise facade, then migrate callback users incrementally. - - Gateway/client `DispatchError` contracts now use `S.TaggedErrorClass`; do - not reopen `S.ErrorClass` unless a new production match appears. - - Gateway `DispatchStream` now uses Effect-native dispatcher streaming - callbacks instead of nested `Effect.runPromiseWith`, and client streaming - facade callbacks now decode the legacy envelope through Schema before - applying service-specific public callback semantics. - - Gateway dispatcher ownership and serialization cleanup is complete: - injected pubsub backends are not closed by the manager, one-shot producers - are acquire/use/release bracketed, and serialization failures are typed - Effect errors. - - `gateway/rpc-protocol.ts` remains a Fastify socket compatibility bridge - while the public Effect RPC server layers require SocketServer or Effect - HTTP routing. The parser decode assertion is complete through Schema - validation; future local cleanups should focus on its connection - collections rather than replacing the whole bridge. - - WebSocket adapter host fallbacks now use `Result.try` and tagged adapter - errors while preserving sync exports. -- RAG/providers/storage: - - RAG and agent requestor bridges are complete: `toPromiseRequestor` has no - remaining `ts/packages` matches. - - Provider SDKs and storage clients should become managed resources where - they have meaningful lifecycle. - - Ollama embeddings now has an effectful canonical layer. There is no - installed Effect AI Ollama provider package, so future Ollama work should - focus on local Effect wrappers/adapters rather than provider replacement. - - Full text-completion provider swaps need parity tests first. OpenAI and - Azure currently use Chat Completions while `@effect/ai-openai` is Responses - API oriented, and no installed Azure/Mistral/Ollama Effect AI provider is - available. Anthropic is the closest direct provider swap, but must preserve - text, token counts, streaming final usage, and rate-limit mapping. The - local provider layer-construction cleanup is complete; remaining provider - work is adapter/parity work, not `Layer.succeed` cleanup. - - The `effect/unstable/ai/LanguageModel` to TrustGraph `LlmProvider` adapter - baseline is complete, and Claude now uses `@effect/ai-anthropic` through - that adapter. Direct OpenAI, Azure, and OpenAI-compatible swaps are no-ops - until Responses-vs-Chat-Completions parity is proven. - - FalkorDB scoped lifecycle is complete for triples query/store. Use the - fakeable client/graph factory pattern from that slice for future storage - client tests. - - Qdrant config/schema/fakeability is complete for graph/doc embedding - store/query modules. Qdrant still has no close/disconnect surface in the - installed client, so do not reopen it as an `acquireRelease` close slice - without new SDK evidence. - - Shared text-completion stream iteration and the Mistral content assertion are - complete, and the Effect AI stream-part adapter now uses `effect/Match`. - The remaining provider-layer item is parity-backed Effect AI adapter work, - not a direct SDK swap. - - AI service modernization remains partially valid: Claude already uses - `@effect/ai-anthropic`, while OpenAI/Azure/OpenAI-compatible/Mistral/Ollama - still need parity-backed Effect AI adapters or provider-specific bridges. -- Scratch-note follow-ups: - - `Term` / compact client term serialization is complete for base schema, - gateway translation, and pure term helper switches. Future work should - only reopen this if client socket schema drift appears or a hidden - consumer needs a different named-graph shape. - - Messaging runtime `Config.duration` cleanup is complete. Internal runtime - config uses `Duration.Duration`; public timeout compatibility inputs and - broker receive/error payload boundaries remain numeric milliseconds. - - CLI modernization remains valid, but the live installed target is - `effect/unstable/cli` rather than an installed `@effect/cli` package. - - Chunking `effect/Chunk` migration is complete: the recursive splitter now - returns `Chunk.Chunk` and converts to arrays only at test/assertion - boundaries. - - Knowledge core internals are largely Effect-native, but public core service - facades still expose Promise methods; migrate tests to Effect-first - methods before shrinking those facades. - - Qdrant graph/doc known-collection caches now use - `MutableHashSet`. Short-lived local traversal sets remain no-ops. - - Gateway dispatcher static service registries, streaming membership, and - scoped requestor cache now use Effect `HashMap`/`HashSet`; gateway - term-bearing service membership sets now use Effect `HashSet` too. - - FlowManager, KnowledgeCore, ConfigService, and Librarian `() => - Effect.gen(...)` factories are normalized to `Effect.fn` / - `Effect.fnUntraced`. - - ConfigService and KnowledgeCore operation dispatch now use `effect/Match` - with `Match.exhaustive`; FlowManager and Librarian operation dispatch now - use `effect/Match` with runtime-preserving `Match.orElse` fallbacks. - - ConfigService nested workspace config state now uses Effect `HashMap`. - JSON and API response helpers intentionally convert back to sorted plain - records/arrays at the boundary. - - Native `switch` statements are now clean in `ts/packages`; future branch - drift should keep service dispatch on `effect/Match` or Schema tagged-union - helpers. - - Client RPC/BaseApi connection-state fanout now uses - `effect/SubscriptionRef`; future gateway/client stream work is broader API - design, not listener bookkeeping. - - Long-lived `Map` / `Set` state in ref-backed services is complete for the - current scratch inventory; local pure traversal maps/sets remain no-ops. - - Fresh strict signal sweep after the 2026-06-04 Predicate cleanup found no - committed production native `Error` construction/inheritance, raw - `try`/`catch`, native `switch`, message-shape assertions, or - Effect-focused type assertions under `ts/packages`. Two `instanceof Error` - hits remain only in the unrelated local dirty - `ts/packages/flow/src/extract/knowledge-extract.ts` file and were left - untouched. - - Remaining production `Effect.gen` matches are one-shot program values, - scoped runtime factories, local command effect values, or public - compatibility callbacks; reusable service helpers from the scratch - inventory are normalized to `Effect.fn` / `Effect.fnUntraced`. - - Flow service persistence helpers now use `effect/FileSystem` with the Bun - FileSystem layer. Remaining direct `Bun.file` / `Bun.write` matches in the - Flow persistence lane are test fixture setup/assertions, not production IO. - - Fresh long-lived native collection targets from the scratch inventory are - complete: Librarian service state, base processor registries, the - standalone Librarian collection manager, prompt template cache, and - workbench explain triples module cache. Local traversal sets and test fakes - remain no-op boundaries. - - Fresh strict error-class sweep after the 2026-06-04 tagged test error - cleanup found no `new Error`, `extends Error`, `export type Error`, - `error?: Error`, or `S.ErrorClass` matches under `ts/packages`. - -## Ranked Findings - -### No-op: Broker Backend Adapter Boundary - -- Status: - - Closed as a P0 rewrite item after the 2026-06-02 broker scout and the - legacy consumer facade slice. -- TrustGraph evidence: - - `ts/packages/base/src/backend/types.ts` - - `ts/packages/base/src/backend/nats.ts` - - `ts/packages/base/src/backend/pubsub.ts` - - `ts/packages/base/src/messaging/runtime.ts` -- Effect primitives: - - `Layer`, `Scope`, `Stream`, `Schedule`, `Queue`, - `Effect.acquireRelease`, and `Effect.tryPromise`. -- Evidence: - - `BackendProducer`, `BackendConsumer`, and `PubSubBackend` are the external - Promise broker adapter contract. `backend/pubsub.ts` wraps that contract in - Effect through `Context.Service`, `Layer`, `Effect.tryPromise`, and scoped - finalizers. - - NATS boundary failures, selective stream/consumer lookup recovery, - producer sends, ack/nak, and close paths are typed with Effect wrappers. - - Producer, consumer, and request-response runtime ownership now live in - scoped Effect factories. -- Rule: - - Keep `PubSubBackend` as the compatibility adapter boundary; Effect native - `PubSub` remains in-process only. - - Treat the producer Promise facade as a completed compatibility wrapper; - avoid reopening it unless backend runtime changes require a narrower - adapter. - - Keep NATS SDK boundary failures typed and avoid catch-all - create-on-failure behavior. Future backend slices should move - connection/stream state into scoped Effect services. - - Treat rate-limit retry timeout semantics as complete; next consumer slices - should focus on blocking compatibility and backend/layer ownership, not - retry policy. - - Treat Effect-native per-worker consumer ownership as complete; do not flag - `makeEffectConsumerFromPubSub` concurrency for shared backend receive - handles. - - Treat request-response pending shutdown semantics as complete; do not flag - `waitForResponse` timeout behavior for stopped runtimes. - - Treat request-response stop idempotence as complete; stop state now uses an - Effect `Ref` and explicit stop plus scoped finalizer closes resources once. - - Treat request-response in-process fanout as complete: response routing now - uses native `effect/PubSub` subscriptions instead of a hand-managed - subscriber map. - - Treat the legacy consumer facade as a completed compatibility wrapper over - `makeEffectConsumerFromPubSub`; do not flag blocking `start()` semantics. - -### No-op: Legacy Promise Facades And One-Shot Program Effects - -- Status: - - Closed as active P0/P1/P2 migration evidence after the final 2026-06-04 - signal pass. -- TrustGraph evidence: - - `ts/packages/base/src/messaging/consumer.ts` - - `ts/packages/base/src/messaging/producer.ts` - - `ts/packages/base/src/messaging/request-response.ts` - - `ts/packages/base/src/processor/flow-processor.ts` - - `ts/packages/flow/src/config/service.ts` - - `ts/packages/flow/src/cores/service.ts` - - `ts/packages/flow/src/flow-manager/service.ts` - - `ts/packages/flow/src/librarian/service.ts` - - `ts/packages/flow/src/gateway/server.ts` - - `ts/packages/mcp/src/server.ts` - - `ts/packages/cli/src/commands/*.ts` -- Effect primitives: - - `Effect.fn`, `ManagedRuntime`, `NodeRuntime.runMain`, `Scope`, - `Layer`, `Effect.tryPromise`, `effect/FileSystem`, Bun/Node FileSystem - provider layers, and tagged error constructors. -- Evidence: - - Internal service handlers and reusable helpers now expose Effect-returning - functions or `Effect.fn` helpers; legacy object methods call - `Effect.runPromise` only where the existing public interface still - requires `Promise` callbacks. - - CLI command actions, Fastify route handlers, MCP SDK callbacks, and client - callback APIs are host/library boundaries that require Promise-returning - callbacks. These boundaries now delegate to Effect programs and map - failures into tagged errors or wire-contract errors. - - Remaining `Effect.gen` matches are one-shot program/runtime values such as - MCP stdio startup, socket RPC protocol construction, agent runtime - construction, or local command effect values; they are not reusable helper - functions that would benefit from `Effect.fn`. - - Flow service file persistence now goes through `effect/FileSystem`; direct - Bun file helpers remain only in tests and fixture setup. -- Rule: - - Do not reopen `Effect.runPromise` matches unless the surrounding public API - is being intentionally redesigned to an Effect-first contract. - - New internal helpers should continue to be written as `Effect.fn` or - direct Effect-returning functions, with Promise compatibility isolated at - the host boundary. - -### No-op: Remaining Effect AI Provider Swaps - -- TrustGraph evidence: - - `ts/packages/flow/src/model/text-completion/*.ts` -- Effect primitives: - - `effect/unstable/ai/LanguageModel`, `effect/unstable/ai/EmbeddingModel`, - Effect AI OpenAI/Anthropic provider layers. -- Rewrite shape: - - Adapter baseline is complete: `makeLanguageModelProvider` bridges - `LanguageModel` into `LlmProvider`. - - Claude migration is complete through `@effect/ai-anthropic`. - - Do not directly swap OpenAI, Azure, or OpenAI-compatible providers yet: - current TrustGraph code uses Chat Completions/local-server semantics while - `@effect/ai-openai` is Responses API backed. - - Do not directly swap Mistral or Ollama until an installed Effect provider - package exists or a parity-backed local Effect wrapper is designed. -- Tests: - - Provider parity for `LlmResult`, final streaming chunk token counts, 429 - mapping, missing-token config failures, and OpenAI-compatible local-server - behavior. - -### No-op: Base Metrics Prometheus Wrapper - -- Status: - - Closed as a scratch-note migration target by the Effect Metrics Prometheus - slice. -- TrustGraph evidence: - - `ts/packages/base/src/metrics/prometheus.ts` - - `ts/packages/flow/src/gateway/server.ts` -- Effect primitives: - - `Metric.counter`, `Metric.histogram`, and - `effect/unstable/observability` `PrometheusMetrics.format`. -- Rule: - - Keep the gateway Fastify route as the external scrape boundary, but record - TrustGraph metrics through Effect `Metric` values. - - Use a fresh `Metric.MetricRegistry` in tests that assert exact scrape - content. - -### Complete: Term And ClientTerm Tagged-Union Normalization - -- TrustGraph evidence: - - `ts/packages/base/src/schema/primitives.ts` - - `ts/packages/flow/src/gateway/dispatch/serialize.ts` - - `ts/packages/client/src/socket/trustgraph-socket.ts` -- Effect primitives: - - `S.toTaggedUnion(...).match` and `effect/Match` discriminator helpers. -- Rewrite shape: - - Add tagged-union helpers for internal `Term` and compact client terms. - - Replace serializer native switches with tagged-union matching or - `Match.discriminatorsExhaustive`. - - Remove unsafe default pass-through casts while preserving compact `g` - string compatibility. -- Tests: - - Base schema tests now cover recursive terms and graph strings. - - Gateway dispatcher tests now cover all compact term variants, nested - triples, compact graph strings, malformed known-tag payloads, and - malformed client triple failures. - -### Complete: Messaging Runtime Duration Config Cleanup - -- TrustGraph evidence: - - `ts/packages/base/src/runtime/messaging-config.ts` - - `ts/packages/base/src/messaging/runtime.ts` -- Effect primitives: - - `Config.duration`, `Duration.Duration`, and existing `Duration.millis` - compatibility conversions. -- Rewrite shape: - - Change internal runtime config fields such as `receiveTimeoutMs`, - `requestTimeoutMs`, `retryDelayMs`, and `rateLimitTimeoutMs` to - `Duration.Duration`. - - Load env-backed values with `Config.duration` while preserving Python-style - millisecond defaults and public numeric compatibility options. - - Keep external `timeoutMs` option names numeric in request/response, - processor, and client boundaries unless their public API is deliberately - changed. -- Tests: - - Base runtime config tests cover legacy millisecond env values and Effect - duration string env values. - - Messaging runtime and consumer tests preserve retry and timeout behavior. - -### Complete: Qdrant Known-Collection MutableHashSet Cleanup - -- TrustGraph evidence: - - `ts/packages/flow/src/storage/embeddings/qdrant-doc.ts` - - `ts/packages/flow/src/storage/embeddings/qdrant-graph.ts` -- Effect primitives: - - `MutableHashSet` from `effect`. -- Rewrite shape: - - Replace long-lived `Set` known-collection caches with - `MutableHashSet` in Qdrant graph/doc embedding stores. - - Keep short-lived local `Set` values for pure query traversal or fixture - assertions as no-op boundaries. -- Tests: - - Qdrant embeddings tests prove graph cache hits skip repeated collection - existence checks and deletion invalidates the cache. - -### No-op: MCP Legacy Compatibility Surface - -- Status: - - MCP now builds under strict tsgo, the stdio server has an Effect-backed - compatibility implementation, and an Effect `McpServer.layerStdio` - entrypoint exists. - - The legacy SDK stdio server remains a compatibility boundary rather than an - active Effect-native rewrite blocker. -- Rule: - - Keep the old SDK/Zod stdio compatibility surface for now. - - Tool-name parity with the canonical Effect toolkit is covered by - `server-effect.test.ts`. - - Prove `tools/call` parity before deleting any public entry point or - dropping `zod`/server-side MCP SDK dependencies. - - Pay special attention to `text_completion`: legacy calls the TrustGraph - gateway, while the Effect server currently uses an Effect AI - `LanguageModel`/OpenAI layer. -- Tests: - - `cd ts && bun run --cwd packages/mcp build` - - Root `cd ts && bun run check` - -### No-op: Workbench Theme And Browser Storage Boundaries - -- Status: migrated and documented no-op for remaining direct browser matches. -- Evidence: - - `settingsAtom`, `themeAtom`, `flowIdAtom`, and `conversationAtom` are - canonical `Atom.kvs` entries backed by - `BrowserKeyValueStore.layerLocalStorage`. - - The only remaining `tg-theme` production read is a legacy migration - fallback. The pre-paint `index.html` script is a host boundary that restores - the canonical key before React and Effect services mount. - - Graph canvas DOM/class inspection remains render-only host behavior. -- Rule: - - Do not reopen workbench `localStorage` as a backend/runtime blocker unless - a canonical `Atom.kvs` entry is bypassed or a legacy migration fallback can - be intentionally removed. - -## Recommended PR Order - -No active PR-sized Effect-native migration remains from the current playbook -signal set. Future work is intentionally API-design or compatibility-policy -work, not a current P0/P1/P2 rewrite blocker: - -1. MCP `tools/call` parity tests before deleting the legacy SDK/Zod stdio - compatibility entrypoint. -2. Flow/client RPC stream API redesign beyond callback/Promise public - compatibility. -3. CLI migration from Commander to `effect/unstable/cli` if the public CLI - surface is intentionally redesigned. -4. Provider-specific Effect AI swaps when installed provider semantics match - TrustGraph's existing Chat Completions/local-server behavior. - -## No-Op Rules - -Do not flag these as rewrite blockers without additional proof: - -- Promise-returning CLI actions, MCP SDK callbacks, client compatibility - methods, and Fastify route handlers at true external boundaries. Boundary - code still must map failures into typed errors or wire errors. -- `try`/`catch` blocks at host/tool boundaries only when the catch maps into a - typed error or a wire-contract error. Internal exception capture should use - `Effect.try`, `Effect.tryPromise`, or `Result.try`. -- `S.Class`, `S.TaggedErrorClass`, `Context.Service`, `Rpc.make`, and - `HttpApi.make` when they are required or idiomatic for the Effect API. -- Plain `Map` usage for local pure transformations, such as graph utility - construction, unless the state is long-lived mutable service state. -- Plain synchronous loops for parsing or tree traversal are not Effect migration - blockers unless they hide async polling, resource ownership, or callback - scheduling. -- JSON stringification in tests or wire-contract fixtures. Production JSON - encode/decode should prefer schema codecs when the encoded form can be - preserved. -- Workbench pre-paint `index.html` storage reads are host-boundary code used - before React and Effect services mount. Keep canonical workbench state in - `Atom.kvs`/`BrowserKeyValueStore`, and treat old-key reads as migration - fallbacks unless backward compatibility is intentionally dropped. -- The old MCP SDK/Zod stdio server is compatibility code until an Effect stdio - entrypoint plus parity tests prove the same public protocol behavior. -- Client `newableFactory` assertions that preserve vendored callable-plus-new - API facades are compatibility boundaries unless the public constructor API is - intentionally redesigned. -- Base `AsyncProcessor`, `Flow`, and `FlowProcessor` callable-plus-newable - export assertions are compatibility boundaries unless the public constructor - API is intentionally redesigned. -- TrustGraph `PubSubBackend` / backend `PubSub` service is a broker adapter - boundary for NATS/Pulsar-style topics, acknowledgement, schema codecs, and - backend lifecycle. Effect's native `PubSub` can replace in-process fanout - helpers, but not the distributed broker abstraction by itself. This was - rechecked against - `/home/elpresidank/YeeBois/projects/beep-effect2/.repos/effect-v4/packages/effect/src/PubSub.ts`, - whose exported API is local publish/subscribe over Effect queues. -- Request-response pending shutdown semantics are complete in - `makeEffectRequestResponseFromPubSub`: pending calls race response waiting - against a `Deferred` stop signal and fail with tagged - `MessagingLifecycleError`; stop idempotence is owned by an Effect `Ref`. -- Legacy `makeConsumer` facade blocking-loop ownership is complete: - `start()` now initializes scoped Effect consumers and returns after startup, - while `stop()` closes the native consumer scope. -- `ts/packages/flow/src/gateway/rpc-protocol.ts` is a Fastify socket - compatibility bridge. Do not treat it as a wholesale server-layer replacement - target until the gateway is ready to move onto Effect SocketServer or Effect - HTTP routing; local parser validation and Effect collection cleanup inside - that bridge are still valid. - -## Acceptance For Final Loop Completion - -The overall playbook loop is complete only when: - -- All remaining playbook signal matches are migrated or documented as no-op - external-boundary cases with concrete evidence. -- No P0/P1/P2 migration item remains in this audit. -- `cd ts && bun run check`, `cd ts && bun run build`, `cd ts && bun run test`, - and `git diff --check` pass after the final migration slice. diff --git a/ts/EFFECT_NATIVE_REWRITE_PLAN.md b/ts/EFFECT_NATIVE_REWRITE_PLAN.md deleted file mode 100644 index 5a7d0b39..00000000 --- a/ts/EFFECT_NATIVE_REWRITE_PLAN.md +++ /dev/null @@ -1,127 +0,0 @@ -# TrustGraph TS Effect-Native Rewrite Plan - -## Summary - -Bring the TypeScript port to a genuinely Effect-native shape while keeping existing TrustGraph capabilities working. The current native MCP rewrite should be preserved. The remaining work is to remove Promise-first TrustGraph APIs and replace manual host/framework plumbing with Effect v4-native backend, messaging, processor, HTTP/RPC, and CLI layers. - -The implementation should proceed in green checkpoints rather than one broad sweep. Each stage should leave the repo closer to Effect-native and should run the smallest relevant gate before moving on. - -## Non-Negotiable Architecture Decisions - -- TrustGraph-authored production APIs should be Effect-first, not Promise-first. -- Remove exported Promise APIs from backend, messaging, processor, flow service, gateway, CLI, and service runner surfaces. -- Do not keep dual Promise and Effect methods on the same TrustGraph objects. -- Do not rely on Fastify, `@fastify/websocket`, Commander, or production `Effect.runPromise` / `ManagedRuntime` compatibility bridges. -- External JavaScript SDKs that are inherently Promise-based may be wrapped with `Effect.tryPromise`, but that Promise shape must stay behind Effect APIs. -- Gateway HTTP and WebSocket behavior should move to Effect v4 native modules: `effect/unstable/httpapi`, `effect/unstable/http`, `effect/unstable/rpc`, `effect/unstable/socket`, and platform Bun/Node HTTP/socket layers. -- CLI behavior should move to `effect/unstable/cli`. -- The MCP package should stay on the native `effect/unstable/ai/McpServer` implementation and must not reintroduce server-side `@modelcontextprotocol/sdk` or `zod`. - -## Current Grounding - -- `bun run check:tsgo` is green. -- `bun run build` is green. -- `bun run --cwd packages/mcp test` is green. -- `bun run test` currently fails only in `@trustgraph/base`, in `packages/base/src/__tests__/nats-backend.test.ts`. -- The NATS failures are a symptom of the backend still sitting between old Promise-shaped abstractions and newer JetStream calls. The rewrite should fix the contract, not only update the mock. -- The Effect v4 source of truth for APIs and examples is available at `/home/elpresidank/YeeBois/projects/beep-effect/.repos/effect-v4`. - -## Stage 1: Backend And NATS - -- Redefine `Message`, `BackendProducer`, `BackendConsumer`, and `PubSubBackend` around `Effect.Effect` return values. -- Make NATS connection, JetStream manager/client, stream initialization, and durable consumer handles scoped resources. -- Replace deprecated receive plumbing with modern JetStream consumer APIs. -- Preserve behavior for stream creation, durable consumer creation, headers, JSON/schema encode/decode, receive timeout returning `null`, ack/nak, close/drain, and tagged `PubSubError` mapping. -- Update NATS tests and in-memory fake backends to the new Effect-native interface. - -Verification: - -- `bun run --cwd packages/base test src/__tests__/nats-backend.test.ts` -- `bun run --cwd packages/base test` -- `bun run check:tsgo` - -## Stage 2: Messaging And Processor Runtime - -- Remove Promise facades from `makeProducer`, `makeConsumer`, `makeRequestResponse`, processor `start` / `stop` / `run`, and flow compatibility wrappers. -- Expose Effect values, scoped constructors, Context services, Layers, and finalizers as the only TrustGraph API shape. -- Convert message handlers, config handlers, shutdown callbacks, producer send/flush/close, consumer receive/ack/nak/close, and request/response operations to Effect functions. -- Remove internal `ManagedRuntime.make(Layer.empty)` compatibility runtimes from production source. - -Verification: - -- `bun run --cwd packages/base test` -- `bun run check:tsgo` - -## Stage 3: Flow Service Call Sites - -- Convert flow service state methods that currently call `Effect.runPromise` into Effect-valued service methods. -- Keep public feature behavior unchanged for config, flow manager, librarian, knowledge cores, retrieval, embeddings, triples, agent, MCP tool, and text-completion services. -- Convert test fakes to the same Effect-native backend/messaging contracts. - -Verification: - -- `bun run --cwd packages/flow test` -- `bun run test` -- `bun run check:tsgo` - -## Stage 4: Gateway HTTP And RPC - -- Remove Fastify and `@fastify/websocket` from `packages/flow`. -- Rebuild the gateway with Effect v4 `HttpApi` groups/endpoints and native RPC/socket layers. -- Preserve existing behavior: - - `POST /api/v1/workbench/dispatch` - - `POST /api/v1/:kind` - - `POST /api/v1/flow/:flow/service/:kind` - - `POST /api/v1/flow/:flow/load` - - `GET /api/v1/rpc` - - `GET /api/v1/metrics` - - bearer auth behavior and RPC token behavior - - existing response/error shapes expected by clients and workbench -- Use native Effect HTTP test utilities or route-level protocol tests instead of Fastify injection. - -Verification: - -- `bun run --cwd packages/flow test` -- `bun run --cwd packages/client test` -- `bun run test` -- `bun run build` - -## Stage 5: CLI And Runner Scripts - -- Remove Commander from `packages/cli`. -- Rebuild CLI commands with `effect/unstable/cli` and Effect-native socket/API clients. -- Convert service runner scripts to launch Effect programs directly with platform runtime `runMain` style execution. -- Remove `run(): Promise` exports from flow services; export Effect programs/layers and Effect-native `runMain` helpers instead. -- Leave script-only demo/seed tools as follow-up only if they are outside production package source, but do not let production packages depend on Promise facades. - -Verification: - -- `bun run --cwd packages/cli test` -- `bun run check:tsgo` -- `bun run build` -- `bun run test` - -## Stage 6: Cleanup And Acceptance - -- Remove obsolete dependencies from package manifests and lockfile: - - `fastify` - - `@fastify/websocket` - - `commander` - - any legacy dependency made unnecessary by the rewrite -- Preserve unrelated dirty work. Do not revert user changes. -- Use parallel agents for bounded audits or disjoint rewrite slices when useful. -- Use Graphiti memory if available; if unavailable, continue safely and report it skipped. - -Final verification: - -- `bun run check:tsgo` -- `bun run build` -- `bun run lint` -- `bun run test` -- `bun run --cwd packages/mcp test` -- `bun run workbench:qa` after installing the matching Playwright browser if needed -- `git diff --check` -- `rg -n "fastify|@fastify/websocket|commander" packages package.json bun.lock` has no production dependency/use hits -- `rg -n "Effect\\.runPromise|ManagedRuntime\\.make|Promise<|async function main" packages/base/src packages/flow/src packages/cli/src scripts -g "*.ts"` has no production-source hits except tests or unavoidable external type declarations that are not TrustGraph APIs - -Do not mark the goal complete until all required gates are green, or until a real external blocker is reported with the exact failing command, error, and smallest next action. diff --git a/ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md b/ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md deleted file mode 100644 index 67048f3e..00000000 --- a/ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md +++ /dev/null @@ -1,225 +0,0 @@ -# TrustGraph Effect-Native Rewrite Playbook - -This playbook is the context packet for read-only sub-agents that audit the -TrustGraph TypeScript port for code that hand-rolls behavior already provided by -Effect v4. It is not an implementation plan for a single rewrite. Its job is to -make future scouts fast, grounded, and allergic to invented APIs. - -## Source Baseline - -Verify these paths at the start of every audit run: - -- TrustGraph TS port: `/home/elpresidank/YeeBois/dev/trustgraph/ts` -- Effect v4 subtree: `/home/elpresidank/YeeBois/projects/beep-effect2/.repos/effect-v4` -- Installed Effect fallback: `ts/node_modules/effect` -- Installed Atom React fallback: `ts/packages/workbench/node_modules/@effect/atom-react` -- Native TypeScript checker: `ts/node_modules/.bin/tsgo` - -The prompt typo path `~/YeeBois/projecects/...` is not valid on this machine. -Use `~/YeeBois/projects/...`. - -When a package is not present in the subtree, verify it from TrustGraph's -installed package sources before proposing a replacement. This matters for -`effect/unstable/reactivity`, `effect/unstable/ai`, `effect/unstable/rpc`, -`effect/unstable/socket`, `effect/unstable/http`, `effect/unstable/httpapi`, -and `@effect/atom-react`. - -Important import rule: the Effect v4 subtree contains separate packages such as -`@effect/ai`, `@effect/rpc`, and `@effect/platform`, but TrustGraph currently -resolves many beta APIs through `effect/unstable/*` import paths. Prefer the -installed TrustGraph import path when it exists; use the subtree package path as -source proof, not as an automatic import recommendation. - -TrustGraph's TypeScript check must use `@typescript/native-preview` patched by -`@effect/tsgo`. The expected local version line is currently: - -```sh -cd ts && ./node_modules/.bin/tsgo --version -# Version 7.0.0-dev+effect-tsgo.0.13.0 -``` - -The root `ts/tsconfig.base.json` language-service plugin should stay aligned -with `/home/elpresidank/YeeBois/projects/beep-effect/tsconfig.base.json`, -using the same diagnostic severities and no test-only diagnostic downgrades. -The repo-specific `namespaceImportPackages` entry should use `@trustgraph/*`. -The root `check` script should delegate to `check:tsgo`, and `check:tsgo` must -run `effect-tsgo patch` before `tsgo -b` so agents get Effect diagnostics -instead of plain TypeScript. Treat every `effect(...)` diagnostic as a hard -blocker. - -## Primitive Map - -Use this map as the starting baseline. A finding is only valid when it cites the -TrustGraph path, the import path, and the Effect source path that proves the -primitive exists. - -| Handrolled pattern | Preferred Effect primitive | Import path | Verified source | -| --- | --- | --- | --- | -| Promise loops, top-level async orchestration | `Effect`, `Effect.fn`, `Effect.scoped`, `Effect.runPromiseWith` at boundaries | `effect` | `packages/effect/src/Effect.ts` | -| Resource construction and teardown | `Layer`, `Scope`, `Effect.acquireRelease`, `Effect.addFinalizer` | `effect` | `packages/effect/src/Layer.ts`, `packages/effect/src/Scope.ts` | -| Mutable service state | `Ref`, `SynchronizedRef`, `SubscriptionRef` | `effect` | `packages/effect/src/Ref.ts`, `packages/effect/src/SynchronizedRef.ts` | -| Long-lived keyed state and set membership | `HashMap`, `MutableHashMap`, `HashSet`, `MutableHashSet` | `effect` | `packages/effect/src/HashMap.ts`, `packages/effect/src/MutableHashMap.ts`, `packages/effect/src/HashSet.ts`, `packages/effect/src/MutableHashSet.ts` | -| Polling, delays, retry/backoff | `Schedule`, `Effect.sleep`, `Effect.retry` | `effect` | `packages/effect/src/Schedule.ts`, `packages/effect/src/Effect.ts` | -| Callback queues and streaming fanout | `Queue`, `PubSub`, `Stream`, `Channel` | `effect` | `packages/effect/src/Queue.ts`, `packages/effect/src/PubSub.ts`, `packages/effect/src/Stream.ts`, `packages/effect/src/Channel.ts` | -| Env/config decoding | `Config`, `ConfigProvider`, platform config providers | `effect`, `effect/ConfigProvider`, provider packages | `packages/effect/src/Config.ts`, `packages/effect/src/ConfigProvider.ts`, `packages/platform/src/PlatformConfigProvider.ts` | -| JSON/wire schemas | `Schema`, `ParseResult`, `RpcSchema` | `effect/Schema`, `effect/unstable/rpc/RpcSchema` | `packages/effect/src/Schema.ts`, `ts/node_modules/effect/src/unstable/rpc/RpcSchema.ts` | -| WebSocket lifecycle and framing | `Socket`, `RpcClient`, `RpcServer`, `RpcSerialization` | `effect/unstable/socket`, `effect/unstable/rpc` | `ts/node_modules/effect/src/unstable/socket/*.ts`, `ts/node_modules/effect/src/unstable/rpc/*.ts` | -| HTTP servers/clients and typed APIs | `HttpApi`, `HttpApiClient`, `HttpApiBuilder`, `HttpClient`, `HttpServer` | `effect/unstable/http`, `effect/unstable/httpapi`, platform providers | `ts/node_modules/effect/src/unstable/http/*.ts`, `ts/node_modules/effect/src/unstable/httpapi/*.ts` | -| File, storage, and process IO | `FileSystem`, `KeyValueStore`, `ChildProcess`, `ChildProcessSpawner` | `effect/FileSystem`, `effect/unstable/persistence/KeyValueStore`, `effect/unstable/process/*`, provider packages | `ts/node_modules/effect/src/FileSystem.ts`, `ts/node_modules/effect/src/unstable/persistence/KeyValueStore.ts`, `ts/node_modules/effect/src/unstable/process/*.ts` | -| Browser local storage and clipboard | `BrowserKeyValueStore`, `Clipboard`, `BrowserHttpClient`, `BrowserSocket` | `@effect/platform-browser/*` | `ts/node_modules/.bun/@effect+platform-browser@4.0.0-beta.75+a5c1409dbf4ddafe/node_modules/@effect/platform-browser/src/*.ts` | -| AI tools, MCP, and model calls | `Tool`, `Toolkit`, `McpServer`, `McpSchema`, `LanguageModel`, provider layers | `effect/unstable/ai`, provider packages such as `@effect/ai-openai` | `ts/node_modules/effect/src/unstable/ai/*.ts`, `packages/ai/ai/src/*.ts` | -| Workbench async state | `Atom`, `AtomRpc`, `AtomHttpApi`, `AsyncResult`, `AtomRegistry`, `Reactivity` | `effect/unstable/reactivity`, `@effect/atom-react` | `ts/node_modules/effect/src/unstable/reactivity/*.ts`, `ts/packages/workbench/node_modules/@effect/atom-react/src/*.ts` | -| Metrics and logs | `Metric`, `Logger`, `Effect.log*` | `effect`, `@effect/opentelemetry` | `packages/effect/src/Metric.ts`, `packages/effect/src/Logger.ts` | -| Normal internal errors | `S.TaggedErrorClass` and existing TrustGraph tagged errors | `effect/Schema`, `@trustgraph/base/errors` | `packages/effect/src/Schema.ts`, `ts/packages/base/src/errors.ts` | -| Imperative exception capture | `Effect.try`, `Effect.tryPromise`, or `Result.try` | `effect`, `effect/Result` | `packages/effect/src/Effect.ts`, `packages/effect/src/Result.ts` | - -Known concrete exports useful to scouts: - -- `Socket.makeWebSocket`, `Socket.fromWebSocket`, `Socket.toChannel`, - `Socket.toChannelString`, `Socket.layerWebSocket`. -- `RpcClient.layerProtocolSocket`, `RpcServer.layerProtocolWebsocket`, - `RpcServer.layerProtocolSocketServer`, `RpcSerialization.layerNdjson`, - `RpcSerialization.layerNdJsonRpc`. -- `BunFileSystem`, `BunSocket`, `BunHttpClient`, `BunHttpServer`, - `BunRuntime`, `BunChildProcessSpawner` from `@effect/platform-bun`. -- `BrowserKeyValueStore.layerLocalStorage`, - `BrowserKeyValueStore.layerSessionStorage`, - `BrowserHttpClient.layerXMLHttpRequest`. -- `Atom.make`, `Atom.runtime`, `Atom.fn`, `Atom.family`, - `Atom.subscriptionRef`, `AtomRegistry.layer`, `Reactivity.layer`, - `Reactivity.query`, `Reactivity.stream`, `Reactivity.mutation`. -- `Tool.make`, `Toolkit.make`, `McpServer.registerToolkit`, - `LanguageModel.generateText`, `LanguageModel.streamText`. -- `S.TaggedErrorClass` for internal/library errors. Treat `new Error` inside - library internals as migration evidence unless it is a host/tool boundary, - test-only helper, or externally mandated error shape. -- `Effect.try`, `Effect.tryPromise`, and `Result.try` for exception capture. - Treat `try`/`catch` blocks as migration evidence unless they are host/tool - boundaries or test-only helpers. -- `S.toTaggedUnion(...).match` and `effect/Match` discriminator helpers for - discriminated unions. Treat native `switch` statements as migration evidence - unless the code is a host parser/traversal boundary with no useful Effect - schema or match primitive. - -## Scout Workflow - -1. Confirm repo state with `git status -sb`. Preserve unrelated files, - especially `.idea/effect.intellij.xml`. -2. Refresh the source baseline above. If a path moved, record the corrected path - in the report before making any recommendations. -3. Verify the tsgo baseline: - - ```sh - cd ts && ./node_modules/.bin/tsgo --version - bun run check - ``` - -4. Run quick signal scans: - - ```sh - rg -n "S\\.ErrorClass|try \\{|catch \\(|new Error|new Promise|setTimeout|while \\(|switch \\(|receive\\(|Effect\\.runPromise|toPromiseRequestor|makeAsyncProcessor|process\\.env|JSON\\.parse|JSON\\.stringify|localStorage|new Map|new Set|Map<|Set<|WebSocket" ts/packages --glob '*.ts' --glob '*.tsx' - ``` - -5. Split scouts by lane. If the thread cannot spawn every scout in parallel, - run them in batches using the same report schema. -6. Every finding must include both: - - Evidence of the handrolled TrustGraph pattern. - - Evidence of the exact Effect primitive that could replace it. -7. Do not rewrite code in this audit. The output is a ranked opportunity map and - a recommended PR order. - -## Agent Lanes - -Use these lane prompts as the durable starting point. - -### Base Messaging And Processor Runtime - -Inspect `ts/packages/base/src/messaging`, `ts/packages/base/src/processor`, and -`ts/packages/base/src/spec`. Find Promise compatibility facades, polling -receivers, manual request/response caches, top-level `Effect.runPromise`, and -mutable maps. Compare against `Queue`, `PubSub`, `Stream`, `Scope`, `Layer`, -`Schedule`, `Ref`, and `SynchronizedRef`. - -### Flow Stateful Services - -Inspect `ts/packages/flow/src/config`, `librarian`, `cores`, `flow-manager`, -`prompt`, and service entrypoints. Find `makeAsyncProcessor` object services, -`while (this.running)`, `sleep`, `receive(2000)`, local persistence, and -process-env config. Compare against scoped layers, state refs, schedules, -`FileSystem`, `KeyValueStore`, `Config`, and schema codecs. In TrustGraph's -installed beta, prefer `effect/FileSystem` and -`effect/unstable/persistence/KeyValueStore` plus runtime-specific provider -packages. - -### Gateway And RPC Boundaries - -Inspect `ts/packages/flow/src/gateway` and `ts/packages/client/src/socket`. -Find manual requestor caches, streaming completion detectors, WebSocket -constructor shims, Promise-returning compatibility APIs, and repeated -`Effect.runPromise`. Compare against `Socket`, `RpcClient`, `RpcServer`, -`RpcSerialization`, `Stream`, `Queue`, and `Scope`. - -### RAG, Agent, Provider, And Storage Layers - -Inspect `ts/packages/flow/src/retrieval`, `agent`, `storage`, `query`, `model`, -and `embeddings`. Find `toPromiseRequestor`, direct SDK resource management, -ambient config, JSON parsing, and manual telemetry. Compare against -`EffectRequestResponse`, `Stream`, provider `Layer`s, `Config`, `Schema`, -`Metric`, `Logger`, and `effect/unstable/ai`. - -### MCP And Workbench - -Inspect `ts/packages/mcp` and `ts/packages/workbench`. Treat these as -lower-priority unless a handrolled pattern clearly remains. Prefer making the -Effect MCP server canonical over rewriting it from scratch. In workbench, -compare local storage and remote state wiring against `BrowserKeyValueStore`, -`AtomRpc`, `AtomHttpApi`, `AsyncResult`, and `Reactivity`. - -## Report Schema - -Each scout must return findings in this shape: - -```md -## Finding: - -- Priority: P0 | P1 | P2 | No-op -- Impact: 1-5 -- Risk: 1-5 -- Confidence: 1-5 -- TrustGraph evidence: and pattern name -- Effect evidence: and source path -- Current behavior: -- Rewrite shape: -- Tests: -- Blockers: -``` - -Scoring: - -- P0: large handrolled lifecycle/concurrency/transport surface with a verified - Effect primitive and clear behavior-preserving route. -- P1: real replacement opportunity, but either risk or compatibility needs a - focused design pass. -- P2: cleanup or local modernization; useful but not strategic. -- No-op: current code is already Effect-native, public-boundary Promise code is - intentional, or the Effect primitive does not actually fit. - -## Acceptance Checks - -For an audit-only pass: - -```sh -git status -sb -git diff --check -- ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md ts/EFFECT_NATIVE_REWRITE_AUDIT.md -``` - -For any later implementation PR, rerun the relevant package tests and at least: - -```sh -cd ts && bun run check -cd ts && bun run build -cd ts && bun run test -``` - -If gateway, live services, or workbench behavior changes, also run the existing -smoke lanes from `ts/CLASS_EFFECT_GOAL.md`. diff --git a/ts/EFFECT_NATIVE_STATUS.md b/ts/EFFECT_NATIVE_STATUS.md new file mode 100644 index 00000000..a6020dff --- /dev/null +++ b/ts/EFFECT_NATIVE_STATUS.md @@ -0,0 +1,80 @@ +# TrustGraph TS Effect-Native Status + +## Current State + +The TypeScript port on `ts-port-effect-v4` is Effect-first across the production +runtime surfaces completed in Phases 0-6. The law baseline is empty: +`scripts/effect-laws.allowlist.json` has no baseline entries and one documented +host-boundary exemption. The native-class inventory reports zero blocking +production native classes. + +The client package's public barrel exports the Effect RPC gateway surface and +shared data models. The legacy `trustgraph-socket.ts` Promise `BaseApi` shim is +kept out of the package barrel and remains only for direct client tests. +Workbench QA uses its own minimal mock API shape. + +## Adapted Effect Laws + +- Data that crosses wires, persistence, or domain boundaries uses `S.Class`, + `S.Struct`, or tagged schema unions. Service contracts and option bags may + remain interfaces. +- Production code reads configuration through `Config`, not `process.env`. +- JSON encode/decode goes through Schema codecs such as + `S.UnknownFromJsonString`. +- HTTP clients use `effect/unstable/http` with platform layers. +- Effects run at entrypoints through platform `runMain`; libraries return + `Effect`, `Stream`, `Layer`, or service values. +- Deterministic ordering uses `effect/Array` and explicit `Order`s. +- File/path work uses platform services rather than `node:fs` or `node:path`. +- Errors owned by TrustGraph code use tagged errors on the Effect channel. + +## Standing Exemptions + +- `packages/workbench/src/main.tsx` may throw if `#root` is missing. This is a + browser host boundary and the app cannot render without the mount point. +- `packages/flow/src/agent/mcp-tool/service.ts` may use the + `@modelcontextprotocol/sdk` client behind Effect wrappers. Effect v4 provides + server-side MCP support here, not a replacement client. +- Provider SDK integrations for OpenAI, Azure OpenAI, OpenAI-compatible, + Mistral, and Ollama stay behind `Effect.tryPromise` and + `makeLanguageModelProvider`. Claude already uses `@effect/ai-anthropic`. +- `PubSubBackend` remains a broker adapter contract. Effect `PubSub` is + in-process only and is not a NATS replacement. +- Tests and QA fixtures are excluded from production law checks. + +## Gate Commands + +Phase 6 validation: + +```sh +bun run check +bun run lint +bunx --bun turbo test build --force +git diff --check +``` + +Final Phase 7 acceptance, all no-cache where applicable: + +```sh +bun run check:tsgo +bunx --bun turbo build lint test --force +bun run --cwd packages/mcp test +bun run workbench:qa +cd deploy && docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build +bun run seed && bun run seed:flows && bun run seed:demo +SKIP_LLM=1 bun run test:pipeline +git diff --check +``` + +Final source audits: + +```sh +bun scripts/check-effect-laws.ts +bun scripts/inventory-native-classes.ts +rg -n "fastify|@fastify/websocket|commander|zod" package.json packages scripts bun.lock +rg -n "@modelcontextprotocol/sdk" packages scripts package.json bun.lock +``` + +Expected result: no law baseline debt, zero blocking native classes, no Fastify, +Commander, Zod, or server-side MCP SDK reintroduction. The MCP SDK client +exemption remains documented above. diff --git a/ts/packages/base/src/schema/primitives.ts b/ts/packages/base/src/schema/primitives.ts index a0c6bb6a..5d628f8f 100644 --- a/ts/packages/base/src/schema/primitives.ts +++ b/ts/packages/base/src/schema/primitives.ts @@ -75,12 +75,12 @@ export const Field = S.Struct({ }); export type Field = typeof Field.Type; -export const RowSchema = S.Struct({ +export const Row = S.Struct({ name: S.String, description: S.optionalKey(S.String), fields: S.Array(Field).pipe(S.mutable), }); -export type RowSchema = typeof RowSchema.Type; +export type Row = typeof Row.Type; export const LlmResult = S.Struct({ text: S.String, diff --git a/ts/packages/client/src/index.ts b/ts/packages/client/src/index.ts index 0a44f4b2..2b805e4b 100644 --- a/ts/packages/client/src/index.ts +++ b/ts/packages/client/src/index.ts @@ -6,8 +6,13 @@ export * from "./models/Triple.js"; export * from "./models/messages.js"; export * from "./models/namespaces.js"; -// Export socket client -export * from "./socket/trustgraph-socket.js"; +// Export retained compatibility types from the legacy socket shim. +export type { + ConnectionState, + ExplainEvent, + GraphRagOptions, + StreamingMetadata, +} from "./socket/trustgraph-socket.js"; export * from "./socket/effect-rpc-client.js"; export * from "./rpc/contract.js"; diff --git a/ts/packages/client/src/socket/trustgraph-socket.ts b/ts/packages/client/src/socket/trustgraph-socket.ts index cfe75370..f2014fc9 100644 --- a/ts/packages/client/src/socket/trustgraph-socket.ts +++ b/ts/packages/client/src/socket/trustgraph-socket.ts @@ -7,7 +7,7 @@ import type { RpcConnectionState, } from "./effect-rpc-client.js"; import { getDefaultSocketUrl, getRandomValues } from "./websocket-adapter.js"; -import { Match, Option, Schema as S } from "effect"; +import { Array as A, Match, Option, Order, Schema as S } from "effect"; import * as Predicate from "effect/Predicate"; // Import all message types for different services @@ -1240,9 +1240,7 @@ export function makeFlowsApi(api: BaseApi) { return this.getConfigAll().then((r) => { const config = r as { config?: { prompt?: Record } }; const promptNs = config.config?.prompt ?? {}; - return Object.keys(promptNs) - .filter((k) => k !== "system") - .sort() + return A.sort(Object.keys(promptNs).filter((k) => k !== "system"), Order.String) .map((id) => ({ id, name: id })); }); }, @@ -2204,9 +2202,7 @@ export function makeConfigApi(api: BaseApi) { return this.getConfigAll().then((r) => { const config = r as { config?: { prompt?: Record } }; const promptNs = config.config?.prompt ?? {}; - return Object.keys(promptNs) - .filter((k) => k !== "system") - .sort() + return A.sort(Object.keys(promptNs).filter((k) => k !== "system"), Order.String) .map((id) => ({ id, name: id })); }); }, diff --git a/ts/packages/flow/src/config/service.ts b/ts/packages/flow/src/config/service.ts index 24fcf4bd..5ac9174d 100644 --- a/ts/packages/flow/src/config/service.ts +++ b/ts/packages/flow/src/config/service.ts @@ -5,7 +5,7 @@ */ import {NodeRuntime} from "@effect/platform-node"; -import {Duration, Effect, HashMap, Match, Option, SynchronizedRef} from "effect"; +import {Array as A, Duration, Effect, HashMap, Match, Option, Order, SynchronizedRef} from "effect"; import * as Predicate from "effect/Predicate"; import * as S from "effect/Schema"; import type { @@ -122,26 +122,27 @@ const initialState = (): ConfigServiceState => ({ const getHashMapValue = (store: HashMap.HashMap, key: K): V | undefined => Option.getOrUndefined(HashMap.get(store, key)); -const compareText = (left: string, right: string): number => - left.localeCompare(right); - -const compareWorkspace = (left: string, right: string): number => +const workspaceOrder = Order.make((left, right) => left === right ? 0 : left === DEFAULT_WORKSPACE ? -1 : right === DEFAULT_WORKSPACE ? 1 - : compareText(left, right); + : Order.String(left, right) +); + +const orderByKey = (order: Order.Order): Order.Order => + Order.mapInput(order, ([key]) => key); const workspaceEntries = (store: ConfigStore): ReadonlyArray => - HashMap.toEntries(store).sort(([left], [right]) => compareWorkspace(left, right)); + A.sort(HashMap.toEntries(store), orderByKey(workspaceOrder)); const namespaceEntries = (store: WorkspaceStore): ReadonlyArray => - HashMap.toEntries(store).sort(([left], [right]) => compareText(left, right)); + A.sort(HashMap.toEntries(store), orderByKey(Order.String)); const valueEntries = (store: NamespaceStore): ReadonlyArray => - HashMap.toEntries(store).sort(([left], [right]) => compareText(left, right)); + A.sort(HashMap.toEntries(store), orderByKey(Order.String)); const toPersistedWorkspaces = ( store: ConfigStore, diff --git a/ts/packages/flow/src/cores/service.ts b/ts/packages/flow/src/cores/service.ts index be38fbd2..b355a56c 100644 --- a/ts/packages/flow/src/cores/service.ts +++ b/ts/packages/flow/src/cores/service.ts @@ -29,7 +29,7 @@ import { processorLifecycleError, topics, } from "@trustgraph/base"; -import {Duration, Effect, HashMap, Match, SynchronizedRef} from "effect"; +import {Array as A, Duration, Effect, HashMap, Match, Order, SynchronizedRef} from "effect"; import * as O from "effect/Option"; import * as S from "effect/Schema"; import {ensureDirectoryEffect, joinPath, readTextFileEffect, writeTextFileEffect} from "../runtime/effect-files.js"; @@ -137,8 +137,11 @@ const cloneKnowledgeCore = (core: KnowledgeCore): KnowledgeCore => ({ })), }); -const sortedEntries = (store: HashMap.HashMap): ReadonlyArray => - HashMap.toEntries(store).sort(([left], [right]) => left.localeCompare(right)); +const sortedEntries = (store: HashMap.HashMap): ReadonlyArray => + A.sort( + HashMap.toEntries(store), + Order.make(([left], [right]) => Order.String(left, right)), + ); const toPersistedSnapshot = (state: KnowledgeCoreServiceState): PersistedKnowledgeSnapshot => { const kg: Record = {}; diff --git a/ts/packages/flow/src/flow-manager/service.ts b/ts/packages/flow/src/flow-manager/service.ts index 66c4136f..6187dd67 100644 --- a/ts/packages/flow/src/flow-manager/service.ts +++ b/ts/packages/flow/src/flow-manager/service.ts @@ -37,7 +37,7 @@ import { import { makeProcessorProgram } from "@trustgraph/base"; import type { Message } from "@trustgraph/base"; import { NodeRuntime } from "@effect/platform-node"; -import { Duration, Effect, HashMap, Match, Option, SynchronizedRef } from "effect"; +import { Array as A, Duration, Effect, HashMap, Match, Option, Order, SynchronizedRef } from "effect"; import * as S from "effect/Schema"; // ---------- Internal state types ---------- @@ -236,8 +236,11 @@ const isStringRecord = (value: unknown): value is Record => const getHashMapValue = (store: HashMap.HashMap, key: K): V | undefined => Option.getOrUndefined(HashMap.get(store, key)); -const sortedEntries = (store: HashMap.HashMap): ReadonlyArray => - HashMap.toEntries(store).sort(([left], [right]) => left.localeCompare(right)); +const sortedEntries = (store: HashMap.HashMap): ReadonlyArray => + A.sort( + HashMap.toEntries(store), + Order.make(([left], [right]) => Order.String(left, right)), + ); const sortedKeys = (store: HashMap.HashMap): Array => sortedEntries(store).map(([key]) => key); diff --git a/ts/packages/flow/src/librarian/service.ts b/ts/packages/flow/src/librarian/service.ts index 642e865d..27bcc2b7 100644 --- a/ts/packages/flow/src/librarian/service.ts +++ b/ts/packages/flow/src/librarian/service.ts @@ -35,7 +35,7 @@ import { } from "@trustgraph/base"; import type { Message } from "@trustgraph/base"; import { NodeRuntime } from "@effect/platform-node"; -import { Clock, Config, DateTime, Duration, Effect, Match, Option, Random, SynchronizedRef } from "effect"; +import { Clock, Config, DateTime, Duration, Effect, Match, Option, Order, Random, SynchronizedRef } from "effect"; import * as A from "effect/Array"; import * as MutableHashMap from "effect/MutableHashMap"; import * as S from "effect/Schema"; @@ -479,7 +479,7 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS if (session === undefined) { return yield* librarianServiceError("get-upload-status", `Upload not found: ${uploadId}`); } - const receivedChunks = Array.from(MutableHashMap.keys(session.chunks)).sort((a, b) => a - b); + const receivedChunks = A.sort(Array.from(MutableHashMap.keys(session.chunks)), Order.Number); const receivedSet = new Set(receivedChunks); const missingChunks = Array.from({ length: session.totalChunks }, (_, i) => i).filter((i) => !receivedSet.has(i)); return { diff --git a/ts/packages/flow/src/retrieval/graph-rag.ts b/ts/packages/flow/src/retrieval/graph-rag.ts index 823c9e1e..790fc656 100644 --- a/ts/packages/flow/src/retrieval/graph-rag.ts +++ b/ts/packages/flow/src/retrieval/graph-rag.ts @@ -20,7 +20,7 @@ import type { TriplesQueryResponse, } from "@trustgraph/base"; import { Triple, errorMessage } from "@trustgraph/base"; -import { Context, Effect, Layer, Match } from "effect"; +import { Array as A, Context, Effect, Layer, Match, Order } from "effect"; import * as O from "effect/Option"; import * as S from "effect/Schema"; @@ -348,8 +348,10 @@ const scoreEdges = Effect.fn("GraphRagEngine.scoreEdges")(function* ( yield* Effect.log(`[GraphRag] Edge scoring LLM response (first 500 chars): ${llmResp.response.slice(0, 500)}`); - const scored = parseScoredEdges(llmResp.response); - scored.sort((a, b) => b.score - a.score); + const scored = A.sort( + parseScoredEdges(llmResp.response), + Order.make((left, right) => Order.Number(right.score, left.score)), + ); const topN = scored.slice(0, config.edgeLimit); const result: Triple[] = []; diff --git a/ts/packages/mcp/src/server-effect.ts b/ts/packages/mcp/src/server-effect.ts index f4b9f087..8ec27a21 100644 --- a/ts/packages/mcp/src/server-effect.ts +++ b/ts/packages/mcp/src/server-effect.ts @@ -9,7 +9,7 @@ import type { import { makeTrustGraphGatewayClientScoped, } from "@trustgraph/client"; -import {Clock, Config, Context, Effect, Layer} from "effect"; +import {Array as A, Clock, Config, Context, Effect, Layer, Order} from "effect"; import * as O from "effect/Option"; import * as Predicate from "effect/Predicate"; import {McpServer, Tool, Toolkit} from "effect/unstable/ai"; @@ -1775,9 +1775,7 @@ export const TrustGraphMcpToolkitLive = TrustGraphMcpToolkit.toLayer( ).pipe( Effect.map((response) => { const promptNs = asRecord(asRecord(response.config).prompt) - const prompts = Object.keys(promptNs) - .filter((key) => key !== "system") - .sort() + const prompts = A.sort(Object.keys(promptNs).filter((key) => key !== "system"), Order.String) .map((id) => ({id, name: id})) return GetPromptsSuccess.make({prompts}) }), diff --git a/ts/packages/workbench/src/atoms/workbench.ts b/ts/packages/workbench/src/atoms/workbench.ts index a5b93f35..a31adc8e 100644 --- a/ts/packages/workbench/src/atoms/workbench.ts +++ b/ts/packages/workbench/src/atoms/workbench.ts @@ -19,8 +19,9 @@ import { GatewayWorkbenchHttpApi, TrustGraphRpcs, } from "@trustgraph/client"; +import type { WorkbenchQaApi } from "@/qa/mock-api"; import type { Scope, } from "effect"; -import { Cause, Clock, Context, Effect, Layer, Match, Metric, Option, Random, Schema as S, Stream } from "effect"; +import { Cause, Clock, Context, Effect, Layer, Match, Metric, Option, Order, Random, Schema as S, Stream } from "effect"; import * as A from "effect/Array"; import * as MutableHashMap from "effect/MutableHashMap"; import * as Predicate from "effect/Predicate"; @@ -802,9 +803,9 @@ function ensureNoGatewayResponseError(operation: string, value: A): Effect.Ef : Effect.fail(WorkbenchPromiseError.make({ cause: value, message: `${operation}: ${message}` })); } -function qaBaseApi(): import("@trustgraph/client").BaseApi | undefined { +function qaBaseApi(): WorkbenchQaApi | undefined { if (typeof window === "undefined") return undefined; - return (window as Window & { __TRUSTGRAPH_WORKBENCH_QA_API__?: import("@trustgraph/client").BaseApi }).__TRUSTGRAPH_WORKBENCH_QA_API__; + return (window as Window & { __TRUSTGRAPH_WORKBENCH_QA_API__?: WorkbenchQaApi }).__TRUSTGRAPH_WORKBENCH_QA_API__; } function makeWorkbenchGatewayApi(settings: Settings) { @@ -897,9 +898,7 @@ function makeWorkbenchGatewayApi(settings: Settings) { Effect.map((response) => { const config = asJsonRecord(response.config); const promptNs = asJsonRecord(config.prompt); - return Object.keys(promptNs) - .filter((key) => key !== "system") - .sort() + return A.sort(Object.keys(promptNs).filter((key) => key !== "system"), Order.String) .map((id) => ({ id, name: id })); }), ), @@ -1826,9 +1825,9 @@ function explainTriplesKey(input: ExplainTriplesInput): string { const graphs = input.events .map((event) => event.explainGraph ?? event.explainId ?? "") .filter((id) => id.length > 0) - .sort() + const sortedGraphs = A.sort(graphs, Order.String) .join(explainGraphSeparator); - return [input.flowId, input.collection, graphs].join(atomFamilyKeySeparator); + return [input.flowId, input.collection, sortedGraphs].join(atomFamilyKeySeparator); } const graphTriplesAtomByKey = Atom.family((key: string) => { diff --git a/ts/packages/workbench/src/components/chat/explain-graph.tsx b/ts/packages/workbench/src/components/chat/explain-graph.tsx index 5a578070..7674e1c0 100644 --- a/ts/packages/workbench/src/components/chat/explain-graph.tsx +++ b/ts/packages/workbench/src/components/chat/explain-graph.tsx @@ -1,5 +1,6 @@ import { lazy, Suspense } from "react"; import { useAtom, useAtomValue } from "@effect/atom-react"; +import { Array as A, Order } from "effect"; import { Network, ChevronRight, ChevronDown, Loader2 } from "lucide-react"; import * as Atom from "effect/unstable/reactivity/Atom"; import { @@ -74,7 +75,7 @@ export function ExplainGraph({ explainEvents, collection }: ExplainGraphProps) { const loading = expanded && resultLoading(result, triples); const error = resultError(result); const { data: graphData, typeMap } = triplesToGraph(triples); - const uniqueTypes = Array.from(new Set(Array.from(typeMap.values()).map(localName))).sort(); + const uniqueTypes = A.sort(Array.from(new Set(Array.from(typeMap.values()).map(localName))), Order.String); return (
diff --git a/ts/packages/workbench/src/pages/graph.tsx b/ts/packages/workbench/src/pages/graph.tsx index 80c6f7cf..8d3f7692 100644 --- a/ts/packages/workbench/src/pages/graph.tsx +++ b/ts/packages/workbench/src/pages/graph.tsx @@ -1,5 +1,6 @@ import { lazy, Suspense } from "react"; import { useAtom, useAtomRefresh, useAtomValue } from "@effect/atom-react"; +import { Array as A, Order } from "effect"; import { Rotate3d, Search, @@ -180,7 +181,7 @@ export default function GraphPage() { const selectedNode = view.selectedNodeId !== null ? data.nodes.find((node) => node.id === view.selectedNodeId) : undefined; - const uniqueTypes = Array.from(new Set(Array.from(typeMap.values()).map(localName))).sort(); + const uniqueTypes = A.sort(Array.from(new Set(Array.from(typeMap.values()).map(localName))), Order.String); return (
diff --git a/ts/packages/workbench/src/qa/initial-values.ts b/ts/packages/workbench/src/qa/initial-values.ts index f67df778..36b0fa71 100644 --- a/ts/packages/workbench/src/qa/initial-values.ts +++ b/ts/packages/workbench/src/qa/initial-values.ts @@ -8,7 +8,7 @@ import { flowIdAtom, settingsAtom, } from "@/atoms/workbench"; -import type { BaseApi } from "@trustgraph/client"; +import type { WorkbenchQaApi } from "@/qa/mock-api"; import { MockWorkbenchFixture, makeMockBaseApi, qaSettingsFromFixture, } from "@/qa/mock-api"; import { Schema as S } from "effect"; @@ -21,7 +21,7 @@ export class WorkbenchQaWindowConfig extends S.Class("W declare global { interface Window { __TRUSTGRAPH_WORKBENCH_QA__?: WorkbenchQaWindowConfig; - __TRUSTGRAPH_WORKBENCH_QA_API__?: BaseApi; + __TRUSTGRAPH_WORKBENCH_QA_API__?: WorkbenchQaApi; } } diff --git a/ts/packages/workbench/src/qa/mock-api.ts b/ts/packages/workbench/src/qa/mock-api.ts index 02dba61a..d68752e4 100644 --- a/ts/packages/workbench/src/qa/mock-api.ts +++ b/ts/packages/workbench/src/qa/mock-api.ts @@ -1,9 +1,26 @@ -import type { BaseApi, DocumentMetadata, ProcessingMetadata, StreamingMetadata, Triple } from "@trustgraph/client"; -import { makeBaseApiWithRpc, } from "@trustgraph/client"; -import { Match, Option, Schema as S } from "effect"; +import type { DocumentMetadata, ProcessingMetadata, StreamingMetadata, Triple } from "@trustgraph/client"; +import { Array as A, Match, Option, Order, Schema as S } from "effect"; type ConfigValues = Record>; +export interface WorkbenchQaApi { + readonly makeRequest: ( + service: string, + request: RequestType, + timeout?: number, + retries?: number, + flow?: string, + ) => Promise; + readonly makeRequestMulti: ( + service: string, + request: RequestType, + receiver: (resp: unknown) => boolean, + timeout?: number, + retries?: number, + flow?: string, + ) => Promise; +} + const UnknownRecord = S.Record(S.String, S.Unknown); const ConfigValuesRecord = S.Record(S.String, UnknownRecord); @@ -358,7 +375,9 @@ function dispatchFlow(state: MockState, request: Record): unkno const id = stringValue(request["flow-id"], "default"); return { flow: encodeJson(state.flows.definitions[id] ?? { id, description: "Mock flow" }) }; } - if (operation === "list-blueprints") return { "blueprint-names": Object.keys(state.flows.blueprints).sort() }; + if (operation === "list-blueprints") { + return { "blueprint-names": A.sort(Object.keys(state.flows.blueprints), Order.String) }; + } if (operation === "get-blueprint") { const name = stringValue(request["blueprint-name"], "qa-blueprint"); return { "blueprint-definition": encodeJson(state.flows.blueprints[name] ?? {}) }; @@ -563,33 +582,37 @@ function dispatchStream( return Promise.resolve({} as ResponseType); } -export function makeMockBaseApi(fixture: MockWorkbenchFixture = {}): BaseApi { +export function makeMockBaseApi(fixture: MockWorkbenchFixture = {}): WorkbenchQaApi { const state = createState(fixture); - const token = state.settings.apiKey.length > 0 ? state.settings.apiKey : undefined; - return makeBaseApiWithRpc(state.settings.user, token, state.settings.gatewayUrl, { - dispatch: (input) => + return { + makeRequest: ( + service: string, + request: RequestType, + _timeout?: number, + _retries?: number, + flow?: string, + ): Promise => Promise.resolve( dispatchRequest( state, - input.service, - input.request, - input.flow, - ), + service, + request as Record, + flow, + ) as ResponseType, ), - dispatchStream: (input, receiver) => - dispatchStream(state, input.service, (message) => { + makeRequestMulti: ( + service: string, + _request: RequestType, + receiver: (resp: unknown) => boolean, + ): Promise => + dispatchStream(state, service, (message) => { const chunk = message as { response?: unknown; complete?: boolean }; return receiver({ response: chunk.response, complete: chunk.complete === true, }); - }).then(() => undefined), - subscribe: (listener) => { - listener({ status: token === undefined ? "connected" : "connected" }); - return () => {}; - }, - close: () => Promise.resolve(), - }); + }).then(() => ({}) as ResponseType), + }; } export function qaSettingsFromFixture(fixture: MockWorkbenchFixture = {}) { diff --git a/ts/scripts/effect-laws.allowlist.json b/ts/scripts/effect-laws.allowlist.json index 60a64de1..e0a2d94d 100644 --- a/ts/scripts/effect-laws.allowlist.json +++ b/ts/scripts/effect-laws.allowlist.json @@ -1 +1 @@ -{"exemptions":[],"baseline":[{"rule":"no-error-throw","path":"packages/workbench/src/main.tsx","count":1},{"rule":"no-native-sort","path":"packages/client/src/socket/trustgraph-socket.ts","count":2},{"rule":"no-native-sort","path":"packages/flow/src/config/service.ts","count":3},{"rule":"no-native-sort","path":"packages/flow/src/cores/service.ts","count":1},{"rule":"no-native-sort","path":"packages/flow/src/flow-manager/service.ts","count":1},{"rule":"no-native-sort","path":"packages/flow/src/librarian/service.ts","count":1},{"rule":"no-native-sort","path":"packages/flow/src/retrieval/graph-rag.ts","count":1},{"rule":"no-native-sort","path":"packages/mcp/src/server-effect.ts","count":1},{"rule":"no-native-sort","path":"packages/workbench/src/atoms/workbench.ts","count":2},{"rule":"no-native-sort","path":"packages/workbench/src/components/chat/explain-graph.tsx","count":1},{"rule":"no-native-sort","path":"packages/workbench/src/pages/graph.tsx","count":1},{"rule":"no-native-sort","path":"packages/workbench/src/qa/mock-api.ts","count":1},{"rule":"no-schema-suffix","path":"packages/base/src/schema/primitives.ts","count":1}]} +{"exemptions":[{"rule":"no-error-throw","path":"packages/workbench/src/main.tsx","reason":"Host boundary: Workbench cannot render without the #root mount element."}],"baseline":[]} diff --git a/ts/scripts/test-pipeline.ts b/ts/scripts/test-pipeline.ts index 91be6441..61256a2a 100644 --- a/ts/scripts/test-pipeline.ts +++ b/ts/scripts/test-pipeline.ts @@ -11,6 +11,7 @@ import { BunRuntime } from "@effect/platform-bun"; import * as BunHttpClient from "@effect/platform-bun/BunHttpClient"; +import { DispatchInput, makeEffectRpcClient } from "@trustgraph/client"; import { Config, Effect, Option as O, Schema as S } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; @@ -19,17 +20,6 @@ const DEFAULT_LLM_MODEL = "qwen2.5:0.5b"; const DEFAULT_FALKORDB_URL = "redis://localhost:6380"; const DEFAULT_PIPELINE_WAIT = 20; -interface RpcSocket { - close: () => void; - makeRequest: ( - service: string, - request: RequestType, - timeout?: number, - retries?: number, - flow?: string, - ) => Promise; -} - interface PipelineConfig { readonly gatewayUrl: string; readonly gatewaySecret: string | undefined; @@ -299,33 +289,24 @@ const testTextCompletion = (config: PipelineConfig) => catchTest("Text completio })); const testWebSocket = (config: PipelineConfig) => catchTest("Effect RPC WebSocket", Effect.gen(function* () { - let socket: RpcSocket | undefined; - return yield* Effect.gen(function* () { - const { createTrustGraphSocket } = yield* Effect.tryPromise({ - try: () => import("../packages/client/src/socket/trustgraph-socket.js"), - catch: (cause) => pipelineError("websocket.import", cause), - }); + const gatewayWsUrl = config.gatewayUrl.replace(/^http/, "ws").replace(/\/$/, ""); + const response = yield* Effect.scoped( + Effect.gen(function*() { + const client = yield* makeEffectRpcClient(`${gatewayWsUrl}/api/v1/rpc`); + return yield* client.dispatch( + DispatchInput.make({ + scope: "global", + service: "config", + request: { operation: "list", keys: [] }, + }), + { timeoutMs: 5_000 }, + ); + }), + ).pipe(Effect.timeout("5 seconds")); - const gatewayWsUrl = config.gatewayUrl.replace(/^http/, "ws").replace(/\/$/, ""); - socket = createTrustGraphSocket( - "pipeline", - config.gatewaySecret, - `${gatewayWsUrl}/api/v1/rpc`, - ); - const response = yield* Effect.tryPromise({ - try: () => - socket?.makeRequest, Record>( - "config", - { operation: "list", keys: [] }, - 5000, - ) ?? Promise.resolve({}), - catch: (cause) => pipelineError("websocket.request", cause), - }).pipe(Effect.timeout("5 seconds")); - - log("websocket/rpc-response", response); - pass("Effect RPC WebSocket round-trip works"); - return true; - }).pipe(Effect.ensuring(Effect.sync(() => socket?.close()))); + log("websocket/rpc-response", response); + pass("Effect RPC WebSocket round-trip works"); + return true; })); // ─── Librarian Tests ──────────────────────────────────────────────────