diff --git a/ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md b/ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md index 96431b93..001ef649 100644 --- a/ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md +++ b/ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md @@ -13,6 +13,7 @@ Verify these paths at the start of every audit run: - Effect v4 subtree: `/home/elpresidank/YeeBois/projects/beep-effect2/.repos/effect-v4` - Installed Effect fallback: `ts/node_modules/effect` - Installed Atom React fallback: `ts/packages/workbench/node_modules/@effect/atom-react` +- Native TypeScript checker: `ts/node_modules/.bin/tsgo` The prompt typo path `~/YeeBois/projecects/...` is not valid on this machine. Use `~/YeeBois/projects/...`. @@ -29,6 +30,23 @@ resolves many beta APIs through `effect/unstable/*` import paths. Prefer the installed TrustGraph import path when it exists; use the subtree package path as source proof, not as an automatic import recommendation. +TrustGraph's TypeScript check must use `@typescript/native-preview` patched by +`@effect/tsgo`. The expected local version line is currently: + +```sh +cd ts && ./node_modules/.bin/tsgo --version +# Version 7.0.0-dev+effect-tsgo.0.13.0 +``` + +The root `ts/tsconfig.base.json` language-service plugin should stay aligned +with `/home/elpresidank/YeeBois/projects/beep-effect/tsconfig.base.json`, +using the same diagnostic severities and no test-only diagnostic downgrades. +The repo-specific `namespaceImportPackages` entry should use `@trustgraph/*`. +The root `check` script should delegate to `check:tsgo`, and `check:tsgo` must +run `effect-tsgo patch` before `tsgo -b` so agents get Effect diagnostics +instead of plain TypeScript. Treat every `effect(...)` diagnostic as a hard +blocker. + ## Primitive Map Use this map as the starting baseline. A finding is only valid when it cites the @@ -84,18 +102,25 @@ Known concrete exports useful to scouts: especially `.idea/effect.intellij.xml`. 2. Refresh the source baseline above. If a path moved, record the corrected path in the report before making any recommendations. -3. Run quick signal scans: +3. Verify the tsgo baseline: + + ```sh + cd ts && ./node_modules/.bin/tsgo --version + bun run check + ``` + +4. Run quick signal scans: ```sh rg -n "try \\{|new Error|new Promise|setTimeout|while \\(|receive\\(|Effect\\.runPromise|toPromiseRequestor|makeAsyncProcessor|process\\.env|JSON\\.parse|JSON\\.stringify|localStorage|new Map|WebSocket" ts/packages --glob '*.ts' --glob '*.tsx' ``` -4. Split scouts by lane. If the thread cannot spawn every scout in parallel, +5. Split scouts by lane. If the thread cannot spawn every scout in parallel, run them in batches using the same report schema. -5. Every finding must include both: +6. Every finding must include both: - Evidence of the handrolled TrustGraph pattern. - Evidence of the exact Effect primitive that could replace it. -6. Do not rewrite code in this audit. The output is a ranked opportunity map and +7. Do not rewrite code in this audit. The output is a ranked opportunity map and a recommended PR order. ## Agent Lanes @@ -186,7 +211,7 @@ git diff --check -- ts/EFFECT_NATIVE_REWRITE_PLAYBOOK.md ts/EFFECT_NATIVE_REWRIT For any later implementation PR, rerun the relevant package tests and at least: ```sh -cd ts && bun run check:tsgo +cd ts && bun run check cd ts && bun run build cd ts && bun run test ``` diff --git a/ts/bun.lock b/ts/bun.lock index 642dbfc2..87261004 100644 --- a/ts/bun.lock +++ b/ts/bun.lock @@ -13,7 +13,7 @@ "@effect/vitest": "4.0.0-beta.75", "@types/bun": "^1.3.13", "@types/node": "^25.7.0", - "@typescript/native-preview": "^7.0.0-dev.20260511.1", + "@typescript/native-preview": "7.0.0-dev.20260511.1", "falkordb": "^5.0.0", "nats": "^2.29.0", "pdf-lib": "^1.17.1", diff --git a/ts/package.json b/ts/package.json index d2654a6b..b3fff7ea 100644 --- a/ts/package.json +++ b/ts/package.json @@ -6,7 +6,8 @@ "dev": "bunx --bun turbo dev", "lint": "bunx --bun turbo lint", "test": "bunx --bun turbo test", - "check:tsgo": "tsgo -b tsconfig.json", + "check": "bun run check:tsgo", + "check:tsgo": "effect-tsgo patch && tsgo -b tsconfig.json", "inventory:classes": "bun scripts/inventory-native-classes.ts", "workbench:qa": "bun run --cwd packages/workbench qa:browser", "prepare": "effect-tsgo patch", @@ -48,7 +49,7 @@ "@effect/vitest": "4.0.0-beta.75", "@types/bun": "^1.3.13", "@types/node": "^25.7.0", - "@typescript/native-preview": "^7.0.0-dev.20260511.1", + "@typescript/native-preview": "7.0.0-dev.20260511.1", "falkordb": "^5.0.0", "nats": "^2.29.0", "pdf-lib": "^1.17.1", diff --git a/ts/packages/flow/src/__tests__/config-service.test.ts b/ts/packages/flow/src/__tests__/config-service.test.ts new file mode 100644 index 00000000..c74f6e62 --- /dev/null +++ b/ts/packages/flow/src/__tests__/config-service.test.ts @@ -0,0 +1,113 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + ConfigServiceError, + makeConfigService, +} from "../config/service.js"; +import type { + BackendConsumer, + BackendProducer, + ConfigRequest, + CreateConsumerOptions, + CreateProducerOptions, + PubSubBackend, +} from "@trustgraph/base"; + +class NoopPubSub implements PubSubBackend { + async createProducer(_options: CreateProducerOptions): Promise> { + return { + send: async () => undefined, + flush: async () => undefined, + close: async () => undefined, + }; + } + + async createConsumer(_options: CreateConsumerOptions): Promise> { + return { + receive: async () => null, + acknowledge: async () => undefined, + negativeAcknowledge: async () => undefined, + unsubscribe: async () => undefined, + close: async () => undefined, + }; + } + + async close(): Promise {} +} + +const makeService = (persistPath?: string) => + makeConfigService({ + id: "config-test", + manageProcessSignals: false, + pubsub: new NoopPubSub(), + ...(persistPath === undefined ? {} : { persistPath }), + }); + +describe("ConfigService operations", () => { + it("uses tagged errors for invalid mutations", async () => { + const service = makeService(); + + const putError = await service.handlePut({ operation: "put" } as ConfigRequest) + .catch((caught: unknown) => caught); + const deleteError = await service.handleDelete({ operation: "delete" } as ConfigRequest) + .catch((caught: unknown) => caught); + + expect(putError).toBeInstanceOf(ConfigServiceError); + expect(putError).toMatchObject({ _tag: "ConfigServiceError", operation: "put" }); + expect(deleteError).toBeInstanceOf(ConfigServiceError); + expect(deleteError).toMatchObject({ _tag: "ConfigServiceError", operation: "delete" }); + }); + + it("persists the workspace-aware config shape through Effect.tryPromise", async () => { + const dir = await mkdtemp(join(tmpdir(), "trustgraph-config-service-")); + const persistPath = join(dir, "config.json"); + const service = makeService(persistPath); + + await service.handlePut({ + operation: "put", + values: [ + { workspace: "alpha", type: "prompt", key: "system", value: "hello" }, + ], + } as ConfigRequest); + + const persisted = await Bun.file(persistPath).json(); + await rm(dir, { recursive: true, force: true }); + + expect(persisted).toEqual({ + version: 1, + workspaces: { + alpha: { + prompt: { + system: "hello", + }, + }, + }, + }); + }); + + it("loads the legacy persisted data shape without try/catch", async () => { + const dir = await mkdtemp(join(tmpdir(), "trustgraph-config-service-")); + const persistPath = join(dir, "config.json"); + await Bun.write( + persistPath, + `{"version":7,"data":{"prompt":{"system":"legacy"}}}`, + ); + const service = makeService(persistPath); + + await service.loadFromDisk(); + const response = service.handleGet({ + operation: "get", + keys: ["prompt", "system"], + } as ConfigRequest); + await rm(dir, { recursive: true, force: true }); + + expect(response).toEqual({ + version: 7, + values: { + system: "legacy", + }, + }); + }); +}); diff --git a/ts/packages/flow/src/agent/mcp-tool/service.ts b/ts/packages/flow/src/agent/mcp-tool/service.ts index ae7f15b1..32ffeb82 100644 --- a/ts/packages/flow/src/agent/mcp-tool/service.ts +++ b/ts/packages/flow/src/agent/mcp-tool/service.ts @@ -75,7 +75,7 @@ const mcpToolError = ( cause: unknown, tool?: string, ): McpToolError => - new McpToolError({ + McpToolError.make({ operation, message: errorMessage(cause), ...(tool === undefined ? {} : { tool }), @@ -166,12 +166,9 @@ const invokeConfiguredTool = Effect.fn("McpToolRuntime.invokeTool")(function* ( const result = yield* Effect.acquireUseRelease( Effect.tryPromise({ - try: async () => { - await client.connect(transport as unknown as Parameters[0]); - return client; - }, + try: () => client.connect(transport as unknown as Parameters[0]), catch: (cause) => mcpToolError("connect", cause, name), - }), + }).pipe(Effect.as(client)), (connectedClient) => Effect.tryPromise({ try: () => @@ -318,6 +315,6 @@ export const program = makeFlowProcessorProgram McpToolRuntimeLive, }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/agent/react/service.ts b/ts/packages/flow/src/agent/react/service.ts index 8faf283a..f6638515 100644 --- a/ts/packages/flow/src/agent/react/service.ts +++ b/ts/packages/flow/src/agent/react/service.ts @@ -132,79 +132,80 @@ const toPromiseRequestor = ( const buildConfiguredTool = ( toolId: string, data: ToolConfigEntry, -): AgentTool | null => { - const implType = data.type ?? ""; - const name = data.name ?? ""; - const description = data.description ?? ""; - const config = { ...data } as Record; +): Effect.Effect => + Effect.gen(function* () { + const implType = data.type ?? ""; + const name = data.name ?? ""; + const description = data.description ?? ""; + const config = { ...data } as Record; - if (name.length === 0) { - console.warn(`[AgentService] Skipping tool with no name: ${toolId}`); - return null; - } - - switch (implType) { - case "knowledge-query": - return { - name, - description: - description.length > 0 - ? description - : "Query the knowledge graph for information about entities and their relationships.", - args: [{ name: "question", type: "string", description: "The question to ask" }], - config, - execute: async () => "", - }; - - case "document-query": - return { - name, - description: - description.length > 0 - ? description - : "Search documents for relevant information.", - args: [{ name: "question", type: "string", description: "The question to search for" }], - config, - execute: async () => "", - }; - - case "triples-query": - return { - name, - description: - description.length > 0 - ? description - : "Query for specific triples in the knowledge graph.", - args: [ - { name: "subject", type: "string", description: "Subject entity (optional)" }, - { name: "predicate", type: "string", description: "Predicate/relationship (optional)" }, - { name: "object", type: "string", description: "Object entity (optional)" }, - ], - config, - execute: async () => "", - }; - - case "mcp-tool": { - const args: ToolArg[] = (data.arguments ?? []).map((arg) => ({ - name: arg.name ?? "", - type: arg.type ?? "string", - description: arg.description ?? "", - })); - - return { - name, - description, - args, - config, - execute: async () => "", - }; + if (name.length === 0) { + yield* Effect.logWarning(`[AgentService] Skipping tool with no name: ${toolId}`); + return null; } - default: - console.warn(`[AgentService] Unknown tool type "${implType}" for ${name}`); - return null; - } -}; + switch (implType) { + case "knowledge-query": + return { + name, + description: + description.length > 0 + ? description + : "Query the knowledge graph for information about entities and their relationships.", + args: [{ name: "question", type: "string", description: "The question to ask" }], + config, + execute: () => Promise.resolve(""), + }; + + case "document-query": + return { + name, + description: + description.length > 0 + ? description + : "Search documents for relevant information.", + args: [{ name: "question", type: "string", description: "The question to search for" }], + config, + execute: () => Promise.resolve(""), + }; + + case "triples-query": + return { + name, + description: + description.length > 0 + ? description + : "Query for specific triples in the knowledge graph.", + args: [ + { name: "subject", type: "string", description: "Subject entity (optional)" }, + { name: "predicate", type: "string", description: "Predicate/relationship (optional)" }, + { name: "object", type: "string", description: "Object entity (optional)" }, + ], + config, + execute: () => Promise.resolve(""), + }; + + case "mcp-tool": { + const args: ToolArg[] = (data.arguments ?? []).map((arg) => ({ + name: arg.name ?? "", + type: arg.type ?? "string", + description: arg.description ?? "", + })); + + return { + name, + description, + args, + config, + execute: () => Promise.resolve(""), + }; + } + + default: + yield* Effect.logWarning(`[AgentService] Unknown tool type "${implType}" for ${name}`); + return null; + } + }); const loadConfiguredTools = Effect.fn("AgentRuntime.loadConfiguredTools")(function* ( config: Record, @@ -231,7 +232,7 @@ const loadConfiguredTools = Effect.fn("AgentRuntime.loadConfiguredTools")(functi continue; } - const tool = buildConfiguredTool(toolId, decoded.value); + const tool = yield* buildConfiguredTool(toolId, decoded.value); if (tool === null) continue; tools.push(tool); @@ -348,7 +349,7 @@ const executeTool = ( ): Effect.Effect => Effect.tryPromise({ try: () => tool.execute(input), - catch: (cause) => new AgentToolExecutionError({ message: errorMessage(cause) }), + catch: (cause) => AgentToolExecutionError.make({ message: errorMessage(cause) }), }).pipe( Effect.catch((error: AgentToolExecutionError) => Effect.succeed(`Error executing tool: ${error.message}`), @@ -473,12 +474,12 @@ const onAgentRequest = Effect.fn("AgentService.onRequest")(function* ( }).pipe( Effect.catch((error: unknown) => Effect.logError(`[AgentService] Error processing request ${requestId}`, { - error: error instanceof Error ? error.message : String(error), + error: errorMessage(error), }).pipe( Effect.flatMap(() => responseProducer.send(requestId, { chunk_type: "error", - content: `Agent error: ${error instanceof Error ? error.message : String(error)}`, + content: `Agent error: ${errorMessage(error)}`, end_of_message: true, end_of_dialog: true, }), @@ -538,7 +539,7 @@ export function makeAgentService(config: ProcessorConfig): AgentService { Effect.provideService(AgentRuntime, runtime), )), ); - console.log("[AgentService] Service initialized"); + Effect.runSync(Effect.log("[AgentService] Service initialized")); return service; } @@ -629,6 +630,6 @@ export const program = makeFlowProcessorProgram AgentRuntimeLive, }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/agent/react/tools.ts b/ts/packages/flow/src/agent/react/tools.ts index 9cc6be11..e2d877d5 100644 --- a/ts/packages/flow/src/agent/react/tools.ts +++ b/ts/packages/flow/src/agent/react/tools.ts @@ -18,9 +18,14 @@ import type { Term, Triple, } from "@trustgraph/base"; +import { Effect } from "effect"; +import * as O from "effect/Option"; +import * as S from "effect/Schema"; import type { AgentTool, ToolArg } from "./types.js"; +const decodeJsonUnknown = S.decodeUnknownOption(S.UnknownFromJsonString); + /** * Format a Term to a human-readable string. */ @@ -41,17 +46,15 @@ function termToString(term: Term): string { * Parse tool input -- accepts either raw JSON or a plain string question. */ function parseQuestion(input: string): string { - try { - const parsed = JSON.parse(input) as Record; - if (typeof parsed === "object" && parsed !== null && "question" in parsed) { - return String(parsed.question); - } - // If it's a string JSON value, use it directly - if (typeof parsed === "string") { - return parsed; - } - } catch { - // Not valid JSON -- treat as plain text + const decoded = decodeJsonUnknown(input); + if (O.isNone(decoded)) return input; + + const parsed = decoded.value; + if (typeof parsed === "object" && parsed !== null && "question" in parsed) { + return String(parsed.question); + } + if (typeof parsed === "string") { + return parsed; } return input; } @@ -83,15 +86,15 @@ export function createKnowledgeQueryTool( description: "The question to ask the knowledge graph", }, ], - async execute(input: string): Promise { + execute: (input: string): Promise => Effect.runPromise(Effect.gen(function* () { const question = parseQuestion(input); - console.log(`[KnowledgeQuery] Executing: "${question.slice(0, 60)}..." collection=${collection}`); + yield* Effect.log(`[KnowledgeQuery] Executing: "${question.slice(0, 60)}..." collection=${collection}`); const request: GraphRagRequest = { query: question, ...(collection !== undefined ? { collection } : {}), }; - const res = await client.request(request); - console.log(`[KnowledgeQuery] Response (${res.response?.length ?? 0} chars): ${res.error !== undefined ? `ERROR: ${res.error.message}` : `${res.response?.slice(0, 300)}...`}`); + const res = yield* Effect.tryPromise(() => client.request(request)); + yield* Effect.log(`[KnowledgeQuery] Response (${res.response?.length ?? 0} chars): ${res.error !== undefined ? `ERROR: ${res.error.message}` : `${res.response?.slice(0, 300)}...`}`); // Extract explain data if embedded in the response const rawRes = res as Record; @@ -100,15 +103,15 @@ export function createKnowledgeQueryTool( rawRes.explain_triples !== undefined && onExplain !== undefined ) { - onExplain({ + yield* Effect.sync(() => onExplain({ explainId: (rawRes.explain_id as string) ?? "", triples: rawRes.explain_triples as Triple[], - }); + })); } if (res.error !== undefined) return `Error: ${res.error.message}`; return res.response; - }, + })), }; } @@ -130,16 +133,16 @@ export function createDocumentQueryTool( description: "The question to search documents for", }, ], - async execute(input: string): Promise { + execute: (input: string): Promise => Effect.runPromise(Effect.gen(function* () { const question = parseQuestion(input); const request: DocumentRagRequest = { query: question, ...(collection !== undefined ? { collection } : {}), }; - const res = await client.request(request); + const res = yield* Effect.tryPromise(() => client.request(request)); if (res.error !== undefined) return `Error: ${res.error.message}`; return res.response; - }, + })), }; } @@ -152,39 +155,42 @@ function parseTriplesInput(input: string): { o?: Term; limit?: number; } { - try { - const parsed = JSON.parse(input) as Record; - - const toTerm = (val: unknown): Term | undefined => { - if (typeof val === "string") { - return { type: "LITERAL", value: val }; - } - if (typeof val === "object" && val !== null && "type" in val) { - return val as Term; - } - return undefined; - }; - - const result: { - s?: Term; - p?: Term; - o?: Term; - limit?: number; - } = {}; - const s = toTerm(parsed.subject ?? parsed.s); - const p = toTerm(parsed.predicate ?? parsed.p); - const o = toTerm(parsed.object ?? parsed.o); - if (s !== undefined) result.s = s; - if (p !== undefined) result.p = p; - if (o !== undefined) result.o = o; - if (typeof parsed.limit === "number") result.limit = parsed.limit; - return result; - } catch { - // If not valid JSON, treat as a subject search + const decoded = decodeJsonUnknown(input); + if ( + O.isNone(decoded) || + typeof decoded.value !== "object" || + decoded.value === null + ) { return { s: { type: "LITERAL", value: input }, }; } + + const parsed = decoded.value as Record; + const toTerm = (val: unknown): Term | undefined => { + if (typeof val === "string") { + return { type: "LITERAL", value: val }; + } + if (typeof val === "object" && val !== null && "type" in val) { + return val as Term; + } + return undefined; + }; + + const result: { + s?: Term; + p?: Term; + o?: Term; + limit?: number; + } = {}; + const s = toTerm(parsed.subject ?? parsed.s); + const p = toTerm(parsed.predicate ?? parsed.p); + const o = toTerm(parsed.object ?? parsed.o); + if (s !== undefined) result.s = s; + if (p !== undefined) result.p = p; + if (o !== undefined) result.o = o; + if (typeof parsed.limit === "number") result.limit = parsed.limit; + return result; } /** @@ -216,7 +222,7 @@ export function createTriplesQueryTool( description: "The object entity to search for (optional)", }, ], - async execute(input: string): Promise { + execute: (input: string): Promise => Effect.runPromise(Effect.gen(function* () { const { s, p, o, limit } = parseTriplesInput(input); const request: TriplesQueryRequest = { limit: limit ?? 20, @@ -225,7 +231,7 @@ export function createTriplesQueryTool( ...(o !== undefined ? { o } : {}), ...(collection !== undefined ? { collection } : {}), }; - const res = await client.request(request); + const res = yield* Effect.tryPromise(() => client.request(request)); if (res.error !== undefined) return `Error: ${res.error.message}`; @@ -238,7 +244,7 @@ export function createTriplesQueryTool( `(${termToString(t.s)}) -[${termToString(t.p)}]-> (${termToString(t.o)})`, ); return lines.join("\n"); - }, + })), }; } @@ -258,12 +264,12 @@ export function createMcpTool( name: toolName, description, args, - async execute(input: string): Promise { - const res = await client.request({ name: toolName, parameters: input }); + execute: (input: string): Promise => Effect.runPromise(Effect.gen(function* () { + const res = yield* Effect.tryPromise(() => client.request({ name: toolName, parameters: input })); if (res.error !== undefined) return `Error: ${res.error.message}`; if (res.text !== undefined) return res.text; if (res.object !== undefined) return res.object; return "No content"; - }, + })), }; } diff --git a/ts/packages/flow/src/chunking/service.ts b/ts/packages/flow/src/chunking/service.ts index 2226cf72..a50c750b 100644 --- a/ts/packages/flow/src/chunking/service.ts +++ b/ts/packages/flow/src/chunking/service.ts @@ -91,7 +91,7 @@ export function makeChunkingService(config: ProcessorConfig): ChunkingService { const service = makeFlowProcessor(config, { specifications: makeChunkingSpecs(), }); - console.log("[ChunkingService] Service initialized"); + Effect.runSync(Effect.log("[ChunkingService] Service initialized")); return service; } @@ -102,6 +102,6 @@ export const program = makeFlowProcessorProgram({ specs: () => makeChunkingSpecs(), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/config/service.ts b/ts/packages/flow/src/config/service.ts index 061a6515..e94163c2 100644 --- a/ts/packages/flow/src/config/service.ts +++ b/ts/packages/flow/src/config/service.ts @@ -11,7 +11,7 @@ * Python reference: trustgraph-flow/trustgraph/config/service/service.py */ -import { Effect } from "effect"; +import { Duration, Effect } from "effect"; import * as S from "effect/Schema"; import { makeAsyncProcessor, @@ -45,6 +45,20 @@ const ConfigPushSchema = S.Struct({ config: S.Record(S.String, S.Unknown), }); +export class ConfigServiceError extends S.TaggedErrorClass()( + "ConfigServiceError", + { + message: S.String, + operation: S.String, + }, +) {} + +const configServiceError = (operation: string, cause: unknown): ConfigServiceError => + ConfigServiceError.make({ + operation, + message: errorMessage(cause), + }); + const DEFAULT_WORKSPACE = "default"; interface ConfigKeyLike { @@ -62,6 +76,14 @@ interface ConfigValueLike { type NamespaceStore = Map; type WorkspaceStore = Map; +const PersistedConfigSchema = S.Struct({ + version: S.optionalKey(S.Number), + data: S.optionalKey(S.Record(S.String, S.Record(S.String, S.Unknown))), + workspaces: S.optionalKey(S.Record(S.String, S.Record(S.String, S.Record(S.String, S.Unknown)))), +}); +const PersistedConfigJsonSchema = PersistedConfigSchema.pipe(S.fromJsonString); +type PersistedConfig = typeof PersistedConfigSchema.Type; + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -70,13 +92,63 @@ function optionalString(value: unknown): string | undefined { return typeof value === "string" && value.length > 0 ? value : undefined; } +function toPersistedWorkspaces( + store: Map, +): Record>> { + const workspaces: Record>> = {}; + + for (const [workspace, ws] of store) { + const workspaceData: Record> = {}; + for (const [namespace, subMap] of ws) { + const obj: Record = {}; + for (const [k, v] of subMap) { + obj[k] = v; + } + workspaceData[namespace] = obj; + } + workspaces[workspace] = workspaceData; + } + + return workspaces; +} + +function hydratePersistedConfig( + store: Map, + parsed: PersistedConfig, +): void { + store.clear(); + + if (parsed.workspaces !== undefined) { + for (const [workspace, namespaces] of Object.entries(parsed.workspaces)) { + const ws = new Map(); + for (const [namespace, obj] of Object.entries(namespaces)) { + const subMap = new Map(); + for (const [k, v] of Object.entries(obj)) { + subMap.set(k, v); + } + ws.set(namespace, subMap); + } + store.set(workspace, ws); + } + return; + } + + const ws = new Map(); + for (const [namespace, obj] of Object.entries(parsed.data ?? {})) { + const subMap = new Map(); + for (const [k, v] of Object.entries(obj)) { + subMap.set(k, v); + } + ws.set(namespace, subMap); + } + store.set(DEFAULT_WORKSPACE, ws); +} + export type ConfigService = AsyncProcessorRuntime & Record; export function makeConfigService(config: ConfigServiceConfig): ConfigService { const service = makeAsyncProcessor(config, { - run: async () => { - await service.run(); - }, + run: () => service.run(), }) as ConfigService; const baseStop = service.stop; service.store = new Map(); @@ -88,115 +160,183 @@ export function makeConfigService(config: ConfigServiceConfig): ConfigService { Object.assign(service, { - run: async function(this: ConfigService): Promise { - // Optionally load persisted state - if (this.persistPath !== null) { - await this.loadFromDisk(); - } + run: function(this: ConfigService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + // Optionally load persisted state + if (service.persistPath !== null) { + yield* Effect.tryPromise({ + try: () => service.loadFromDisk(), + catch: (cause) => configServiceError("load", cause), + }); + } - // Create producers - this.responseProducer = await this.pubsub.createProducer({ - topic: topics.configResponse, - schema: ConfigResponseSchema, - }); - this.pushProducer = await this.pubsub.createProducer({ - topic: topics.configPush, - schema: ConfigPushSchema, - }); + // Create producers + service.responseProducer = yield* Effect.tryPromise({ + try: () => + service.pubsub.createProducer({ + topic: topics.configResponse, + schema: ConfigResponseSchema, + }), + catch: (cause) => configServiceError("response-producer", cause), + }); + service.pushProducer = yield* Effect.tryPromise({ + try: () => + service.pubsub.createProducer({ + topic: topics.configPush, + schema: ConfigPushSchema, + }), + catch: (cause) => configServiceError("push-producer", cause), + }); - // Create consumer for config requests - this.consumer = await this.pubsub.createConsumer({ - topic: topics.configRequest, - subscription: `${this.config.id}-config-request`, - schema: ConfigRequestSchema, - }); + // Create consumer for config requests + service.consumer = yield* Effect.tryPromise({ + try: () => + service.pubsub.createConsumer({ + topic: topics.configRequest, + subscription: `${service.config.id}-config-request`, + schema: ConfigRequestSchema, + }), + catch: (cause) => configServiceError("consumer", cause), + }); - // Push initial config - await this.pushConfig(); + // Push initial config + yield* Effect.tryPromise({ + try: () => service.pushConfig(), + catch: (cause) => configServiceError("push-initial-config", cause), + }); - console.log(`[ConfigService] Listening on ${topics.configRequest}`); + yield* Effect.log(`[ConfigService] Listening on ${topics.configRequest}`); - // Main consume loop - while (this.running) { - try { - const consumer = this.consumer; - if (consumer === null) throw new Error("Config consumer not started"); + // Main consume loop + while (service.running) { + const shouldContinue = yield* Effect.gen(function* () { + const consumer = service.consumer; + if (consumer === null) { + return yield* configServiceError("consume", "Config consumer not started"); + } - const msg = await consumer.receive(2000); - if (msg === null) continue; + const msg = yield* Effect.tryPromise({ + try: () => consumer.receive(2000), + catch: (cause) => configServiceError("consume-receive", cause), + }); + if (msg === null) return true; - await this.handleMessage(msg); - await consumer.acknowledge(msg); - } catch (err) { - if (!this.running) break; - console.error("[ConfigService] Error in consume loop:", err); - await sleep(1000); - } - } + yield* Effect.tryPromise({ + try: () => service.handleMessage(msg), + catch: (cause) => configServiceError("consume-handle", cause), + }); + yield* Effect.tryPromise({ + try: () => consumer.acknowledge(msg), + catch: (cause) => configServiceError("consume-acknowledge", cause), + }); + + return true; + }).pipe( + Effect.catch((err) => { + if (!service.running) return Effect.succeed(false); + return Effect.logError("[ConfigService] Error in consume loop", { error: err.message }).pipe( + Effect.flatMap(() => Effect.sleep(Duration.millis(1000))), + Effect.as(true), + ); + }), + ); + if (!shouldContinue) break; + } + }), + ); }, - handleMessage: async function(this: ConfigService, msg: Message): Promise { - const request = await Effect.runPromise(S.decodeUnknownEffect(ConfigRequestSchema)(msg.value())); - const props = msg.properties(); - const requestId = props.id; + handleMessage: function(this: ConfigService, msg: Message): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const request = yield* S.decodeUnknownEffect(ConfigRequestSchema)(msg.value()).pipe( + Effect.mapError((cause) => configServiceError("decode", cause)), + ); + const props = msg.properties(); + const requestId = props.id; - if (requestId === undefined || requestId.length === 0) { - console.warn("[ConfigService] Received request without id, ignoring"); - return; - } + if (requestId === undefined || requestId.length === 0) { + yield* Effect.logWarning("[ConfigService] Received request without id, ignoring"); + return; + } - try { - const response = await this.handleOperation(request); - const responseProducer = this.responseProducer; - if (responseProducer === null) throw new Error("Config response producer not started"); - await responseProducer.send(response, { id: requestId }); - } catch (err) { - const message = errorMessage(err); - const responseProducer = this.responseProducer; - if (responseProducer === null) throw new Error("Config response producer not started"); - await responseProducer.send( - { - error: { type: "config-error", message }, - }, - { id: requestId }, - ); - } + const sendResponse = (response: ConfigResponse): Effect.Effect => + Effect.gen(function* () { + const responseProducer = service.responseProducer; + if (responseProducer === null) { + return yield* configServiceError("respond", "Config response producer not started"); + } + yield* Effect.tryPromise({ + try: () => responseProducer.send(response, { id: requestId }), + catch: (cause) => configServiceError("respond", cause), + }); + }); + + yield* Effect.gen(function* () { + const response = yield* Effect.tryPromise({ + try: () => service.handleOperation(request), + catch: (cause) => configServiceError("operation", cause), + }); + yield* sendResponse(response); + }).pipe( + Effect.catch((err) => + sendResponse({ + error: { type: "config-error", message: err.message }, + }), + ), + ); + }), + ); }, - handleOperation: async function(this: ConfigService, request: ConfigRequest): Promise { - const op: ConfigOperation = request.operation; + handleOperation: function(this: ConfigService, request: ConfigRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const op: ConfigOperation = request.operation; - switch (op) { - case "get": - return this.handleGet(request); + switch (op) { + case "get": + return service.handleGet(request); - case "put": - return await this.handlePut(request); + case "put": + return yield* Effect.tryPromise({ + try: () => service.handlePut(request), + catch: (cause) => configServiceError("put", cause), + }); - case "delete": - return await this.handleDelete(request); + case "delete": + return yield* Effect.tryPromise({ + try: () => service.handleDelete(request), + catch: (cause) => configServiceError("delete", cause), + }); - case "list": - return this.handleList(request); + case "list": + return service.handleList(request); - case "config": - return this.handleConfigDump(request); + case "config": + return service.handleConfigDump(request); - case "getvalues": - return this.handleGetValues(request); + case "getvalues": + return service.handleGetValues(request); - case "getvalues-all-ws": - return this.handleGetValuesAllWorkspaces(request); + case "getvalues-all-ws": + return service.handleGetValuesAllWorkspaces(request); - default: - throw new Error(`Unknown config operation: ${op as string}`); - } + default: + return yield* configServiceError("operation", `Unknown config operation: ${op as string}`); + } + }), + ); }, @@ -364,76 +504,104 @@ export function makeConfigService(config: ConfigServiceConfig): ConfigService { - handlePut: async function(this: ConfigService, request: ConfigRequest): Promise { - const values = this.configValues(request); - if (values.length === 0) throw new Error("Put requires config values"); + handlePut: function(this: ConfigService, request: ConfigRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const values = service.configValues(request); + if (values.length === 0) return yield* configServiceError("put", "Put requires config values"); - for (const item of values) { - this.namespaceStore(item.workspace ?? DEFAULT_WORKSPACE, item.type, true)?.set(item.key, item.value); - } + for (const item of values) { + service.namespaceStore(item.workspace ?? DEFAULT_WORKSPACE, item.type, true)?.set(item.key, item.value); + } - this.version++; - await this.persist(); - await this.pushConfig(); + service.version++; + yield* Effect.tryPromise({ + try: () => service.persist(), + catch: (cause) => configServiceError("persist", cause), + }); + yield* Effect.tryPromise({ + try: () => service.pushConfig(), + catch: (cause) => configServiceError("push-config", cause), + }); - return { version: this.version }; + return { version: service.version }; + }), + ); }, - handleDelete: async function(this: ConfigService, request: ConfigRequest): Promise { - const workspace = this.workspaceFor(request); - const objectKeys = this.objectKeys(request); - if (objectKeys.length > 0) { - for (const key of objectKeys) { - const ws = this.workspaceStore(workspace, false); - if (ws === undefined) continue; - if (key.key === undefined) { - ws.delete(key.type); - } else { - const ns = ws.get(key.type); - ns?.delete(key.key); - if (ns !== undefined && ns.size === 0) ws.delete(key.type); + handleDelete: function(this: ConfigService, request: ConfigRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const workspace = service.workspaceFor(request); + const objectKeys = service.objectKeys(request); + if (objectKeys.length > 0) { + for (const key of objectKeys) { + const ws = service.workspaceStore(workspace, false); + if (ws === undefined) continue; + if (key.key === undefined) { + ws.delete(key.type); + } else { + const ns = ws.get(key.type); + ns?.delete(key.key); + if (ns !== undefined && ns.size === 0) ws.delete(key.type); + } + } + + service.version++; + yield* Effect.tryPromise({ + try: () => service.persist(), + catch: (cause) => configServiceError("persist", cause), + }); + yield* Effect.tryPromise({ + try: () => service.pushConfig(), + catch: (cause) => configServiceError("push-config", cause), + }); + return { version: service.version }; } - } - this.version++; - await this.persist(); - await this.pushConfig(); - return { version: this.version }; - } - - const keys = this.stringKeys(request); - if (keys.length === 0) { - throw new Error("Delete requires at least one key"); - } - - const namespace = keys[0]; - const ws = this.workspaceStore(workspace, false); - if (ws === undefined) return { version: this.version }; - - if (keys.length === 1) { - // Delete entire namespace - ws.delete(namespace); - } else { - // Delete specific keys within namespace - const subMap = ws.get(namespace); - if (subMap !== undefined) { - for (let i = 1; i < keys.length; i++) { - subMap.delete(keys[i]); + const keys = service.stringKeys(request); + if (keys.length === 0) { + return yield* configServiceError("delete", "Delete requires at least one key"); } - if (subMap.size === 0) { + + const namespace = keys[0]; + const ws = service.workspaceStore(workspace, false); + if (ws === undefined) return { version: service.version }; + + if (keys.length === 1) { + // Delete entire namespace ws.delete(namespace); + } else { + // Delete specific keys within namespace + const subMap = ws.get(namespace); + if (subMap !== undefined) { + for (let i = 1; i < keys.length; i++) { + subMap.delete(keys[i]); + } + if (subMap.size === 0) { + ws.delete(namespace); + } + } } - } - } - this.version++; - await this.persist(); - await this.pushConfig(); + service.version++; + yield* Effect.tryPromise({ + try: () => service.persist(), + catch: (cause) => configServiceError("persist", cause), + }); + yield* Effect.tryPromise({ + try: () => service.pushConfig(), + catch: (cause) => configServiceError("push-config", cause), + }); - return { version: this.version }; + return { version: service.version }; + }), + ); }, @@ -528,130 +696,140 @@ export function makeConfigService(config: ConfigServiceConfig): ConfigService { - pushConfig: async function(this: ConfigService): Promise { - const pushProducer = this.pushProducer; - if (pushProducer === null) return; + pushConfig: function(this: ConfigService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const pushProducer = service.pushProducer; + if (pushProducer === null) return; - const config: Record = {}; - const ws = this.workspaceStore(DEFAULT_WORKSPACE, false); - for (const [namespace, subMap] of ws ?? new Map()) { - const obj: Record = {}; - for (const [k, v] of subMap) { - obj[k] = v; - } - config[namespace] = obj; - } - - await pushProducer.send({ - version: this.version, - config, - }); - - console.log(`[ConfigService] Pushed configuration version ${this.version}`); - - }, - - - - persist: async function(this: ConfigService): Promise { - const persistPath = this.persistPath; - if (persistPath === null) return; - - try { - const workspaces: Record>> = {}; - - for (const [workspace, ws] of this.store) { - const workspaceData: Record> = {}; - for (const [namespace, subMap] of ws) { + const config: Record = {}; + const ws = service.workspaceStore(DEFAULT_WORKSPACE, false); + for (const [namespace, subMap] of ws ?? new Map()) { const obj: Record = {}; for (const [k, v] of subMap) { obj[k] = v; } - workspaceData[namespace] = obj; + config[namespace] = obj; } - workspaces[workspace] = workspaceData; - } - const json = JSON.stringify( - { version: this.version, workspaces }, - null, - 2, - ); + yield* Effect.tryPromise({ + try: () => + pushProducer.send({ + version: service.version, + config, + }), + catch: (cause) => configServiceError("push-config", cause), + }); - await writeTextFile(persistPath, json); - } catch (err) { - await Effect.runPromise(Effect.logError("[ConfigService] Failed to persist config", { error: errorMessage(err) })); - } + yield* Effect.log(`[ConfigService] Pushed configuration version ${service.version}`); + }), + ); }, - loadFromDisk: async function(this: ConfigService): Promise { - const persistPath = this.persistPath; - if (persistPath === null) return; + persist: function(this: ConfigService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const persistPath = service.persistPath; + if (persistPath === null) return; + const payload = { + version: service.version, + workspaces: toPersistedWorkspaces(service.store), + }; - try { - const raw = await readTextFile(persistPath); - const parsed = JSON.parse(raw) as { - version: number; - data?: Record>; - workspaces?: Record>>; - }; + const json = yield* S.encodeUnknownEffect(S.UnknownFromJsonString)(payload).pipe( + Effect.mapError((cause) => configServiceError("persist-encode", cause)), + ); - this.version = parsed.version ?? 0; - this.store.clear(); - - if (parsed.workspaces !== undefined) { - for (const [workspace, namespaces] of Object.entries(parsed.workspaces)) { - const ws = new Map(); - for (const [namespace, obj] of Object.entries(namespaces)) { - const subMap = new Map(); - for (const [k, v] of Object.entries(obj)) { - subMap.set(k, v); - } - ws.set(namespace, subMap); - } - this.store.set(workspace, ws); - } - } else { - const ws = new Map(); - for (const [namespace, obj] of Object.entries(parsed.data ?? {})) { - const subMap = new Map(); - for (const [k, v] of Object.entries(obj)) { - subMap.set(k, v); - } - ws.set(namespace, subMap); - } - this.store.set(DEFAULT_WORKSPACE, ws); - } - - console.log( - `[ConfigService] Loaded persisted config (version=${this.version}, workspaces=${this.store.size})`, - ); - } catch { - // File doesn't exist yet or is invalid — start fresh - await Effect.runPromise(Effect.log("[ConfigService] No persisted config found, starting fresh")); - } + yield* Effect.tryPromise({ + try: () => writeTextFile(persistPath, json), + catch: (cause) => configServiceError("persist-write", cause), + }); + }).pipe( + Effect.catch((err) => + Effect.logError("[ConfigService] Failed to persist config", { error: err.message }), + ), + ), + ); }, - stop: async function(this: ConfigService): Promise { - if (this.consumer !== null) { - await this.consumer.close(); - this.consumer = null; - } - if (this.responseProducer !== null) { - await this.responseProducer.close(); - this.responseProducer = null; - } - if (this.pushProducer !== null) { - await this.pushProducer.close(); - this.pushProducer = null; - } - await baseStop(); + loadFromDisk: function(this: ConfigService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const persistPath = service.persistPath; + if (persistPath === null) return; + + const parsed = yield* Effect.gen(function* () { + const raw = yield* Effect.tryPromise({ + try: () => readTextFile(persistPath), + catch: (cause) => configServiceError("persist-read", cause), + }); + return yield* S.decodeUnknownEffect(PersistedConfigJsonSchema)(raw).pipe( + Effect.mapError((cause) => configServiceError("persist-decode", cause)), + ); + }).pipe( + Effect.catch(() => + Effect.log("[ConfigService] No persisted config found, starting fresh").pipe( + Effect.as(null as PersistedConfig | null), + ), + ), + ); + + if (parsed === null) return; + + service.version = parsed.version ?? 0; + hydratePersistedConfig(service.store, parsed); + + yield* Effect.log(`[ConfigService] Loaded persisted config (version=${service.version}, workspaces=${service.store.size})`); + }), + ); + + }, + + + + stop: function(this: ConfigService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const consumer = service.consumer; + if (consumer !== null) { + yield* Effect.tryPromise({ + try: () => consumer.close(), + catch: (cause) => configServiceError("close-consumer", cause), + }); + service.consumer = null; + } + const responseProducer = service.responseProducer; + if (responseProducer !== null) { + yield* Effect.tryPromise({ + try: () => responseProducer.close(), + catch: (cause) => configServiceError("close-response-producer", cause), + }); + service.responseProducer = null; + } + const pushProducer = service.pushProducer; + if (pushProducer !== null) { + yield* Effect.tryPromise({ + try: () => pushProducer.close(), + catch: (cause) => configServiceError("close-push-producer", cause), + }); + service.pushProducer = null; + } + yield* Effect.tryPromise({ + try: () => baseStop(), + catch: (cause) => configServiceError("stop", cause), + }); + }), + ); } }); @@ -660,10 +838,6 @@ export function makeConfigService(config: ConfigServiceConfig): ConfigService { export const ConfigService = makeConfigService; -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - export const loadConfigServiceRuntimeConfig = Effect.fn("loadConfigServiceRuntimeConfig")(function* () { const processorConfig = yield* loadProcessorRuntimeConfig("config-svc", { manageProcessSignals: false, @@ -681,6 +855,6 @@ export const program = makeProcessorProgram({ make: (config) => makeConfigService(config), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/cores/service.ts b/ts/packages/flow/src/cores/service.ts index 321b0343..e87431c8 100644 --- a/ts/packages/flow/src/cores/service.ts +++ b/ts/packages/flow/src/cores/service.ts @@ -12,6 +12,7 @@ import { makeAsyncProcessor, + makeProcessorProgram, type ProcessorConfig, type AsyncProcessorRuntime, topics, @@ -19,10 +20,11 @@ import { type KnowledgeResponse, type Triple, type Term, + errorMessage, } from "@trustgraph/base"; -import { makeProcessorProgram } from "@trustgraph/base"; import type { Message } from "@trustgraph/base"; -import { Effect } from "effect"; +import { Config, Duration, Effect } from "effect"; +import * as S from "effect/Schema"; import { ensureDirectory, joinPath, readTextFile, writeTextFile } from "../runtime/effect-files.js"; export interface KnowledgeCoreServiceConfig extends ProcessorConfig { @@ -42,18 +44,88 @@ interface DocumentEmbeddingsCore { export type KnowledgeCoreService = AsyncProcessorRuntime & Record; +export class KnowledgeCoreServiceError extends S.TaggedErrorClass()( + "KnowledgeCoreServiceError", + { + message: S.String, + operation: S.String, + }, +) {} + +interface KnowledgeResponseProducer { + send(response: KnowledgeResponse, properties: { id: string }): Promise; + close(): Promise; +} + +interface CloseableResource { + close(): Promise; +} + +const knowledgeCoreServiceError = (operation: string, cause: unknown): KnowledgeCoreServiceError => + KnowledgeCoreServiceError.make({ + operation, + message: errorMessage(cause), + }); + +const tryPromise = ( + operation: string, + evaluate: () => Promise, +): Effect.Effect => + Effect.tryPromise({ + try: evaluate, + catch: (cause) => knowledgeCoreServiceError(operation, cause), + }); + +const trySync = ( + operation: string, + evaluate: () => A, +): Effect.Effect => + Effect.try({ + try: evaluate, + catch: (cause) => knowledgeCoreServiceError(operation, cause), + }); + +const failPromise = (operation: string, cause: unknown): Promise => + Effect.runPromise(Effect.fail(knowledgeCoreServiceError(operation, cause))); + +const sendResponse = ( + service: KnowledgeCoreService, + response: KnowledgeResponse, + requestId: string, + operation = "respond", +): Effect.Effect => + Effect.gen(function* () { + const responseProducer = service.responseProducer as KnowledgeResponseProducer | null | undefined; + if (responseProducer === null || responseProducer === undefined) { + return yield* knowledgeCoreServiceError(operation, "Knowledge response producer not started"); + } + + yield* tryPromise(operation, () => responseProducer.send(response, { id: requestId })); + }); + +const closeResource = ( + resource: CloseableResource, + operation: string, +): Effect.Effect => + tryPromise(operation, () => resource.close()).pipe( + Effect.catch((error) => + Effect.logError("[KnowledgeCoreService] Failed to close resource", { + error: error.message, + operation: error.operation, + }), + ), + ); + export function makeKnowledgeCoreService(config: KnowledgeCoreServiceConfig): KnowledgeCoreService { const service = makeAsyncProcessor(config, { - run: async () => { - await service.run(); - }, + run: () => service.run(), }) as KnowledgeCoreService; const baseStop = service.stop; service.cores = new Map(); service.deCores = new Map(); service.consumer = null; service.responseProducer = null; - const dataDir = config.dataDir ?? process.env.KNOWLEDGE_DATA_DIR ?? "./data/knowledge"; + const dataDir = config.dataDir ?? "./data/knowledge"; service.dataDir = dataDir; service.persistPath = joinPath(dataDir, "knowledge-state.json"); Object.assign(service, { @@ -66,68 +138,106 @@ export function makeKnowledgeCoreService(config: KnowledgeCoreServiceConfig): Kn - run: async function(this: KnowledgeCoreService): Promise { - await ensureDirectory(this.dataDir); - // Load persisted state - await this.loadFromDisk(); + run: function(this: KnowledgeCoreService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + if (config.dataDir === undefined) { + const configuredDataDir = yield* Config.string("KNOWLEDGE_DATA_DIR").pipe( + Config.withDefault("./data/knowledge"), + ); + service.dataDir = configuredDataDir; + service.persistPath = joinPath(configuredDataDir, "knowledge-state.json"); + } - // Create producer - this.responseProducer = await this.pubsub.createProducer({ - topic: topics.knowledgeResponse, - }); + yield* tryPromise("ensure-directory", () => ensureDirectory(service.dataDir)); + // Load persisted state + yield* tryPromise("load", () => service.loadFromDisk()); - // Create consumer - this.consumer = await this.pubsub.createConsumer({ - topic: topics.knowledgeRequest, - subscription: `${this.config.id}-knowledge-request`, - }); + // Create producer + service.responseProducer = yield* tryPromise("response-producer", () => + service.pubsub.createProducer({ + topic: topics.knowledgeResponse, + }), + ); - console.log(`[KnowledgeCoreService] Listening on ${topics.knowledgeRequest}`); + // Create consumer + service.consumer = yield* tryPromise("consumer", () => + service.pubsub.createConsumer({ + topic: topics.knowledgeRequest, + subscription: `${service.config.id}-knowledge-request`, + }), + ); - // Main consume loop - while (this.running) { - try { - const msg = await this.consumer.receive(2000); - if (msg === null) continue; + yield* Effect.log(`[KnowledgeCoreService] Listening on ${topics.knowledgeRequest}`); - await this.handleMessage(msg); - await this.consumer.acknowledge(msg); - } catch (err) { - if (!this.running) break; - console.error("[KnowledgeCoreService] Error in consume loop:", err); - await sleep(1000); - } - } + // Main consume loop + while (service.running) { + const shouldContinue = yield* Effect.gen(function* () { + const consumer = service.consumer; + if (consumer === null || consumer === undefined) { + return yield* knowledgeCoreServiceError("consume", "Knowledge request consumer not started"); + } + + const msg = yield* tryPromise("consume-receive", () => consumer.receive(2000)); + if (msg === null) return true; + + yield* tryPromise("consume-handle", () => service.handleMessage(msg)); + yield* tryPromise("consume-acknowledge", () => consumer.acknowledge(msg)); + + return true; + }).pipe( + Effect.catch((error) => { + if (!service.running) return Effect.succeed(false); + return Effect.logError("[KnowledgeCoreService] Error in consume loop", { + error: error.message, + }).pipe( + Effect.flatMap(() => Effect.sleep(Duration.millis(1000))), + Effect.as(true), + ); + }), + ); + + if (!shouldContinue) break; + } + }), + ); }, - handleMessage: async function(this: KnowledgeCoreService, msg: Message): Promise { - const request = msg.value(); - const props = msg.properties(); - const requestId = props.id; + handleMessage: function(this: KnowledgeCoreService, msg: Message): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const request = msg.value(); + const props = msg.properties(); + const requestId = props.id; - if (requestId === undefined || requestId.length === 0) { - console.warn("[KnowledgeCoreService] Received request without id, ignoring"); - return; - } + if (requestId === undefined || requestId.length === 0) { + yield* Effect.logWarning("[KnowledgeCoreService] Received request without id, ignoring"); + return; + } - try { - await this.handleOperation(request, requestId); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - await this.responseProducer!.send( - { error: { type: "knowledge-error", message } }, - { id: requestId }, - ); - } + yield* tryPromise("operation", () => service.handleOperation(request, requestId)).pipe( + Effect.catch((error) => + sendResponse( + service, + { error: { type: "knowledge-error", message: error.message } }, + requestId, + "respond-error", + ), + ), + ); + }), + ); }, - handleOperation: async function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { + handleOperation: function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { switch (request.operation) { case "list-kg-cores": return this.listKgCores(request, requestId); @@ -152,7 +262,7 @@ export function makeKnowledgeCoreService(config: KnowledgeCoreServiceConfig): Kn case "load-de-core": return this.loadDeCore(request, requestId); default: - throw new Error(`Unknown knowledge operation: ${request.operation as string}`); + return failPromise("operation", `Unknown knowledge operation: ${request.operation as string}`); } }, @@ -185,231 +295,297 @@ export function makeKnowledgeCoreService(config: KnowledgeCoreServiceConfig): Kn - listKgCores: async function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { - const user = request.user ?? ""; - const prefix = user.length > 0 ? `${user}:` : ""; + listKgCores: function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const user = request.user ?? ""; + const prefix = user.length > 0 ? `${user}:` : ""; - const ids: string[] = []; - for (const key of (this.cores as Map).keys()) { - if (prefix.length === 0 || key.startsWith(prefix)) { - // Extract the ID portion after the user prefix - const id = key.slice(prefix.length); - ids.push(id); - } - } + const ids: string[] = []; + for (const key of (service.cores as Map).keys()) { + if (prefix.length === 0 || key.startsWith(prefix)) { + // Extract the ID portion after the user prefix + const id = key.slice(prefix.length); + ids.push(id); + } + } - await this.responseProducer!.send({ ids }, { id: requestId }); - - }, - - - - getKgCore: async function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { - const user = request.user ?? ""; - const coreId = request.id ?? ""; - const key = this.coreKey(user, coreId); - - const core = this.cores.get(key); - if (core === undefined) { - throw new Error(`Knowledge core not found: ${key}`); - } - - // Send triples and embeddings in batches - const BATCH_SIZE = 100; - - // Send triples in batches - for (let i = 0; i < core.triples.length; i += BATCH_SIZE) { - const batch = core.triples.slice(i, i + BATCH_SIZE); - const isLast = i + BATCH_SIZE >= core.triples.length && core.graphEmbeddings.length === 0; - - await this.responseProducer!.send( - { triples: batch, eos: isLast }, - { id: requestId }, - ); - } - - // Send graph embeddings in batches - for (let i = 0; i < core.graphEmbeddings.length; i += BATCH_SIZE) { - const batch = core.graphEmbeddings.slice(i, i + BATCH_SIZE); - const isLast = i + BATCH_SIZE >= core.graphEmbeddings.length; - - await this.responseProducer!.send( - { graphEmbeddings: batch, "graph-embeddings": batch, eos: isLast } as KnowledgeResponse, - { id: requestId }, - ); - } - - // If core was empty, send a final eos - if (core.triples.length === 0 && core.graphEmbeddings.length === 0) { - await this.responseProducer!.send({ eos: true }, { id: requestId }); - } - - }, - - - - deleteKgCore: async function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { - const user = request.user ?? ""; - const coreId = request.id ?? ""; - const key = this.coreKey(user, coreId); - - this.cores.delete(key); - await this.persist(); - - console.log(`[KnowledgeCoreService] Deleted core: ${key}`); - await this.responseProducer!.send({}, { id: requestId }); - - }, - - - - putKgCore: async function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { - const user = request.user ?? ""; - const coreId = request.id ?? ""; - const key = this.coreKey(user, coreId); - - let core = this.cores.get(key); - if (core === undefined) { - core = { triples: [], graphEmbeddings: [] }; - this.cores.set(key, core); - } - - // Append triples if provided - if (request.triples !== undefined && request.triples.length > 0) { - core.triples.push(...request.triples); - } - - // Append graph embeddings if provided - const graphEmbeddings = this.graphEmbeddings(request); - if (graphEmbeddings.length > 0) { - core.graphEmbeddings.push(...graphEmbeddings); - } - - await this.persist(); - - console.log( - `[KnowledgeCoreService] Updated core ${key}: triples=${core.triples.length}, embeddings=${core.graphEmbeddings.length}`, + yield* sendResponse(service, { ids }, requestId); + }), ); - await this.responseProducer!.send({}, { id: requestId }); }, - loadKgCore: async function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { - const user = request.user ?? ""; - const coreId = request.id ?? ""; - const key = this.coreKey(user, coreId); + getKgCore: function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const user = request.user ?? ""; + const coreId = request.id ?? ""; + const key = service.coreKey(user, coreId); - const core = this.cores.get(key); - if (core === undefined) { - throw new Error(`Knowledge core not found: ${key}`); - } + const core = service.cores.get(key); + if (core === undefined) { + return yield* knowledgeCoreServiceError("get-kg-core", `Knowledge core not found: ${key}`); + } - if (core.triples.length > 0) { - const producer = await this.pubsub.createProducer({ topic: "tg.flow.triples" }); - try { - await producer.send({ - metadata: { - id: coreId, - root: coreId, - user, - collection: request.collection ?? "default", - }, - triples: core.triples, - }); - } finally { - await producer.close(); - } - } + // Send triples and embeddings in batches + const BATCH_SIZE = 100; - console.log( - `[KnowledgeCoreService] Loaded core ${key} (triples=${core.triples.length}, embeddings=${core.graphEmbeddings.length})`, + // Send triples in batches + for (let i = 0; i < core.triples.length; i += BATCH_SIZE) { + const batch = core.triples.slice(i, i + BATCH_SIZE); + const isLast = i + BATCH_SIZE >= core.triples.length && core.graphEmbeddings.length === 0; + + yield* sendResponse( + service, + { triples: batch, eos: isLast }, + requestId, + "respond-kg-triples", + ); + } + + // Send graph embeddings in batches + for (let i = 0; i < core.graphEmbeddings.length; i += BATCH_SIZE) { + const batch = core.graphEmbeddings.slice(i, i + BATCH_SIZE); + const isLast = i + BATCH_SIZE >= core.graphEmbeddings.length; + + yield* sendResponse( + service, + { graphEmbeddings: batch, "graph-embeddings": batch, eos: isLast } as KnowledgeResponse, + requestId, + "respond-kg-embeddings", + ); + } + + // If core was empty, send a final eos + if (core.triples.length === 0 && core.graphEmbeddings.length === 0) { + yield* sendResponse(service, { eos: true }, requestId, "respond-kg-empty"); + } + }), ); - await this.responseProducer!.send({}, { id: requestId }); }, - unloadKgCore: async function(this: KnowledgeCoreService, _request: KnowledgeRequest, requestId: string): Promise { - await this.responseProducer!.send({}, { id: requestId }); + deleteKgCore: function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const user = request.user ?? ""; + const coreId = request.id ?? ""; + const key = service.coreKey(user, coreId); + + service.cores.delete(key); + yield* tryPromise("persist-delete-kg-core", () => service.persist()); + + yield* Effect.log(`[KnowledgeCoreService] Deleted core: ${key}`); + yield* sendResponse(service, {}, requestId); + }), + ); }, - listDeCores: async function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { - const user = request.user ?? ""; - const prefix = user.length > 0 ? `${user}:` : ""; - const ids = [...this.deCores.keys()] - .filter((key) => prefix.length === 0 || key.startsWith(prefix)) - .map((key) => key.slice(prefix.length)); - await this.responseProducer!.send({ ids }, { id: requestId }); + putKgCore: function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const user = request.user ?? ""; + const coreId = request.id ?? ""; + const key = service.coreKey(user, coreId); + + let core = service.cores.get(key); + if (core === undefined) { + core = { triples: [], graphEmbeddings: [] }; + service.cores.set(key, core); + } + + // Append triples if provided + if (request.triples !== undefined && request.triples.length > 0) { + core.triples.push(...request.triples); + } + + // Append graph embeddings if provided + const graphEmbeddings = service.graphEmbeddings(request); + if (graphEmbeddings.length > 0) { + core.graphEmbeddings.push(...graphEmbeddings); + } + + yield* tryPromise("persist-put-kg-core", () => service.persist()); + + yield* Effect.log( + `[KnowledgeCoreService] Updated core ${key}: triples=${core.triples.length}, embeddings=${core.graphEmbeddings.length}`, + ); + yield* sendResponse(service, {}, requestId); + }), + ); }, - getDeCore: async function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { - const user = request.user ?? ""; - const coreId = request.id ?? ""; - const key = this.coreKey(user, coreId); - const core = this.deCores.get(key); - if (core === undefined) throw new Error(`Document embeddings core not found: ${key}`); + loadKgCore: function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const user = request.user ?? ""; + const coreId = request.id ?? ""; + const key = service.coreKey(user, coreId); - for (let i = 0; i < core.length; i++) { - const isLast = i === core.length - 1; - await this.responseProducer!.send( - { - documentEmbeddings: core[i], - "document-embeddings": core[i], - eos: isLast, - } as KnowledgeResponse, - { id: requestId }, - ); - } - if (core.length === 0) { - await this.responseProducer!.send({ eos: true }, { id: requestId }); - } + const core = service.cores.get(key); + if (core === undefined) { + return yield* knowledgeCoreServiceError("load-kg-core", `Knowledge core not found: ${key}`); + } + + if (core.triples.length > 0) { + yield* Effect.acquireUseRelease( + tryPromise("triples-producer", () => + service.pubsub.createProducer({ topic: "tg.flow.triples" }), + ), + (producer) => + tryPromise("send-triples", () => + producer.send({ + metadata: { + id: coreId, + root: coreId, + user, + collection: request.collection ?? "default", + }, + triples: core.triples, + }), + ), + (producer) => closeResource(producer, "close-triples-producer"), + ); + } + + yield* Effect.log( + `[KnowledgeCoreService] Loaded core ${key} (triples=${core.triples.length}, embeddings=${core.graphEmbeddings.length})`, + ); + yield* sendResponse(service, {}, requestId); + }), + ); }, - deleteDeCore: async function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { - const user = request.user ?? ""; - const coreId = request.id ?? ""; - this.deCores.delete(this.coreKey(user, coreId)); - await this.persist(); - await this.responseProducer!.send({}, { id: requestId }); + unloadKgCore: function(this: KnowledgeCoreService, _request: KnowledgeRequest, requestId: string): Promise { + return Effect.runPromise(sendResponse(this, {}, requestId)); }, - putDeCore: async function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { - const user = request.user ?? ""; - const coreId = request.id ?? ""; - const key = this.coreKey(user, coreId); - const item = this.documentEmbeddings(request); - if (item === undefined) throw new Error("put-de-core requires document-embeddings"); - const core = this.deCores.get(key) ?? []; - core.push(item); - this.deCores.set(key, core); - await this.persist(); - await this.responseProducer!.send({}, { id: requestId }); + listDeCores: function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const user = request.user ?? ""; + const prefix = user.length > 0 ? `${user}:` : ""; + const ids = [...service.deCores.keys()] + .filter((key) => prefix.length === 0 || key.startsWith(prefix)) + .map((key) => key.slice(prefix.length)); + yield* sendResponse(service, { ids }, requestId); + }), + ); }, - loadDeCore: async function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { - const user = request.user ?? ""; - const coreId = request.id ?? ""; - const key = this.coreKey(user, coreId); - if (!(this.deCores as Map).has(key)) throw new Error(`Document embeddings core not found: ${key}`); - await this.responseProducer!.send({}, { id: requestId }); + getDeCore: function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const user = request.user ?? ""; + const coreId = request.id ?? ""; + const key = service.coreKey(user, coreId); + const core = service.deCores.get(key); + if (core === undefined) { + return yield* knowledgeCoreServiceError("get-de-core", `Document embeddings core not found: ${key}`); + } + + for (let i = 0; i < core.length; i++) { + const isLast = i === core.length - 1; + yield* sendResponse( + service, + { + documentEmbeddings: core[i], + "document-embeddings": core[i], + eos: isLast, + } as KnowledgeResponse, + requestId, + "respond-de-core", + ); + } + if (core.length === 0) { + yield* sendResponse(service, { eos: true }, requestId, "respond-de-empty"); + } + }), + ); + + }, + + + + deleteDeCore: function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const user = request.user ?? ""; + const coreId = request.id ?? ""; + service.deCores.delete(service.coreKey(user, coreId)); + yield* tryPromise("persist-delete-de-core", () => service.persist()); + yield* sendResponse(service, {}, requestId); + }), + ); + + }, + + + + putDeCore: function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const user = request.user ?? ""; + const coreId = request.id ?? ""; + const key = service.coreKey(user, coreId); + const item = service.documentEmbeddings(request); + if (item === undefined) { + return yield* knowledgeCoreServiceError("put-de-core", "put-de-core requires document-embeddings"); + } + const core = service.deCores.get(key) ?? []; + core.push(item); + service.deCores.set(key, core); + yield* tryPromise("persist-put-de-core", () => service.persist()); + yield* sendResponse(service, {}, requestId); + }), + ); + + }, + + + + loadDeCore: function(this: KnowledgeCoreService, request: KnowledgeRequest, requestId: string): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const user = request.user ?? ""; + const coreId = request.id ?? ""; + const key = service.coreKey(user, coreId); + if (!(service.deCores as Map).has(key)) { + return yield* knowledgeCoreServiceError("load-de-core", `Document embeddings core not found: ${key}`); + } + yield* sendResponse(service, {}, requestId); + }), + ); }, @@ -417,69 +593,88 @@ export function makeKnowledgeCoreService(config: KnowledgeCoreServiceConfig): Kn // ---------- Persistence ---------- - persist: async function(this: KnowledgeCoreService): Promise { - try { - // Serialize Map to object - const data: { - kg: Record; - de: Record; - } = { kg: {}, de: {} }; - for (const [key, core] of this.cores) { - data.kg[key] = core; - } - for (const [key, core] of this.deCores) { - data.de[key] = core; - } - - const json = JSON.stringify(data, null, 2); - await writeTextFile(this.persistPath, json); - } catch (err) { - console.error("[KnowledgeCoreService] Failed to persist state:", err); - } - - }, - - - - loadFromDisk: async function(this: KnowledgeCoreService): Promise { - try { - const raw = await readTextFile(this.persistPath); - const parsed = JSON.parse(raw) as Record | { - kg?: Record; - de?: Record; - }; - - this.cores.clear(); - this.deCores.clear(); - const kg = "kg" in parsed && parsed.kg !== undefined ? parsed.kg : parsed as Record; - for (const [key, core] of Object.entries(kg)) { - this.cores.set(key, core); - } - if ("de" in parsed && parsed.de !== undefined) { - for (const [key, core] of Object.entries(parsed.de)) { - this.deCores.set(key, core); + persist: function(this: KnowledgeCoreService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + // Serialize Map to object + const data: { + kg: Record; + de: Record; + } = { kg: {}, de: {} }; + for (const [key, core] of service.cores) { + data.kg[key] = core; + } + for (const [key, core] of service.deCores) { + data.de[key] = core; } - } - console.log(`[KnowledgeCoreService] Loaded persisted state (kg=${this.cores.size}, de=${this.deCores.size})`); - } catch { - console.log("[KnowledgeCoreService] No persisted state found, starting fresh"); - } + const json = yield* trySync("persist-serialize", () => JSON.stringify(data, null, 2)); + yield* tryPromise("persist-write", () => writeTextFile(service.persistPath, json)); + }).pipe( + Effect.catch((error) => + Effect.logError("[KnowledgeCoreService] Failed to persist state", { + error: error.message, + }), + ), + ), + ); }, - stop: async function(this: KnowledgeCoreService): Promise { - if (this.consumer !== null) { - await this.consumer.close(); - this.consumer = null; - } - if (this.responseProducer !== null) { - await this.responseProducer.close(); - this.responseProducer = null; - } - await baseStop(); + loadFromDisk: function(this: KnowledgeCoreService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const raw = yield* tryPromise("load-read", () => readTextFile(service.persistPath)); + const parsed = yield* trySync("load-parse", () => + JSON.parse(raw) as Record | { + kg?: Record; + de?: Record; + }, + ); + + service.cores.clear(); + service.deCores.clear(); + const kg = "kg" in parsed && parsed.kg !== undefined ? parsed.kg : parsed as Record; + for (const [key, core] of Object.entries(kg)) { + service.cores.set(key, core); + } + if ("de" in parsed && parsed.de !== undefined) { + for (const [key, core] of Object.entries(parsed.de)) { + service.deCores.set(key, core); + } + } + + yield* Effect.log(`[KnowledgeCoreService] Loaded persisted state (kg=${service.cores.size}, de=${service.deCores.size})`); + }).pipe( + Effect.catch(() => + Effect.log("[KnowledgeCoreService] No persisted state found, starting fresh"), + ), + ), + ); + + }, + + + + stop: function(this: KnowledgeCoreService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + if (service.consumer !== null) { + yield* tryPromise("close-consumer", () => service.consumer.close()); + service.consumer = null; + } + if (service.responseProducer !== null) { + yield* tryPromise("close-response-producer", () => service.responseProducer.close()); + service.responseProducer = null; + } + yield* tryPromise("base-stop", () => baseStop()); + }), + ); } }); @@ -488,15 +683,11 @@ export function makeKnowledgeCoreService(config: KnowledgeCoreServiceConfig): Kn export const KnowledgeCoreService = makeKnowledgeCoreService; -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - export const program = makeProcessorProgram({ id: "knowledge-svc", make: (config) => makeKnowledgeCoreService(config), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/decoding/pdf-decoder.ts b/ts/packages/flow/src/decoding/pdf-decoder.ts index 96c579fb..6055be37 100644 --- a/ts/packages/flow/src/decoding/pdf-decoder.ts +++ b/ts/packages/flow/src/decoding/pdf-decoder.ts @@ -37,7 +37,7 @@ import { errorMessage, } from "@trustgraph/base"; import { makeFlowProcessorProgram } from "@trustgraph/base"; -import { Effect } from "effect"; +import { Clock, Effect } from "effect"; import * as S from "effect/Schema"; export class PdfDecoderError extends S.TaggedErrorClass()( @@ -63,7 +63,7 @@ const pdfDecoderError = ( documentId: string, cause: unknown, ) => - new PdfDecoderError({ + PdfDecoderError.make({ operation, documentId, message: errorMessage(cause), @@ -76,18 +76,24 @@ const loadPdf = (documentId: string, pdfBuffer: Buffer) => catch: (cause) => pdfDecoderError("load-pdf", documentId, cause), }); -const loadPageText = (documentId: string, pageNumber: number, pdf: PdfDocument) => - Effect.tryPromise({ - try: async () => { - const page = await pdf.getPage(pageNumber); - const textContent = await page.getTextContent(); - return textContent.items - .filter((item): item is TextItem => "str" in item) - .map((item) => item.str) - .join(" "); - }, - catch: (cause) => pdfDecoderError("load-page-text", documentId, cause), - }); +const loadPageText = Effect.fn("loadPageText")(function*( + documentId: string, + pageNumber: number, + pdf: PdfDocument, +) { + const page = yield* Effect.tryPromise({ + try: () => pdf.getPage(pageNumber), + catch: (cause) => pdfDecoderError("load-page", documentId, cause), + }); + const textContent = yield* Effect.tryPromise({ + try: () => page.getTextContent(), + catch: (cause) => pdfDecoderError("load-page-text", documentId, cause), + }); + return textContent.items + .filter((item): item is TextItem => "str" in item) + .map((item) => item.str) + .join(" "); +}); const onPdfDecodeMessage = Effect.fn("PdfDecoderService.onMessage")(function* ( msg: Document, @@ -156,6 +162,7 @@ const onPdfDecodeMessage = Effect.fn("PdfDecoderService.onMessage")(function* ( continue; } + const timestamp = yield* Clock.currentTimeMillis; const childResp = yield* librarian.request({ operation: "add-child-document", documentMetadata: { @@ -165,7 +172,7 @@ const onPdfDecodeMessage = Effect.fn("PdfDecoderService.onMessage")(function* ( title: `Page ${i}`, parentId: documentId, documentType: "page", - time: Date.now(), + time: timestamp, comments: "", tags: [], }, @@ -226,7 +233,7 @@ export function makePdfDecoderService(config: ProcessorConfig): PdfDecoderServic const service = makeFlowProcessor(config, { specifications: makePdfDecoderSpecs(), }); - console.log("[PdfDecoder] Service initialized"); + Effect.runSync(Effect.log("[PdfDecoder] Service initialized")); return service; } @@ -245,6 +252,6 @@ export const program = makeFlowProcessorProgram({ specs: () => makePdfDecoderSpecs(), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/embeddings/ollama.ts b/ts/packages/flow/src/embeddings/ollama.ts index 65925008..3413cdca 100644 --- a/ts/packages/flow/src/embeddings/ollama.ts +++ b/ts/packages/flow/src/embeddings/ollama.ts @@ -4,11 +4,13 @@ * Python reference: trustgraph-flow/trustgraph/embeddings/ollama/processor.py */ -import { Effect, Layer } from "effect"; +import { Config, Effect, Layer } from "effect"; +import * as O from "effect/Option"; import * as S from "effect/Schema"; import { Embeddings, - embeddingsError, + EmbeddingsError, + errorMessage, makeEmbeddingsService, makeEmbeddingsSpecs, type EmbeddingsServiceShape, @@ -22,18 +24,58 @@ export interface OllamaEmbeddingsConfig extends ProcessorConfig { fetch?: typeof fetch; } -interface OllamaEmbedResponse { - embeddings: number[][]; +const EmbeddingVector = S.Array(S.Number); + +const OllamaEmbedResponse = S.Struct({ + embeddings: S.Array(EmbeddingVector), +}); + +type OllamaEmbedResponse = typeof OllamaEmbedResponse.Type; + +interface ResolvedOllamaEmbeddingsConfig { + readonly defaultModel: string; + readonly ollamaHost: string; + readonly fetchImpl: typeof fetch; } +const ollamaEmbeddingsError = (operation: string, cause: unknown): EmbeddingsError => + EmbeddingsError.make({ + operation, + message: errorMessage(cause), + provider: "ollama", + }); + +const ollamaEmbeddingsMessageError = (operation: string, message: string): EmbeddingsError => + EmbeddingsError.make({ + operation, + message, + provider: "ollama", + }); + +const optionalStringConfig = Effect.fn("OllamaEmbeddings.optionalStringConfig")(function*(name: string) { + return O.getOrUndefined(yield* Config.string(name).pipe(Config.option)); +}); + +const loadOllamaEmbeddingsConfig = Effect.fn("OllamaEmbeddings.loadConfig")(function*( + config: OllamaEmbeddingsConfig, +) { + return { + defaultModel: config.model ?? "mxbai-embed-large", + ollamaHost: + config.ollamaHost ?? + (yield* optionalStringConfig("OLLAMA_URL")) ?? + (yield* optionalStringConfig("OLLAMA_HOST")) ?? + "http://localhost:11434", + fetchImpl: config.fetch ?? globalThis.fetch, + } satisfies ResolvedOllamaEmbeddingsConfig; +}); + export function makeOllamaEmbeddings(config: OllamaEmbeddingsConfig): EmbeddingsServiceShape { - const defaultModel = config.model ?? "mxbai-embed-large"; - const ollamaHost = - config.ollamaHost ?? - process.env.OLLAMA_URL ?? - process.env.OLLAMA_HOST ?? - "http://localhost:11434"; - const fetchImpl = config.fetch ?? globalThis.fetch; + const { + defaultModel, + ollamaHost, + fetchImpl, + } = Effect.runSync(loadOllamaEmbeddingsConfig(config)) satisfies ResolvedOllamaEmbeddingsConfig; return { embed: Effect.fn("OllamaEmbeddings.embed")((texts: ReadonlyArray, model?: string) => { @@ -49,29 +91,38 @@ export function makeOllamaEmbeddings(config: OllamaEmbeddingsConfig): Embeddings model: useModel, input: Array.from(texts), }).pipe( - Effect.mapError((error) => embeddingsError("ollama.encode-request", error, "ollama")), + Effect.mapError((error) => ollamaEmbeddingsError("ollama.encode-request", error)) ); - return yield* Effect.tryPromise({ - try: async () => { - const response = await fetchImpl(url, { + const response = yield* Effect.tryPromise({ + try: () => + fetchImpl(url, { method: "POST", headers: { "Content-Type": "application/json" }, body, - }); - - if (!response.ok) { - const errorBody = await response.text(); - throw new Error( - `Ollama embeddings request failed (${response.status}): ${errorBody}`, - ); - } - - const data = (await response.json()) as OllamaEmbedResponse; - return data.embeddings; - }, - catch: (error) => embeddingsError("ollama.embed", error, "ollama"), + }), + catch: (error) => ollamaEmbeddingsError("ollama.fetch", error), }); + + if (!response.ok) { + const errorBody = yield* Effect.tryPromise({ + try: () => response.text(), + catch: (error) => ollamaEmbeddingsError("ollama.error-body", error), + }); + return yield* ollamaEmbeddingsMessageError( + "ollama.embed", + `Ollama embeddings request failed (${response.status}): ${errorBody}`, + ); + } + + const data = yield* Effect.tryPromise({ + try: () => response.json() as Promise, + catch: (error) => ollamaEmbeddingsError("ollama.response-json", error), + }); + const decoded = yield* S.decodeUnknownEffect(OllamaEmbedResponse)(data).pipe( + Effect.mapError((error) => ollamaEmbeddingsError("ollama.decode-response", error)) + ); + return Array.from(decoded.embeddings, (vector) => Array.from(vector)); }); }), }; @@ -88,9 +139,10 @@ export type OllamaEmbeddingsProcessor = ReturnType OllamaEmbeddingsLive(config), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/extract/knowledge-extract.ts b/ts/packages/flow/src/extract/knowledge-extract.ts index 63a4d3c1..43808a5b 100644 --- a/ts/packages/flow/src/extract/knowledge-extract.ts +++ b/ts/packages/flow/src/extract/knowledge-extract.ts @@ -289,7 +289,7 @@ export function makeKnowledgeExtractService(config: ProcessorConfig): KnowledgeE const service = makeFlowProcessor(config, { specifications: makeKnowledgeExtractSpecs(), }); - console.log("[KnowledgeExtract] Service initialized"); + Effect.runSync(Effect.log("[KnowledgeExtract] Service initialized")); return service; } @@ -324,7 +324,9 @@ export function parseJsonResponse(raw: string): T | null { if (O.isSome(decoded)) return decoded.value as T; } - console.warn("[KnowledgeExtract] Failed to parse JSON from LLM response:", raw.slice(0, 300)); + Effect.runSync(Effect.logWarning("[KnowledgeExtract] Failed to parse JSON from LLM response", { + response: raw.slice(0, 300), + })); return null; } @@ -333,7 +335,9 @@ function parseRelationshipsResponse(raw: string): ReadonlyArray makeKnowledgeExtractSpecs(), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/flow-manager/service.ts b/ts/packages/flow/src/flow-manager/service.ts index 077710c4..8d6084aa 100644 --- a/ts/packages/flow/src/flow-manager/service.ts +++ b/ts/packages/flow/src/flow-manager/service.ts @@ -22,10 +22,12 @@ import { makeRequestResponse, type ConfigRequest, type ConfigResponse, + errorMessage, } from "@trustgraph/base"; import { makeProcessorProgram } from "@trustgraph/base"; import type { Message } from "@trustgraph/base"; -import { Effect } from "effect"; +import { Duration, Effect, Option } from "effect"; +import * as S from "effect/Schema"; // ---------- Internal state types ---------- @@ -49,6 +51,27 @@ interface ConfigValueEntry { value: unknown; } +export class FlowManagerError extends S.TaggedErrorClass()( + "FlowManagerError", + { + message: S.String, + operation: S.String, + }, +) {} + +const flowManagerError = (operation: string, cause: unknown): FlowManagerError => + FlowManagerError.make({ + operation, + message: errorMessage(cause), + }); + +const decodeJsonUnknown = S.decodeUnknownOption(S.UnknownFromJsonString); + +const encodeJson = (value: unknown, operation: string): Effect.Effect => + S.encodeUnknownEffect(S.UnknownFromJsonString)(value).pipe( + Effect.mapError((cause) => flowManagerError(operation, cause)), + ); + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -69,12 +92,10 @@ function configValues(response: ConfigResponse): ConfigValueEntry[] { } function parseConfigRecord(value: unknown): Record | undefined { - try { - const parsed = typeof value === "string" ? JSON.parse(value) as unknown : value; - return isRecord(parsed) ? parsed : undefined; - } catch { - return undefined; - } + const parsed = typeof value === "string" + ? Option.getOrUndefined(decodeJsonUnknown(value)) + : value; + return isRecord(parsed) ? parsed : undefined; } // ---------- Default blueprint ---------- @@ -137,9 +158,7 @@ export type FlowManagerService = AsyncProcessorRuntime & Record; export function makeFlowManagerService(config: ProcessorConfig): FlowManagerService { const service = makeAsyncProcessor(config, { - run: async () => { - await service.run(); - }, + run: () => service.run(), }) as FlowManagerService; const baseStop = service.stop; service.flows = new Map(); @@ -151,203 +170,341 @@ export function makeFlowManagerService(config: ProcessorConfig): FlowManagerServ Object.assign(service, { - run: async function(this: FlowManagerService): Promise { - // Create config client for pushing flow configs to the config service - this.configClient = makeRequestResponse({ - pubsub: this.pubsub, - requestTopic: topics.configRequest, - responseTopic: topics.configResponse, - subscription: `${this.config.id}-config-client`, - }); - await this.configClient.start(); - await this.ensureDefaultBlueprint(); - await this.refreshBlueprintsFromConfig(); - - // Create producer for flow-response topic - this.responseProducer = await this.pubsub.createProducer>({ - topic: topics.flowResponse, - }); - - // Create consumer for flow-request topic - this.consumer = await this.pubsub.createConsumer>({ - topic: topics.flowRequest, - subscription: `${this.config.id}-flow-request`, - }); - - console.log(`[FlowManager] Listening on ${topics.flowRequest}`); - - // Main consume loop (same pattern as ConfigService) - while (this.running) { - try { - const msg = await this.consumer.receive(2000); - if (msg === null) continue; - - await this.handleMessage(msg); - await this.consumer.acknowledge(msg); - } catch (err) { - if (!this.running) break; - console.error("[FlowManager] Error in consume loop:", err); - await sleep(1000); - } - } - - }, - - - - handleMessage: async function(this: FlowManagerService, msg: Message>): Promise { - const request = msg.value(); - const props = msg.properties(); - const requestId = props.id; - - if (requestId === undefined || requestId.length === 0) { - console.warn("[FlowManager] Received request without id, ignoring"); - return; - } - - try { - const response = await this.handleOperation(request); - await this.responseProducer!.send(response, { id: requestId }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - await this.responseProducer!.send( - { - error: { type: "flow-error", message }, - }, - { id: requestId }, - ); - } - - }, - - - - configRequest: async function(this: FlowManagerService, request: ConfigRequest): Promise { - if (this.configClient === null) throw new Error("Config client not started"); - return this.configClient.request(request); - - }, - - - - ensureDefaultBlueprint: async function(this: FlowManagerService): Promise { - const response = await this.configRequest({ - operation: "getvalues", - type: "flow-blueprint", - }); - if (configValues(response).some((value) => value.key === "default")) { - return; - } - - await this.configRequest({ - operation: "put", - keys: ["flow-blueprint"], - values: { - default: JSON.stringify(DEFAULT_BLUEPRINT), - }, - }); - - }, - - - - refreshBlueprintsFromConfig: async function(this: FlowManagerService): Promise { - const response = await this.configRequest({ - operation: "getvalues", - type: "flow-blueprint", - }); - const next = new Map(); - - for (const item of configValues(response)) { - const parsed = parseConfigRecord(item.value); - if (parsed === undefined) continue; - next.set(item.key, parsed as Blueprint); - } - - if (!next.has("default")) { - next.set("default", DEFAULT_BLUEPRINT); - } - this.blueprints = next; - - }, - - - - refreshFlowsFromConfig: async function(this: FlowManagerService): Promise { - const response = await this.configRequest({ - operation: "getvalues", - type: "flow", - }); - const next = new Map(); - - for (const item of configValues(response)) { - const parsed = parseConfigRecord(item.value); - if (parsed === undefined) continue; - const parameters = isRecord(parsed.parameters) ? parsed.parameters : {}; - next.set(item.key, { - id: item.key, - blueprintName: optionalString(parsed["blueprint-name"]) ?? optionalString(parsed.blueprintName) ?? "default", - description: optionalString(parsed.description) ?? "", - parameters, - status: "running", - }); - } - - if (next.size === 0) { - const flowsResponse = await this.configRequest({ - operation: "getvalues", - type: "flows", - }); - for (const item of configValues(flowsResponse)) { - next.set(item.key, { - id: item.key, - blueprintName: "default", - description: "", - parameters: {}, - status: "running", + run: function(this: FlowManagerService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + // Create config client for pushing flow configs to the config service + service.configClient = makeRequestResponse({ + pubsub: service.pubsub, + requestTopic: topics.configRequest, + responseTopic: topics.configResponse, + subscription: `${service.config.id}-config-client`, + }); + yield* Effect.tryPromise({ + try: () => service.configClient.start(), + catch: (cause) => flowManagerError("config-client-start", cause), + }); + yield* Effect.tryPromise({ + try: () => service.ensureDefaultBlueprint(), + catch: (cause) => flowManagerError("ensure-default-blueprint", cause), + }); + yield* Effect.tryPromise({ + try: () => service.refreshBlueprintsFromConfig(), + catch: (cause) => flowManagerError("refresh-blueprints", cause), }); - } - } - this.flows = next; + // Create producer for flow-response topic + service.responseProducer = yield* Effect.tryPromise({ + try: () => + service.pubsub.createProducer>({ + topic: topics.flowResponse, + }), + catch: (cause) => flowManagerError("response-producer", cause), + }); + + // Create consumer for flow-request topic + service.consumer = yield* Effect.tryPromise({ + try: () => + service.pubsub.createConsumer>({ + topic: topics.flowRequest, + subscription: `${service.config.id}-flow-request`, + }), + catch: (cause) => flowManagerError("consumer", cause), + }); + + yield* Effect.log(`[FlowManager] Listening on ${topics.flowRequest}`); + + // Main consume loop (same pattern as ConfigService) + while (service.running) { + const shouldContinue = yield* Effect.gen(function* () { + const consumer = service.consumer; + if (consumer === null) { + return yield* flowManagerError("consume", "Flow request consumer not started"); + } + + const msg = yield* Effect.tryPromise({ + try: () => consumer.receive(2000), + catch: (cause) => flowManagerError("consume-receive", cause), + }); + if (msg === null) return true; + + yield* Effect.tryPromise({ + try: () => service.handleMessage(msg), + catch: (cause) => flowManagerError("consume-handle", cause), + }); + yield* Effect.tryPromise({ + try: () => consumer.acknowledge(msg), + catch: (cause) => flowManagerError("consume-acknowledge", cause), + }); + + return true; + }).pipe( + Effect.catch((err) => { + if (!service.running) return Effect.succeed(false); + return Effect.logError("[FlowManager] Error in consume loop", { error: err.message }).pipe( + Effect.flatMap(() => Effect.sleep(Duration.millis(1000))), + Effect.as(true), + ); + }), + ); + if (!shouldContinue) break; + } + }), + ); }, - handleOperation: async function(this: FlowManagerService, request: Record): Promise> { - const op = request.operation as string; - await this.refreshBlueprintsFromConfig(); - await this.refreshFlowsFromConfig(); + handleMessage: function(this: FlowManagerService, msg: Message>): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const request = msg.value(); + const props = msg.properties(); + const requestId = props.id; - switch (op) { - case "list-blueprints": - return this.handleListBlueprints(); + if (requestId === undefined || requestId.length === 0) { + yield* Effect.logWarning("[FlowManager] Received request without id, ignoring"); + return; + } - case "put-blueprint": - return await this.handlePutBlueprint(request); + const sendResponse = (response: Record): Effect.Effect => + Effect.gen(function* () { + const responseProducer = service.responseProducer; + if (responseProducer === null) { + return yield* flowManagerError("respond", "Flow response producer not started"); + } + yield* Effect.tryPromise({ + try: () => responseProducer.send(response, { id: requestId }), + catch: (cause) => flowManagerError("respond", cause), + }); + }); - case "get-blueprint": - return this.handleGetBlueprint(request); + yield* Effect.gen(function* () { + const response = yield* Effect.tryPromise, FlowManagerError>({ + try: () => service.handleOperation(request), + catch: (cause) => flowManagerError("operation", cause), + }); + yield* sendResponse(response); + }).pipe( + Effect.catch((err) => + sendResponse({ + error: { type: "flow-error", message: err.message }, + }), + ), + ); + }), + ); - case "delete-blueprint": - return this.handleDeleteBlueprint(request); + }, - case "list-flows": - return this.handleListFlows(); - case "get-flow": - return this.handleGetFlow(request); - case "start-flow": - return await this.handleStartFlow(request); + configRequest: function(this: FlowManagerService, request: ConfigRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const configClient = service.configClient; + if (configClient === null) { + return yield* flowManagerError("config-request", "Config client not started"); + } + return yield* Effect.tryPromise({ + try: () => configClient.request(request), + catch: (cause) => flowManagerError("config-request", cause), + }); + }), + ); - case "stop-flow": - return await this.handleStopFlow(request); + }, - default: - throw new Error(`Unknown flow operation: ${op}`); - } + + + ensureDefaultBlueprint: function(this: FlowManagerService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const response = yield* Effect.tryPromise({ + try: () => + service.configRequest({ + operation: "getvalues", + type: "flow-blueprint", + }), + catch: (cause) => flowManagerError("get-default-blueprint", cause), + }); + if (configValues(response).some((value) => value.key === "default")) { + return; + } + + const defaultBlueprint = yield* encodeJson(DEFAULT_BLUEPRINT, "encode-default-blueprint"); + + yield* Effect.tryPromise({ + try: () => + service.configRequest({ + operation: "put", + keys: ["flow-blueprint"], + values: { + default: defaultBlueprint, + }, + }), + catch: (cause) => flowManagerError("put-default-blueprint", cause), + }); + }), + ); + + }, + + + + refreshBlueprintsFromConfig: function(this: FlowManagerService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const response = yield* Effect.tryPromise({ + try: () => + service.configRequest({ + operation: "getvalues", + type: "flow-blueprint", + }), + catch: (cause) => flowManagerError("refresh-blueprints", cause), + }); + const next = new Map(); + + for (const item of configValues(response)) { + const parsed = parseConfigRecord(item.value); + if (parsed === undefined) continue; + next.set(item.key, parsed as Blueprint); + } + + if (!next.has("default")) { + next.set("default", DEFAULT_BLUEPRINT); + } + service.blueprints = next; + }), + ); + + }, + + + + refreshFlowsFromConfig: function(this: FlowManagerService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const response = yield* Effect.tryPromise({ + try: () => + service.configRequest({ + operation: "getvalues", + type: "flow", + }), + catch: (cause) => flowManagerError("refresh-flows", cause), + }); + const next = new Map(); + + for (const item of configValues(response)) { + const parsed = parseConfigRecord(item.value); + if (parsed === undefined) continue; + const parameters = isRecord(parsed.parameters) ? parsed.parameters : {}; + next.set(item.key, { + id: item.key, + blueprintName: optionalString(parsed["blueprint-name"]) ?? optionalString(parsed.blueprintName) ?? "default", + description: optionalString(parsed.description) ?? "", + parameters, + status: "running", + }); + } + + if (next.size === 0) { + const flowsResponse = yield* Effect.tryPromise({ + try: () => + service.configRequest({ + operation: "getvalues", + type: "flows", + }), + catch: (cause) => flowManagerError("refresh-legacy-flows", cause), + }); + for (const item of configValues(flowsResponse)) { + next.set(item.key, { + id: item.key, + blueprintName: "default", + description: "", + parameters: {}, + status: "running", + }); + } + } + + service.flows = next; + }), + ); + + }, + + + + handleOperation: function(this: FlowManagerService, request: Record): Promise> { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const op = optionalString(request.operation); + yield* Effect.tryPromise({ + try: () => service.refreshBlueprintsFromConfig(), + catch: (cause) => flowManagerError("refresh-blueprints", cause), + }); + yield* Effect.tryPromise({ + try: () => service.refreshFlowsFromConfig(), + catch: (cause) => flowManagerError("refresh-flows", cause), + }); + + switch (op) { + case "list-blueprints": + return service.handleListBlueprints(); + + case "put-blueprint": + return yield* Effect.tryPromise, FlowManagerError>({ + try: () => service.handlePutBlueprint(request), + catch: (cause) => flowManagerError("put-blueprint", cause), + }); + + case "get-blueprint": + return yield* Effect.tryPromise, FlowManagerError>({ + try: () => service.handleGetBlueprint(request), + catch: (cause) => flowManagerError("get-blueprint", cause), + }); + + case "delete-blueprint": + return yield* Effect.tryPromise, FlowManagerError>({ + try: () => service.handleDeleteBlueprint(request), + catch: (cause) => flowManagerError("delete-blueprint", cause), + }); + + case "list-flows": + return service.handleListFlows(); + + case "get-flow": + return yield* Effect.tryPromise, FlowManagerError>({ + try: () => service.handleGetFlow(request), + catch: (cause) => flowManagerError("get-flow", cause), + }); + + case "start-flow": + return yield* Effect.tryPromise, FlowManagerError>({ + try: () => service.handleStartFlow(request), + catch: (cause) => flowManagerError("start-flow", cause), + }); + + case "stop-flow": + return yield* Effect.tryPromise, FlowManagerError>({ + try: () => service.handleStopFlow(request), + catch: (cause) => flowManagerError("stop-flow", cause), + }); + + default: + return yield* flowManagerError("operation", `Unknown flow operation: ${op ?? ""}`); + } + }), + ); }, @@ -364,67 +521,94 @@ export function makeFlowManagerService(config: ProcessorConfig): FlowManagerServ - handleGetBlueprint: function(this: FlowManagerService, request: Record): Record { - const name = request["blueprint-name"] as string | undefined; - if (name === undefined || name.length === 0) { - throw new Error("Missing blueprint-name"); - } + handleGetBlueprint: function(this: FlowManagerService, request: Record): Promise> { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const name = optionalString(request["blueprint-name"]); + if (name === undefined) { + return yield* flowManagerError("get-blueprint", "Missing blueprint-name"); + } - const blueprint = this.blueprints.get(name); - if (blueprint === undefined) { - throw new Error(`Blueprint not found: ${name}`); - } + const blueprint = service.blueprints.get(name); + if (blueprint === undefined) { + return yield* flowManagerError("get-blueprint", `Blueprint not found: ${name}`); + } - return { - "blueprint-definition": JSON.stringify(blueprint), - }; + const definition = yield* encodeJson(blueprint, "encode-blueprint"); + return { + "blueprint-definition": definition, + }; + }), + ); }, - handlePutBlueprint: async function(this: FlowManagerService, request: Record): Promise> { - const name = request["blueprint-name"] as string | undefined; - if (name === undefined || name.length === 0) { - throw new Error("Missing blueprint-name"); - } - const rawDefinition = request["blueprint-definition"]; - if (rawDefinition === undefined) { - throw new Error("Missing blueprint-definition"); - } - const definition = typeof rawDefinition === "string" - ? rawDefinition - : JSON.stringify(rawDefinition); + handlePutBlueprint: function(this: FlowManagerService, request: Record): Promise> { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const name = optionalString(request["blueprint-name"]); + if (name === undefined) { + return yield* flowManagerError("put-blueprint", "Missing blueprint-name"); + } + const rawDefinition = request["blueprint-definition"]; + if (rawDefinition === undefined) { + return yield* flowManagerError("put-blueprint", "Missing blueprint-definition"); + } + const definition = typeof rawDefinition === "string" + ? rawDefinition + : yield* encodeJson(rawDefinition, "encode-blueprint"); - await this.configRequest({ - operation: "put", - keys: ["flow-blueprint"], - values: { [name]: definition }, - }); - await this.refreshBlueprintsFromConfig(); - return {}; + yield* Effect.tryPromise({ + try: () => + service.configRequest({ + operation: "put", + keys: ["flow-blueprint"], + values: { [name]: definition }, + }), + catch: (cause) => flowManagerError("put-blueprint-config", cause), + }); + yield* Effect.tryPromise({ + try: () => service.refreshBlueprintsFromConfig(), + catch: (cause) => flowManagerError("refresh-blueprints", cause), + }); + return {}; + }), + ); }, - handleDeleteBlueprint: async function(this: FlowManagerService, request: Record): Promise> { - const name = request["blueprint-name"] as string | undefined; - if (name === undefined || name.length === 0) { - throw new Error("Missing blueprint-name"); - } + handleDeleteBlueprint: function(this: FlowManagerService, request: Record): Promise> { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const name = optionalString(request["blueprint-name"]); + if (name === undefined) { + return yield* flowManagerError("delete-blueprint", "Missing blueprint-name"); + } - if (name === "default") { - throw new Error("Cannot delete the default blueprint"); - } + if (name === "default") { + return yield* flowManagerError("delete-blueprint", "Cannot delete the default blueprint"); + } - await this.configRequest({ - operation: "delete", - keys: ["flow-blueprint", name], - }); - this.blueprints.delete(name); + yield* Effect.tryPromise({ + try: () => + service.configRequest({ + operation: "delete", + keys: ["flow-blueprint", name], + }), + catch: (cause) => flowManagerError("delete-blueprint-config", cause), + }); + service.blueprints.delete(name); - return {}; + return {}; + }), + ); }, @@ -441,92 +625,119 @@ export function makeFlowManagerService(config: ProcessorConfig): FlowManagerServ - handleGetFlow: function(this: FlowManagerService, request: Record): Record { - const id = request["flow-id"] as string | undefined; - if (id === undefined || id.length === 0) { - throw new Error("Missing flow-id"); - } + handleGetFlow: function(this: FlowManagerService, request: Record): Promise> { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const id = optionalString(request["flow-id"]); + if (id === undefined) { + return yield* flowManagerError("get-flow", "Missing flow-id"); + } - const inst = this.flows.get(id); - if (inst === undefined) { - throw new Error(`Flow not found: ${id}`); - } + const inst = service.flows.get(id); + if (inst === undefined) { + return yield* flowManagerError("get-flow", `Flow not found: ${id}`); + } - return { - flow: JSON.stringify({ - "blueprint-name": inst.blueprintName, - description: inst.description, - parameters: inst.parameters, + const flow = yield* encodeJson( + { + "blueprint-name": inst.blueprintName, + description: inst.description, + parameters: inst.parameters, + }, + "encode-flow", + ); + + return { flow }; }), - }; - - }, - - - - handleStartFlow: async function(this: FlowManagerService, request: Record): Promise> { - const id = request["flow-id"] as string | undefined; - const blueprintName = (request["blueprint-name"] as string) ?? "default"; - const description = (request["description"] as string) ?? ""; - const parameters = (request["parameters"] as Record) ?? {}; - - if (id === undefined || id.length === 0) { - throw new Error("Missing flow-id"); - } - - if ((this.flows as Map).has(id)) { - throw new Error(`Flow already exists: ${id}`); - } - - const blueprint = this.blueprints.get(blueprintName); - if (blueprint === undefined) { - throw new Error(`Blueprint not found: ${blueprintName}`); - } - - // Create the flow instance - const inst: FlowInstance = { - id, - blueprintName, - description, - parameters, - status: "running", - }; - this.flows.set(id, inst); - - console.log( - `[FlowManager] Started flow "${id}" with blueprint "${blueprintName}"`, ); - // Push updated flows config to the config service - await this.pushFlowsConfig(); + }, - return {}; + + + handleStartFlow: function(this: FlowManagerService, request: Record): Promise> { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const id = optionalString(request["flow-id"]); + const blueprintName = optionalString(request["blueprint-name"]) ?? "default"; + const description = optionalString(request.description) ?? ""; + const parameters = isRecord(request.parameters) ? request.parameters : {}; + + if (id === undefined) { + return yield* flowManagerError("start-flow", "Missing flow-id"); + } + + if ((service.flows as Map).has(id)) { + return yield* flowManagerError("start-flow", `Flow already exists: ${id}`); + } + + const blueprint = service.blueprints.get(blueprintName); + if (blueprint === undefined) { + return yield* flowManagerError("start-flow", `Blueprint not found: ${blueprintName}`); + } + + // Create the flow instance + const inst: FlowInstance = { + id, + blueprintName, + description, + parameters, + status: "running", + }; + service.flows.set(id, inst); + + yield* Effect.log( + `[FlowManager] Started flow "${id}" with blueprint "${blueprintName}"`, + ); + + // Push updated flows config to the config service + yield* Effect.tryPromise({ + try: () => service.pushFlowsConfig(), + catch: (cause) => flowManagerError("push-flows-config", cause), + }); + + return {}; + }), + ); }, - handleStopFlow: async function(this: FlowManagerService, request: Record): Promise> { - const id = request["flow-id"] as string | undefined; - if (id === undefined || id.length === 0) { - throw new Error("Missing flow-id"); - } + handleStopFlow: function(this: FlowManagerService, request: Record): Promise> { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const id = optionalString(request["flow-id"]); + if (id === undefined) { + return yield* flowManagerError("stop-flow", "Missing flow-id"); + } - const inst = this.flows.get(id); - if (inst === undefined) { - throw new Error(`Flow not found: ${id}`); - } + const inst = service.flows.get(id); + if (inst === undefined) { + return yield* flowManagerError("stop-flow", `Flow not found: ${id}`); + } - this.flows.delete(id); + service.flows.delete(id); - console.log(`[FlowManager] Stopped flow "${id}"`); + yield* Effect.log(`[FlowManager] Stopped flow "${id}"`); - await this.deleteFlowConfig(id); + yield* Effect.tryPromise({ + try: () => service.deleteFlowConfig(id), + catch: (cause) => flowManagerError("delete-flow-config", cause), + }); - // Push updated flows config (without the removed flow) - await this.pushFlowsConfig(); + // Push updated flows config (without the removed flow) + yield* Effect.tryPromise({ + try: () => service.pushFlowsConfig(), + catch: (cause) => flowManagerError("push-flows-config", cause), + }); - return {}; + return {}; + }), + ); }, @@ -538,55 +749,88 @@ export function makeFlowManagerService(config: ProcessorConfig): FlowManagerServ * Build the flows config object from all running flows and push it * to the config service via a PUT operation. */ - pushFlowsConfig: async function(this: FlowManagerService): Promise { - if (this.configClient === null) return; + pushFlowsConfig: function(this: FlowManagerService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const configClient = service.configClient; + if (configClient === null) return; - const flowsConfig: Record }> = {}; - const flowRecords: Record = {}; - for (const [id, inst] of this.flows) { - const blueprint = this.blueprints.get(inst.blueprintName); - if (blueprint !== undefined) { - flowsConfig[id] = { topics: blueprint.topics }; - flowRecords[id] = JSON.stringify({ - "blueprint-name": inst.blueprintName, - description: inst.description, - parameters: inst.parameters, - }); - } - } + const flowsConfig: Record }> = {}; + const flowRecords: Record = {}; + for (const [id, inst] of service.flows) { + const blueprint = service.blueprints.get(inst.blueprintName); + if (blueprint !== undefined) { + flowsConfig[id] = { topics: blueprint.topics }; + flowRecords[id] = yield* encodeJson( + { + "blueprint-name": inst.blueprintName, + description: inst.description, + parameters: inst.parameters, + }, + "encode-flow-config", + ); + } + } - try { - await this.configClient.request({ - operation: "put", - keys: ["flows"], - values: flowsConfig, - }); - await this.configClient.request({ - operation: "put", - keys: ["flow"], - values: flowRecords, - }); - console.log( - `[FlowManager] Pushed flows config (${this.flows.size} active flows)`, - ); - } catch (err) { - console.error("[FlowManager] Failed to push flows config:", err); - } + yield* Effect.gen(function* () { + yield* Effect.tryPromise({ + try: () => + configClient.request({ + operation: "put", + keys: ["flows"], + values: flowsConfig, + }), + catch: (cause) => flowManagerError("put-flows-config", cause), + }); + yield* Effect.tryPromise({ + try: () => + configClient.request({ + operation: "put", + keys: ["flow"], + values: flowRecords, + }), + catch: (cause) => flowManagerError("put-flow-records", cause), + }); + yield* Effect.log( + `[FlowManager] Pushed flows config (${service.flows.size} active flows)`, + ); + }).pipe( + Effect.catch((err) => + Effect.logError("[FlowManager] Failed to push flows config", { error: err.message }), + ), + ); + }), + ); }, - deleteFlowConfig: async function(this: FlowManagerService, id: string): Promise { - if (this.configClient === null) return; - await this.configClient.request({ - operation: "delete", - keys: ["flows", id], - }); - await this.configClient.request({ - operation: "delete", - keys: ["flow", id], - }); + deleteFlowConfig: function(this: FlowManagerService, id: string): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const configClient = service.configClient; + if (configClient === null) return; + yield* Effect.tryPromise({ + try: () => + configClient.request({ + operation: "delete", + keys: ["flows", id], + }), + catch: (cause) => flowManagerError("delete-flows-config", cause), + }); + yield* Effect.tryPromise({ + try: () => + configClient.request({ + operation: "delete", + keys: ["flow", id], + }), + catch: (cause) => flowManagerError("delete-flow-record", cause), + }); + }), + ); }, @@ -594,20 +838,40 @@ export function makeFlowManagerService(config: ProcessorConfig): FlowManagerServ // ---------- Lifecycle ---------- - stop: async function(this: FlowManagerService): Promise { - if (this.consumer !== null) { - await this.consumer.close(); - this.consumer = null; - } - if (this.responseProducer !== null) { - await this.responseProducer.close(); - this.responseProducer = null; - } - if (this.configClient !== null) { - await this.configClient.stop(); - this.configClient = null; - } - await baseStop(); + stop: function(this: FlowManagerService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + if (service.consumer !== null) { + const consumer = service.consumer; + yield* Effect.tryPromise({ + try: () => consumer.close(), + catch: (cause) => flowManagerError("consumer-close", cause), + }); + service.consumer = null; + } + if (service.responseProducer !== null) { + const responseProducer = service.responseProducer; + yield* Effect.tryPromise({ + try: () => responseProducer.close(), + catch: (cause) => flowManagerError("response-producer-close", cause), + }); + service.responseProducer = null; + } + if (service.configClient !== null) { + const configClient = service.configClient; + yield* Effect.tryPromise({ + try: () => configClient.stop(), + catch: (cause) => flowManagerError("config-client-stop", cause), + }); + service.configClient = null; + } + yield* Effect.tryPromise({ + try: () => baseStop(), + catch: (cause) => flowManagerError("base-stop", cause), + }); + }), + ); } }); @@ -616,15 +880,11 @@ export function makeFlowManagerService(config: ProcessorConfig): FlowManagerServ export const FlowManagerService = makeFlowManagerService; -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - export const program = makeProcessorProgram({ id: "flow-manager", make: (config) => makeFlowManagerService(config), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/gateway/dispatch/manager.ts b/ts/packages/flow/src/gateway/dispatch/manager.ts index fcbc81b5..81f111b3 100644 --- a/ts/packages/flow/src/gateway/dispatch/manager.ts +++ b/ts/packages/flow/src/gateway/dispatch/manager.ts @@ -8,7 +8,7 @@ * Python reference: trustgraph-flow/trustgraph/gateway/dispatch/manager.py */ -import { Effect, Exit, Scope, SynchronizedRef } from "effect"; +import { Clock, Effect, Exit, Random, Scope, SynchronizedRef } from "effect"; import { loadMessagingRuntimeConfig, makeNatsBackend, @@ -146,13 +146,13 @@ export function makeDispatcherManager(config: GatewayConfig): DispatcherManager const pubsub: PubSubBackend = config.pubsub ?? makeNatsBackend(config.natsUrl ?? "nats://localhost:4222"); let runtime: DispatcherRuntime | null = null; - const start = async (): Promise => { - if (runtime !== null) return; + const start = (): Promise => { + if (runtime !== null) return Promise.resolve(); - runtime = await Effect.runPromise( + return Effect.runPromise( Effect.gen(function* () { const scope = yield* Scope.make(); - return yield* Effect.gen(function* () { + const nextRuntime = yield* Effect.gen(function* () { const messagingConfig = yield* loadMessagingRuntimeConfig(); const requestors = yield* SynchronizedRef.make(new Map()); return { @@ -163,62 +163,79 @@ export function makeDispatcherManager(config: GatewayConfig): DispatcherManager }).pipe( Effect.onError((cause) => Scope.close(scope, Exit.failCause(cause))), ); + runtime = nextRuntime; }), ); }; - const stop = async (): Promise => { - const current = runtime; - runtime = null; + const stop = (): Promise => + Effect.runPromise( + Effect.gen(function* () { + const current = runtime; + runtime = null; - if (current !== null) { - await Effect.runPromise(Scope.close(current.scope, Exit.void)); - } + if (current !== null) { + yield* Scope.close(current.scope, Exit.void); + } - await pubsub.close(); - }; + yield* Effect.tryPromise({ + try: () => pubsub.close(), + catch: (cause) => messagingLifecycleError("gateway-dispatcher", "close-pubsub", cause), + }); + }), + ); // ---------- Internal helpers ---------- - const ensureRuntime = async (): Promise => { - if (runtime === null) { - await start(); - } - if (runtime === null) { - throw messagingLifecycleError("gateway-dispatcher", "start", "Dispatcher manager failed to start"); - } - return runtime; - }; + const ensureRuntime = (): Promise => + Effect.runPromise( + Effect.gen(function* () { + if (runtime === null) { + yield* Effect.tryPromise({ + try: () => start(), + catch: (cause) => messagingLifecycleError("gateway-dispatcher", "start", cause), + }); + } + if (runtime === null) { + return yield* messagingLifecycleError("gateway-dispatcher", "start", "Dispatcher manager failed to start"); + } + return runtime; + }), + ); - const getRequestor = async ( + const getRequestor = ( requestTopic: string, responseTopic: string, key: string, - ): Promise> => { - const current = await ensureRuntime(); + ): Promise> => + Effect.runPromise( + Effect.gen(function* () { + const current = yield* Effect.tryPromise({ + try: () => ensureRuntime(), + catch: (cause) => messagingLifecycleError("gateway-dispatcher", "ensure-runtime", cause), + }); - return await Effect.runPromise( - SynchronizedRef.modifyEffect(current.requestors, (requestors) => { - const cached = requestors.get(key); - if (cached !== undefined) { - return Effect.succeed([cached, requestors] as const); - } + return yield* SynchronizedRef.modifyEffect(current.requestors, (requestors) => { + const cached = requestors.get(key); + if (cached !== undefined) { + return Effect.succeed([cached, requestors] as const); + } - return current.factory.make({ - requestTopic, - responseTopic, - subscription: `gateway-${key}`, - }).pipe( - Scope.provide(current.scope), - Effect.map((requestor) => { - const next = new Map(requestors); - next.set(key, requestor); - return [requestor, next] as const; - }), - ); + return current.factory.make({ + requestTopic, + responseTopic, + subscription: `gateway-${key}`, + }).pipe( + Scope.provide(current.scope), + Effect.map((requestor) => { + const next = new Map(requestors); + next.set(key, requestor); + return [requestor, next] as const; + }), + ); + }); }), ); - }; const resolveGlobalTopics = ( kind: string, @@ -256,93 +273,107 @@ export function makeDispatcherManager(config: GatewayConfig): DispatcherManager // ---------- Global service dispatch ---------- - const dispatchGlobalService = async ( + const dispatchGlobalService = ( kind: string, request: Record, - ): Promise => { - const { requestTopic, responseTopic } = resolveGlobalTopics(kind); - const rr = await getRequestor(requestTopic, responseTopic, `global:${kind}`); + ): Promise => + Effect.runPromise( + Effect.gen(function* () { + const { requestTopic, responseTopic } = resolveGlobalTopics(kind); + const rr = yield* Effect.tryPromise({ + try: () => getRequestor(requestTopic, responseTopic, `global:${kind}`), + catch: (cause) => messagingLifecycleError("gateway-dispatcher", "get-requestor", cause), + }); - const translated = translateRequest(kind, request); - const response = await Effect.runPromise(rr.request(translated)); - return translateResponse(kind, response); - }; + const translated = translateRequest(kind, request); + const response = yield* rr.request(translated); + return translateResponse(kind, response); + }), + ); - const dispatchGlobalServiceStreaming = async ( + const dispatchGlobalServiceStreaming = ( kind: string, request: Record, responder: Responder, - ): Promise => { - const { requestTopic, responseTopic } = resolveGlobalTopics(kind); - const rr = await getRequestor(requestTopic, responseTopic, `global:${kind}`); - const translated = translateRequest(kind, request); + ): Promise => + Effect.runPromise( + Effect.gen(function* () { + const { requestTopic, responseTopic } = resolveGlobalTopics(kind); + const rr = yield* Effect.tryPromise({ + try: () => getRequestor(requestTopic, responseTopic, `global:${kind}`), + catch: (cause) => messagingLifecycleError("gateway-dispatcher", "get-requestor", cause), + }); + const translated = translateRequest(kind, request); - await Effect.runPromise( - rr.request(translated, { - recipient: (response) => { - const translatedRes = translateResponse(kind, response); - const complete = dispatcherManagerIsCompleteResponse(translatedRes); - return Effect.tryPromise({ - try: async () => { - await responder(translatedRes, complete); - return complete; - }, - catch: (error) => messagingDeliveryError(responseTopic, "stream-responder", error), - }); - }, + yield* rr.request(translated, { + recipient: (response) => { + const translatedRes = translateResponse(kind, response); + const complete = dispatcherManagerIsCompleteResponse(translatedRes); + return Effect.tryPromise({ + try: () => responder(translatedRes, complete).then(() => complete), + catch: (error) => messagingDeliveryError(responseTopic, "stream-responder", error), + }); + }, + }); }), ); - }; // ---------- Flow-scoped service dispatch ---------- - const dispatchFlowService = async ( + const dispatchFlowService = ( flow: string, kind: string, request: Record, - ): Promise => { - const { requestTopic, responseTopic } = resolveFlowTopics(kind); - const rr = await getRequestor( - requestTopic, - responseTopic, - `flow:${flow}:${kind}`, + ): Promise => + Effect.runPromise( + Effect.gen(function* () { + const { requestTopic, responseTopic } = resolveFlowTopics(kind); + const rr = yield* Effect.tryPromise({ + try: () => getRequestor( + requestTopic, + responseTopic, + `flow:${flow}:${kind}`, + ), + catch: (cause) => messagingLifecycleError("gateway-dispatcher", "get-requestor", cause), + }); + + const translated = translateRequest(kind, request); + const response = yield* rr.request(translated); + return translateResponse(kind, response); + }), ); - const translated = translateRequest(kind, request); - const response = await Effect.runPromise(rr.request(translated)); - return translateResponse(kind, response); - }; - - const dispatchFlowServiceStreaming = async ( + const dispatchFlowServiceStreaming = ( flow: string, kind: string, request: Record, responder: Responder, - ): Promise => { - const { requestTopic, responseTopic } = resolveFlowTopics(kind); - const rr = await getRequestor( - requestTopic, - responseTopic, - `flow:${flow}:${kind}`, - ); - const translated = translateRequest(kind, request); + ): Promise => + Effect.runPromise( + Effect.gen(function* () { + const { requestTopic, responseTopic } = resolveFlowTopics(kind); + const rr = yield* Effect.tryPromise({ + try: () => getRequestor( + requestTopic, + responseTopic, + `flow:${flow}:${kind}`, + ), + catch: (cause) => messagingLifecycleError("gateway-dispatcher", "get-requestor", cause), + }); + const translated = translateRequest(kind, request); - await Effect.runPromise( - rr.request(translated, { - recipient: (response) => { - const translatedRes = translateResponse(kind, response); - const complete = dispatcherManagerIsCompleteResponse(translatedRes); - return Effect.tryPromise({ - try: async () => { - await responder(translatedRes, complete); - return complete; - }, - catch: (error) => messagingDeliveryError(responseTopic, "stream-responder", error), - }); - }, + yield* rr.request(translated, { + recipient: (response) => { + const translatedRes = translateResponse(kind, response); + const complete = dispatcherManagerIsCompleteResponse(translatedRes); + return Effect.tryPromise({ + try: () => responder(translatedRes, complete).then(() => complete), + catch: (error) => messagingDeliveryError(responseTopic, "stream-responder", error), + }); + }, + }); }), ); - }; // ---------- Fire-and-forget publish ---------- @@ -350,12 +381,27 @@ export function makeDispatcherManager(config: GatewayConfig): DispatcherManager * Publish a single message to an arbitrary topic (no request/response). * Used for injecting documents into the processing pipeline. */ - const publishToTopic = async (topic: string, message: unknown, id?: string): Promise => { - const producer = await pubsub.createProducer({ topic }); - const messageId = id ?? `pub-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; - await producer.send(message, { id: messageId }); - await producer.close(); - }; + const publishToTopic = (topic: string, message: unknown, id?: string): Promise => + Effect.runPromise( + Effect.gen(function* () { + const producer = yield* Effect.tryPromise({ + try: () => pubsub.createProducer({ topic }), + catch: (cause) => messagingDeliveryError(topic, "create-producer", cause), + }); + const timestamp = yield* Clock.currentTimeMillis; + const suffix = yield* Random.nextIntBetween(0, 36 ** 6, { halfOpen: true }); + const messageId = id ?? `pub-${timestamp}-${suffix.toString(36).padStart(6, "0")}`; + + yield* Effect.tryPromise({ + try: () => producer.send(message, { id: messageId }), + catch: (cause) => messagingDeliveryError(topic, "send", cause), + }); + yield* Effect.tryPromise({ + try: () => producer.close(), + catch: (cause) => messagingDeliveryError(topic, "close-producer", cause), + }); + }), + ); return { start, diff --git a/ts/packages/flow/src/gateway/dispatch/serialize.ts b/ts/packages/flow/src/gateway/dispatch/serialize.ts index 18a4f53d..f502b7e3 100644 --- a/ts/packages/flow/src/gateway/dispatch/serialize.ts +++ b/ts/packages/flow/src/gateway/dispatch/serialize.ts @@ -19,6 +19,7 @@ */ import type { Term, Triple } from "@trustgraph/base"; +import * as S from "effect/Schema"; // ---------- Client wire format type definitions ---------- @@ -46,6 +47,14 @@ interface ClientTripleTerm { type ClientTerm = ClientIriTerm | ClientBlankTerm | ClientLiteralTerm | ClientTripleTerm; +export class DispatchSerializationError extends S.TaggedErrorClass()( + "DispatchSerializationError", + { + message: S.String, + operation: S.String, + }, +) {} + interface ClientTriple { s: ClientTerm; p: ClientTerm; @@ -70,7 +79,10 @@ export function clientTermToInternal(wire: ClientTerm): Term { }; case "t": { if (wire.tr === undefined) { - throw new Error("Client triple term is missing tr"); + throw DispatchSerializationError.make({ + operation: "client-term-to-internal", + message: "Client triple term is missing tr", + }); } return { type: "TRIPLE", diff --git a/ts/packages/flow/src/gateway/rpc-protocol.ts b/ts/packages/flow/src/gateway/rpc-protocol.ts index 62484588..c037ea86 100644 --- a/ts/packages/flow/src/gateway/rpc-protocol.ts +++ b/ts/packages/flow/src/gateway/rpc-protocol.ts @@ -34,37 +34,44 @@ export const makeSocketRpcProtocol = Effect.gen(function* () { }); const writeRaw = yield* socket.writer; - const write = (response: RpcMessage.FromServerEncoded) => { - try { - const encoded = parser.encode(response); - if (encoded === undefined) return Effect.void; - return Effect.orDie(writeRaw(encoded)); - } catch (cause) { - return Effect.orDie( - writeRaw(parser.encode(RpcMessage.ResponseDefectEncoded(cause))!), - ); - } - }; + const encodeDefect = (cause: unknown) => + Effect.sync(() => parser.encode(RpcMessage.ResponseDefectEncoded(cause))!); + const write = (response: RpcMessage.FromServerEncoded) => + Effect.sync(() => parser.encode(response)).pipe( + Effect.flatMap((encoded) => + encoded === undefined ? Effect.void : Effect.orDie(writeRaw(encoded)), + ), + Effect.catchDefect((cause: unknown) => + encodeDefect(cause).pipe( + Effect.flatMap((encoded) => Effect.orDie(writeRaw(encoded))), + Effect.orDie, + ), + ), + ); clients.set(clientId, { write }); clientIds.add(clientId); - yield* socket.runRaw((data) => { - try { - const decoded = parser.decode(data) as ReadonlyArray; - return Effect.forEach(decoded, (message) => { - if (message._tag === "Request" && headers !== undefined) { - return writeRequest(clientId, { - ...message, - headers: headers.concat(message.headers), - }); - } - return writeRequest(clientId, message); - }, { discard: true }); - } catch (cause) { - return writeRaw(parser.encode(RpcMessage.ResponseDefectEncoded(cause))!); - } - }).pipe( + yield* socket.runRaw((data) => + Effect.sync(() => parser.decode(data) as ReadonlyArray).pipe( + Effect.flatMap((decoded) => + Effect.forEach(decoded, (message) => { + if (message._tag === "Request" && headers !== undefined) { + return writeRequest(clientId, { + ...message, + headers: headers.concat(message.headers), + }); + } + return writeRequest(clientId, message); + }, { discard: true }), + ), + Effect.catchDefect((cause: unknown) => + encodeDefect(cause).pipe( + Effect.flatMap((encoded) => writeRaw(encoded)), + ), + ), + ) + ).pipe( Effect.catchReason("SocketError", "SocketCloseError", () => Effect.void), Effect.orDie, ); diff --git a/ts/packages/flow/src/gateway/rpc-server.ts b/ts/packages/flow/src/gateway/rpc-server.ts index a4e82c36..f1787957 100644 --- a/ts/packages/flow/src/gateway/rpc-server.ts +++ b/ts/packages/flow/src/gateway/rpc-server.ts @@ -42,7 +42,7 @@ const makeGatewayRpcHandlers = (dispatcher: DispatcherManager) => Dispatch: (payload) => Effect.tryPromise({ try: () => dispatchOne(dispatcher, payload), - catch: (cause) => new DispatchError({ message: errorMessage(cause) }), + catch: (cause) => DispatchError.make({ message: errorMessage(cause) }), }), DispatchStream: Effect.fn("GatewayRpc.DispatchStream")(function* (payload) { const context = yield* Effect.context(); @@ -52,11 +52,10 @@ const makeGatewayRpcHandlers = (dispatcher: DispatcherManager) => yield* Effect.tryPromise({ try: () => - dispatchStream(dispatcher, payload, async (response, complete) => { - await runPromise(Queue.offer(queue, new DispatchStreamChunk({ response, complete }))); - return complete; - }), - catch: (cause) => new DispatchError({ message: errorMessage(cause) }), + dispatchStream(dispatcher, payload, (response, complete) => + runPromise(Queue.offer(queue, DispatchStreamChunk.make({ response, complete }))).then(() => complete), + ), + catch: (cause) => DispatchError.make({ message: errorMessage(cause) }), }).pipe( Effect.flatMap(() => Queue.end(queue)), Effect.catch((error) => Queue.fail(queue, error)), @@ -82,26 +81,24 @@ function dispatchOne( return dispatcher.dispatchGlobalService(payload.service, payload.request); } -async function dispatchStream( +function dispatchStream( dispatcher: DispatcherManager, payload: DispatchPayload, responder: (response: unknown, complete: boolean) => Promise, ): Promise { - const send = async (response: unknown, complete: boolean) => { - await responder(response, complete); - }; + const send = (response: unknown, complete: boolean): Promise => + responder(response, complete).then(() => undefined); if (payload.scope === "flow") { - await dispatcher.dispatchFlowServiceStreaming( + return dispatcher.dispatchFlowServiceStreaming( payload.flow ?? "default", payload.service, payload.request, send, ); - return; } - await dispatcher.dispatchGlobalServiceStreaming( + return dispatcher.dispatchGlobalServiceStreaming( payload.service, payload.request, send, diff --git a/ts/packages/flow/src/gateway/server.ts b/ts/packages/flow/src/gateway/server.ts index 6d8d7b5f..f2943fe5 100644 --- a/ts/packages/flow/src/gateway/server.ts +++ b/ts/packages/flow/src/gateway/server.ts @@ -7,13 +7,13 @@ * Python reference: trustgraph-flow/trustgraph/gateway/service.py */ -import Fastify from "fastify"; +import Fastify, { type FastifyReply } from "fastify"; import websocketPlugin from "@fastify/websocket"; -import { Config, Effect, Exit, Scope } from "effect"; +import { Clock, Config, Effect, Exit, Random, Scope } from "effect"; import * as O from "effect/Option"; import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"; import * as EffectSocket from "effect/unstable/socket/Socket"; -import { optionalStringConfig, registry, toTgError, type PubSubBackend } from "@trustgraph/base"; +import { messagingLifecycleError, optionalStringConfig, registry, toTgError, type PubSubBackend } from "@trustgraph/base"; import { makeDispatcherManager } from "./dispatch/manager.js"; import { makeGatewayRpcServer } from "./rpc-server.js"; @@ -25,187 +25,227 @@ export interface GatewayConfig { pubsub?: PubSubBackend; } -export async function createGateway(config: GatewayConfig) { +export function createGateway(config: GatewayConfig) { const app = Fastify({ logger: true }); - await app.register(websocketPlugin); - const dispatcher = makeDispatcherManager(config); - await dispatcher.start(); - const rpcScope = await Effect.runPromise(Scope.make()); - const rpcServer = await Effect.runPromise( - makeGatewayRpcServer(dispatcher).pipe( - Effect.provideService(RpcSerialization.RpcSerialization, RpcSerialization.ndjson), - Scope.provide(rpcScope), - ), - ); - // Authentication middleware - app.addHook("onRequest", async (request, reply) => { - if (request.url === "/api/v1/metrics") return; - if (request.url.startsWith("/api/v1/rpc")) return; // RPC socket auth via query param - - if (config.secret !== undefined && config.secret.length > 0) { - const auth = request.headers.authorization; - if (auth === undefined || auth !== `Bearer ${config.secret}`) { - reply.code(401).send({ error: "Unauthorized" }); - } + const sendDispatchResult = (reply: FastifyReply, result: unknown): unknown => { + const err = (result as Record)?.error as { type?: string; message?: string } | undefined; + if (err !== undefined) { + const statusCode = err.type === "not-found" ? 404 : 400; + return reply.code(statusCode).send(result); } - }); - - app.post<{ - Body: { - scope?: string; - service?: string; - flow?: string; - request?: Record; - }; - }>("/api/v1/workbench/dispatch", async (request, reply) => { - const body = request.body; - const service = body.service; - const payload = body.request; - if (service === undefined || service.length === 0 || payload === undefined) { - return reply.code(400).send({ - error: { type: "bad-request", message: "service and request are required" }, - }); - } - - try { - const result = body.scope === "flow" - ? await dispatcher.dispatchFlowService(body.flow ?? "default", service, payload) - : await dispatcher.dispatchGlobalService(service, payload); - const err = (result as Record)?.error as { type?: string; message?: string } | undefined; - if (err !== undefined) { - const statusCode = err.type === "not-found" ? 404 : 400; - return reply.code(statusCode).send(result); - } - return result; - } catch (err) { - reply.code(500).send({ error: toTgError(err) }); - } - }); - - // REST endpoint: POST /api/v1/:kind (global services) - app.post<{ Params: { kind: string } }>("/api/v1/:kind", async (request, reply) => { - const { kind } = request.params; - const body = request.body as Record; - - try { - const result = await dispatcher.dispatchGlobalService(kind, body) as Record; - const err = result?.error as { type?: string; message?: string } | undefined; - if (err !== undefined) { - const statusCode = err.type === "not-found" ? 404 : 400; - return reply.code(statusCode).send(result); - } - return result; - } catch (err) { - reply.code(500).send({ error: toTgError(err) }); - } - }); - - // REST endpoint: POST /api/v1/flow/:flow/service/:kind (flow-scoped services) - app.post<{ Params: { flow: string; kind: string } }>( - "/api/v1/flow/:flow/service/:kind", - async (request, reply) => { - const { flow, kind } = request.params; - const body = request.body as Record; - - try { - const result = await dispatcher.dispatchFlowService(flow, kind, body) as Record; - const err = result?.error as { type?: string; message?: string } | undefined; - if (err !== undefined) { - const statusCode = err.type === "not-found" ? 404 : 400; - return reply.code(statusCode).send(result); - } - return result; - } catch (err) { - reply.code(500).send({ error: toTgError(err) }); - } - }, - ); - - // REST endpoint: POST /api/v1/flow/:flow/load (trigger document processing) - app.post<{ Params: { flow: string } }>( - "/api/v1/flow/:flow/load", - async (request, reply) => { - const { flow } = request.params; - const body = request.body as { - documentId?: string; - user?: string; - collection?: string; - }; - - if (body.documentId === undefined || body.documentId.length === 0) { - return reply.code(400).send({ - error: { type: "bad-request", message: "documentId is required" }, - }); - } - - try { - const user = body.user ?? "default"; - const collection = body.collection ?? "default"; - const documentId = body.documentId; - - // Publish Document message to the decode-input topic - const topic = "tg.flow.document"; - const metadata = { - id: `load-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, - root: documentId, - user, - collection, - }; - - await dispatcher.publishToTopic(topic, { metadata, documentId }); - - return { status: "processing", documentId, flow }; - } catch (err) { - reply.code(500).send({ - error: toTgError(err), - }); - } - }, - ); - - // Effect RPC WebSocket endpoint: /api/v1/rpc - app.get("/api/v1/rpc", { websocket: true }, (socket, request) => { - const url = new URL(request.url, `http://${request.headers.host}`); - const token = url.searchParams.get("token"); - if (config.secret !== undefined && config.secret.length > 0 && token !== config.secret) { - socket.close(4001, "Unauthorized"); - return; - } - - const program = Effect.scoped( - Effect.gen(function* () { - const effectSocket = yield* EffectSocket.fromWebSocket( - Effect.succeed(socket as unknown as globalThis.WebSocket), - { closeCodeIsError: (code) => code !== 1000 }, - ); - yield* rpcServer.onSocket(effectSocket, headersFrom(request.headers)); - }), - ); - - Effect.runPromise(program.pipe(Scope.provide(rpcScope))).catch((err) => { - console.error("[Gateway] RPC WebSocket error:", err); - if (socket.readyState === 1) { - socket.close(1011, "Internal server error"); - } - }); - }); - - // Metrics endpoint — returns Prometheus metrics from prom-client - app.get("/api/v1/metrics", async (_, reply) => { - reply.header("content-type", registry.contentType); - return registry.metrics(); - }); - - return { - start: () => app.listen({ port: config.port, host: "0.0.0.0" }), - stop: async () => { - await app.close(); - await Effect.runPromise(Scope.close(rpcScope, Exit.void)); - await dispatcher.stop(); - }, + return result; }; + + const sendDispatchError = (reply: FastifyReply, error: unknown): unknown => + reply.code(500).send({ error: toTgError(error) }); + + return Effect.runPromise( + Effect.gen(function* () { + yield* Effect.tryPromise({ + try: () => app.register(websocketPlugin), + catch: (cause) => messagingLifecycleError("gateway", "register-websocket", cause), + }); + + yield* Effect.tryPromise({ + try: () => dispatcher.start(), + catch: (cause) => messagingLifecycleError("gateway", "dispatcher-start", cause), + }); + + const rpcScope = yield* Scope.make(); + const rpcServer = yield* makeGatewayRpcServer(dispatcher).pipe( + Effect.provideService(RpcSerialization.RpcSerialization, RpcSerialization.ndjson), + Scope.provide(rpcScope), + ); + + return { rpcScope, rpcServer }; + }), + ).then(({ rpcScope, rpcServer }) => { + // Authentication middleware + app.addHook("onRequest", (request, reply) => { + if (request.url === "/api/v1/metrics") return; + if (request.url.startsWith("/api/v1/rpc")) return; // RPC socket auth via query param + + if (config.secret !== undefined && config.secret.length > 0) { + const auth = request.headers.authorization; + if (auth === undefined || auth !== `Bearer ${config.secret}`) { + reply.code(401).send({ error: "Unauthorized" }); + } + } + }); + + app.post<{ + Body: { + scope?: string; + service?: string; + flow?: string; + request?: Record; + }; + }>("/api/v1/workbench/dispatch", (request, reply) => { + const body = request.body; + const service = body.service; + const payload = body.request; + if (service === undefined || service.length === 0 || payload === undefined) { + return reply.code(400).send({ + error: { type: "bad-request", message: "service and request are required" }, + }); + } + + return Effect.runPromise( + Effect.tryPromise({ + try: () => + body.scope === "flow" + ? dispatcher.dispatchFlowService(body.flow ?? "default", service, payload) + : dispatcher.dispatchGlobalService(service, payload), + catch: (cause) => messagingLifecycleError("gateway", "workbench-dispatch", cause), + }).pipe( + Effect.map((result) => sendDispatchResult(reply, result)), + Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))), + ), + ); + }); + + // REST endpoint: POST /api/v1/:kind (global services) + app.post<{ Params: { kind: string } }>("/api/v1/:kind", (request, reply) => { + const { kind } = request.params; + const body = request.body as Record; + + return Effect.runPromise( + Effect.tryPromise({ + try: () => dispatcher.dispatchGlobalService(kind, body), + catch: (cause) => messagingLifecycleError("gateway", "global-dispatch", cause), + }).pipe( + Effect.map((result) => sendDispatchResult(reply, result)), + Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))), + ), + ); + }); + + // REST endpoint: POST /api/v1/flow/:flow/service/:kind (flow-scoped services) + app.post<{ Params: { flow: string; kind: string } }>( + "/api/v1/flow/:flow/service/:kind", + (request, reply) => { + const { flow, kind } = request.params; + const body = request.body as Record; + + return Effect.runPromise( + Effect.tryPromise({ + try: () => dispatcher.dispatchFlowService(flow, kind, body), + catch: (cause) => messagingLifecycleError("gateway", "flow-dispatch", cause), + }).pipe( + Effect.map((result) => sendDispatchResult(reply, result)), + Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))), + ), + ); + }, + ); + + // REST endpoint: POST /api/v1/flow/:flow/load (trigger document processing) + app.post<{ Params: { flow: string } }>( + "/api/v1/flow/:flow/load", + (request, reply) => { + const { flow } = request.params; + const body = request.body as { + documentId?: string; + user?: string; + collection?: string; + }; + + if (body.documentId === undefined || body.documentId.length === 0) { + return reply.code(400).send({ + error: { type: "bad-request", message: "documentId is required" }, + }); + } + + return Effect.runPromise( + Effect.gen(function* () { + const user = body.user ?? "default"; + const collection = body.collection ?? "default"; + const documentId = body.documentId; + const timestamp = yield* Clock.currentTimeMillis; + const suffix = yield* Random.nextIntBetween(0, 36 ** 6, { halfOpen: true }); + + // Publish Document message to the decode-input topic + const topic = "tg.flow.document"; + const metadata = { + id: `load-${timestamp}-${suffix.toString(36).padStart(6, "0")}`, + root: documentId, + user, + collection, + }; + + yield* Effect.tryPromise({ + try: () => dispatcher.publishToTopic(topic, { metadata, documentId }), + catch: (cause) => messagingLifecycleError("gateway", "publish-load", cause), + }); + + return { status: "processing", documentId, flow }; + }).pipe( + Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))), + ), + ); + }, + ); + + // Effect RPC WebSocket endpoint: /api/v1/rpc + app.get("/api/v1/rpc", { websocket: true }, (socket, request) => { + const url = new URL(request.url, `http://${request.headers.host}`); + const token = url.searchParams.get("token"); + if (config.secret !== undefined && config.secret.length > 0 && token !== config.secret) { + socket.close(4001, "Unauthorized"); + return; + } + + const program = Effect.scoped( + Effect.gen(function* () { + const effectSocket = yield* EffectSocket.fromWebSocket( + Effect.succeed(socket as unknown as globalThis.WebSocket), + { closeCodeIsError: (code) => code !== 1000 }, + ); + yield* rpcServer.onSocket(effectSocket, headersFrom(request.headers)); + }), + ); + + void Effect.runPromise(program.pipe(Scope.provide(rpcScope))).catch((error) => { + void Effect.runPromise( + Effect.logError("[Gateway] RPC WebSocket error", { error: toTgError(error).message }).pipe( + Effect.flatMap(() => + Effect.sync(() => { + if (socket.readyState === 1) { + socket.close(1011, "Internal server error"); + } + }), + ), + ), + ); + }); + }); + + // Metrics endpoint — returns Prometheus metrics from prom-client + app.get("/api/v1/metrics", (_, reply) => { + reply.header("content-type", registry.contentType); + return registry.metrics(); + }); + + return { + start: () => app.listen({ port: config.port, host: "0.0.0.0" }), + stop: () => + Effect.runPromise( + Effect.gen(function* () { + yield* Effect.tryPromise({ + try: () => app.close(), + catch: (cause) => messagingLifecycleError("gateway", "app-close", cause), + }); + yield* Scope.close(rpcScope, Exit.void); + yield* Effect.tryPromise({ + try: () => dispatcher.stop(), + catch: (cause) => messagingLifecycleError("gateway", "dispatcher-stop", cause), + }); + }), + ), + }; + }); } function headersFrom(headers: Record): ReadonlyArray<[string, string]> { @@ -217,8 +257,8 @@ function headersFrom(headers: Record { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } export const loadGatewayConfig = Effect.fn("loadGatewayConfig")(function* () { diff --git a/ts/packages/flow/src/librarian/service.ts b/ts/packages/flow/src/librarian/service.ts index 1a4e4523..1403d39a 100644 --- a/ts/packages/flow/src/librarian/service.ts +++ b/ts/packages/flow/src/librarian/service.ts @@ -11,7 +11,9 @@ */ import { + errorMessage, makeAsyncProcessor, + makeProcessorProgram, type ProcessorConfig, type AsyncProcessorRuntime, topics, @@ -22,9 +24,9 @@ import { type DocumentMetadata, type ProcessingMetadata, } from "@trustgraph/base"; -import { makeProcessorProgram } from "@trustgraph/base"; import type { Message } from "@trustgraph/base"; -import { Effect } from "effect"; +import { Clock, Config, DateTime, Duration, Effect, Random } from "effect"; +import * as S from "effect/Schema"; import { makeCollectionManager } from "./collection-manager.js"; import { ensureDirectory, @@ -51,6 +53,20 @@ interface UploadSession { user: string; } +type PersistedCollection = { + user: string; + collection: string; + name: string; + description: string; + tags: string[]; +}; + +type PersistedLibrarianState = { + documents?: Record; + processing?: Record; + collections?: PersistedCollection[]; +}; + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -59,13 +75,71 @@ function optionalString(value: unknown): string | undefined { return typeof value === "string" && value.length > 0 ? value : undefined; } +export class LibrarianServiceError extends S.TaggedErrorClass()( + "LibrarianServiceError", + { + message: S.String, + operation: S.String, + }, +) {} + +const librarianServiceError = (operation: string, cause: unknown): LibrarianServiceError => + LibrarianServiceError.make({ + operation, + message: errorMessage(cause), + }); + +function throwLibrarianServiceError(operation: string, cause: unknown): never { + throw librarianServiceError(operation, cause); +} + +function resolveDataDir(config: LibrarianServiceConfig): string { + return config.dataDir ?? Effect.runSync( + Config.string("LIBRARIAN_DATA_DIR").pipe(Config.withDefault("./data/librarian")), + ); +} + +const currentEpochSeconds: Effect.Effect = Clock.currentTimeMillis.pipe( + Effect.map((millis) => Math.floor(millis / 1000)), +); + +const currentIsoString: Effect.Effect = DateTime.now.pipe(Effect.map(DateTime.formatIso)); + +const encodeJsonString = (operation: string, value: unknown): Effect.Effect => + S.encodeUnknownEffect(S.UnknownFromJsonString)(value).pipe( + Effect.mapError((cause) => librarianServiceError(operation, cause)), + ); + +const decodeJsonString = (operation: string, value: string): Effect.Effect => + S.decodeUnknownEffect(S.UnknownFromJsonString)(value).pipe( + Effect.map((decoded) => decoded as A), + Effect.mapError((cause) => librarianServiceError(operation, cause)), + ); + +const randomUuid: Effect.Effect = Effect.gen(function* () { + const bytes: number[] = []; + for (let index = 0; index < 16; index += 1) { + bytes.push(yield* Random.nextIntBetween(0, 255)); + } + + bytes[6] = (bytes[6]! & 0x0f) | 0x40; + bytes[8] = (bytes[8]! & 0x3f) | 0x80; + + const hex = bytes.map((byte) => byte.toString(16).padStart(2, "0")); + return [ + hex.slice(0, 4).join(""), + hex.slice(4, 6).join(""), + hex.slice(6, 8).join(""), + hex.slice(8, 10).join(""), + hex.slice(10, 16).join(""), + ].join("-"); +}); + export type LibrarianService = AsyncProcessorRuntime & Record; export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianService { const service = makeAsyncProcessor(config, { - run: async () => { - await service.run(); - }, + run: () => service.run(), }) as LibrarianService; const baseStop = service.stop; service.documents = new Map(); @@ -76,60 +150,115 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS service.libProducer = null; service.colConsumer = null; service.colProducer = null; - service.dataDir = config.dataDir ?? process.env.LIBRARIAN_DATA_DIR ?? "./data/librarian"; + service.dataDir = resolveDataDir(config); service.persistPath = joinPath(service.dataDir, "librarian-state.json"); Object.assign(service, { - run: async function(this: LibrarianService): Promise { - // Ensure directories exist - await ensureDirectory(joinPath(this.dataDir, "docs")); + run: function(this: LibrarianService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + // Ensure directories exist + yield* Effect.tryPromise({ + try: () => ensureDirectory(joinPath(service.dataDir, "docs")), + catch: (cause) => librarianServiceError("ensure-data-dir", cause), + }); - // Load persisted state - await this.loadFromDisk(); + // Load persisted state + yield* Effect.tryPromise({ + try: () => service.loadFromDisk(), + catch: (cause) => librarianServiceError("load", cause), + }); - // Create producers - this.libProducer = await this.pubsub.createProducer({ - topic: topics.librarianResponse, - }); - this.colProducer = await this.pubsub.createProducer({ - topic: topics.collectionManagementResponse, - }); + // Create producers + service.libProducer = yield* Effect.tryPromise({ + try: () => service.pubsub.createProducer({ + topic: topics.librarianResponse, + }), + catch: (cause) => librarianServiceError("librarian-producer", cause), + }); + service.colProducer = yield* Effect.tryPromise({ + try: () => service.pubsub.createProducer({ + topic: topics.collectionManagementResponse, + }), + catch: (cause) => librarianServiceError("collection-producer", cause), + }); - // Create consumers - this.libConsumer = await this.pubsub.createConsumer({ - topic: topics.librarianRequest, - subscription: `${this.config.id}-librarian-request`, - }); - this.colConsumer = await this.pubsub.createConsumer({ - topic: topics.collectionManagementRequest, - subscription: `${this.config.id}-collection-management-request`, - }); + // Create consumers + service.libConsumer = yield* Effect.tryPromise({ + try: () => service.pubsub.createConsumer({ + topic: topics.librarianRequest, + subscription: `${service.config.id}-librarian-request`, + }), + catch: (cause) => librarianServiceError("librarian-consumer", cause), + }); + service.colConsumer = yield* Effect.tryPromise({ + try: () => service.pubsub.createConsumer({ + topic: topics.collectionManagementRequest, + subscription: `${service.config.id}-collection-management-request`, + }), + catch: (cause) => librarianServiceError("collection-consumer", cause), + }); - console.log(`[LibrarianService] Listening on ${topics.librarianRequest} and ${topics.collectionManagementRequest}`); + yield* Effect.log(`[LibrarianService] Listening on ${topics.librarianRequest} and ${topics.collectionManagementRequest}`); - // Main consume loop — poll both consumers - while (this.running) { - try { - // Poll librarian requests - const libMsg = await this.libConsumer.receive(2000); - if (libMsg !== null) { - await this.handleLibrarianMessage(libMsg); - await this.libConsumer.acknowledge(libMsg); + // Main consume loop -- poll both consumers + while (service.running) { + const shouldContinue = yield* Effect.gen(function* () { + const libConsumer = service.libConsumer; + if (libConsumer === null) { + return yield* librarianServiceError("consume", "Librarian consumer not started"); + } + const colConsumer = service.colConsumer; + if (colConsumer === null) { + return yield* librarianServiceError("consume", "Collection consumer not started"); + } + + const libMsg = yield* Effect.tryPromise({ + try: () => libConsumer.receive(2000), + catch: (cause) => librarianServiceError("librarian-receive", cause), + }); + if (libMsg !== null) { + yield* Effect.tryPromise({ + try: () => service.handleLibrarianMessage(libMsg), + catch: (cause) => librarianServiceError("librarian-handle", cause), + }); + yield* Effect.tryPromise({ + try: () => libConsumer.acknowledge(libMsg), + catch: (cause) => librarianServiceError("librarian-acknowledge", cause), + }); + } + + const colMsg = yield* Effect.tryPromise({ + try: () => colConsumer.receive(2000), + catch: (cause) => librarianServiceError("collection-receive", cause), + }); + if (colMsg !== null) { + yield* Effect.tryPromise({ + try: () => service.handleCollectionMessage(colMsg), + catch: (cause) => librarianServiceError("collection-handle", cause), + }); + yield* Effect.tryPromise({ + try: () => colConsumer.acknowledge(colMsg), + catch: (cause) => librarianServiceError("collection-acknowledge", cause), + }); + } + + return true; + }).pipe( + Effect.catch((err) => { + if (!service.running) return Effect.succeed(false); + return Effect.logError("[LibrarianService] Error in consume loop", { error: err.message }).pipe( + Effect.flatMap(() => Effect.sleep(Duration.millis(1000))), + Effect.as(true), + ); + }), + ); + if (!shouldContinue) break; } - - // Poll collection management requests - const colMsg = await this.colConsumer.receive(2000); - if (colMsg !== null) { - await this.handleCollectionMessage(colMsg); - await this.colConsumer.acknowledge(colMsg); - } - } catch (err) { - if (!this.running) break; - console.error("[LibrarianService] Error in consume loop:", err); - await sleep(1000); - } - } + }), + ); }, @@ -160,52 +289,65 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS - documentMetadata: function(this: LibrarianService, request: LibrarianRequest): DocumentMetadata | undefined { + documentMetadata: function(this: LibrarianService, request: LibrarianRequest): Promise { const req = this.requestRecord(request); const value = req.documentMetadata ?? req["document-metadata"]; - return isRecord(value) ? this.normaliseDocumentMetadata(value) : undefined; + if (!isRecord(value)) return Promise.resolve(undefined); + return this.normaliseDocumentMetadata(value); }, - processingMetadata: function(this: LibrarianService, request: LibrarianRequest): ProcessingMetadata | undefined { - const req = this.requestRecord(request); - const value = req.processingMetadata ?? req["processing-metadata"]; - if (!isRecord(value)) return undefined; - const documentId = optionalString(value.documentId) ?? optionalString(value["document-id"]) ?? ""; - return { - id: optionalString(value.id) ?? crypto.randomUUID(), - documentId, - "document-id": documentId, - time: typeof value.time === "number" ? value.time : Math.floor(Date.now() / 1000), - flow: optionalString(value.flow) ?? "default", - user: optionalString(value.user) ?? optionalString(this.requestRecord(request).user) ?? "default", - collection: optionalString(value.collection) ?? optionalString(this.requestRecord(request).collection) ?? "default", - tags: Array.isArray(value.tags) ? value.tags.filter((tag): tag is string => typeof tag === "string") : [], - }; + processingMetadata: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const req = service.requestRecord(request); + const value = req.processingMetadata ?? req["processing-metadata"]; + if (!isRecord(value)) return undefined; + const documentId = optionalString(value.documentId) ?? optionalString(value["document-id"]) ?? ""; + const id = optionalString(value.id) ?? (yield* randomUuid); + const time = typeof value.time === "number" ? value.time : yield* currentEpochSeconds; + return { + id, + documentId, + "document-id": documentId, + time, + flow: optionalString(value.flow) ?? "default", + user: optionalString(value.user) ?? optionalString(service.requestRecord(request).user) ?? "default", + collection: optionalString(value.collection) ?? optionalString(service.requestRecord(request).collection) ?? "default", + tags: Array.isArray(value.tags) ? value.tags.filter((tag): tag is string => typeof tag === "string") : [], + }; + }), + ); }, - normaliseDocumentMetadata: function(this: LibrarianService, value: Record): DocumentMetadata { - const id = optionalString(value.id) ?? crypto.randomUUID(); - const parentId = optionalString(value.parentId) ?? optionalString(value["parent-id"]); - const documentType = optionalString(value.documentType) ?? optionalString(value["document-type"]) ?? "source"; - return { - id, - time: typeof value.time === "number" ? value.time : Math.floor(Date.now() / 1000), - kind: optionalString(value.kind) ?? "application/octet-stream", - title: optionalString(value.title) ?? "", - comments: optionalString(value.comments) ?? "", - user: optionalString(value.user) ?? "default", - tags: Array.isArray(value.tags) ? value.tags.filter((tag): tag is string => typeof tag === "string") : [], - ...(parentId !== undefined ? { parentId, "parent-id": parentId } : {}), - documentType, - "document-type": documentType, - ...(Array.isArray(value.metadata) ? { metadata: value.metadata as NonNullable } : {}), - }; + normaliseDocumentMetadata: function(this: LibrarianService, value: Record): Promise { + return Effect.runPromise( + Effect.gen(function* () { + const id = optionalString(value.id) ?? (yield* randomUuid); + const parentId = optionalString(value.parentId) ?? optionalString(value["parent-id"]); + const documentType = optionalString(value.documentType) ?? optionalString(value["document-type"]) ?? "source"; + const time = typeof value.time === "number" ? value.time : yield* currentEpochSeconds; + return { + id, + time, + kind: optionalString(value.kind) ?? "application/octet-stream", + title: optionalString(value.title) ?? "", + comments: optionalString(value.comments) ?? "", + user: optionalString(value.user) ?? "default", + tags: Array.isArray(value.tags) ? value.tags.filter((tag): tag is string => typeof tag === "string") : [], + ...(parentId !== undefined ? { parentId, "parent-id": parentId } : {}), + documentType, + "document-type": documentType, + ...(Array.isArray(value.metadata) ? { metadata: value.metadata as NonNullable } : {}), + }; + }), + ); }, @@ -271,181 +413,293 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS - handleLibrarianMessage: async function(this: LibrarianService, msg: Message): Promise { - const request = msg.value(); - const props = msg.properties(); - const requestId = props.id; + handleLibrarianMessage: function(this: LibrarianService, msg: Message): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const request = msg.value(); + const props = msg.properties(); + const requestId = props.id; - if (requestId === undefined || requestId.length === 0) { - console.warn("[LibrarianService] Received request without id, ignoring"); - return; - } - - try { - if (request.operation === "stream-document") { - for (const response of await this.streamDocument(request)) { - await this.libProducer!.send(response, { id: requestId }); + if (requestId === undefined || requestId.length === 0) { + yield* Effect.logWarning("[LibrarianService] Received request without id, ignoring"); + return; } - return; - } - const response = await this.handleLibrarianOperation(request); - await this.libProducer!.send(response, { id: requestId }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - await this.libProducer!.send( - { error: { type: "librarian-error", message } }, - { id: requestId }, - ); - } + + const sendResponse = (response: LibrarianResponse): Effect.Effect => + Effect.gen(function* () { + const producer = service.libProducer; + if (producer === null) { + return yield* librarianServiceError("librarian-respond", "Librarian producer not started"); + } + yield* Effect.tryPromise({ + try: () => producer.send(response, { id: requestId }), + catch: (cause) => librarianServiceError("librarian-respond", cause), + }); + }); + + yield* Effect.gen(function* () { + if (request.operation === "stream-document") { + const responses = yield* Effect.tryPromise({ + try: () => service.streamDocument(request), + catch: (cause) => librarianServiceError("stream-document", cause), + }); + for (const response of responses) { + yield* sendResponse(response); + } + return; + } + + const response = yield* Effect.tryPromise({ + try: () => service.handleLibrarianOperation(request), + catch: (cause) => librarianServiceError("librarian-operation", cause), + }); + yield* sendResponse(response); + }).pipe( + Effect.catch((err) => + sendResponse({ + error: { type: "librarian-error", message: err.message }, + }), + ), + ); + }), + ); }, - handleLibrarianOperation: async function(this: LibrarianService, request: LibrarianRequest): Promise { - switch (request.operation) { - case "add-document": - return this.addDocument(request); - case "remove-document": - return this.removeDocument(request); - case "update-document": - return this.updateDocument(request); - case "list-documents": - return this.listDocuments(request); - case "get-document-metadata": - return this.getDocumentMetadata(request); - case "get-document-content": - return this.getDocumentContent(request); - case "add-child-document": - return this.addChildDocument(request); - case "list-children": - return this.listChildren(request); - case "add-processing": - return this.addProcessing(request); - case "remove-processing": - return this.removeProcessing(request); - case "list-processing": - return this.listProcessing(request); - case "begin-upload": - return this.beginUpload(request); - case "upload-chunk": - return this.uploadChunk(request); - case "complete-upload": - return this.completeUpload(request); - case "get-upload-status": - return this.getUploadStatus(request); - case "abort-upload": - return this.abortUpload(request); - case "list-uploads": - return this.listUploads(request); - case "stream-document": - throw new Error("stream-document must be handled as a streaming operation"); - default: - throw new Error(`Unknown librarian operation: ${request.operation as string}`); - } + handleLibrarianOperation: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + switch (request.operation) { + case "add-document": + return yield* Effect.tryPromise({ + try: () => service.addDocument(request), + catch: (cause) => librarianServiceError("add-document", cause), + }); + case "remove-document": + return yield* Effect.tryPromise({ + try: () => service.removeDocument(request), + catch: (cause) => librarianServiceError("remove-document", cause), + }); + case "update-document": + return yield* Effect.tryPromise({ + try: () => service.updateDocument(request), + catch: (cause) => librarianServiceError("update-document", cause), + }); + case "list-documents": + return yield* Effect.try({ + try: () => service.listDocuments(request), + catch: (cause) => librarianServiceError("list-documents", cause), + }); + case "get-document-metadata": + return yield* Effect.try({ + try: () => service.getDocumentMetadata(request), + catch: (cause) => librarianServiceError("get-document-metadata", cause), + }); + case "get-document-content": + return yield* Effect.tryPromise({ + try: () => service.getDocumentContent(request), + catch: (cause) => librarianServiceError("get-document-content", cause), + }); + case "add-child-document": + return yield* Effect.tryPromise({ + try: () => service.addChildDocument(request), + catch: (cause) => librarianServiceError("add-child-document", cause), + }); + case "list-children": + return yield* Effect.try({ + try: () => service.listChildren(request), + catch: (cause) => librarianServiceError("list-children", cause), + }); + case "add-processing": + return yield* Effect.tryPromise({ + try: () => service.addProcessing(request), + catch: (cause) => librarianServiceError("add-processing", cause), + }); + case "remove-processing": + return yield* Effect.tryPromise({ + try: () => service.removeProcessing(request), + catch: (cause) => librarianServiceError("remove-processing", cause), + }); + case "list-processing": + return yield* Effect.try({ + try: () => service.listProcessing(request), + catch: (cause) => librarianServiceError("list-processing", cause), + }); + case "begin-upload": + return yield* Effect.tryPromise({ + try: () => service.beginUpload(request), + catch: (cause) => librarianServiceError("begin-upload", cause), + }); + case "upload-chunk": + return yield* Effect.try({ + try: () => service.uploadChunk(request), + catch: (cause) => librarianServiceError("upload-chunk", cause), + }); + case "complete-upload": + return yield* Effect.tryPromise({ + try: () => service.completeUpload(request), + catch: (cause) => librarianServiceError("complete-upload", cause), + }); + case "get-upload-status": + return yield* Effect.try({ + try: () => service.getUploadStatus(request), + catch: (cause) => librarianServiceError("get-upload-status", cause), + }); + case "abort-upload": + return yield* Effect.try({ + try: () => service.abortUpload(request), + catch: (cause) => librarianServiceError("abort-upload", cause), + }); + case "list-uploads": + return yield* Effect.tryPromise({ + try: () => service.listUploads(request), + catch: (cause) => librarianServiceError("list-uploads", cause), + }); + case "stream-document": + return yield* librarianServiceError("stream-document", "stream-document must be handled as a streaming operation"); + default: + return yield* librarianServiceError("operation", `Unknown librarian operation: ${request.operation as string}`); + } + }), + ); }, - addDocument: async function(this: LibrarianService, request: LibrarianRequest): Promise { - const meta = this.documentMetadata(request); - if (meta === undefined) throw new Error("add-document requires documentMetadata"); + addDocument: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const meta = yield* Effect.tryPromise({ + try: () => service.documentMetadata(request), + catch: (cause) => librarianServiceError("add-document-metadata", cause), + }); + if (meta === undefined) return yield* librarianServiceError("add-document", "add-document requires documentMetadata"); - const id = meta.id; - const now = Math.floor(Date.now() / 1000); + const id = meta.id; + const now = yield* currentEpochSeconds; - const doc: DocumentMetadata = { - ...meta, - id, - time: now, - }; + const doc: DocumentMetadata = { + ...meta, + id, + time: now, + }; - this.documents.set(id, doc); + service.documents.set(id, doc); - // Store file content if provided - if (request.content !== undefined && request.content.length > 0) { - const filePath = joinPath(this.dataDir, "docs", `${id}.bin`); - const buf = Buffer.from(request.content, "base64"); - await writeBinaryFile(filePath, buf); - } + // Store file content if provided + if (request.content !== undefined && request.content.length > 0) { + const filePath = joinPath(service.dataDir, "docs", `${id}.bin`); + const buf = Buffer.from(request.content, "base64"); + yield* Effect.tryPromise({ + try: () => writeBinaryFile(filePath, buf), + catch: (cause) => librarianServiceError("add-document-write", cause), + }); + } - await this.persist(); - console.log(`[LibrarianService] Added document ${id}: ${doc.title}`); + yield* Effect.tryPromise({ + try: () => service.persist(), + catch: (cause) => librarianServiceError("add-document-persist", cause), + }); + yield* Effect.log(`[LibrarianService] Added document ${id}: ${doc.title}`); - return this.documentResponse(doc); + return service.documentResponse(doc); + }), + ); }, - removeDocument: async function(this: LibrarianService, request: LibrarianRequest): Promise { - const id = this.documentId(request); - if (id === undefined || id.length === 0) { - throw new Error("remove-document requires documentId"); - } + removeDocument: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const id = service.documentId(request); + if (id === undefined || id.length === 0) { + return yield* librarianServiceError("remove-document", "remove-document requires documentId"); + } - // Remove the document itself - this.documents.delete(id); + // Remove the document itself + service.documents.delete(id); - // Remove the file - try { - await removePath(joinPath(this.dataDir, "docs", `${id}.bin`)); - } catch { - // File may not exist — that's fine - } + // Remove the file + yield* Effect.tryPromise({ + try: () => removePath(joinPath(service.dataDir, "docs", `${id}.bin`)), + catch: (cause) => librarianServiceError("remove-document-file", cause), + }).pipe(Effect.orElseSucceed(() => undefined)); - // Cascade: remove children - const childIds = [...this.documents.entries()] - .filter(([, doc]) => doc.parentId === id) - .map(([childId]) => childId); + // Cascade: remove children + const childIds = [...service.documents.entries()] + .filter(([, doc]) => doc.parentId === id) + .map(([childId]) => childId); - for (const childId of childIds) { - this.documents.delete(childId); - try { - await removePath(joinPath(this.dataDir, "docs", `${childId}.bin`)); - } catch { - // ignore - } - } + for (const childId of childIds) { + service.documents.delete(childId); + yield* Effect.tryPromise({ + try: () => removePath(joinPath(service.dataDir, "docs", `${childId}.bin`)), + catch: (cause) => librarianServiceError("remove-child-file", cause), + }).pipe(Effect.orElseSucceed(() => undefined)); + } - // Remove associated processing records - const procIds = [...this.processing.entries()] - .filter(([, proc]) => proc.documentId === id) - .map(([procId]) => procId); + // Remove associated processing records + const procIds = [...service.processing.entries()] + .filter(([, proc]) => proc.documentId === id) + .map(([procId]) => procId); - for (const procId of procIds) { - this.processing.delete(procId); - } + for (const procId of procIds) { + service.processing.delete(procId); + } - await this.persist(); - console.log(`[LibrarianService] Removed document ${id} (cascade: ${childIds.length} children, ${procIds.length} processing)`); + yield* Effect.tryPromise({ + try: () => service.persist(), + catch: (cause) => librarianServiceError("remove-document-persist", cause), + }); + yield* Effect.log(`[LibrarianService] Removed document ${id} (cascade: ${childIds.length} children, ${procIds.length} processing)`); - return {}; + return {}; + }), + ); }, - updateDocument: async function(this: LibrarianService, request: LibrarianRequest): Promise { - const id = this.documentId(request) ?? this.documentMetadata(request)?.id; - if (id === undefined || id.length === 0) { - throw new Error("update-document requires documentId"); - } - const existing = this.documents.get(id); - if (existing === undefined) throw new Error(`Document not found: ${id}`); - const meta = this.documentMetadata(request); - if (meta === undefined) throw new Error("update-document requires documentMetadata"); + updateDocument: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const meta = yield* Effect.tryPromise({ + try: () => service.documentMetadata(request), + catch: (cause) => librarianServiceError("update-document-metadata", cause), + }); + const id = service.documentId(request) ?? meta?.id; + if (id === undefined || id.length === 0) { + return yield* librarianServiceError("update-document", "update-document requires documentId"); + } + const existing = service.documents.get(id); + if (existing === undefined) return yield* librarianServiceError("update-document", `Document not found: ${id}`); + if (meta === undefined) return yield* librarianServiceError("update-document", "update-document requires documentMetadata"); - const doc: DocumentMetadata = this.publicDocument({ - ...existing, - ...meta, - id, - time: meta.time ?? existing.time, - }); - this.documents.set(id, doc); - await this.persist(); - return this.documentResponse(doc); + const doc: DocumentMetadata = service.publicDocument({ + ...existing, + ...meta, + id, + time: meta.time ?? existing.time, + }); + service.documents.set(id, doc); + yield* Effect.tryPromise({ + try: () => service.persist(), + catch: (cause) => librarianServiceError("update-document-persist", cause), + }); + return service.documentResponse(doc); + }), + ); }, @@ -473,11 +727,11 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS getDocumentMetadata: function(this: LibrarianService, request: LibrarianRequest): LibrarianResponse { const id = this.documentId(request); if (id === undefined || id.length === 0) { - throw new Error("get-document-metadata requires documentId"); + throwLibrarianServiceError("get-document-metadata", "get-document-metadata requires documentId"); } const doc = this.documents.get(id); - if (doc === undefined) throw new Error(`Document not found: ${id}`); + if (doc === undefined) throwLibrarianServiceError("get-document-metadata", `Document not found: ${id}`); return this.documentResponse(doc); @@ -485,64 +739,82 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS - getDocumentContent: async function(this: LibrarianService, request: LibrarianRequest): Promise { - const id = this.documentId(request); - if (id === undefined || id.length === 0) { - throw new Error("get-document-content requires documentId"); - } + getDocumentContent: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const id = service.documentId(request); + if (id === undefined || id.length === 0) { + return yield* librarianServiceError("get-document-content", "get-document-content requires documentId"); + } - const doc = this.documents.get(id); - if (doc === undefined) throw new Error(`Document not found: ${id}`); + const doc = service.documents.get(id); + if (doc === undefined) return yield* librarianServiceError("get-document-content", `Document not found: ${id}`); - try { - const filePath = joinPath(this.dataDir, "docs", `${id}.bin`); - const buf = await readBinaryFile(filePath); - const content = Buffer.from(buf).toString("base64"); - return { ...this.documentResponse(doc), content }; - } catch { - throw new Error(`Document content not found on disk: ${id}`); - } + const filePath = joinPath(service.dataDir, "docs", `${id}.bin`); + const buf = yield* Effect.tryPromise({ + try: () => readBinaryFile(filePath), + catch: () => librarianServiceError("get-document-content", `Document content not found on disk: ${id}`), + }); + const content = Buffer.from(buf).toString("base64"); + return { ...service.documentResponse(doc), content }; + }), + ); }, - addChildDocument: async function(this: LibrarianService, request: LibrarianRequest): Promise { - const meta = this.documentMetadata(request); - if (meta === undefined) { - throw new Error("add-child-document requires documentMetadata"); - } - if (meta.parentId === undefined || meta.parentId.length === 0) { - throw new Error("add-child-document requires parentId in metadata"); - } + addChildDocument: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const meta = yield* Effect.tryPromise({ + try: () => service.documentMetadata(request), + catch: (cause) => librarianServiceError("add-child-document-metadata", cause), + }); + if (meta === undefined) { + return yield* librarianServiceError("add-child-document", "add-child-document requires documentMetadata"); + } + if (meta.parentId === undefined || meta.parentId.length === 0) { + return yield* librarianServiceError("add-child-document", "add-child-document requires parentId in metadata"); + } - // Verify parent exists - if (!(this.documents as Map).has(meta.parentId)) { - throw new Error(`Parent document not found: ${meta.parentId}`); - } + // Verify parent exists + if (!(service.documents as Map).has(meta.parentId)) { + return yield* librarianServiceError("add-child-document", `Parent document not found: ${meta.parentId}`); + } - const id = meta.id; - const now = Math.floor(Date.now() / 1000); + const id = meta.id; + const now = yield* currentEpochSeconds; - const doc: DocumentMetadata = { - ...meta, - id, - time: now, - }; + const doc: DocumentMetadata = { + ...meta, + id, + time: now, + }; - this.documents.set(id, doc); + service.documents.set(id, doc); - // Store file content if provided - if (request.content !== undefined && request.content.length > 0) { - const filePath = joinPath(this.dataDir, "docs", `${id}.bin`); - const buf = Buffer.from(request.content, "base64"); - await writeBinaryFile(filePath, buf); - } + // Store file content if provided + if (request.content !== undefined && request.content.length > 0) { + const filePath = joinPath(service.dataDir, "docs", `${id}.bin`); + const buf = Buffer.from(request.content, "base64"); + yield* Effect.tryPromise({ + try: () => writeBinaryFile(filePath, buf), + catch: (cause) => librarianServiceError("add-child-document-write", cause), + }); + } - await this.persist(); - console.log(`[LibrarianService] Added child document ${id} (parent: ${meta.parentId})`); + yield* Effect.tryPromise({ + try: () => service.persist(), + catch: (cause) => librarianServiceError("add-child-document-persist", cause), + }); + yield* Effect.log(`[LibrarianService] Added child document ${id} (parent: ${meta.parentId})`); - return this.documentResponse(doc); + return service.documentResponse(doc); + }), + ); }, @@ -551,7 +823,7 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS listChildren: function(this: LibrarianService, request: LibrarianRequest): LibrarianResponse { const parentId = this.documentId(request); if (parentId === undefined || parentId.length === 0) { - throw new Error("list-children requires documentId"); + throwLibrarianServiceError("list-children", "list-children requires documentId"); } const children: DocumentMetadata[] = []; @@ -567,39 +839,58 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS - addProcessing: async function(this: LibrarianService, request: LibrarianRequest): Promise { - const proc = this.processingMetadata(request); - if (proc === undefined) throw new Error("add-processing requires processingMetadata"); + addProcessing: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const proc = yield* Effect.tryPromise({ + try: () => service.processingMetadata(request), + catch: (cause) => librarianServiceError("add-processing-metadata", cause), + }); + if (proc === undefined) return yield* librarianServiceError("add-processing", "add-processing requires processingMetadata"); - const id = proc.id; - const now = Math.floor(Date.now() / 1000); + const id = proc.id; + const now = yield* currentEpochSeconds; - const record: ProcessingMetadata = { - ...proc, - id, - time: now, - }; + const record: ProcessingMetadata = { + ...proc, + id, + time: now, + }; - this.processing.set(id, record); - await this.persist(); + service.processing.set(id, record); + yield* Effect.tryPromise({ + try: () => service.persist(), + catch: (cause) => librarianServiceError("add-processing-persist", cause), + }); - console.log(`[LibrarianService] Added processing ${id} for document ${proc.documentId}`); - return this.processingResponse([record]); + yield* Effect.log(`[LibrarianService] Added processing ${id} for document ${proc.documentId}`); + return service.processingResponse([record]); + }), + ); }, - removeProcessing: async function(this: LibrarianService, request: LibrarianRequest): Promise { - const id = this.processingId(request); - if (id === undefined || id.length === 0) { - throw new Error("remove-processing requires processingId"); - } + removeProcessing: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const id = service.processingId(request); + if (id === undefined || id.length === 0) { + return yield* librarianServiceError("remove-processing", "remove-processing requires processingId"); + } - this.processing.delete(id); - await this.persist(); + service.processing.delete(id); + yield* Effect.tryPromise({ + try: () => service.persist(), + catch: (cause) => librarianServiceError("remove-processing-persist", cause), + }); - return {}; + return {}; + }), + ); }, @@ -623,34 +914,43 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS - beginUpload: function(this: LibrarianService, request: LibrarianRequest): LibrarianResponse { - const meta = this.documentMetadata(request); - if (meta === undefined) throw new Error("begin-upload requires documentMetadata"); - const req = this.requestRecord(request); - const totalSize = typeof req["total-size"] === "number" ? req["total-size"] : 0; - if (totalSize <= 0) throw new Error("begin-upload requires total-size"); - const chunkSize = typeof req["chunk-size"] === "number" && req["chunk-size"] > 0 - ? req["chunk-size"] - : 3 * 1024 * 1024; - const totalChunks = Math.max(1, Math.ceil(totalSize / chunkSize)); - const uploadId = crypto.randomUUID(); + beginUpload: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const meta = yield* Effect.tryPromise({ + try: () => service.documentMetadata(request), + catch: (cause) => librarianServiceError("begin-upload-metadata", cause), + }); + if (meta === undefined) return yield* librarianServiceError("begin-upload", "begin-upload requires documentMetadata"); + const req = service.requestRecord(request); + const totalSize = typeof req["total-size"] === "number" ? req["total-size"] : 0; + if (totalSize <= 0) return yield* librarianServiceError("begin-upload", "begin-upload requires total-size"); + const chunkSize = typeof req["chunk-size"] === "number" && req["chunk-size"] > 0 + ? req["chunk-size"] + : 3 * 1024 * 1024; + const totalChunks = Math.max(1, Math.ceil(totalSize / chunkSize)); + const uploadId = yield* randomUuid; + const createdAt = yield* currentIsoString; - this.uploads.set(uploadId, { - id: uploadId, - documentMetadata: meta, - totalSize, - chunkSize, - totalChunks, - createdAt: new Date().toISOString(), - chunks: new Map(), - user: meta.user ?? optionalString(req.user) ?? "default", - }); + service.uploads.set(uploadId, { + id: uploadId, + documentMetadata: meta, + totalSize, + chunkSize, + totalChunks, + createdAt, + chunks: new Map(), + user: meta.user ?? optionalString(req.user) ?? "default", + }); - return { - "upload-id": uploadId, - "chunk-size": chunkSize, - "total-chunks": totalChunks, - } as LibrarianResponse; + return { + "upload-id": uploadId, + "chunk-size": chunkSize, + "total-chunks": totalChunks, + } as LibrarianResponse; + }), + ); }, @@ -659,15 +959,15 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS uploadChunk: function(this: LibrarianService, request: LibrarianRequest): LibrarianResponse { const req = this.requestRecord(request); const uploadId = optionalString(req["upload-id"]); - if (uploadId === undefined) throw new Error("upload-chunk requires upload-id"); + if (uploadId === undefined) throwLibrarianServiceError("upload-chunk", "upload-chunk requires upload-id"); const session = this.uploads.get(uploadId); - if (session === undefined) throw new Error(`Upload not found: ${uploadId}`); + if (session === undefined) throwLibrarianServiceError("upload-chunk", `Upload not found: ${uploadId}`); const chunkIndex = typeof req["chunk-index"] === "number" ? req["chunk-index"] : -1; if (!Number.isInteger(chunkIndex) || chunkIndex < 0 || chunkIndex >= session.totalChunks) { - throw new Error("upload-chunk requires a valid chunk-index"); + throwLibrarianServiceError("upload-chunk", "upload-chunk requires a valid chunk-index"); } const content = optionalString(req.content); - if (content === undefined) throw new Error("upload-chunk requires content"); + if (content === undefined) throwLibrarianServiceError("upload-chunk", "upload-chunk requires content"); session.chunks.set(chunkIndex, content); const bytesReceived = [...session.chunks.values()].reduce((sum, chunk) => sum + chunk.length, 0); @@ -684,30 +984,38 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS - completeUpload: async function(this: LibrarianService, request: LibrarianRequest): Promise { - const uploadId = optionalString(this.requestRecord(request)["upload-id"]); - if (uploadId === undefined) throw new Error("complete-upload requires upload-id"); - const session = this.uploads.get(uploadId); - if (session === undefined) throw new Error(`Upload not found: ${uploadId}`); - if (session.chunks.size !== session.totalChunks) { - throw new Error(`Upload incomplete: ${session.chunks.size}/${session.totalChunks} chunks received`); - } + completeUpload: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const uploadId = optionalString(service.requestRecord(request)["upload-id"]); + if (uploadId === undefined) return yield* librarianServiceError("complete-upload", "complete-upload requires upload-id"); + const session = service.uploads.get(uploadId); + if (session === undefined) return yield* librarianServiceError("complete-upload", `Upload not found: ${uploadId}`); + if (session.chunks.size !== session.totalChunks) { + return yield* librarianServiceError("complete-upload", `Upload incomplete: ${session.chunks.size}/${session.totalChunks} chunks received`); + } - const content = Array.from({ length: session.totalChunks }, (_, i) => session.chunks.get(i) ?? "").join(""); - const response = await this.addDocument({ - operation: "add-document", - documentMetadata: session.documentMetadata, - "document-metadata": session.documentMetadata, - content, - user: session.user, - } as LibrarianRequest); - this.uploads.delete(uploadId); - const documentId = response.documentMetadata?.id ?? response["document-metadata"]?.id ?? session.documentMetadata.id; - return { - ...response, - "document-id": documentId, - "object-id": documentId, - } as LibrarianResponse; + const content = Array.from({ length: session.totalChunks }, (_, i) => session.chunks.get(i) ?? "").join(""); + const response = yield* Effect.tryPromise({ + try: () => service.addDocument({ + operation: "add-document", + documentMetadata: session.documentMetadata, + "document-metadata": session.documentMetadata, + content, + user: session.user, + } as LibrarianRequest), + catch: (cause) => librarianServiceError("complete-upload-add-document", cause), + }); + service.uploads.delete(uploadId); + const documentId = response.documentMetadata?.id ?? response["document-metadata"]?.id ?? session.documentMetadata.id; + return { + ...response, + "document-id": documentId, + "object-id": documentId, + } as LibrarianResponse; + }), + ); }, @@ -715,9 +1023,9 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS getUploadStatus: function(this: LibrarianService, request: LibrarianRequest): LibrarianResponse { const uploadId = optionalString(this.requestRecord(request)["upload-id"]); - if (uploadId === undefined) throw new Error("get-upload-status requires upload-id"); + if (uploadId === undefined) throwLibrarianServiceError("get-upload-status", "get-upload-status requires upload-id"); const session = this.uploads.get(uploadId); - if (session === undefined) throw new Error(`Upload not found: ${uploadId}`); + if (session === undefined) throwLibrarianServiceError("get-upload-status", `Upload not found: ${uploadId}`); const receivedChunks = [...session.chunks.keys()].sort((a, b) => a - b); const receivedSet = new Set(receivedChunks); const missingChunks = Array.from({ length: session.totalChunks }, (_, i) => i).filter((i) => !receivedSet.has(i)); @@ -739,7 +1047,7 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS abortUpload: function(this: LibrarianService, request: LibrarianRequest): LibrarianResponse { const uploadId = optionalString(this.requestRecord(request)["upload-id"]); - if (uploadId === undefined) throw new Error("abort-upload requires upload-id"); + if (uploadId === undefined) throwLibrarianServiceError("abort-upload", "abort-upload requires upload-id"); this.uploads.delete(uploadId); return {}; @@ -747,47 +1055,66 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS - listUploads: function(this: LibrarianService, request: LibrarianRequest): LibrarianResponse { - const user = optionalString(this.requestRecord(request).user); - const sessions = [...this.uploads.values()] - .filter((session) => user === undefined || session.user === user) - .map((session) => ({ - "upload-id": session.id, - "document-id": session.documentMetadata.id, - "document-metadata-json": JSON.stringify(this.publicDocument(session.documentMetadata)), - "total-size": session.totalSize, - "chunk-size": session.chunkSize, - "total-chunks": session.totalChunks, - "chunks-received": session.chunks.size, - "created-at": session.createdAt, - })); - return { "upload-sessions": sessions } as LibrarianResponse; + listUploads: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const user = optionalString(service.requestRecord(request).user); + const sessions = []; + for (const session of service.uploads.values()) { + if (user !== undefined && session.user !== user) continue; + const documentMetadataJson = yield* encodeJsonString( + "list-uploads-document-metadata", + service.publicDocument(session.documentMetadata), + ); + sessions.push({ + "upload-id": session.id, + "document-id": session.documentMetadata.id, + "document-metadata-json": documentMetadataJson, + "total-size": session.totalSize, + "chunk-size": session.chunkSize, + "total-chunks": session.totalChunks, + "chunks-received": session.chunks.size, + "created-at": session.createdAt, + }); + } + return { "upload-sessions": sessions } as LibrarianResponse; + }), + ); }, - streamDocument: async function(this: LibrarianService, request: LibrarianRequest): Promise { - const id = this.documentId(request); - if (id === undefined) throw new Error("stream-document requires documentId"); - const req = this.requestRecord(request); - const chunkSize = typeof req["chunk-size"] === "number" && req["chunk-size"] > 0 - ? req["chunk-size"] - : 1024 * 1024; - const filePath = joinPath(this.dataDir, "docs", `${id}.bin`); - const buf = await readBinaryFile(filePath); - const base64 = Buffer.from(buf).toString("base64"); - const totalChunks = Math.max(1, Math.ceil(base64.length / chunkSize)); - return Array.from({ length: totalChunks }, (_, index) => { - const start = index * chunkSize; - const content = base64.slice(start, start + chunkSize); - return { - content, - "chunk-index": index, - "total-chunks": totalChunks, - eos: index === totalChunks - 1, - } as LibrarianResponse; - }); + streamDocument: function(this: LibrarianService, request: LibrarianRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const id = service.documentId(request); + if (id === undefined) return yield* librarianServiceError("stream-document", "stream-document requires documentId"); + const req = service.requestRecord(request); + const chunkSize = typeof req["chunk-size"] === "number" && req["chunk-size"] > 0 + ? req["chunk-size"] + : 1024 * 1024; + const filePath = joinPath(service.dataDir, "docs", `${id}.bin`); + const buf = yield* Effect.tryPromise({ + try: () => readBinaryFile(filePath), + catch: (cause) => librarianServiceError("stream-document-read", cause), + }); + const base64 = Buffer.from(buf).toString("base64"); + const totalChunks = Math.max(1, Math.ceil(base64.length / chunkSize)); + return Array.from({ length: totalChunks }, (_, index) => { + const start = index * chunkSize; + const content = base64.slice(start, start + chunkSize); + return { + content, + "chunk-index": index, + "total-chunks": totalChunks, + eos: index === totalChunks - 1, + } as LibrarianResponse; + }); + }), + ); }, @@ -795,67 +1122,97 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS // ---------- Collection management ---------- - handleCollectionMessage: async function(this: LibrarianService, msg: Message): Promise { - const request = msg.value(); - const props = msg.properties(); - const requestId = props.id; + handleCollectionMessage: function(this: LibrarianService, msg: Message): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const request = msg.value(); + const props = msg.properties(); + const requestId = props.id; - if (requestId === undefined || requestId.length === 0) { - console.warn("[LibrarianService] Received collection request without id, ignoring"); - return; - } + if (requestId === undefined || requestId.length === 0) { + yield* Effect.logWarning("[LibrarianService] Received collection request without id, ignoring"); + return; + } - try { - const response = this.handleCollectionOperation(request); - await this.colProducer!.send(response, { id: requestId }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - await this.colProducer!.send( - { error: { type: "collection-error", message } }, - { id: requestId }, - ); - } + const sendResponse = (response: CollectionManagementResponse): Effect.Effect => + Effect.gen(function* () { + const producer = service.colProducer; + if (producer === null) { + return yield* librarianServiceError("collection-respond", "Collection producer not started"); + } + yield* Effect.tryPromise({ + try: () => producer.send(response, { id: requestId }), + catch: (cause) => librarianServiceError("collection-respond", cause), + }); + }); + + yield* Effect.gen(function* () { + const response = yield* Effect.tryPromise({ + try: () => service.handleCollectionOperation(request), + catch: (cause) => librarianServiceError("collection-operation", cause), + }); + yield* sendResponse(response); + }).pipe( + Effect.catch((err) => + sendResponse({ + error: { type: "collection-error", message: err.message }, + }), + ), + ); + }), + ); }, - handleCollectionOperation: function(this: LibrarianService, request: CollectionManagementRequest): CollectionManagementResponse { - switch (request.operation) { - case "list-collections": { - const user = request.user ?? ""; - const collections = this.collectionManager.listCollections(user); - return { collections }; - } + handleCollectionOperation: function(this: LibrarianService, request: CollectionManagementRequest): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + switch (request.operation) { + case "list-collections": { + const user = request.user ?? ""; + const collections = service.collectionManager.listCollections(user); + return { collections }; + } - case "update-collection": { - const user = request.user ?? ""; - const collection = request.collection ?? ""; - const name = request.name ?? collection; - const description = request.description ?? ""; - const tags = request.tags ?? []; + case "update-collection": { + const user = request.user ?? ""; + const collection = request.collection ?? ""; + const name = request.name ?? collection; + const description = request.description ?? ""; + const tags = request.tags ?? []; - this.collectionManager.updateCollection(user, collection, name, description, tags); - // Persist after mutation - this.persist().catch((err: unknown) => console.error("[LibrarianService] Persist failed:", err)); + service.collectionManager.updateCollection(user, collection, name, description, tags); + yield* Effect.tryPromise({ + try: () => service.persist(), + catch: (cause) => librarianServiceError("update-collection-persist", cause), + }); - const collections = this.collectionManager.listCollections(user); - return { collections }; - } + const collections = service.collectionManager.listCollections(user); + return { collections }; + } - case "delete-collection": { - const user = request.user ?? ""; - const collection = request.collection ?? ""; + case "delete-collection": { + const user = request.user ?? ""; + const collection = request.collection ?? ""; - this.collectionManager.deleteCollection(user, collection); - this.persist().catch((err: unknown) => console.error("[LibrarianService] Persist failed:", err)); + service.collectionManager.deleteCollection(user, collection); + yield* Effect.tryPromise({ + try: () => service.persist(), + catch: (cause) => librarianServiceError("delete-collection-persist", cause), + }); - return {}; - } + return {}; + } - default: - throw new Error(`Unknown collection operation: ${request.operation as string}`); - } + default: + return yield* librarianServiceError("collection-operation", `Unknown collection operation: ${request.operation as string}`); + } + }), + ); }, @@ -863,80 +1220,122 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS // ---------- Persistence ---------- - persist: async function(this: LibrarianService): Promise { - try { - const data = { - documents: Object.fromEntries(this.documents), - processing: Object.fromEntries(this.processing), - collections: this.collectionManager.toJSON(), - }; + persist: function(this: LibrarianService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const data = { + documents: Object.fromEntries(service.documents), + processing: Object.fromEntries(service.processing), + collections: service.collectionManager.toJSON(), + }; - const json = JSON.stringify(data, null, 2); - await writeTextFile(this.persistPath, json); - } catch (err) { - console.error("[LibrarianService] Failed to persist state:", err); - } + const json = yield* encodeJsonString("persist-encode", data); + yield* Effect.tryPromise({ + try: () => writeTextFile(service.persistPath, json), + catch: (cause) => librarianServiceError("persist-write", cause), + }); + }).pipe( + Effect.catch((err) => + Effect.logError("[LibrarianService] Failed to persist state", { error: err.message }), + ), + ), + ); }, - loadFromDisk: async function(this: LibrarianService): Promise { - try { - const raw = await readTextFile(this.persistPath); - const parsed = JSON.parse(raw) as { - documents?: Record; - processing?: Record; - collections?: Array<{ user: string; collection: string; name: string; description: string; tags: string[] }>; - }; + loadFromDisk: function(this: LibrarianService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const parsed = yield* Effect.gen(function* () { + const raw = yield* Effect.tryPromise({ + try: () => readTextFile(service.persistPath), + catch: (cause) => librarianServiceError("persist-read", cause), + }); + return yield* decodeJsonString("persist-decode", raw); + }).pipe( + Effect.catch(() => + Effect.log("[LibrarianService] No persisted state found, starting fresh").pipe( + Effect.as(null as PersistedLibrarianState | null), + ), + ), + ); - this.documents.clear(); - if (parsed.documents !== undefined) { - for (const [id, doc] of Object.entries(parsed.documents)) { - this.documents.set(id, this.publicDocument(doc)); + if (parsed === null) return; + + service.documents.clear(); + if (parsed.documents !== undefined) { + for (const [id, doc] of Object.entries(parsed.documents)) { + service.documents.set(id, service.publicDocument(doc)); + } } - } - this.processing.clear(); - if (parsed.processing !== undefined) { - for (const [id, proc] of Object.entries(parsed.processing)) { - this.processing.set(id, this.publicProcessing(proc)); + service.processing.clear(); + if (parsed.processing !== undefined) { + for (const [id, proc] of Object.entries(parsed.processing)) { + service.processing.set(id, service.publicProcessing(proc)); + } } - } - if (parsed.collections !== undefined) { - this.collectionManager.loadFromJSON(parsed.collections); - } + if (parsed.collections !== undefined) { + service.collectionManager.loadFromJSON(parsed.collections); + } - console.log( - `[LibrarianService] Loaded persisted state (documents=${this.documents.size}, processing=${this.processing.size})`, - ); - } catch { - console.log("[LibrarianService] No persisted state found, starting fresh"); - } + yield* Effect.log( + `[LibrarianService] Loaded persisted state (documents=${service.documents.size}, processing=${service.processing.size})`, + ); + }), + ); }, - stop: async function(this: LibrarianService): Promise { - if (this.libConsumer !== null) { - await this.libConsumer.close(); - this.libConsumer = null; - } - if (this.libProducer !== null) { - await this.libProducer.close(); - this.libProducer = null; - } - if (this.colConsumer !== null) { - await this.colConsumer.close(); - this.colConsumer = null; - } - if (this.colProducer !== null) { - await this.colProducer.close(); - this.colProducer = null; - } - await baseStop(); + stop: function(this: LibrarianService): Promise { + const service = this; + return Effect.runPromise( + Effect.gen(function* () { + const libConsumer = service.libConsumer; + if (libConsumer !== null) { + yield* Effect.tryPromise({ + try: () => libConsumer.close(), + catch: (cause) => librarianServiceError("close-librarian-consumer", cause), + }); + service.libConsumer = null; + } + const libProducer = service.libProducer; + if (libProducer !== null) { + yield* Effect.tryPromise({ + try: () => libProducer.close(), + catch: (cause) => librarianServiceError("close-librarian-producer", cause), + }); + service.libProducer = null; + } + const colConsumer = service.colConsumer; + if (colConsumer !== null) { + yield* Effect.tryPromise({ + try: () => colConsumer.close(), + catch: (cause) => librarianServiceError("close-collection-consumer", cause), + }); + service.colConsumer = null; + } + const colProducer = service.colProducer; + if (colProducer !== null) { + yield* Effect.tryPromise({ + try: () => colProducer.close(), + catch: (cause) => librarianServiceError("close-collection-producer", cause), + }); + service.colProducer = null; + } + yield* Effect.tryPromise({ + try: () => baseStop(), + catch: (cause) => librarianServiceError("stop", cause), + }); + }), + ); } }); @@ -945,15 +1344,11 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS export const LibrarianService = makeLibrarianService; -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - export const program = makeProcessorProgram({ id: "librarian-svc", make: (config) => makeLibrarianService(config), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/model/text-completion/azure-openai.ts b/ts/packages/flow/src/model/text-completion/azure-openai.ts index cf4f598a..b6112e23 100644 --- a/ts/packages/flow/src/model/text-completion/azure-openai.ts +++ b/ts/packages/flow/src/model/text-completion/azure-openai.ts @@ -19,9 +19,15 @@ import { type ProcessorConfig, type LlmResult, type LlmChunk, - tooManyRequestsError, } from "@trustgraph/base"; -import { Effect, Layer } from "effect"; +import { Effect, Layer, Stream } from "effect"; +import { + optionalStringConfig, + providerStatusError, + requiredString, + toAsyncGenerator, + type TextCompletionRuntimeError, +} from "./common.ts"; export type AzureOpenAIProcessorConfig = ProcessorConfig & { model?: string; @@ -32,32 +38,65 @@ export type AzureOpenAIProcessorConfig = ProcessorConfig & { maxOutput?: number; }; -export function makeAzureOpenAIProvider(config: AzureOpenAIProcessorConfig): LlmProvider { - const defaultModel = config.model ?? process.env.AZURE_MODEL ?? "gpt-4o"; - const defaultTemperature = config.temperature ?? 0.0; - const maxOutput = config.maxOutput ?? 4096; - - const apiKey = config.apiKey ?? process.env.AZURE_TOKEN; - if (apiKey === undefined || apiKey.length === 0) { - throw new Error("Azure OpenAI API key not specified"); - } - - const endpoint = config.endpoint ?? process.env.AZURE_ENDPOINT; - if (endpoint === undefined || endpoint.length === 0) { - throw new Error("Azure OpenAI endpoint not specified"); - } +type ResolvedAzureOpenAIConfig = { + readonly defaultModel: string; + readonly defaultTemperature: number; + readonly maxOutput: number; + readonly apiKey: string; + readonly endpoint: string; + readonly apiVersion: string; +}; +const loadAzureOpenAIConfig = Effect.fn("loadAzureOpenAIConfig")(function* ( + config: AzureOpenAIProcessorConfig, +) { + const defaultModel = + config.model ?? (yield* optionalStringConfig("AzureOpenAI", "AZURE_MODEL")) ?? "gpt-4o"; + const apiKey = yield* requiredString( + config.apiKey ?? (yield* optionalStringConfig("AzureOpenAI", "AZURE_TOKEN")), + "AzureOpenAI", + "AZURE_TOKEN", + "Azure OpenAI API key not specified", + ); + const endpoint = yield* requiredString( + config.endpoint ?? (yield* optionalStringConfig("AzureOpenAI", "AZURE_ENDPOINT")), + "AzureOpenAI", + "AZURE_ENDPOINT", + "Azure OpenAI endpoint not specified", + ); const apiVersion = config.apiVersion ?? - process.env.AZURE_API_VERSION ?? + (yield* optionalStringConfig("AzureOpenAI", "AZURE_API_VERSION")) ?? "2024-12-01-preview"; + return { + defaultModel, + defaultTemperature: config.temperature ?? 0.0, + maxOutput: config.maxOutput ?? 4096, + apiKey, + endpoint, + apiVersion, + }; +}); + +const mapAzureOpenAIError = (error: unknown): TextCompletionRuntimeError => + providerStatusError("AzureOpenAI", error); + +export function makeAzureOpenAIProvider(config: AzureOpenAIProcessorConfig): LlmProvider { + const { + defaultModel, + defaultTemperature, + maxOutput, + apiKey, + endpoint, + apiVersion, + } = Effect.runSync(loadAzureOpenAIConfig(config)) satisfies ResolvedAzureOpenAIConfig; const client = new AzureOpenAI({ apiKey, apiVersion, endpoint }); - console.log("[AzureOpenAI] LLM service initialized"); + Effect.runSync(Effect.log("[AzureOpenAI] LLM service initialized")); return { - generateContent: async ( + generateContent: ( system: string, prompt: string, model?: string, @@ -66,87 +105,106 @@ export function makeAzureOpenAIProvider(config: AzureOpenAIProcessorConfig): Llm const modelName = model ?? defaultModel; const temp = temperature ?? defaultTemperature; - try { - const resp = await client.chat.completions.create({ - model: modelName, - messages: [ - { role: "system", content: system }, - { role: "user", content: prompt }, - ], - temperature: temp, - max_completion_tokens: maxOutput, - }); - - return { - text: resp.choices[0].message.content ?? "", - inToken: resp.usage?.prompt_tokens ?? 0, - outToken: resp.usage?.completion_tokens ?? 0, - model: modelName, - }; - } catch (err) { - if ((err as any)?.status === 429) { - throw tooManyRequestsError(); - } - throw err; - } + return Effect.runPromise( + Effect.tryPromise({ + try: () => + client.chat.completions.create({ + model: modelName, + messages: [ + { role: "system", content: system }, + { role: "user", content: prompt }, + ], + temperature: temp, + max_completion_tokens: maxOutput, + }), + catch: mapAzureOpenAIError, + }).pipe( + Effect.map((resp): LlmResult => ({ + text: resp.choices[0].message.content ?? "", + inToken: resp.usage?.prompt_tokens ?? 0, + outToken: resp.usage?.completion_tokens ?? 0, + model: modelName, + })), + ), + ); }, supportsStreaming: () => true, - generateContentStream: async function* ( + generateContentStream: ( system: string, prompt: string, model?: string, temperature?: number, - ): AsyncGenerator { + ): AsyncGenerator => { const modelName = model ?? defaultModel; const temp = temperature ?? defaultTemperature; - try { - const stream = await client.chat.completions.create({ - model: modelName, - messages: [ - { role: "system", content: system }, - { role: "user", content: prompt }, - ], - temperature: temp, - max_completion_tokens: maxOutput, - stream: true, - stream_options: { include_usage: true }, - }); - + const stream = Stream.fromEffect( + Effect.tryPromise({ + try: () => + client.chat.completions.create({ + model: modelName, + messages: [ + { role: "system", content: system }, + { role: "user", content: prompt }, + ], + temperature: temp, + max_completion_tokens: maxOutput, + stream: true, + stream_options: { include_usage: true }, + }), + catch: mapAzureOpenAIError, + }), + ).pipe( + Stream.flatMap((openAIStream) => { + const iterator = openAIStream[Symbol.asyncIterator](); let totalInputTokens = 0; let totalOutputTokens = 0; - for await (const chunk of stream) { - const content = chunk.choices[0]?.delta?.content; - if (content !== null && content !== undefined && content.length > 0) { - yield { - text: content, - inToken: null, - outToken: null, - model: modelName, - isFinal: false, - }; - } + return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>( + "pulling", + (state) => { + if (state === "done") return Effect.void as Effect.Effect; - if (chunk.usage !== null && chunk.usage !== undefined) { - totalInputTokens = chunk.usage.prompt_tokens; - totalOutputTokens = chunk.usage.completion_tokens; - } - } + return Effect.gen(function* () { + while (true) { + const next = yield* Effect.tryPromise({ + try: () => iterator.next(), + catch: mapAzureOpenAIError, + }); - yield { - text: "", - inToken: totalInputTokens, - outToken: totalOutputTokens, - model: modelName, - isFinal: true, - }; - } catch (err) { - if ((err as any)?.status === 429) { - throw tooManyRequestsError(); - } - throw err; - } + if (next.done === true) { + return [{ + text: "", + inToken: totalInputTokens, + outToken: totalOutputTokens, + model: modelName, + isFinal: true, + }, "done"] as const; + } + + const chunk = next.value; + const content = chunk.choices[0]?.delta?.content; + if (chunk.usage !== null && chunk.usage !== undefined) { + totalInputTokens = chunk.usage.prompt_tokens; + totalOutputTokens = chunk.usage.completion_tokens; + } + if (content !== null && content !== undefined && content.length > 0) { + return [{ + text: content, + inToken: null, + outToken: null, + model: modelName, + isFinal: false, + }, "pulling"] as const; + } + } + }); + }, + ); + }), + ); + + return toAsyncGenerator(Stream.toAsyncIterable(stream), mapAzureOpenAIError); }, }; } @@ -171,6 +229,6 @@ export const program = makeFlowProcessorProgram({ ), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/model/text-completion/claude.ts b/ts/packages/flow/src/model/text-completion/claude.ts index 4d43dd4c..786ebcfe 100644 --- a/ts/packages/flow/src/model/text-completion/claude.ts +++ b/ts/packages/flow/src/model/text-completion/claude.ts @@ -15,9 +15,15 @@ import { type ProcessorConfig, type LlmResult, type LlmChunk, - tooManyRequestsError, } from "@trustgraph/base"; -import { Effect, Layer } from "effect"; +import { Effect, Layer, Stream } from "effect"; +import { + optionalStringConfig, + providerStatusError, + requiredString, + toAsyncGenerator, + type TextCompletionRuntimeError, +} from "./common.ts"; export type ClaudeProcessorConfig = ProcessorConfig & { model?: string; @@ -26,21 +32,46 @@ export type ClaudeProcessorConfig = ProcessorConfig & { maxOutput?: number; }; +type ResolvedClaudeConfig = { + readonly defaultModel: string; + readonly defaultTemperature: number; + readonly maxOutput: number; + readonly apiKey: string; +}; + +const loadClaudeConfig = Effect.fn("loadClaudeConfig")(function*(config: ClaudeProcessorConfig) { + const apiKey = yield* requiredString( + config.apiKey ?? (yield* optionalStringConfig("Claude", "CLAUDE_KEY")), + "Claude", + "CLAUDE_KEY", + "Claude API key not specified", + ); + + return { + defaultModel: config.model ?? "claude-sonnet-4-20250514", + defaultTemperature: config.temperature ?? 0.0, + maxOutput: config.maxOutput ?? 8192, + apiKey, + } satisfies ResolvedClaudeConfig; +}); + +const mapClaudeError = (error: unknown): TextCompletionRuntimeError => + providerStatusError("Claude", error); + export function makeClaudeProvider(config: ClaudeProcessorConfig): LlmProvider { - const defaultModel = config.model ?? "claude-sonnet-4-20250514"; - const defaultTemperature = config.temperature ?? 0.0; - const maxOutput = config.maxOutput ?? 8192; - const apiKey = config.apiKey ?? process.env.CLAUDE_KEY; - if (apiKey === undefined || apiKey.length === 0) { - throw new Error("Claude API key not specified"); - } + const { + defaultModel, + defaultTemperature, + maxOutput, + apiKey, + } = Effect.runSync(loadClaudeConfig(config)) satisfies ResolvedClaudeConfig; const client = new Anthropic({ apiKey }); - console.log("[Claude] LLM service initialized"); + Effect.runSync(Effect.log("[Claude] LLM service initialized")); return { - generateContent: async ( + generateContent: ( system: string, prompt: string, model?: string, @@ -49,88 +80,120 @@ export function makeClaudeProvider(config: ClaudeProcessorConfig): LlmProvider { const modelName = model ?? defaultModel; const temp = temperature ?? defaultTemperature; - try { - const response = await client.messages.create({ - model: modelName, - max_tokens: maxOutput, - temperature: temp, - system, - messages: [ - { role: "user", content: prompt }, - ], - }); + return Effect.runPromise( + Effect.tryPromise({ + try: () => + client.messages.create({ + model: modelName, + max_tokens: maxOutput, + temperature: temp, + system, + messages: [ + { role: "user", content: prompt }, + ], + }), + catch: mapClaudeError, + }).pipe( + Effect.map((response): LlmResult => { + const firstContent = response.content[0]; + const text = firstContent?.type === "text" + ? firstContent.text + : ""; - const text = response.content[0].type === "text" - ? response.content[0].text - : ""; - - return { - text, - inToken: response.usage.input_tokens, - outToken: response.usage.output_tokens, - model: modelName, - }; - } catch (err) { - if (err instanceof Anthropic.RateLimitError) { - throw tooManyRequestsError(); - } - throw err; - } + return { + text, + inToken: response.usage.input_tokens, + outToken: response.usage.output_tokens, + model: modelName, + }; + }), + ), + ); }, supportsStreaming: () => true, - generateContentStream: async function* ( + generateContentStream: ( system: string, prompt: string, model?: string, temperature?: number, - ): AsyncGenerator { + ): AsyncGenerator => { const modelName = model ?? defaultModel; const temp = temperature ?? defaultTemperature; - try { - const stream = client.messages.stream({ - model: modelName, - max_tokens: maxOutput, - temperature: temp, - system, - messages: [ - { role: "user", content: prompt }, - ], - }); - - for await (const event of stream) { - if (event.type === "content_block_delta" && event.delta.type === "text_delta") { - yield { - text: event.delta.text, - inToken: null, - outToken: null, + const stream = Stream.fromEffect( + Effect.try({ + try: () => + client.messages.stream({ model: modelName, - isFinal: false, - }; - } - } + max_tokens: maxOutput, + temperature: temp, + system, + messages: [ + { role: "user", content: prompt }, + ], + }), + catch: mapClaudeError, + }), + ).pipe( + Stream.flatMap((anthropicStream) => { + const iterator = anthropicStream[Symbol.asyncIterator](); - const finalMessage = await stream.finalMessage(); - yield { - text: "", - inToken: finalMessage.usage.input_tokens, - outToken: finalMessage.usage.output_tokens, - model: modelName, - isFinal: true, - }; - } catch (err) { - if (err instanceof Anthropic.RateLimitError) { - throw tooManyRequestsError(); - } - throw err; - } + return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>( + "pulling", + (state) => { + if (state === "done") return Effect.void as Effect.Effect; + + return Effect.gen(function* () { + while (true) { + const next = yield* Effect.tryPromise({ + try: () => iterator.next(), + catch: mapClaudeError, + }); + + if (next.done === true) { + const finalMessage = yield* Effect.tryPromise({ + try: () => anthropicStream.finalMessage(), + catch: mapClaudeError, + }); + return [{ + text: "", + inToken: finalMessage.usage.input_tokens, + outToken: finalMessage.usage.output_tokens, + model: modelName, + isFinal: true, + }, "done"] as const; + } + + const event = next.value; + if ( + event.type === "content_block_delta" && + event.delta.type === "text_delta" + ) { + return [{ + text: event.delta.text, + inToken: null, + outToken: null, + model: modelName, + isFinal: false, + }, "pulling"] as const; + } + } + }); + }, + ); + }), + ); + + return toAsyncGenerator(Stream.toAsyncIterable(stream), mapClaudeError); }, }; } export type ClaudeProcessor = ReturnType; -export function makeClaudeProcessor(config: ClaudeProcessorConfig): ReturnType { +export function makeClaudeProcessor( + config: ClaudeProcessorConfig, +): ReturnType { return makeLlmService(config, makeClaudeProvider(config)); } @@ -146,6 +209,6 @@ export const program = makeFlowProcessorProgram({ ), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/model/text-completion/common.ts b/ts/packages/flow/src/model/text-completion/common.ts new file mode 100644 index 00000000..a4532e9f --- /dev/null +++ b/ts/packages/flow/src/model/text-completion/common.ts @@ -0,0 +1,101 @@ +import { + TooManyRequestsError, + errorMessage, + type LlmChunk, +} from "@trustgraph/base"; +import { Config, Effect } from "effect"; +import * as O from "effect/Option"; +import * as S from "effect/Schema"; + +export class TextCompletionConfigError extends S.TaggedErrorClass()( + "TextCompletionConfigError", + { + message: S.String, + provider: S.String, + key: S.String, + }, +) {} + +export class TextCompletionProviderError extends S.TaggedErrorClass()( + "TextCompletionProviderError", + { + message: S.String, + provider: S.String, + }, +) {} + +export type TextCompletionRuntimeError = + | TextCompletionProviderError + | TooManyRequestsError; + +export const optionalStringConfig = Effect.fn("TextCompletion.optionalStringConfig")(function*( + provider: string, + name: string, +) { + const value = yield* Config.string(name).pipe( + Config.option, + Effect.mapError((cause) => + TextCompletionConfigError.make({ + provider, + key: name, + message: errorMessage(cause), + }) + ), + ); + return O.getOrUndefined(value); +}); + +export const requiredString = ( + value: string | undefined, + provider: string, + key: string, + message: string, +) => + value !== undefined && value.length > 0 + ? Effect.succeed(value) + : Effect.fail(TextCompletionConfigError.make({ provider, key, message })); + +export const providerRuntimeError = ( + provider: string, + error: unknown, +): TextCompletionRuntimeError => + TextCompletionProviderError.make({ + provider, + message: errorMessage(error), + }); + +export const providerStatusError = ( + provider: string, + error: unknown, +): TextCompletionRuntimeError => { + const status = typeof error === "object" && error !== null && "status" in error + ? (error as { readonly status?: unknown }).status + : undefined; + const statusCode = typeof error === "object" && error !== null && "statusCode" in error + ? (error as { readonly statusCode?: unknown }).statusCode + : undefined; + return status === 429 || statusCode === 429 + ? TooManyRequestsError.make({ message: "Rate limit exceeded" }) + : providerRuntimeError(provider, error); +}; + +export const toAsyncGenerator = ( + iterable: AsyncIterable, + mapError: (error: unknown) => TextCompletionRuntimeError, +): AsyncGenerator => { + const iterator = iterable[Symbol.asyncIterator](); + let generator: AsyncGenerator; + generator = { + next: (value?: unknown) => iterator.next(value as never), + return: (value?: unknown) => + iterator.return === undefined + ? Promise.resolve({ done: true, value: value as LlmChunk }) + : iterator.return(value as never) as Promise>, + throw: (error?: unknown) => + iterator.throw === undefined + ? Effect.runPromise(Effect.fail(mapError(error))) as Promise> + : iterator.throw(error) as Promise>, + [Symbol.asyncIterator]: () => generator, + } as AsyncGenerator; + return generator; +}; diff --git a/ts/packages/flow/src/model/text-completion/mistral.ts b/ts/packages/flow/src/model/text-completion/mistral.ts index 8968919a..90d33c0b 100644 --- a/ts/packages/flow/src/model/text-completion/mistral.ts +++ b/ts/packages/flow/src/model/text-completion/mistral.ts @@ -17,9 +17,15 @@ import { type ProcessorConfig, type LlmResult, type LlmChunk, - tooManyRequestsError, } from "@trustgraph/base"; -import { Effect, Layer } from "effect"; +import { Effect, Layer, Stream } from "effect"; +import { + optionalStringConfig, + providerStatusError, + requiredString, + toAsyncGenerator, + type TextCompletionRuntimeError, +} from "./common.ts"; export type MistralProcessorConfig = ProcessorConfig & { model?: string; @@ -28,22 +34,49 @@ export type MistralProcessorConfig = ProcessorConfig & { maxOutput?: number; }; +type ResolvedMistralConfig = { + readonly defaultModel: string; + readonly defaultTemperature: number; + readonly maxOutput: number; + readonly apiKey: string; +}; + +const loadMistralConfig = Effect.fn("loadMistralConfig")(function*(config: MistralProcessorConfig) { + const apiKey = yield* requiredString( + config.apiKey ?? (yield* optionalStringConfig("Mistral", "MISTRAL_TOKEN")), + "Mistral", + "MISTRAL_TOKEN", + "Mistral API key not specified", + ); + + return { + defaultModel: + config.model ?? + (yield* optionalStringConfig("Mistral", "MISTRAL_MODEL")) ?? + "ministral-8b-latest", + defaultTemperature: config.temperature ?? 0.0, + maxOutput: config.maxOutput ?? 4096, + apiKey, + } satisfies ResolvedMistralConfig; +}); + +const mapMistralError = (error: unknown): TextCompletionRuntimeError => + providerStatusError("Mistral", error); + export function makeMistralProvider(config: MistralProcessorConfig): LlmProvider { - const defaultModel = - config.model ?? process.env.MISTRAL_MODEL ?? "ministral-8b-latest"; - const defaultTemperature = config.temperature ?? 0.0; - const maxOutput = config.maxOutput ?? 4096; - const apiKey = config.apiKey ?? process.env.MISTRAL_TOKEN; - if (apiKey === undefined || apiKey.length === 0) { - throw new Error("Mistral API key not specified"); - } + const { + defaultModel, + defaultTemperature, + maxOutput, + apiKey, + } = Effect.runSync(loadMistralConfig(config)) satisfies ResolvedMistralConfig; const client = new Mistral({ apiKey }); - console.log("[Mistral] LLM service initialized"); + Effect.runSync(Effect.log("[Mistral] LLM service initialized")); return { - generateContent: async ( + generateContent: ( system: string, prompt: string, model?: string, @@ -52,93 +85,114 @@ export function makeMistralProvider(config: MistralProcessorConfig): LlmProvider const modelName = model ?? defaultModel; const temp = temperature ?? defaultTemperature; - try { - const resp = await client.chat.complete({ - model: modelName, - messages: [ - { role: "system", content: system }, - { role: "user", content: prompt }, - ], - temperature: temp, - maxTokens: maxOutput, - }); - - return { - text: (resp.choices?.[0]?.message?.content as string) ?? "", - inToken: resp.usage?.promptTokens ?? 0, - outToken: resp.usage?.completionTokens ?? 0, - model: modelName, - }; - } catch (err) { - if ((err as any)?.statusCode === 429 || (err as any)?.status === 429) { - throw tooManyRequestsError(); - } - throw err; - } + return Effect.runPromise( + Effect.tryPromise({ + try: () => + client.chat.complete({ + model: modelName, + messages: [ + { role: "system", content: system }, + { role: "user", content: prompt }, + ], + temperature: temp, + maxTokens: maxOutput, + }), + catch: mapMistralError, + }).pipe( + Effect.map((resp): LlmResult => ({ + text: (resp.choices?.[0]?.message?.content as string) ?? "", + inToken: resp.usage?.promptTokens ?? 0, + outToken: resp.usage?.completionTokens ?? 0, + model: modelName, + })), + ), + ); }, supportsStreaming: () => true, - generateContentStream: async function* ( + generateContentStream: ( system: string, prompt: string, model?: string, temperature?: number, - ): AsyncGenerator { + ): AsyncGenerator => { const modelName = model ?? defaultModel; const temp = temperature ?? defaultTemperature; - try { - const stream = await client.chat.stream({ - model: modelName, - messages: [ - { role: "system", content: system }, - { role: "user", content: prompt }, - ], - temperature: temp, - maxTokens: maxOutput, - }); - + const stream = Stream.fromEffect( + Effect.tryPromise({ + try: () => + client.chat.stream({ + model: modelName, + messages: [ + { role: "system", content: system }, + { role: "user", content: prompt }, + ], + temperature: temp, + maxTokens: maxOutput, + }), + catch: mapMistralError, + }), + ).pipe( + Stream.flatMap((mistralStream) => { + const iterator = mistralStream[Symbol.asyncIterator](); let totalInputTokens = 0; let totalOutputTokens = 0; - for await (const chunk of stream) { - const delta = chunk.data?.choices?.[0]?.delta; - const content = delta?.content; - if (typeof content === "string" && content.length > 0) { - yield { - text: content, - inToken: null, - outToken: null, - model: modelName, - isFinal: false, - }; - } + return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>( + "pulling", + (state) => { + if (state === "done") return Effect.void as Effect.Effect; - if (chunk.data?.usage !== undefined) { - totalInputTokens = chunk.data.usage.promptTokens ?? 0; - totalOutputTokens = chunk.data.usage.completionTokens ?? 0; - } - } + return Effect.gen(function* () { + while (true) { + const next = yield* Effect.tryPromise({ + try: () => iterator.next(), + catch: mapMistralError, + }); - yield { - text: "", - inToken: totalInputTokens, - outToken: totalOutputTokens, - model: modelName, - isFinal: true, - }; - } catch (err) { - if ((err as any)?.statusCode === 429 || (err as any)?.status === 429) { - throw tooManyRequestsError(); - } - throw err; - } + if (next.done === true) { + return [{ + text: "", + inToken: totalInputTokens, + outToken: totalOutputTokens, + model: modelName, + isFinal: true, + }, "done"] as const; + } + + const chunk = next.value; + const delta = chunk.data?.choices?.[0]?.delta; + const content = delta?.content; + if (chunk.data?.usage !== undefined) { + totalInputTokens = chunk.data.usage.promptTokens ?? 0; + totalOutputTokens = chunk.data.usage.completionTokens ?? 0; + } + if (typeof content === "string" && content.length > 0) { + return [{ + text: content, + inToken: null, + outToken: null, + model: modelName, + isFinal: false, + }, "pulling"] as const; + } + } + }); + }, + ); + }), + ); + + return toAsyncGenerator(Stream.toAsyncIterable(stream), mapMistralError); }, }; } export type MistralProcessor = ReturnType; -export function makeMistralProcessor(config: MistralProcessorConfig): ReturnType { +export function makeMistralProcessor( + config: MistralProcessorConfig, +): ReturnType { return makeLlmService(config, makeMistralProvider(config)); } @@ -154,6 +208,6 @@ export const program = makeFlowProcessorProgram({ ), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/model/text-completion/ollama.ts b/ts/packages/flow/src/model/text-completion/ollama.ts index 5ba792b7..7358f3af 100644 --- a/ts/packages/flow/src/model/text-completion/ollama.ts +++ b/ts/packages/flow/src/model/text-completion/ollama.ts @@ -18,32 +18,51 @@ import { type LlmResult, type LlmChunk, } from "@trustgraph/base"; -import { Effect, Layer } from "effect"; +import { Effect, Layer, Stream } from "effect"; +import { + optionalStringConfig, + providerRuntimeError, + toAsyncGenerator, + type TextCompletionRuntimeError, +} from "./common.ts"; export type OllamaProcessorConfig = ProcessorConfig & { model?: string; ollamaUrl?: string; }; -export function makeOllamaProvider(config: OllamaProcessorConfig): LlmProvider { - const defaultModel = - config.model ?? - process.env.OLLAMA_MODEL ?? - "qwen2.5:0.5b"; +type ResolvedOllamaConfig = { + readonly defaultModel: string; + readonly host: string; +}; - const host = - config.ollamaUrl ?? - process.env.OLLAMA_URL ?? - "http://localhost:11434"; +const loadOllamaConfig = Effect.fn("loadOllamaConfig")(function*(config: OllamaProcessorConfig) { + return { + defaultModel: + config.model ?? + (yield* optionalStringConfig("Ollama", "OLLAMA_MODEL")) ?? + "qwen2.5:0.5b", + host: + config.ollamaUrl ?? + (yield* optionalStringConfig("Ollama", "OLLAMA_URL")) ?? + "http://localhost:11434", + } satisfies ResolvedOllamaConfig; +}); + +const mapOllamaError = (error: unknown): TextCompletionRuntimeError => + providerRuntimeError("Ollama", error); + +export function makeOllamaProvider(config: OllamaProcessorConfig): LlmProvider { + const { defaultModel, host } = Effect.runSync(loadOllamaConfig(config)) satisfies ResolvedOllamaConfig; const client = new Ollama({ host }); - console.log( + Effect.runSync(Effect.log( `[Ollama] LLM service initialized (host=${host}, model=${defaultModel})`, - ); + )); return { - generateContent: async ( + generateContent: ( system: string, prompt: string, model?: string, @@ -52,73 +71,107 @@ export function makeOllamaProvider(config: OllamaProcessorConfig): LlmProvider { const modelName = model ?? defaultModel; const fullPrompt = system + "\n\n" + prompt; - const resp = await client.generate({ - model: modelName, - prompt: fullPrompt, - stream: false, - }); - - return { - text: resp.response, - inToken: resp.prompt_eval_count ?? 0, - outToken: resp.eval_count ?? 0, - model: modelName, - }; + return Effect.runPromise( + Effect.tryPromise({ + try: () => + client.generate({ + model: modelName, + prompt: fullPrompt, + stream: false, + }), + catch: mapOllamaError, + }).pipe( + Effect.map((resp): LlmResult => ({ + text: resp.response, + inToken: resp.prompt_eval_count ?? 0, + outToken: resp.eval_count ?? 0, + model: modelName, + })), + ), + ); }, supportsStreaming: () => true, - generateContentStream: async function* ( + generateContentStream: ( system: string, prompt: string, model?: string, _temperature?: number, - ): AsyncGenerator { + ): AsyncGenerator => { const modelName = model ?? defaultModel; const fullPrompt = system + "\n\n" + prompt; - const stream = await client.generate({ - model: modelName, - prompt: fullPrompt, - stream: true, - }); - + const stream = Stream.fromEffect( + Effect.tryPromise({ + try: () => + client.generate({ + model: modelName, + prompt: fullPrompt, + stream: true, + }), + catch: mapOllamaError, + }), + ).pipe( + Stream.flatMap((ollamaStream) => { + const iterator = ollamaStream[Symbol.asyncIterator](); let totalInputTokens = 0; let totalOutputTokens = 0; - for await (const chunk of stream) { - // Token counts accumulate across chunks; keep the latest values - if (chunk.prompt_eval_count !== undefined) { - totalInputTokens = chunk.prompt_eval_count; - } - if (chunk.eval_count !== undefined) { - totalOutputTokens = chunk.eval_count; - } + return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>( + "pulling", + (state) => { + if (state === "done") return Effect.void as Effect.Effect; - if (chunk.response.length > 0) { - yield { - text: chunk.response, - inToken: null, - outToken: null, - model: modelName, - isFinal: false, - }; - } - } + return Effect.gen(function* () { + while (true) { + const next = yield* Effect.tryPromise({ + try: () => iterator.next(), + catch: mapOllamaError, + }); - // Final chunk with accumulated token counts - yield { - text: "", - inToken: totalInputTokens, - outToken: totalOutputTokens, - model: modelName, - isFinal: true, - }; + if (next.done === true) { + return [{ + text: "", + inToken: totalInputTokens, + outToken: totalOutputTokens, + model: modelName, + isFinal: true, + }, "done"] as const; + } + + const chunk = next.value; + if (chunk.prompt_eval_count !== undefined) { + totalInputTokens = chunk.prompt_eval_count; + } + if (chunk.eval_count !== undefined) { + totalOutputTokens = chunk.eval_count; + } + + if (chunk.response.length > 0) { + return [{ + text: chunk.response, + inToken: null, + outToken: null, + model: modelName, + isFinal: false, + }, "pulling"] as const; + } + } + }); + }, + ); + }), + ); + + return toAsyncGenerator(Stream.toAsyncIterable(stream), mapOllamaError); }, }; } export type OllamaProcessor = ReturnType; -export function makeOllamaProcessor(config: OllamaProcessorConfig): ReturnType { +export function makeOllamaProcessor( + config: OllamaProcessorConfig, +): ReturnType { return makeLlmService(config, makeOllamaProvider(config)); } @@ -134,6 +187,6 @@ export const program = makeFlowProcessorProgram({ ), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/model/text-completion/openai-compatible.ts b/ts/packages/flow/src/model/text-completion/openai-compatible.ts index a5e4755b..2a36377c 100644 --- a/ts/packages/flow/src/model/text-completion/openai-compatible.ts +++ b/ts/packages/flow/src/model/text-completion/openai-compatible.ts @@ -21,7 +21,14 @@ import { type LlmResult, type LlmChunk, } from "@trustgraph/base"; -import { Effect, Layer } from "effect"; +import { Effect, Layer, Stream } from "effect"; +import { + optionalStringConfig, + providerStatusError, + requiredString, + toAsyncGenerator, + type TextCompletionRuntimeError, +} from "./common.ts"; export type OpenAICompatibleProcessorConfig = ProcessorConfig & { model?: string; @@ -31,30 +38,57 @@ export type OpenAICompatibleProcessorConfig = ProcessorConfig & { maxOutput?: number; }; +type ResolvedOpenAICompatibleConfig = { + readonly defaultModel: string; + readonly defaultTemperature: number; + readonly maxOutput: number; + readonly apiKey: string; + readonly baseURL: string; +}; + +const loadOpenAICompatibleConfig = Effect.fn("loadOpenAICompatibleConfig")(function*( + config: OpenAICompatibleProcessorConfig, +) { + const defaultModel = + config.model ?? (yield* optionalStringConfig("OpenAI-Compatible", "OPENAI_COMPAT_MODEL")) ?? "default"; + const baseURL = yield* requiredString( + config.baseUrl ?? (yield* optionalStringConfig("OpenAI-Compatible", "OPENAI_COMPAT_URL")), + "OpenAI-Compatible", + "OPENAI_COMPAT_URL", + "OpenAI-compatible server URL not specified (set OPENAI_COMPAT_URL)", + ); + const apiKey = + config.apiKey ?? (yield* optionalStringConfig("OpenAI-Compatible", "OPENAI_COMPAT_KEY")) ?? "sk-no-key-required"; + + return { + defaultModel, + defaultTemperature: config.temperature ?? 0.0, + maxOutput: config.maxOutput ?? 4096, + apiKey, + baseURL, + } satisfies ResolvedOpenAICompatibleConfig; +}); + +const mapOpenAICompatibleError = (error: unknown): TextCompletionRuntimeError => + providerStatusError("OpenAI-Compatible", error); + export function makeOpenAICompatibleProvider( config: OpenAICompatibleProcessorConfig, ): LlmProvider { - const defaultModel = - config.model ?? process.env.OPENAI_COMPAT_MODEL ?? "default"; - const defaultTemperature = config.temperature ?? 0.0; - const maxOutput = config.maxOutput ?? 4096; - - const baseURL = config.baseUrl ?? process.env.OPENAI_COMPAT_URL; - if (baseURL === undefined || baseURL.length === 0) { - throw new Error( - "OpenAI-compatible server URL not specified (set OPENAI_COMPAT_URL)", - ); - } - - const apiKey = - config.apiKey ?? process.env.OPENAI_COMPAT_KEY ?? "sk-no-key-required"; + const { + defaultModel, + defaultTemperature, + maxOutput, + apiKey, + baseURL, + } = Effect.runSync(loadOpenAICompatibleConfig(config)) satisfies ResolvedOpenAICompatibleConfig; const client = new OpenAI({ baseURL, apiKey }); - console.log("[OpenAI-Compatible] LLM service initialized"); + Effect.runSync(Effect.log("[OpenAI-Compatible] LLM service initialized")); return { - generateContent: async ( + generateContent: ( system: string, prompt: string, model?: string, @@ -63,72 +97,105 @@ export function makeOpenAICompatibleProvider( const modelName = model ?? defaultModel; const temp = temperature ?? defaultTemperature; - const resp = await client.chat.completions.create({ - model: modelName, - messages: [ - { role: "system", content: system }, - { role: "user", content: prompt }, - ], - temperature: temp, - max_tokens: maxOutput, - }); - - return { - text: resp.choices[0].message.content ?? "", - inToken: resp.usage?.prompt_tokens ?? 0, - outToken: resp.usage?.completion_tokens ?? 0, - model: modelName, - }; + return Effect.runPromise( + Effect.tryPromise({ + try: () => + client.chat.completions.create({ + model: modelName, + messages: [ + { role: "system", content: system }, + { role: "user", content: prompt }, + ], + temperature: temp, + max_tokens: maxOutput, + }), + catch: mapOpenAICompatibleError, + }).pipe( + Effect.map((resp): LlmResult => ({ + text: resp.choices[0].message.content ?? "", + inToken: resp.usage?.prompt_tokens ?? 0, + outToken: resp.usage?.completion_tokens ?? 0, + model: modelName, + })), + ), + ); }, supportsStreaming: () => true, - generateContentStream: async function* ( + generateContentStream: ( system: string, prompt: string, model?: string, temperature?: number, - ): AsyncGenerator { + ): AsyncGenerator => { const modelName = model ?? defaultModel; const temp = temperature ?? defaultTemperature; - const stream = await client.chat.completions.create({ - model: modelName, - messages: [ - { role: "system", content: system }, - { role: "user", content: prompt }, - ], - temperature: temp, - max_tokens: maxOutput, - stream: true, - }); - + const stream = Stream.fromEffect( + Effect.tryPromise({ + try: () => + client.chat.completions.create({ + model: modelName, + messages: [ + { role: "system", content: system }, + { role: "user", content: prompt }, + ], + temperature: temp, + max_tokens: maxOutput, + stream: true, + }), + catch: mapOpenAICompatibleError, + }), + ).pipe( + Stream.flatMap((openAIStream) => { + const iterator = openAIStream[Symbol.asyncIterator](); let totalInputTokens = 0; let totalOutputTokens = 0; - for await (const chunk of stream) { - const content = chunk.choices[0]?.delta?.content; - if (content !== null && content !== undefined && content.length > 0) { - yield { - text: content, - inToken: null, - outToken: null, - model: modelName, - isFinal: false, - }; - } + return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>( + "pulling", + (state) => { + if (state === "done") return Effect.void as Effect.Effect; - if (chunk.usage !== null && chunk.usage !== undefined) { - totalInputTokens = chunk.usage.prompt_tokens; - totalOutputTokens = chunk.usage.completion_tokens; - } - } + return Effect.gen(function* () { + while (true) { + const next = yield* Effect.tryPromise({ + try: () => iterator.next(), + catch: mapOpenAICompatibleError, + }); - yield { - text: "", - inToken: totalInputTokens, - outToken: totalOutputTokens, - model: modelName, - isFinal: true, - }; + if (next.done === true) { + return [{ + text: "", + inToken: totalInputTokens, + outToken: totalOutputTokens, + model: modelName, + isFinal: true, + }, "done"] as const; + } + + const chunk = next.value; + const content = chunk.choices[0]?.delta?.content; + if (chunk.usage !== null && chunk.usage !== undefined) { + totalInputTokens = chunk.usage.prompt_tokens; + totalOutputTokens = chunk.usage.completion_tokens; + } + if (content !== null && content !== undefined && content.length > 0) { + return [{ + text: content, + inToken: null, + outToken: null, + model: modelName, + isFinal: false, + }, "pulling"] as const; + } + } + }); + }, + ); + }), + ); + + return toAsyncGenerator(Stream.toAsyncIterable(stream), mapOpenAICompatibleError); }, }; } @@ -153,6 +220,6 @@ export const program = makeFlowProcessorProgram({ ), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/model/text-completion/openai.ts b/ts/packages/flow/src/model/text-completion/openai.ts index a6547915..b4885448 100644 --- a/ts/packages/flow/src/model/text-completion/openai.ts +++ b/ts/packages/flow/src/model/text-completion/openai.ts @@ -15,9 +15,15 @@ import { type ProcessorConfig, type LlmResult, type LlmChunk, - tooManyRequestsError, } from "@trustgraph/base"; -import { Effect, Layer } from "effect"; +import { Effect, Layer, Stream } from "effect"; +import { + optionalStringConfig, + providerStatusError, + requiredString, + toAsyncGenerator, + type TextCompletionRuntimeError, +} from "./common.ts"; export type OpenAIProcessorConfig = ProcessorConfig & { model?: string; @@ -27,24 +33,52 @@ export type OpenAIProcessorConfig = ProcessorConfig & { maxOutput?: number; }; +type ResolvedOpenAIConfig = { + readonly defaultModel: string; + readonly defaultTemperature: number; + readonly maxOutput: number; + readonly apiKey: string; + readonly baseURL: string | undefined; +}; + +const loadOpenAIConfig = Effect.fn("loadOpenAIConfig")(function*(config: OpenAIProcessorConfig) { + const apiKey = yield* requiredString( + config.apiKey ?? (yield* optionalStringConfig("OpenAI", "OPENAI_TOKEN")), + "OpenAI", + "OPENAI_TOKEN", + "OpenAI API key not specified", + ); + + return { + defaultModel: config.model ?? "gpt-4o", + defaultTemperature: config.temperature ?? 0.0, + maxOutput: config.maxOutput ?? 4096, + apiKey, + baseURL: config.baseUrl ?? (yield* optionalStringConfig("OpenAI", "OPENAI_BASE_URL")), + } satisfies ResolvedOpenAIConfig; +}); + +const mapOpenAIError = (error: unknown): TextCompletionRuntimeError => + providerStatusError("OpenAI", error); + export function makeOpenAIProvider(config: OpenAIProcessorConfig): LlmProvider { - const defaultModel = config.model ?? "gpt-4o"; - const defaultTemperature = config.temperature ?? 0.0; - const maxOutput = config.maxOutput ?? 4096; - const apiKey = config.apiKey ?? process.env.OPENAI_TOKEN; - if (apiKey === undefined || apiKey.length === 0) { - throw new Error("OpenAI API key not specified"); - } + const { + defaultModel, + defaultTemperature, + maxOutput, + apiKey, + baseURL, + } = Effect.runSync(loadOpenAIConfig(config)) satisfies ResolvedOpenAIConfig; const client = new OpenAI({ - apiKey, - baseURL: config.baseUrl ?? process.env.OPENAI_BASE_URL, - }); + apiKey, + baseURL, + }); - console.log("[OpenAI] LLM service initialized"); + Effect.runSync(Effect.log("[OpenAI] LLM service initialized")); return { - generateContent: async ( + generateContent: ( system: string, prompt: string, model?: string, @@ -53,94 +87,115 @@ export function makeOpenAIProvider(config: OpenAIProcessorConfig): LlmProvider { const modelName = model ?? defaultModel; const temp = temperature ?? defaultTemperature; - try { - const resp = await client.chat.completions.create({ - model: modelName, - messages: [ - { role: "system", content: system }, - { role: "user", content: prompt }, - ], - temperature: temp, - max_completion_tokens: maxOutput, - }); - - return { - text: resp.choices[0].message.content ?? "", - inToken: resp.usage?.prompt_tokens ?? 0, - outToken: resp.usage?.completion_tokens ?? 0, - model: modelName, - }; - } catch (err) { - if (err instanceof OpenAI.RateLimitError) { - throw tooManyRequestsError(); - } - throw err; - } + return Effect.runPromise( + Effect.tryPromise({ + try: () => + client.chat.completions.create({ + model: modelName, + messages: [ + { role: "system", content: system }, + { role: "user", content: prompt }, + ], + temperature: temp, + max_completion_tokens: maxOutput, + }), + catch: mapOpenAIError, + }).pipe( + Effect.map((resp): LlmResult => ({ + text: resp.choices[0].message.content ?? "", + inToken: resp.usage?.prompt_tokens ?? 0, + outToken: resp.usage?.completion_tokens ?? 0, + model: modelName, + })), + ), + ); }, supportsStreaming: () => true, - generateContentStream: async function* ( + generateContentStream: ( system: string, prompt: string, model?: string, temperature?: number, - ): AsyncGenerator { + ): AsyncGenerator => { const modelName = model ?? defaultModel; const temp = temperature ?? defaultTemperature; - try { - const stream = await client.chat.completions.create({ - model: modelName, - messages: [ - { role: "system", content: system }, - { role: "user", content: prompt }, - ], - temperature: temp, - max_completion_tokens: maxOutput, - stream: true, - stream_options: { include_usage: true }, - }); - + const stream = Stream.fromEffect( + Effect.tryPromise({ + try: () => + client.chat.completions.create({ + model: modelName, + messages: [ + { role: "system", content: system }, + { role: "user", content: prompt }, + ], + temperature: temp, + max_completion_tokens: maxOutput, + stream: true, + stream_options: { include_usage: true }, + }), + catch: mapOpenAIError, + }), + ).pipe( + Stream.flatMap((openAIStream) => { + const iterator = openAIStream[Symbol.asyncIterator](); let totalInputTokens = 0; let totalOutputTokens = 0; - for await (const chunk of stream) { - const content = chunk.choices[0]?.delta?.content; - if (content !== null && content !== undefined && content.length > 0) { - yield { - text: content, - inToken: null, - outToken: null, - model: modelName, - isFinal: false, - }; - } + return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>( + "pulling", + (state) => { + if (state === "done") return Effect.void as Effect.Effect; - if (chunk.usage !== null && chunk.usage !== undefined) { - totalInputTokens = chunk.usage.prompt_tokens; - totalOutputTokens = chunk.usage.completion_tokens; - } - } + return Effect.gen(function* () { + while (true) { + const next = yield* Effect.tryPromise({ + try: () => iterator.next(), + catch: mapOpenAIError, + }); - yield { - text: "", - inToken: totalInputTokens, - outToken: totalOutputTokens, - model: modelName, - isFinal: true, - }; - } catch (err) { - if (err instanceof OpenAI.RateLimitError) { - throw tooManyRequestsError(); - } - throw err; - } + if (next.done === true) { + return [{ + text: "", + inToken: totalInputTokens, + outToken: totalOutputTokens, + model: modelName, + isFinal: true, + }, "done"] as const; + } + + const chunk = next.value; + const content = chunk.choices[0]?.delta?.content; + if (chunk.usage !== null && chunk.usage !== undefined) { + totalInputTokens = chunk.usage.prompt_tokens; + totalOutputTokens = chunk.usage.completion_tokens; + } + if (content !== null && content !== undefined && content.length > 0) { + return [{ + text: content, + inToken: null, + outToken: null, + model: modelName, + isFinal: false, + }, "pulling"] as const; + } + } + }); + }, + ); + }), + ); + + return toAsyncGenerator(Stream.toAsyncIterable(stream), mapOpenAIError); }, }; } export type OpenAIProcessor = ReturnType; -export function makeOpenAIProcessor(config: OpenAIProcessorConfig): ReturnType { +export function makeOpenAIProcessor( + config: OpenAIProcessorConfig, +): ReturnType { return makeLlmService(config, makeOpenAIProvider(config)); } @@ -156,6 +211,6 @@ export const program = makeFlowProcessorProgram({ ), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/prompt/template.ts b/ts/packages/flow/src/prompt/template.ts index 4c5b1873..7111878e 100644 --- a/ts/packages/flow/src/prompt/template.ts +++ b/ts/packages/flow/src/prompt/template.ts @@ -167,7 +167,7 @@ export function makePromptTemplateService(config: PromptTemplateConfig): PromptT Effect.runPromise(handler(pushedConfig, version)), ); } - console.log("[PromptTemplate] Service initialized"); + Effect.runSync(Effect.log("[PromptTemplate] Service initialized")); return service; } @@ -195,6 +195,6 @@ export const program = makeFlowProcessorProgram({ configHandlers: (config: PromptTemplateConfig) => promptTemplateRuntime(config).configHandlers, }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/query/embeddings/qdrant-doc-service.ts b/ts/packages/flow/src/query/embeddings/qdrant-doc-service.ts index 6666d17c..ee1713db 100644 --- a/ts/packages/flow/src/query/embeddings/qdrant-doc-service.ts +++ b/ts/packages/flow/src/query/embeddings/qdrant-doc-service.ts @@ -101,7 +101,7 @@ export function makeDocEmbeddingsQueryService(config: ProcessorConfig): DocEmbed ), ), }); - console.log("[DocEmbeddingsQuery] Service initialized"); + Effect.runSync(Effect.log("[DocEmbeddingsQuery] Service initialized")); return service; } @@ -113,6 +113,6 @@ export const program = makeFlowProcessorProgram QdrantDocEmbeddingsQueryLive(config), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/query/embeddings/qdrant-doc.ts b/ts/packages/flow/src/query/embeddings/qdrant-doc.ts index 8aa91387..83d8369d 100644 --- a/ts/packages/flow/src/query/embeddings/qdrant-doc.ts +++ b/ts/packages/flow/src/query/embeddings/qdrant-doc.ts @@ -9,7 +9,8 @@ import { QdrantClient } from "@qdrant/js-client-rest"; import { errorMessage } from "@trustgraph/base"; -import { Context, Effect, Layer } from "effect"; +import { Config, Context, Effect, Layer } from "effect"; +import * as O from "effect/Option"; import * as S from "effect/Schema"; export interface QdrantDocQueryConfig { @@ -30,24 +31,58 @@ export interface DocEmbeddingsQueryRequest { limit: number; } +export class QdrantDocEmbeddingsQueryError extends S.TaggedErrorClass()( + "QdrantDocEmbeddingsQueryError", + { + message: S.String, + operation: S.String, + cause: S.DefectWithStack, + }, +) {} + +const qdrantDocEmbeddingsQueryError = (operation: string, cause: unknown) => + QdrantDocEmbeddingsQueryError.make({ + operation, + message: errorMessage(cause), + cause, + }); + +interface ResolvedQdrantDocQueryConfig { + readonly url: string; + readonly apiKey?: string; +} + +const loadQdrantDocQueryConfig = Effect.fn("QdrantDocEmbeddingsQuery.loadConfig")(function* ( + config: QdrantDocQueryConfig, +) { + const envApiKey = O.getOrUndefined(yield* Config.string("QDRANT_API_KEY").pipe(Config.option)); + const apiKey = config.apiKey ?? envApiKey; + return { + url: config.url ?? (yield* Config.string("QDRANT_URL").pipe(Config.withDefault("http://localhost:6333"))), + ...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}), + } satisfies ResolvedQdrantDocQueryConfig; +}); + export interface QdrantDocEmbeddingsQuery { readonly query: (request: DocEmbeddingsQueryRequest) => Promise; + readonly queryEffect: ( + request: DocEmbeddingsQueryRequest, + ) => Effect.Effect, QdrantDocEmbeddingsQueryError>; } export function makeQdrantDocEmbeddingsQuery( config: QdrantDocQueryConfig = {}, ): QdrantDocEmbeddingsQuery { - const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333"; - const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY; + const resolved = Effect.runSync(loadQdrantDocQueryConfig(config)); const client = new QdrantClient({ - url, - ...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}), + url: resolved.url, + ...(resolved.apiKey !== undefined ? { apiKey: resolved.apiKey } : {}), }); - console.log("[QdrantDocQuery] Query service initialized"); + Effect.runSync(Effect.log("[QdrantDocQuery] Query service initialized")); - const query = async (request: DocEmbeddingsQueryRequest): Promise => { + const queryEffect = Effect.fn("QdrantDocEmbeddingsQuery.query")(function* (request: DocEmbeddingsQueryRequest) { const { vector, user, collection, limit } = request; if (vector.length === 0) { @@ -58,18 +93,25 @@ export function makeQdrantDocEmbeddingsQuery( const collectionName = `d_${user}_${collection}_${dim}`; // Check if collection exists -- return empty if not - const exists = await client.collectionExists(collectionName); + const exists = yield* Effect.tryPromise({ + try: () => client.collectionExists(collectionName), + catch: (cause) => qdrantDocEmbeddingsQueryError("collection-exists", cause), + }); if (!exists.exists) { - console.log( + yield* Effect.log( `[QdrantDocQuery] Collection ${collectionName} does not exist, returning empty results`, ); return []; } - const searchResult = await client.search(collectionName, { - vector, - limit, - with_payload: true, + const searchResult = yield* Effect.tryPromise({ + try: () => + client.search(collectionName, { + vector, + limit, + with_payload: true, + }), + catch: (cause) => qdrantDocEmbeddingsQueryError("search", cause), }); const chunks: ChunkMatch[] = []; @@ -86,20 +128,14 @@ export function makeQdrantDocEmbeddingsQuery( } return chunks; + }); + + return { + query: (request) => Effect.runPromise(queryEffect(request)), + queryEffect, }; - - return { query }; } -export class QdrantDocEmbeddingsQueryError extends S.TaggedErrorClass()( - "QdrantDocEmbeddingsQueryError", - { - message: S.String, - operation: S.String, - cause: S.DefectWithStack, - }, -) {} - export interface QdrantDocEmbeddingsQueryServiceShape { readonly query: ( request: DocEmbeddingsQueryRequest, @@ -113,24 +149,12 @@ export class QdrantDocEmbeddingsQueryService extends Context.Service< "@trustgraph/flow/query/embeddings/qdrant-doc/QdrantDocEmbeddingsQueryService", ) {} -const qdrantDocEmbeddingsQueryError = (operation: string, cause: unknown) => - new QdrantDocEmbeddingsQueryError({ - operation, - message: errorMessage(cause), - cause, - }); - export const makeQdrantDocEmbeddingsQueryService = ( config: QdrantDocQueryConfig = {}, ): QdrantDocEmbeddingsQueryServiceShape => { const query = makeQdrantDocEmbeddingsQuery(config); return { - query: Effect.fn("QdrantDocEmbeddingsQuery.query")(function* (request) { - return yield* Effect.tryPromise({ - try: () => query.query(request), - catch: (cause) => qdrantDocEmbeddingsQueryError("query", cause), - }); - }), + query: query.queryEffect, }; }; diff --git a/ts/packages/flow/src/query/embeddings/qdrant-graph-service.ts b/ts/packages/flow/src/query/embeddings/qdrant-graph-service.ts index 7140f30e..0a74e041 100644 --- a/ts/packages/flow/src/query/embeddings/qdrant-graph-service.ts +++ b/ts/packages/flow/src/query/embeddings/qdrant-graph-service.ts @@ -102,7 +102,7 @@ export function makeGraphEmbeddingsQueryService(config: ProcessorConfig): GraphE ), ), }); - console.log("[GraphEmbeddingsQuery] Service initialized"); + Effect.runSync(Effect.log("[GraphEmbeddingsQuery] Service initialized")); return service; } @@ -114,6 +114,6 @@ export const program = makeFlowProcessorProgram QdrantGraphEmbeddingsQueryLive(config), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/query/embeddings/qdrant-graph.ts b/ts/packages/flow/src/query/embeddings/qdrant-graph.ts index 5a71d1c7..e309d08e 100644 --- a/ts/packages/flow/src/query/embeddings/qdrant-graph.ts +++ b/ts/packages/flow/src/query/embeddings/qdrant-graph.ts @@ -12,7 +12,8 @@ import { QdrantClient } from "@qdrant/js-client-rest"; import { errorMessage, type Term } from "@trustgraph/base"; -import { Context, Effect, Layer } from "effect"; +import { Config, Context, Effect, Layer } from "effect"; +import * as O from "effect/Option"; import * as S from "effect/Schema"; export interface QdrantGraphQueryConfig { @@ -32,6 +33,38 @@ export interface GraphEmbeddingsQueryRequest { limit: number; } +export class QdrantGraphEmbeddingsQueryError extends S.TaggedErrorClass()( + "QdrantGraphEmbeddingsQueryError", + { + message: S.String, + operation: S.String, + cause: S.DefectWithStack, + }, +) {} + +const qdrantGraphEmbeddingsQueryError = (operation: string, cause: unknown) => + QdrantGraphEmbeddingsQueryError.make({ + operation, + message: errorMessage(cause), + cause, + }); + +interface ResolvedQdrantGraphQueryConfig { + readonly url: string; + readonly apiKey?: string; +} + +const loadQdrantGraphQueryConfig = Effect.fn("QdrantGraphEmbeddingsQuery.loadConfig")(function* ( + config: QdrantGraphQueryConfig, +) { + const envApiKey = O.getOrUndefined(yield* Config.string("QDRANT_API_KEY").pipe(Config.option)); + const apiKey = config.apiKey ?? envApiKey; + return { + url: config.url ?? (yield* Config.string("QDRANT_URL").pipe(Config.withDefault("http://localhost:6333"))), + ...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}), + } satisfies ResolvedQdrantGraphQueryConfig; +}); + function createTerm(value: string): Term { if (value.startsWith("http://") || value.startsWith("https://")) { return { type: "IRI", iri: value }; @@ -41,22 +74,26 @@ function createTerm(value: string): Term { export interface QdrantGraphEmbeddingsQuery { readonly query: (request: GraphEmbeddingsQueryRequest) => Promise; + readonly queryEffect: ( + request: GraphEmbeddingsQueryRequest, + ) => Effect.Effect, QdrantGraphEmbeddingsQueryError>; } export function makeQdrantGraphEmbeddingsQuery( config: QdrantGraphQueryConfig = {}, ): QdrantGraphEmbeddingsQuery { - const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333"; - const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY; + const resolved = Effect.runSync(loadQdrantGraphQueryConfig(config)); const client = new QdrantClient({ - url, - ...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}), + url: resolved.url, + ...(resolved.apiKey !== undefined ? { apiKey: resolved.apiKey } : {}), }); - console.log("[QdrantGraphQuery] Query service initialized"); + Effect.runSync(Effect.log("[QdrantGraphQuery] Query service initialized")); - const query = async (request: GraphEmbeddingsQueryRequest): Promise => { + const queryEffect = Effect.fn("QdrantGraphEmbeddingsQuery.query")(function* ( + request: GraphEmbeddingsQueryRequest, + ) { const { vector, user, collection, limit } = request; if (vector.length === 0) { @@ -67,9 +104,12 @@ export function makeQdrantGraphEmbeddingsQuery( const collectionName = `t_${user}_${collection}_${dim}`; // Check if collection exists -- return empty if not - const exists = await client.collectionExists(collectionName); + const exists = yield* Effect.tryPromise({ + try: () => client.collectionExists(collectionName), + catch: (cause) => qdrantGraphEmbeddingsQueryError("collection-exists", cause), + }); if (!exists.exists) { - console.log( + yield* Effect.log( `[QdrantGraphQuery] Collection ${collectionName} does not exist, returning empty results`, ); return []; @@ -77,10 +117,14 @@ export function makeQdrantGraphEmbeddingsQuery( // Query 2x the limit so we have a better chance of getting `limit` // unique entities after deduplication (same heuristic as Python impl) - const searchResult = await client.search(collectionName, { - vector, - limit: limit * 2, - with_payload: true, + const searchResult = yield* Effect.tryPromise({ + try: () => + client.search(collectionName, { + vector, + limit: limit * 2, + with_payload: true, + }), + catch: (cause) => qdrantGraphEmbeddingsQueryError("search", cause), }); const entitySet = new Set(); @@ -106,20 +150,14 @@ export function makeQdrantGraphEmbeddingsQuery( } return entities; + }); + + return { + query: (request) => Effect.runPromise(queryEffect(request)), + queryEffect, }; - - return { query }; } -export class QdrantGraphEmbeddingsQueryError extends S.TaggedErrorClass()( - "QdrantGraphEmbeddingsQueryError", - { - message: S.String, - operation: S.String, - cause: S.DefectWithStack, - }, -) {} - export interface QdrantGraphEmbeddingsQueryServiceShape { readonly query: ( request: GraphEmbeddingsQueryRequest, @@ -133,24 +171,12 @@ export class QdrantGraphEmbeddingsQueryService extends Context.Service< "@trustgraph/flow/query/embeddings/qdrant-graph/QdrantGraphEmbeddingsQueryService", ) {} -const qdrantGraphEmbeddingsQueryError = (operation: string, cause: unknown) => - new QdrantGraphEmbeddingsQueryError({ - operation, - message: errorMessage(cause), - cause, - }); - export const makeQdrantGraphEmbeddingsQueryService = ( config: QdrantGraphQueryConfig = {}, ): QdrantGraphEmbeddingsQueryServiceShape => { const query = makeQdrantGraphEmbeddingsQuery(config); return { - query: Effect.fn("QdrantGraphEmbeddingsQuery.query")(function* (request) { - return yield* Effect.tryPromise({ - try: () => query.query(request), - catch: (cause) => qdrantGraphEmbeddingsQueryError("query", cause), - }); - }), + query: query.queryEffect, }; }; diff --git a/ts/packages/flow/src/query/triples/falkordb-service.ts b/ts/packages/flow/src/query/triples/falkordb-service.ts index 1be6a406..5c612474 100644 --- a/ts/packages/flow/src/query/triples/falkordb-service.ts +++ b/ts/packages/flow/src/query/triples/falkordb-service.ts @@ -88,7 +88,7 @@ export function makeTriplesQueryService(config: ProcessorConfig): TriplesQuerySe ), ), }); - console.log("[TriplesQuery] Service initialized"); + void Effect.runPromise(Effect.log("[TriplesQuery] Service initialized")); return service; } @@ -100,6 +100,6 @@ export const program = makeFlowProcessorProgram FalkorDBTriplesQueryLive(config), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/query/triples/falkordb.ts b/ts/packages/flow/src/query/triples/falkordb.ts index ec4322ed..8aa1558d 100644 --- a/ts/packages/flow/src/query/triples/falkordb.ts +++ b/ts/packages/flow/src/query/triples/falkordb.ts @@ -1,5 +1,5 @@ /** - * FalkorDB triples query service — queries RDF triples from FalkorDB. + * FalkorDB triples query service - queries RDF triples from FalkorDB. * * Implements all SPO query patterns (S, P, O, SP, SO, PO, SPO, *). * @@ -8,7 +8,7 @@ import { createClient, Graph } from "falkordb"; import { errorMessage, type Term, type Triple } from "@trustgraph/base"; -import { Context, Effect, Layer } from "effect"; +import { Config, Context, Effect, Layer } from "effect"; import * as S from "effect/Schema"; export interface FalkorDBQueryConfig { @@ -19,10 +19,14 @@ export interface FalkorDBQueryConfig { function termToValue(term: Term | undefined): string | null { if (term === undefined) return null; switch (term.type) { - case "IRI": return term.iri; - case "LITERAL": return term.value; - case "BLANK": return term.id; - default: return null; + case "IRI": + return term.iri; + case "LITERAL": + return term.value; + case "BLANK": + return term.id; + default: + return null; } } @@ -36,7 +40,6 @@ function createTerm(value: string): Term { return { type: "LITERAL", value }; } -/** Extract a string field from a FalkorDB result row (returns object with named keys). */ function field(row: unknown, key: string): string { return (row as Record)?.[key] as string ?? ""; } @@ -50,231 +53,6 @@ export interface FalkorDBTriplesQuery { ) => Promise; } -export function makeFalkorDBTriplesQuery( - config: FalkorDBQueryConfig = {}, -): FalkorDBTriplesQuery { - const url = config.url ?? process.env.FALKORDB_URL ?? "redis://localhost:6379"; - const database = config.database ?? "falkordb"; - - const client = createClient({ url }); - const graph = new Graph(client, database); - const connectPromise = client.connect().then(() => { - console.log(`[FalkorDBTriplesQuery] Connected to ${url}, graph: ${database}`); - }).catch((err) => { - console.error(`[FalkorDBTriplesQuery] Connection failed:`, err); - throw err; - }); - - const ensureConnected = async (): Promise => { - await connectPromise; - }; - - const matchPattern = async ( - out: [string, string, string][], - sv: string, pv: string, ov: string, limit: number, - ): Promise => { - for (const destType of ["Literal", "Node"] as const) { - const destKey = destType === "Literal" ? "value" : "uri"; - const result = await graph.query( - `MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:${destType} {${destKey}: $dest}) ` + - `RETURN src.uri LIMIT ${limit}`, - { params: { src: sv, rel: pv, dest: ov } }, - ); - for (const _rec of (result.data ?? [])) { - out.push([sv, pv, ov]); - } - } - }; - - const matchSP = async ( - out: [string, string, string][], - sv: string, pv: string, limit: number, - ): Promise => { - // Literals - const litResult = await graph.query( - `MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal) ` + - `RETURN dest.value as dest LIMIT ${limit}`, - { params: { src: sv, rel: pv } }, - ); - for (const rec of (litResult.data ?? [])) { - out.push([sv, pv, field(rec, "dest")]); - } - // Nodes - const nodeResult = await graph.query( - `MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node) ` + - `RETURN dest.uri as dest LIMIT ${limit}`, - { params: { src: sv, rel: pv } }, - ); - for (const rec of (nodeResult.data ?? [])) { - out.push([sv, pv, field(rec, "dest")]); - } - }; - - const matchSO = async ( - out: [string, string, string][], - sv: string, ov: string, limit: number, - ): Promise => { - for (const [destType, destKey] of [["Literal", "value"], ["Node", "uri"]] as const) { - const result = await graph.query( - `MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:${destType} {${destKey}: $dest}) ` + - `RETURN rel.uri as rel LIMIT ${limit}`, - { params: { src: sv, dest: ov } }, - ); - for (const rec of (result.data ?? [])) { - out.push([sv, field(rec, "rel"), ov]); - } - } - }; - - const matchPO = async ( - out: [string, string, string][], - pv: string, ov: string, limit: number, - ): Promise => { - for (const [destType, destKey] of [["Literal", "value"], ["Node", "uri"]] as const) { - const result = await graph.query( - `MATCH (src:Node)-[rel:Rel {uri: $rel}]->(dest:${destType} {${destKey}: $dest}) ` + - `RETURN src.uri as src LIMIT ${limit}`, - { params: { rel: pv, dest: ov } }, - ); - for (const rec of (result.data ?? [])) { - out.push([field(rec, "src"), pv, ov]); - } - } - }; - - const matchS = async ( - out: [string, string, string][], - sv: string, limit: number, - ): Promise => { - const litResult = await graph.query( - `MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal) ` + - `RETURN rel.uri as rel, dest.value as dest LIMIT ${limit}`, - { params: { src: sv } }, - ); - for (const rec of (litResult.data ?? [])) { - out.push([sv, field(rec, "rel"), field(rec, "dest")]); - } - const nodeResult = await graph.query( - `MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node) ` + - `RETURN rel.uri as rel, dest.uri as dest LIMIT ${limit}`, - { params: { src: sv } }, - ); - for (const rec of (nodeResult.data ?? [])) { - out.push([sv, field(rec, "rel"), field(rec, "dest")]); - } - }; - - const matchP = async ( - out: [string, string, string][], - pv: string, limit: number, - ): Promise => { - const litResult = await graph.query( - `MATCH (src:Node)-[rel:Rel {uri: $rel}]->(dest:Literal) ` + - `RETURN src.uri as src, dest.value as dest LIMIT ${limit}`, - { params: { rel: pv } }, - ); - for (const rec of (litResult.data ?? [])) { - out.push([field(rec, "src"), pv, field(rec, "dest")]); - } - const nodeResult = await graph.query( - `MATCH (src:Node)-[rel:Rel {uri: $rel}]->(dest:Node) ` + - `RETURN src.uri as src, dest.uri as dest LIMIT ${limit}`, - { params: { rel: pv } }, - ); - for (const rec of (nodeResult.data ?? [])) { - out.push([field(rec, "src"), pv, field(rec, "dest")]); - } - }; - - const matchO = async ( - out: [string, string, string][], - ov: string, limit: number, - ): Promise => { - for (const [destType, destKey] of [["Literal", "value"], ["Node", "uri"]] as const) { - const result = await graph.query( - `MATCH (src:Node)-[rel:Rel]->(dest:${destType} {${destKey}: $dest}) ` + - `RETURN src.uri as src, rel.uri as rel LIMIT ${limit}`, - { params: { dest: ov } }, - ); - for (const rec of (result.data ?? [])) { - out.push([field(rec, "src"), field(rec, "rel"), ov]); - } - } - }; - - const matchAll = async ( - out: [string, string, string][], - limit: number, - ): Promise => { - const litResult = await graph.query( - `MATCH (src:Node)-[rel:Rel]->(dest:Literal) ` + - `RETURN src.uri as src, rel.uri as rel, dest.value as dest LIMIT ${limit}`, - ); - for (const rec of (litResult.data ?? [])) { - out.push([field(rec, "src"), field(rec, "rel"), field(rec, "dest")]); - } - const nodeResult = await graph.query( - `MATCH (src:Node)-[rel:Rel]->(dest:Node) ` + - `RETURN src.uri as src, rel.uri as rel, dest.uri as dest LIMIT ${limit}`, - ); - for (const rec of (nodeResult.data ?? [])) { - out.push([field(rec, "src"), field(rec, "rel"), field(rec, "dest")]); - } - }; - - const queryTriples = async ( - s?: Term, - p?: Term, - o?: Term, - limit = 100, - ): Promise => { - await ensureConnected(); - const sv = termToValue(s); - const pv = termToValue(p); - const ov = termToValue(o); - - const rawTriples: [string, string, string][] = []; - - // Query both Node and Literal targets for each pattern - if (sv !== null && pv !== null && ov !== null) { - // SPO — exact match - await matchPattern(rawTriples, sv, pv, ov, limit); - } else if (sv !== null && pv !== null) { - // SP — known subject + predicate - await matchSP(rawTriples, sv, pv, limit); - } else if (sv !== null && ov !== null) { - // SO — known subject + object - await matchSO(rawTriples, sv, ov, limit); - } else if (pv !== null && ov !== null) { - // PO — known predicate + object - await matchPO(rawTriples, pv, ov, limit); - } else if (sv !== null) { - // S only - await matchS(rawTriples, sv, limit); - } else if (pv !== null) { - // P only - await matchP(rawTriples, pv, limit); - } else if (ov !== null) { - // O only - await matchO(rawTriples, ov, limit); - } else { - // Wildcard — all triples - await matchAll(rawTriples, limit); - } - - return rawTriples - .filter(([s, p, o]) => s !== null && p !== null && o !== null) - .slice(0, limit) - .map(([s, p, o]) => ({ - s: createTerm(s), - p: createTerm(p), - o: createTerm(o), - })); - }; - - return { queryTriples }; -} - export class FalkorDBTriplesQueryError extends S.TaggedErrorClass()( "FalkorDBTriplesQueryError", { @@ -300,31 +78,358 @@ export class FalkorDBTriplesQueryService extends Context.Service< "@trustgraph/flow/query/triples/falkordb/FalkorDBTriplesQueryService", ) {} -const falkorDBTriplesQueryError = (operation: string, cause: unknown) => - new FalkorDBTriplesQueryError({ +const falkorDBTriplesQueryError = (operation: string, cause: unknown): FalkorDBTriplesQueryError => + FalkorDBTriplesQueryError.make({ operation, message: errorMessage(cause), cause, }); -export const makeFalkorDBTriplesQueryService = ( +interface FalkorDBQueryConnection { + readonly graph: Graph; +} + +type FalkorDBQueryOptions = Parameters[1]; + +const resolveFalkorDBQueryConfig = Effect.fn("FalkorDBTriplesQuery.resolveConfig")(function* ( + config: FalkorDBQueryConfig, +) { + const url = config.url ?? (yield* Config.string("FALKORDB_URL").pipe( + Config.withDefault("redis://localhost:6379"), + Effect.mapError((cause) => falkorDBTriplesQueryError("config", cause)), + )); + return { + url, + database: config.database ?? "falkordb", + }; +}); + +const connectFalkorDBTriplesQuery = ( + config: FalkorDBQueryConfig, +): Effect.Effect => + Effect.gen(function* () { + const { url, database } = yield* resolveFalkorDBQueryConfig(config); + const { client, graph } = yield* Effect.try({ + try: () => { + const client = createClient({ url }); + return { client, graph: new Graph(client, database) }; + }, + catch: (cause) => falkorDBTriplesQueryError("create-client", cause), + }); + + yield* Effect.tryPromise({ + try: () => client.connect(), + catch: (cause) => falkorDBTriplesQueryError("connect", cause), + }).pipe( + Effect.tapError((error) => + Effect.logError("[FalkorDBTriplesQuery] Connection failed", { + error: error.message, + operation: error.operation, + }) + ), + ); + + yield* Effect.log(`[FalkorDBTriplesQuery] Connected to ${url}, graph: ${database}`); + return { graph }; + }); + +const queryRows = ( + graph: Graph, + operation: string, + query: string, + options?: FalkorDBQueryOptions, +): Effect.Effect, FalkorDBTriplesQueryError> => + Effect.tryPromise({ + try: () => graph.query(query, options), + catch: (cause) => falkorDBTriplesQueryError(operation, cause), + }).pipe( + Effect.map((result) => result.data ?? []), + ); + +const matchPattern = ( + graph: Graph, + out: [string, string, string][], + sv: string, + pv: string, + ov: string, + limit: number, +): Effect.Effect => + Effect.gen(function* () { + for (const destType of ["Literal", "Node"] as const) { + const destKey = destType === "Literal" ? "value" : "uri"; + const rows = yield* queryRows( + graph, + "match-spo", + `MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:${destType} {${destKey}: $dest}) ` + + `RETURN src.uri LIMIT ${limit}`, + { params: { src: sv, rel: pv, dest: ov } }, + ); + for (const _rec of rows) { + out.push([sv, pv, ov]); + } + } + }); + +const matchSP = ( + graph: Graph, + out: [string, string, string][], + sv: string, + pv: string, + limit: number, +): Effect.Effect => + Effect.gen(function* () { + const litRows = yield* queryRows( + graph, + "match-sp-literal", + `MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal) ` + + `RETURN dest.value as dest LIMIT ${limit}`, + { params: { src: sv, rel: pv } }, + ); + for (const rec of litRows) { + out.push([sv, pv, field(rec, "dest")]); + } + + const nodeRows = yield* queryRows( + graph, + "match-sp-node", + `MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node) ` + + `RETURN dest.uri as dest LIMIT ${limit}`, + { params: { src: sv, rel: pv } }, + ); + for (const rec of nodeRows) { + out.push([sv, pv, field(rec, "dest")]); + } + }); + +const matchSO = ( + graph: Graph, + out: [string, string, string][], + sv: string, + ov: string, + limit: number, +): Effect.Effect => + Effect.gen(function* () { + for (const [destType, destKey] of [["Literal", "value"], ["Node", "uri"]] as const) { + const rows = yield* queryRows( + graph, + "match-so", + `MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:${destType} {${destKey}: $dest}) ` + + `RETURN rel.uri as rel LIMIT ${limit}`, + { params: { src: sv, dest: ov } }, + ); + for (const rec of rows) { + out.push([sv, field(rec, "rel"), ov]); + } + } + }); + +const matchPO = ( + graph: Graph, + out: [string, string, string][], + pv: string, + ov: string, + limit: number, +): Effect.Effect => + Effect.gen(function* () { + for (const [destType, destKey] of [["Literal", "value"], ["Node", "uri"]] as const) { + const rows = yield* queryRows( + graph, + "match-po", + `MATCH (src:Node)-[rel:Rel {uri: $rel}]->(dest:${destType} {${destKey}: $dest}) ` + + `RETURN src.uri as src LIMIT ${limit}`, + { params: { rel: pv, dest: ov } }, + ); + for (const rec of rows) { + out.push([field(rec, "src"), pv, ov]); + } + } + }); + +const matchS = ( + graph: Graph, + out: [string, string, string][], + sv: string, + limit: number, +): Effect.Effect => + Effect.gen(function* () { + const litRows = yield* queryRows( + graph, + "match-s-literal", + `MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal) ` + + `RETURN rel.uri as rel, dest.value as dest LIMIT ${limit}`, + { params: { src: sv } }, + ); + for (const rec of litRows) { + out.push([sv, field(rec, "rel"), field(rec, "dest")]); + } + + const nodeRows = yield* queryRows( + graph, + "match-s-node", + `MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node) ` + + `RETURN rel.uri as rel, dest.uri as dest LIMIT ${limit}`, + { params: { src: sv } }, + ); + for (const rec of nodeRows) { + out.push([sv, field(rec, "rel"), field(rec, "dest")]); + } + }); + +const matchP = ( + graph: Graph, + out: [string, string, string][], + pv: string, + limit: number, +): Effect.Effect => + Effect.gen(function* () { + const litRows = yield* queryRows( + graph, + "match-p-literal", + `MATCH (src:Node)-[rel:Rel {uri: $rel}]->(dest:Literal) ` + + `RETURN src.uri as src, dest.value as dest LIMIT ${limit}`, + { params: { rel: pv } }, + ); + for (const rec of litRows) { + out.push([field(rec, "src"), pv, field(rec, "dest")]); + } + + const nodeRows = yield* queryRows( + graph, + "match-p-node", + `MATCH (src:Node)-[rel:Rel {uri: $rel}]->(dest:Node) ` + + `RETURN src.uri as src, dest.uri as dest LIMIT ${limit}`, + { params: { rel: pv } }, + ); + for (const rec of nodeRows) { + out.push([field(rec, "src"), pv, field(rec, "dest")]); + } + }); + +const matchO = ( + graph: Graph, + out: [string, string, string][], + ov: string, + limit: number, +): Effect.Effect => + Effect.gen(function* () { + for (const [destType, destKey] of [["Literal", "value"], ["Node", "uri"]] as const) { + const rows = yield* queryRows( + graph, + "match-o", + `MATCH (src:Node)-[rel:Rel]->(dest:${destType} {${destKey}: $dest}) ` + + `RETURN src.uri as src, rel.uri as rel LIMIT ${limit}`, + { params: { dest: ov } }, + ); + for (const rec of rows) { + out.push([field(rec, "src"), field(rec, "rel"), ov]); + } + } + }); + +const matchAll = ( + graph: Graph, + out: [string, string, string][], + limit: number, +): Effect.Effect => + Effect.gen(function* () { + const litRows = yield* queryRows( + graph, + "match-all-literal", + `MATCH (src:Node)-[rel:Rel]->(dest:Literal) ` + + `RETURN src.uri as src, rel.uri as rel, dest.value as dest LIMIT ${limit}`, + ); + for (const rec of litRows) { + out.push([field(rec, "src"), field(rec, "rel"), field(rec, "dest")]); + } + + const nodeRows = yield* queryRows( + graph, + "match-all-node", + `MATCH (src:Node)-[rel:Rel]->(dest:Node) ` + + `RETURN src.uri as src, rel.uri as rel, dest.uri as dest LIMIT ${limit}`, + ); + for (const rec of nodeRows) { + out.push([field(rec, "src"), field(rec, "rel"), field(rec, "dest")]); + } + }); + +const queryTriplesEffect = ( + getConnection: () => Effect.Effect, + s: Term | undefined, + p: Term | undefined, + o: Term | undefined, + limit: number, +): Effect.Effect, FalkorDBTriplesQueryError> => + Effect.gen(function* () { + const { graph } = yield* getConnection(); + const sv = termToValue(s); + const pv = termToValue(p); + const ov = termToValue(o); + const rawTriples: [string, string, string][] = []; + + if (sv !== null && pv !== null && ov !== null) { + yield* matchPattern(graph, rawTriples, sv, pv, ov, limit); + } else if (sv !== null && pv !== null) { + yield* matchSP(graph, rawTriples, sv, pv, limit); + } else if (sv !== null && ov !== null) { + yield* matchSO(graph, rawTriples, sv, ov, limit); + } else if (pv !== null && ov !== null) { + yield* matchPO(graph, rawTriples, pv, ov, limit); + } else if (sv !== null) { + yield* matchS(graph, rawTriples, sv, limit); + } else if (pv !== null) { + yield* matchP(graph, rawTriples, pv, limit); + } else if (ov !== null) { + yield* matchO(graph, rawTriples, ov, limit); + } else { + yield* matchAll(graph, rawTriples, limit); + } + + return rawTriples + .slice(0, limit) + .map(([subject, predicate, object]) => ({ + s: createTerm(subject), + p: createTerm(predicate), + o: createTerm(object), + })); + }); + +const makeFalkorDBTriplesQueryEffect = ( config: FalkorDBQueryConfig = {}, ): FalkorDBTriplesQueryServiceShape => { - const query = makeFalkorDBTriplesQuery(config); + let cachedConnection: Effect.Effect | undefined; + + const getConnection = Effect.fn("FalkorDBTriplesQuery.connection")(function* () { + if (cachedConnection === undefined) { + cachedConnection = yield* Effect.cached(connectFalkorDBTriplesQuery(config)); + } + return yield* cachedConnection; + }); + return { queryTriples: Effect.fn("FalkorDBTriplesQuery.queryTriples")(( s: Term | undefined, p: Term | undefined, o: Term | undefined, limit: number, - ) => - Effect.tryPromise({ - try: () => query.queryTriples(s, p, o, limit), - catch: (cause) => falkorDBTriplesQueryError("query-triples", cause), - })), + ) => queryTriplesEffect(getConnection, s, p, o, limit)), }; }; +export function makeFalkorDBTriplesQuery( + config: FalkorDBQueryConfig = {}, +): FalkorDBTriplesQuery { + const query = makeFalkorDBTriplesQueryEffect(config); + return { + queryTriples: (s, p, o, limit = 100) => + Effect.runPromise(query.queryTriples(s, p, o, limit)).then((triples) => Array.from(triples)), + }; +} + +export const makeFalkorDBTriplesQueryService = ( + config: FalkorDBQueryConfig = {}, +): FalkorDBTriplesQueryServiceShape => makeFalkorDBTriplesQueryEffect(config); + export const FalkorDBTriplesQueryLive = ( config: FalkorDBQueryConfig = {}, ): Layer.Layer => diff --git a/ts/packages/flow/src/retrieval/document-rag-service.ts b/ts/packages/flow/src/retrieval/document-rag-service.ts index fd7d372d..ddb3bc27 100644 --- a/ts/packages/flow/src/retrieval/document-rag-service.ts +++ b/ts/packages/flow/src/retrieval/document-rag-service.ts @@ -161,6 +161,6 @@ export const program = makeFlowProcessorProgram({ layer: () => DocumentRagLive, }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/retrieval/document-rag.ts b/ts/packages/flow/src/retrieval/document-rag.ts index d09137c1..d614032f 100644 --- a/ts/packages/flow/src/retrieval/document-rag.ts +++ b/ts/packages/flow/src/retrieval/document-rag.ts @@ -56,7 +56,7 @@ export class DocumentRagEngine extends Context.Service - new DocumentRagEngineError({ + DocumentRagEngineError.make({ operation, cause, message: errorMessage(cause), @@ -68,11 +68,7 @@ export function makeDocumentRagEngine(): DocumentRagEngineShape { clients: DocumentRagClients, queryText: string, options?: DocumentRagQueryOptions, - ) => - Effect.tryPromise({ - try: () => queryDocumentRag(clients, queryText, options), - catch: (cause) => documentRagError("query", cause), - }), + ) => queryDocumentRag(clients, queryText, options), ), }; } @@ -97,40 +93,54 @@ export function makeDocumentRag(clients: DocumentRagClients): DocumentRag { }; } -async function queryDocumentRag( +function queryDocumentRag( clients: DocumentRagClients, queryText: string, options?: DocumentRagQueryOptions, -): Promise { - const collection = options?.collection ?? "default"; +): Effect.Effect { + return Effect.gen(function* () { + const collection = options?.collection ?? "default"; - const embResp = await clients.embeddings.request({ text: [queryText] }); - const vectors = embResp.vectors; + const embResp = yield* Effect.tryPromise({ + try: () => clients.embeddings.request({ text: [queryText] }), + catch: (cause) => documentRagError("embeddings", cause), + }); + const vectors = embResp.vectors; - const docResp = await clients.docEmbeddings.request({ - vectors, - limit: 10, - collection, - user: "default", + const docResp = yield* Effect.tryPromise({ + try: () => clients.docEmbeddings.request({ + vectors, + limit: 10, + collection, + user: "default", + }), + catch: (cause) => documentRagError("document-embeddings", cause), + }); + const chunks = docResp.chunks ?? []; + yield* Effect.log(`[DocumentRag] Found ${chunks.length} matching chunks`); + + const context = chunks + .flatMap((chunk) => + chunk.content !== undefined && chunk.content.length > 0 ? [chunk.content] : [], + ) + .join("\n\n---\n\n"); + + const promptResp = yield* Effect.tryPromise({ + try: () => clients.prompt.request({ + name: "document-rag-synthesize", + variables: { query: queryText, context }, + }), + catch: (cause) => documentRagError("prompt", cause), + }); + + const resp = yield* Effect.tryPromise({ + try: () => clients.llm.request({ + system: promptResp.system, + prompt: promptResp.prompt, + }), + catch: (cause) => documentRagError("llm", cause), + }); + + return resp.response; }); - const chunks = docResp.chunks ?? []; - console.log(`[DocumentRag] Found ${chunks.length} matching chunks`); - - const context = chunks - .flatMap((chunk) => - chunk.content !== undefined && chunk.content.length > 0 ? [chunk.content] : [], - ) - .join("\n\n---\n\n"); - - const promptResp = await clients.prompt.request({ - name: "document-rag-synthesize", - variables: { query: queryText, context }, - }); - - const resp = await clients.llm.request({ - system: promptResp.system, - prompt: promptResp.prompt, - }); - - return resp.response; } diff --git a/ts/packages/flow/src/retrieval/graph-rag-service.ts b/ts/packages/flow/src/retrieval/graph-rag-service.ts index 48b4051e..b1d72285 100644 --- a/ts/packages/flow/src/retrieval/graph-rag-service.ts +++ b/ts/packages/flow/src/retrieval/graph-rag-service.ts @@ -192,6 +192,6 @@ export const program = makeFlowProcessorProgram({ layer: () => GraphRagLive, }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/retrieval/graph-rag.ts b/ts/packages/flow/src/retrieval/graph-rag.ts index 11e24a19..a56c8fcd 100644 --- a/ts/packages/flow/src/retrieval/graph-rag.ts +++ b/ts/packages/flow/src/retrieval/graph-rag.ts @@ -86,7 +86,7 @@ export class GraphRagEngine extends Context.Service - new GraphRagEngineError({ + GraphRagEngineError.make({ operation, cause, message: errorMessage(cause), @@ -110,11 +110,7 @@ export function makeGraphRagEngine(): GraphRagEngineShape { queryText: string, options?: GraphRagQueryOptions, config?: GraphRagConfig, - ) => - Effect.tryPromise({ - try: () => queryGraphRag(clients, queryText, options, config), - catch: (cause) => graphRagError("query", cause), - }), + ) => queryGraphRag(clients, queryText, options, config), ), }; } @@ -142,239 +138,283 @@ export function makeGraphRag( }; } -async function queryGraphRag( +function queryGraphRag( clients: GraphRagClients, queryText: string, options?: GraphRagQueryOptions, rawConfig?: GraphRagConfig, -): Promise { - const config = normalizeGraphRagConfig(rawConfig); - console.log(`[GraphRag] Query: "${queryText.slice(0, 80)}..."`); +): Effect.Effect { + return Effect.gen(function* () { + const config = normalizeGraphRagConfig(rawConfig); + yield* Effect.log(`[GraphRag] Query: "${queryText.slice(0, 80)}..."`); - const concepts = await extractConcepts(clients, queryText); - console.log(`[GraphRag] Step 1: extracted ${concepts.length} concepts: ${concepts.slice(0, 5).join(", ")}`); + const concepts = yield* extractConcepts(clients, queryText); + yield* Effect.log(`[GraphRag] Step 1: extracted ${concepts.length} concepts: ${concepts.slice(0, 5).join(", ")}`); - const vectors = await getVectors(clients, concepts); - console.log(`[GraphRag] Step 2: got ${vectors.length} vectors (dim=${vectors[0]?.length ?? 0})`); + const vectors = yield* getVectors(clients, concepts); + yield* Effect.log(`[GraphRag] Step 2: got ${vectors.length} vectors (dim=${vectors[0]?.length ?? 0})`); - const entities = await getEntities(clients, config, vectors, options?.collection); - console.log(`[GraphRag] Step 3: found ${entities.length} matching entities`); + const entities = yield* getEntities(clients, config, vectors, options?.collection); + yield* Effect.log(`[GraphRag] Step 3: found ${entities.length} matching entities`); - const subgraph = await followEdges(clients, config, entities, options?.collection); - console.log(`[GraphRag] Step 4: traversed graph, ${subgraph.length} triples in subgraph`); + const subgraph = yield* followEdges(clients, config, entities, options?.collection); + yield* Effect.log(`[GraphRag] Step 4: traversed graph, ${subgraph.length} triples in subgraph`); - const scoredEdges = await scoreEdges(clients, config, queryText, subgraph); - console.log(`[GraphRag] Step 5: scored down to ${scoredEdges.length} edges`); + const scoredEdges = yield* scoreEdges(clients, config, queryText, subgraph); + yield* Effect.log(`[GraphRag] Step 5: scored down to ${scoredEdges.length} edges`); - console.log(`[GraphRag] Step 6: synthesizing answer from ${scoredEdges.length} edges...`); - const answer = await synthesize( - clients, - queryText, - scoredEdges, - options?.chunkCallback, - ); - console.log(`[GraphRag] Step 6: done (${answer.length} chars)`); + yield* Effect.log(`[GraphRag] Step 6: synthesizing answer from ${scoredEdges.length} edges...`); + const answer = yield* synthesize( + clients, + queryText, + scoredEdges, + options?.chunkCallback, + ); + yield* Effect.log(`[GraphRag] Step 6: done (${answer.length} chars)`); - return { answer, subgraph: scoredEdges }; -} - -async function extractConcepts(clients: GraphRagClients, query: string): Promise { - const promptResp = await clients.prompt.request({ - name: "extract-concepts", - variables: { query }, + return { answer, subgraph: scoredEdges }; }); +} - const llmResp = await clients.llm.request({ - system: promptResp.system, - prompt: promptResp.prompt, +function extractConcepts(clients: GraphRagClients, query: string): Effect.Effect { + return Effect.gen(function* () { + const promptResp = yield* Effect.tryPromise({ + try: () => clients.prompt.request({ + name: "extract-concepts", + variables: { query }, + }), + catch: (cause) => graphRagError("extract-concepts-prompt", cause), + }); + + const llmResp = yield* Effect.tryPromise({ + try: () => clients.llm.request({ + system: promptResp.system, + prompt: promptResp.prompt, + }), + catch: (cause) => graphRagError("extract-concepts-llm", cause), + }); + + return llmResp.response + .split("\n") + .map((concept) => concept.trim()) + .filter((concept) => concept.length > 0); }); - - return llmResp.response - .split("\n") - .map((concept) => concept.trim()) - .filter((concept) => concept.length > 0); } -async function getVectors(clients: GraphRagClients, concepts: string[]): Promise { - const resp = await clients.embeddings.request({ text: concepts }); - return resp.vectors; +function getVectors(clients: GraphRagClients, concepts: string[]): Effect.Effect { + return Effect.gen(function* () { + const resp = yield* Effect.tryPromise({ + try: () => clients.embeddings.request({ text: concepts }), + catch: (cause) => graphRagError("get-vectors", cause), + }); + return resp.vectors; + }); } -async function getEntities( +function getEntities( clients: GraphRagClients, config: NormalizedGraphRagConfig, vectors: number[][], collection?: string, -): Promise { - const resp = await clients.graphEmbeddings.request({ - vectors, - user: "default", - collection: collection ?? "default", - limit: config.entityLimit, +): Effect.Effect { + return Effect.gen(function* () { + const resp = yield* Effect.tryPromise({ + try: () => clients.graphEmbeddings.request({ + vectors, + user: "default", + collection: collection ?? "default", + limit: config.entityLimit, + }), + catch: (cause) => graphRagError("get-entities", cause), + }); + return resp.entities; }); - return resp.entities; } -async function followEdges( +function followEdges( clients: GraphRagClients, config: NormalizedGraphRagConfig, entities: Term[], collection?: string, -): Promise { - const visited = new Set(); - const subgraph: Triple[] = []; - let currentLevel = new Set( - entities.map((entity) => termToString(entity)), - ); +): Effect.Effect { + return Effect.gen(function* () { + const visited = new Set(); + const subgraph: Triple[] = []; + let currentLevel = new Set( + entities.map((entity) => termToString(entity)), + ); - for (let depth = 0; depth < config.maxPathLength; depth++) { - if (currentLevel.size === 0 || subgraph.length >= config.maxSubgraphSize) { - break; - } + for (let depth = 0; depth < config.maxPathLength; depth++) { + if (currentLevel.size === 0 || subgraph.length >= config.maxSubgraphSize) { + break; + } - const unvisited = [...currentLevel].filter((entity) => !visited.has(entity)); - if (unvisited.length === 0) break; + const unvisited = [...currentLevel].filter((entity) => !visited.has(entity)); + if (unvisited.length === 0) break; - const queries = unvisited.map((entityStr) => { - const term = stringToTerm(entityStr); - const request: TriplesQueryRequest = { - s: term, - limit: config.tripleLimit, - ...(collection !== undefined ? { collection } : {}), - }; - return clients.triples.request(request); - }); + const queries = unvisited.map((entityStr) => { + const term = stringToTerm(entityStr); + const request: TriplesQueryRequest = { + s: term, + limit: config.tripleLimit, + ...(collection !== undefined ? { collection } : {}), + }; + return Effect.tryPromise({ + try: () => clients.triples.request(request), + catch: (cause) => graphRagError("follow-edges-query", cause), + }); + }); - const results = await Promise.all(queries); - const nextLevel = new Set(); + const results = yield* Effect.all(queries); + const nextLevel = new Set(); - for (const result of results) { - for (const triple of result.triples) { - subgraph.push(triple); + for (const result of results) { + for (const triple of result.triples) { + subgraph.push(triple); - if (depth < config.maxPathLength - 1) { - const objStr = termToString(triple.o); - if (!visited.has(objStr)) { - nextLevel.add(objStr); + if (depth < config.maxPathLength - 1) { + const objStr = termToString(triple.o); + if (!visited.has(objStr)) { + nextLevel.add(objStr); + } + } + + if (subgraph.length >= config.maxSubgraphSize) { + return subgraph; } } - - if (subgraph.length >= config.maxSubgraphSize) { - return subgraph; - } } + + for (const entity of currentLevel) { + visited.add(entity); + } + currentLevel = nextLevel; } - for (const entity of currentLevel) { - visited.add(entity); - } - currentLevel = nextLevel; - } - - return subgraph.slice(0, config.maxSubgraphSize); + return subgraph.slice(0, config.maxSubgraphSize); + }); } -async function scoreEdges( +function scoreEdges( clients: GraphRagClients, config: NormalizedGraphRagConfig, query: string, triples: Triple[], -): Promise { - if (triples.length === 0) return []; +): Effect.Effect { + return Effect.gen(function* () { + if (triples.length === 0) return []; - if (triples.length <= 500) { - console.log(`[GraphRag] Skipping edge scoring - ${triples.length} triples fits in context directly`); - return triples; - } - - const edgeDescriptions = triples.map((triple, index) => ({ - id: String(index), - s: termToString(triple.s), - p: termToString(triple.p), - o: termToString(triple.o), - })); - - const toScore = edgeDescriptions.slice(0, config.edgeScoreLimit); - const knowledgeJson = JSON.stringify(toScore, null, 2); - - const promptResp = await clients.prompt.request({ - name: "kg-edge-scoring", - variables: { - query, - knowledge: knowledgeJson, - }, - }); - - const llmResp = await clients.llm.request({ - system: promptResp.system, - prompt: promptResp.prompt, - }); - - console.log(`[GraphRag] Edge scoring LLM response (first 500 chars): ${llmResp.response.slice(0, 500)}`); - - const scored = parseScoredEdges(llmResp.response); - scored.sort((a, b) => b.score - a.score); - const topN = scored.slice(0, config.edgeLimit); - - const result: Triple[] = []; - for (const entry of topN) { - const idx = Number.parseInt(entry.id, 10); - if (!Number.isNaN(idx) && idx >= 0 && idx < triples.length) { - result.push(triples[idx]); + if (triples.length <= 500) { + yield* Effect.log(`[GraphRag] Skipping edge scoring - ${triples.length} triples fits in context directly`); + return triples; } - } - console.log(`[GraphRag] Edge scoring: LLM returned ${scored.length} scores, keeping top ${topN.length}, mapped ${result.length} triples`); + const edgeDescriptions = triples.map((triple, index) => ({ + id: String(index), + s: termToString(triple.s), + p: termToString(triple.p), + o: termToString(triple.o), + })); - if (result.length === 0) { - return triples.slice(0, config.edgeLimit); - } + const toScore = edgeDescriptions.slice(0, config.edgeScoreLimit); + const knowledgeJson = yield* S.encodeUnknownEffect(S.UnknownFromJsonString)(toScore).pipe( + Effect.mapError((cause) => graphRagError("edge-score-encode", cause)), + ); - return result; + const promptResp = yield* Effect.tryPromise({ + try: () => clients.prompt.request({ + name: "kg-edge-scoring", + variables: { + query, + knowledge: knowledgeJson, + }, + }), + catch: (cause) => graphRagError("edge-score-prompt", cause), + }); + + const llmResp = yield* Effect.tryPromise({ + try: () => clients.llm.request({ + system: promptResp.system, + prompt: promptResp.prompt, + }), + catch: (cause) => graphRagError("edge-score-llm", cause), + }); + + yield* Effect.log(`[GraphRag] Edge scoring LLM response (first 500 chars): ${llmResp.response.slice(0, 500)}`); + + const scored = parseScoredEdges(llmResp.response); + scored.sort((a, b) => b.score - a.score); + const topN = scored.slice(0, config.edgeLimit); + + const result: Triple[] = []; + for (const entry of topN) { + const idx = Number.parseInt(entry.id, 10); + if (!Number.isNaN(idx) && idx >= 0 && idx < triples.length) { + result.push(triples[idx]); + } + } + + yield* Effect.log(`[GraphRag] Edge scoring: LLM returned ${scored.length} scores, keeping top ${topN.length}, mapped ${result.length} triples`); + + if (result.length === 0) { + return triples.slice(0, config.edgeLimit); + } + + return result; + }); } -async function synthesize( +function synthesize( clients: GraphRagClients, query: string, edges: Triple[], chunkCallback?: ChunkCallback, -): Promise { - const context = edges - .map((triple) => `${termToString(triple.s)} -> ${termToString(triple.p)} -> ${termToString(triple.o)}`) - .join("\n"); +): Effect.Effect { + return Effect.gen(function* () { + const context = edges + .map((triple) => `${termToString(triple.s)} -> ${termToString(triple.p)} -> ${termToString(triple.o)}`) + .join("\n"); - const promptResp = await clients.prompt.request({ - name: "graph-rag-synthesize", - variables: { query, context }, - }); + const promptResp = yield* Effect.tryPromise({ + try: () => clients.prompt.request({ + name: "graph-rag-synthesize", + variables: { query, context }, + }), + catch: (cause) => graphRagError("synthesize-prompt", cause), + }); - if (chunkCallback !== undefined) { - let fullText = ""; - await clients.llm.request( - { + if (chunkCallback !== undefined) { + let fullText = ""; + yield* Effect.tryPromise({ + try: () => clients.llm.request( + { + system: promptResp.system, + prompt: promptResp.prompt, + streaming: true, + }, + { + recipient: (resp) => { + if (resp.response.length === 0) return Promise.resolve(resp.endOfStream === true); + fullText += resp.response; + return chunkCallback(resp.response, resp.endOfStream === true).then(() => resp.endOfStream === true); + }, + }, + ), + catch: (cause) => graphRagError("synthesize-stream", cause), + }); + return fullText; + } + + const resp = yield* Effect.tryPromise({ + try: () => clients.llm.request({ system: promptResp.system, prompt: promptResp.prompt, - streaming: true, - }, - { - recipient: async (resp) => { - if (resp.response.length > 0) { - fullText += resp.response; - await chunkCallback(resp.response, resp.endOfStream === true); - } - return resp.endOfStream === true; - }, - }, - ); - return fullText; - } + }), + catch: (cause) => graphRagError("synthesize-llm", cause), + }); - const resp = await clients.llm.request({ - system: promptResp.system, - prompt: promptResp.prompt, + return resp.response; }); - - return resp.response; } const ScoredEdge = S.Struct({ diff --git a/ts/packages/flow/src/runtime/effect-files.ts b/ts/packages/flow/src/runtime/effect-files.ts index 66f09044..684f5f84 100644 --- a/ts/packages/flow/src/runtime/effect-files.ts +++ b/ts/packages/flow/src/runtime/effect-files.ts @@ -23,8 +23,8 @@ export function readTextFile(path: string): Promise { return Bun.file(path).text(); } -export async function readBinaryFile(path: string): Promise { - return new Uint8Array(await Bun.file(path).arrayBuffer()); +export function readBinaryFile(path: string): Promise { + return Bun.file(path).arrayBuffer().then((buffer) => new Uint8Array(buffer)); } export function writeTextFile(path: string, data: string): Promise { diff --git a/ts/packages/flow/src/storage/embeddings/graph-embeddings-service.ts b/ts/packages/flow/src/storage/embeddings/graph-embeddings-service.ts index 55eb0b8a..06c5e370 100644 --- a/ts/packages/flow/src/storage/embeddings/graph-embeddings-service.ts +++ b/ts/packages/flow/src/storage/embeddings/graph-embeddings-service.ts @@ -103,7 +103,7 @@ export function makeGraphEmbeddingsStoreService(config: ProcessorConfig): GraphE ), ), }); - console.log("[GraphEmbeddingsStore] Service initialized"); + Effect.runSync(Effect.log("[GraphEmbeddingsStore] Service initialized")); return service; } @@ -119,6 +119,6 @@ export const program = makeFlowProcessorProgram< layer: (config) => QdrantGraphEmbeddingsStoreLive(config), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/storage/embeddings/qdrant-doc.ts b/ts/packages/flow/src/storage/embeddings/qdrant-doc.ts index 662c0120..c8182fdd 100644 --- a/ts/packages/flow/src/storage/embeddings/qdrant-doc.ts +++ b/ts/packages/flow/src/storage/embeddings/qdrant-doc.ts @@ -9,6 +9,10 @@ */ import { QdrantClient } from "@qdrant/js-client-rest"; +import { errorMessage } from "@trustgraph/base"; +import { Config, Effect, Random } from "effect"; +import * as O from "effect/Option"; +import * as S from "effect/Schema"; export interface QdrantDocEmbeddingsConfig { url?: string; @@ -27,43 +31,110 @@ export interface DocEmbeddingsMessage { chunks: DocEmbeddingChunk[]; } +export class QdrantDocEmbeddingsStoreError extends S.TaggedErrorClass()( + "QdrantDocEmbeddingsStoreError", + { + message: S.String, + operation: S.String, + cause: S.DefectWithStack, + }, +) {} + +const qdrantDocEmbeddingsStoreError = (operation: string, cause: unknown) => + QdrantDocEmbeddingsStoreError.make({ + operation, + message: errorMessage(cause), + cause, + }); + +interface ResolvedQdrantDocEmbeddingsConfig { + readonly url: string; + readonly apiKey?: string; +} + +const loadQdrantDocEmbeddingsConfig = Effect.fn("QdrantDocEmbeddings.loadConfig")(function* ( + config: QdrantDocEmbeddingsConfig, +) { + const envApiKey = O.getOrUndefined(yield* Config.string("QDRANT_API_KEY").pipe(Config.option)); + const apiKey = config.apiKey ?? envApiKey; + return { + url: config.url ?? (yield* Config.string("QDRANT_URL").pipe(Config.withDefault("http://localhost:6333"))), + ...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}), + } satisfies ResolvedQdrantDocEmbeddingsConfig; +}); + +const randomHex = Effect.fn("QdrantDocEmbeddings.randomHex")(function* (digits: number) { + let result = ""; + for (let index = 0; index < digits; index++) { + const value = yield* Random.nextIntBetween(0, 16); + result += value.toString(16); + } + return result; +}); + +const randomPointId = Effect.fn("QdrantDocEmbeddings.randomPointId")(function* () { + const part1 = yield* randomHex(8); + const part2 = yield* randomHex(4); + const versionRest = yield* randomHex(3); + const variant = yield* Random.nextIntBetween(8, 12); + const variantRest = yield* randomHex(3); + const part5 = yield* randomHex(12); + return `${part1}-${part2}-4${versionRest}-${variant.toString(16)}${variantRest}-${part5}`; +}); + export interface QdrantDocEmbeddingsStore { readonly store: (message: DocEmbeddingsMessage) => Promise; readonly deleteCollection: (user: string, collection: string) => Promise; + readonly storeEffect: ( + message: DocEmbeddingsMessage, + ) => Effect.Effect; + readonly deleteCollectionEffect: ( + user: string, + collection: string, + ) => Effect.Effect; } export function makeQdrantDocEmbeddingsStore( config: QdrantDocEmbeddingsConfig = {}, ): QdrantDocEmbeddingsStore { - const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333"; - const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY; + const resolved = Effect.runSync(loadQdrantDocEmbeddingsConfig(config)); const client = new QdrantClient({ - url, - ...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}), + url: resolved.url, + ...(resolved.apiKey !== undefined ? { apiKey: resolved.apiKey } : {}), }); const knownCollections = new Set(); - console.log("[QdrantDocEmbeddings] Store initialized"); + Effect.runSync(Effect.log("[QdrantDocEmbeddings] Store initialized")); const collectionName = (user: string, collection: string, dim: number): string => `d_${user}_${collection}_${dim}`; - const ensureCollection = async (name: string, dim: number): Promise => { + const ensureCollectionEffect = Effect.fn("QdrantDocEmbeddings.ensureCollection")(function* ( + name: string, + dim: number, + ) { if (knownCollections.has(name)) return; - const exists = await client.collectionExists(name); + const exists = yield* Effect.tryPromise({ + try: () => client.collectionExists(name), + catch: (cause) => qdrantDocEmbeddingsStoreError("collection-exists", cause), + }); if (!exists.exists) { - console.log(`[QdrantDocEmbeddings] Creating collection ${name} (dim=${dim})`); - await client.createCollection(name, { - vectors: { size: dim, distance: "Cosine" }, + yield* Effect.log(`[QdrantDocEmbeddings] Creating collection ${name} (dim=${dim})`); + yield* Effect.tryPromise({ + try: () => + client.createCollection(name, { + vectors: { size: dim, distance: "Cosine" }, + }), + catch: (cause) => qdrantDocEmbeddingsStoreError("create-collection", cause), }); } knownCollections.add(name); - }; + }); - const store = async (message: DocEmbeddingsMessage): Promise => { + const storeEffect = Effect.fn("QdrantDocEmbeddings.store")(function* (message: DocEmbeddingsMessage) { for (const chunk of message.chunks) { if (chunk.chunkId.length === 0) continue; if (chunk.vector.length === 0) continue; @@ -71,48 +142,68 @@ export function makeQdrantDocEmbeddingsStore( const dim = chunk.vector.length; const name = collectionName(message.user, message.collection, dim); - await ensureCollection(name, dim); + yield* ensureCollectionEffect(name, dim); - await client.upsert(name, { - points: [ - { - id: crypto.randomUUID(), - vector: chunk.vector, - payload: { - chunk_id: chunk.chunkId, - ...(chunk.content !== undefined && chunk.content.length > 0 - ? { content: chunk.content } - : {}), - }, - }, - ], + const id = yield* randomPointId(); + yield* Effect.tryPromise({ + try: () => + client.upsert(name, { + points: [ + { + id, + vector: chunk.vector, + payload: { + chunk_id: chunk.chunkId, + ...(chunk.content !== undefined && chunk.content.length > 0 + ? { content: chunk.content } + : {}), + }, + }, + ], + }), + catch: (cause) => qdrantDocEmbeddingsStoreError("upsert", cause), }); } - }; + }); - const deleteCollection = async (user: string, collection: string): Promise => { + const deleteCollectionEffect = Effect.fn("QdrantDocEmbeddings.deleteCollection")(function* ( + user: string, + collection: string, + ) { const prefix = `d_${user}_${collection}_`; - const allCollections = await client.getCollections(); + const allCollections = yield* Effect.tryPromise({ + try: () => client.getCollections(), + catch: (cause) => qdrantDocEmbeddingsStoreError("get-collections", cause), + }); const matching = allCollections.collections.filter((c) => c.name.startsWith(prefix), ); if (matching.length === 0) { - console.log(`[QdrantDocEmbeddings] No collections matching prefix ${prefix}`); + yield* Effect.log(`[QdrantDocEmbeddings] No collections matching prefix ${prefix}`); return; } for (const coll of matching) { - await client.deleteCollection(coll.name); + yield* Effect.tryPromise({ + try: () => client.deleteCollection(coll.name), + catch: (cause) => qdrantDocEmbeddingsStoreError("delete-collection", cause), + }); knownCollections.delete(coll.name); - console.log(`[QdrantDocEmbeddings] Deleted collection: ${coll.name}`); + yield* Effect.log(`[QdrantDocEmbeddings] Deleted collection: ${coll.name}`); } - console.log( + yield* Effect.log( `[QdrantDocEmbeddings] Deleted ${matching.length} collection(s) for ${user}/${collection}`, ); - }; + }); - return { store, deleteCollection }; + return { + store: (message) => Effect.runPromise(storeEffect(message)), + deleteCollection: (user, collection) => + Effect.runPromise(deleteCollectionEffect(user, collection)), + storeEffect, + deleteCollectionEffect, + }; } diff --git a/ts/packages/flow/src/storage/embeddings/qdrant-graph.ts b/ts/packages/flow/src/storage/embeddings/qdrant-graph.ts index c1e3bbe2..36075a61 100644 --- a/ts/packages/flow/src/storage/embeddings/qdrant-graph.ts +++ b/ts/packages/flow/src/storage/embeddings/qdrant-graph.ts @@ -10,7 +10,8 @@ import { QdrantClient } from "@qdrant/js-client-rest"; import { errorMessage, type Term } from "@trustgraph/base"; -import { Context, Effect, Layer } from "effect"; +import { Config, Context, Effect, Layer, Random } from "effect"; +import * as O from "effect/Option"; import * as S from "effect/Schema"; export interface QdrantGraphEmbeddingsConfig { @@ -30,6 +31,57 @@ export interface GraphEmbeddingsMessage { entities: GraphEmbeddingEntity[]; } +export class QdrantGraphEmbeddingsStoreError extends S.TaggedErrorClass()( + "QdrantGraphEmbeddingsStoreError", + { + message: S.String, + operation: S.String, + cause: S.DefectWithStack, + }, +) {} + +const qdrantGraphEmbeddingsStoreError = (operation: string, cause: unknown) => + QdrantGraphEmbeddingsStoreError.make({ + operation, + message: errorMessage(cause), + cause, + }); + +interface ResolvedQdrantGraphEmbeddingsConfig { + readonly url: string; + readonly apiKey?: string; +} + +const loadQdrantGraphEmbeddingsConfig = Effect.fn("QdrantGraphEmbeddings.loadConfig")(function* ( + config: QdrantGraphEmbeddingsConfig, +) { + const envApiKey = O.getOrUndefined(yield* Config.string("QDRANT_API_KEY").pipe(Config.option)); + const apiKey = config.apiKey ?? envApiKey; + return { + url: config.url ?? (yield* Config.string("QDRANT_URL").pipe(Config.withDefault("http://localhost:6333"))), + ...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}), + } satisfies ResolvedQdrantGraphEmbeddingsConfig; +}); + +const randomHex = Effect.fn("QdrantGraphEmbeddings.randomHex")(function* (digits: number) { + let result = ""; + for (let index = 0; index < digits; index++) { + const value = yield* Random.nextIntBetween(0, 16); + result += value.toString(16); + } + return result; +}); + +const randomPointId = Effect.fn("QdrantGraphEmbeddings.randomPointId")(function* () { + const part1 = yield* randomHex(8); + const part2 = yield* randomHex(4); + const versionRest = yield* randomHex(3); + const variant = yield* Random.nextIntBetween(8, 12); + const variantRest = yield* randomHex(3); + const part5 = yield* randomHex(12); + return `${part1}-${part2}-4${versionRest}-${variant.toString(16)}${variantRest}-${part5}`; +}); + function getTermValue(term: Term): string | null { switch (term.type) { case "IRI": @@ -46,40 +98,56 @@ function getTermValue(term: Term): string | null { export interface QdrantGraphEmbeddingsStore { readonly store: (message: GraphEmbeddingsMessage) => Promise; readonly deleteCollection: (user: string, collection: string) => Promise; + readonly storeEffect: ( + message: GraphEmbeddingsMessage, + ) => Effect.Effect; + readonly deleteCollectionEffect: ( + user: string, + collection: string, + ) => Effect.Effect; } export function makeQdrantGraphEmbeddingsStore( config: QdrantGraphEmbeddingsConfig = {}, ): QdrantGraphEmbeddingsStore { - const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333"; - const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY; + const resolved = Effect.runSync(loadQdrantGraphEmbeddingsConfig(config)); const client = new QdrantClient({ - url, - ...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}), + url: resolved.url, + ...(resolved.apiKey !== undefined ? { apiKey: resolved.apiKey } : {}), }); const knownCollections = new Set(); - console.log("[QdrantGraphEmbeddings] Store initialized"); + Effect.runSync(Effect.log("[QdrantGraphEmbeddings] Store initialized")); const collectionName = (user: string, collection: string, dim: number): string => `t_${user}_${collection}_${dim}`; - const ensureCollection = async (name: string, dim: number): Promise => { + const ensureCollectionEffect = Effect.fn("QdrantGraphEmbeddings.ensureCollection")(function* ( + name: string, + dim: number, + ) { if (knownCollections.has(name)) return; - const exists = await client.collectionExists(name); + const exists = yield* Effect.tryPromise({ + try: () => client.collectionExists(name), + catch: (cause) => qdrantGraphEmbeddingsStoreError("collection-exists", cause), + }); if (!exists.exists) { - console.log(`[QdrantGraphEmbeddings] Creating collection ${name} (dim=${dim})`); - await client.createCollection(name, { - vectors: { size: dim, distance: "Cosine" }, + yield* Effect.log(`[QdrantGraphEmbeddings] Creating collection ${name} (dim=${dim})`); + yield* Effect.tryPromise({ + try: () => + client.createCollection(name, { + vectors: { size: dim, distance: "Cosine" }, + }), + catch: (cause) => qdrantGraphEmbeddingsStoreError("create-collection", cause), }); } knownCollections.add(name); - }; + }); - const store = async (message: GraphEmbeddingsMessage): Promise => { + const storeEffect = Effect.fn("QdrantGraphEmbeddings.store")(function* (message: GraphEmbeddingsMessage) { for (const entry of message.entities) { const entityValue = getTermValue(entry.entity); if (entityValue === null || entityValue.length === 0) continue; @@ -88,61 +156,72 @@ export function makeQdrantGraphEmbeddingsStore( const dim = entry.vector.length; const name = collectionName(message.user, message.collection, dim); - await ensureCollection(name, dim); + yield* ensureCollectionEffect(name, dim); const payload: Record = { entity: entityValue }; if (entry.chunkId !== undefined && entry.chunkId.length > 0) { payload.chunk_id = entry.chunkId; } - await client.upsert(name, { - points: [ - { - id: crypto.randomUUID(), - vector: entry.vector, - payload, - }, - ], + const id = yield* randomPointId(); + yield* Effect.tryPromise({ + try: () => + client.upsert(name, { + points: [ + { + id, + vector: entry.vector, + payload, + }, + ], + }), + catch: (cause) => qdrantGraphEmbeddingsStoreError("upsert", cause), }); } - }; + }); - const deleteCollection = async (user: string, collection: string): Promise => { + const deleteCollectionEffect = Effect.fn("QdrantGraphEmbeddings.deleteCollection")(function* ( + user: string, + collection: string, + ) { const prefix = `t_${user}_${collection}_`; - const allCollections = await client.getCollections(); + const allCollections = yield* Effect.tryPromise({ + try: () => client.getCollections(), + catch: (cause) => qdrantGraphEmbeddingsStoreError("get-collections", cause), + }); const matching = allCollections.collections.filter((c) => c.name.startsWith(prefix), ); if (matching.length === 0) { - console.log(`[QdrantGraphEmbeddings] No collections matching prefix ${prefix}`); + yield* Effect.log(`[QdrantGraphEmbeddings] No collections matching prefix ${prefix}`); return; } for (const coll of matching) { - await client.deleteCollection(coll.name); + yield* Effect.tryPromise({ + try: () => client.deleteCollection(coll.name), + catch: (cause) => qdrantGraphEmbeddingsStoreError("delete-collection", cause), + }); knownCollections.delete(coll.name); - console.log(`[QdrantGraphEmbeddings] Deleted collection: ${coll.name}`); + yield* Effect.log(`[QdrantGraphEmbeddings] Deleted collection: ${coll.name}`); } - console.log( + yield* Effect.log( `[QdrantGraphEmbeddings] Deleted ${matching.length} collection(s) for ${user}/${collection}`, ); + }); + + return { + store: (message) => Effect.runPromise(storeEffect(message)), + deleteCollection: (user, collection) => + Effect.runPromise(deleteCollectionEffect(user, collection)), + storeEffect, + deleteCollectionEffect, }; - - return { store, deleteCollection }; } -export class QdrantGraphEmbeddingsStoreError extends S.TaggedErrorClass()( - "QdrantGraphEmbeddingsStoreError", - { - message: S.String, - operation: S.String, - cause: S.DefectWithStack, - }, -) {} - export interface QdrantGraphEmbeddingsStoreServiceShape { readonly store: ( message: GraphEmbeddingsMessage, @@ -160,33 +239,13 @@ export class QdrantGraphEmbeddingsStoreService extends Context.Service< "@trustgraph/flow/storage/embeddings/qdrant-graph/QdrantGraphEmbeddingsStoreService", ) {} -const qdrantGraphEmbeddingsStoreError = (operation: string, cause: unknown) => - new QdrantGraphEmbeddingsStoreError({ - operation, - message: errorMessage(cause), - cause, - }); - export const makeQdrantGraphEmbeddingsStoreService = ( config: QdrantGraphEmbeddingsConfig = {}, ): QdrantGraphEmbeddingsStoreServiceShape => { const store = makeQdrantGraphEmbeddingsStore(config); return { - store: Effect.fn("QdrantGraphEmbeddingsStore.store")(function* (message) { - return yield* Effect.tryPromise({ - try: () => store.store(message), - catch: (cause) => qdrantGraphEmbeddingsStoreError("store", cause), - }); - }), - deleteCollection: Effect.fn("QdrantGraphEmbeddingsStore.deleteCollection")(function* ( - user, - collection, - ) { - return yield* Effect.tryPromise({ - try: () => store.deleteCollection(user, collection), - catch: (cause) => qdrantGraphEmbeddingsStoreError("delete-collection", cause), - }); - }), + store: store.storeEffect, + deleteCollection: store.deleteCollectionEffect, }; }; diff --git a/ts/packages/flow/src/storage/triples/falkordb-service.ts b/ts/packages/flow/src/storage/triples/falkordb-service.ts index 713deaa9..11a5b4df 100644 --- a/ts/packages/flow/src/storage/triples/falkordb-service.ts +++ b/ts/packages/flow/src/storage/triples/falkordb-service.ts @@ -66,7 +66,7 @@ export function makeTriplesStoreService(config: ProcessorConfig): TriplesStoreSe ), ), }); - console.log("[TriplesStore] Service initialized"); + void Effect.runPromise(Effect.log("[TriplesStore] Service initialized")); return service; } @@ -78,6 +78,6 @@ export const program = makeFlowProcessorProgram FalkorDBTriplesStoreLive(config), }); -export async function run(): Promise { - await Effect.runPromise(program); +export function run(): Promise { + return Effect.runPromise(program); } diff --git a/ts/packages/flow/src/storage/triples/falkordb.ts b/ts/packages/flow/src/storage/triples/falkordb.ts index 38a162c1..3f052c2a 100644 --- a/ts/packages/flow/src/storage/triples/falkordb.ts +++ b/ts/packages/flow/src/storage/triples/falkordb.ts @@ -1,5 +1,5 @@ /** - * FalkorDB triples store — writes RDF triples to a FalkorDB graph. + * FalkorDB triples store - writes RDF triples to a FalkorDB graph. * * FalkorDB is Redis-based and uses Cypher queries, same as the Python impl. * Pairs well with Graphiti which also uses FalkorDB as its backend. @@ -9,7 +9,7 @@ import { createClient, Graph } from "falkordb"; import { errorMessage, type Term, type Triple } from "@trustgraph/base"; -import { Context, Effect, Layer } from "effect"; +import { Config, Context, Effect, Layer } from "effect"; import * as S from "effect/Schema"; export interface FalkorDBConfig { @@ -26,7 +26,7 @@ function getTermValue(term: Term): string { case "BLANK": return term.id; case "TRIPLE": - return getTermValue(term.triple.s); // fallback + return getTermValue(term.triple.s); } } @@ -55,113 +55,6 @@ export interface FalkorDBTriplesStore { readonly deleteCollection: (user: string, collection: string) => Promise; } -export function makeFalkorDBTriplesStore(config: FalkorDBConfig = {}): FalkorDBTriplesStore { - const url = config.url ?? process.env.FALKORDB_URL ?? "redis://localhost:6379"; - const database = config.database ?? "falkordb"; - - const client = createClient({ url }); - const graph = new Graph(client, database); - const connectPromise = client.connect().then(() => { - console.log(`[FalkorDBTriplesStore] Connected to ${url}, graph: ${database}`); - }).catch((err) => { - console.error(`[FalkorDBTriplesStore] Connection failed:`, err); - throw err; - }); - - const ensureConnected = async (): Promise => { - await connectPromise; - }; - - const createNode = async (uri: string, user: string, collection: string): Promise => { - await ensureConnected(); - await graph.query( - "MERGE (n:Node {uri: $uri, user: $user, collection: $collection})", - { params: { uri, user, collection } }, - ); - }; - - const createLiteral = async (value: string, user: string, collection: string): Promise => { - await ensureConnected(); - await graph.query( - "MERGE (n:Literal {value: $value, user: $user, collection: $collection})", - { params: { value, user, collection } }, - ); - }; - - const relateNode = async ( - src: string, uri: string, dest: string, - user: string, collection: string, - ): Promise => { - await ensureConnected(); - await graph.query( - "MATCH (src:Node {uri: $src, user: $user, collection: $collection}) " + - "MATCH (dest:Node {uri: $dest, user: $user, collection: $collection}) " + - "MERGE (src)-[:Rel {uri: $uri, user: $user, collection: $collection}]->(dest)", - { params: { src, dest, uri, user, collection } }, - ); - }; - - const relateLiteral = async ( - src: string, uri: string, dest: string, - user: string, collection: string, - ): Promise => { - await ensureConnected(); - await graph.query( - "MATCH (src:Node {uri: $src, user: $user, collection: $collection}) " + - "MATCH (dest:Literal {value: $dest, user: $user, collection: $collection}) " + - "MERGE (src)-[:Rel {uri: $uri, user: $user, collection: $collection}]->(dest)", - { params: { src, dest, uri, user, collection } }, - ); - }; - - const storeTriples = async ( - triples: Triple[], - user = "default", - collection = "default", - ): Promise => { - for (const t of triples) { - const s = getTermValue(t.s); - const p = getTermValue(t.p); - const o = getTermValue(t.o); - - await createNode(s, user, collection); - - if (t.o.type === "IRI") { - await createNode(o, user, collection); - await relateNode(s, p, o, user, collection); - } else { - await createLiteral(o, user, collection); - await relateLiteral(s, p, o, user, collection); - } - } - }; - - const deleteCollection = async (user: string, collection: string): Promise => { - await ensureConnected(); - await graph.query( - "MATCH (n:Node {user: $user, collection: $collection}) DETACH DELETE n", - { params: { user, collection } }, - ); - await graph.query( - "MATCH (n:Literal {user: $user, collection: $collection}) DETACH DELETE n", - { params: { user, collection } }, - ); - await graph.query( - "MATCH (c:CollectionMetadata {user: $user, collection: $collection}) DELETE c", - { params: { user, collection } }, - ); - }; - - return { - createNode, - createLiteral, - relateNode, - relateLiteral, - storeTriples, - deleteCollection, - }; -} - export class FalkorDBTriplesStoreError extends S.TaggedErrorClass()( "FalkorDBTriplesStoreError", { @@ -190,35 +83,268 @@ export class FalkorDBTriplesStoreService extends Context.Service< "@trustgraph/flow/storage/triples/falkordb/FalkorDBTriplesStoreService", ) {} -const falkorDBTriplesStoreError = (operation: string, cause: unknown) => - new FalkorDBTriplesStoreError({ +const falkorDBTriplesStoreError = (operation: string, cause: unknown): FalkorDBTriplesStoreError => + FalkorDBTriplesStoreError.make({ operation, message: errorMessage(cause), cause, }); +interface FalkorDBStoreConnection { + readonly graph: Graph; +} + +type FalkorDBQueryOptions = Parameters[1]; + +interface FalkorDBTriplesStoreEffectShape { + readonly createNode: ( + uri: string, + user: string, + collection: string, + ) => Effect.Effect; + readonly createLiteral: ( + value: string, + user: string, + collection: string, + ) => Effect.Effect; + readonly relateNode: ( + src: string, + uri: string, + dest: string, + user: string, + collection: string, + ) => Effect.Effect; + readonly relateLiteral: ( + src: string, + uri: string, + dest: string, + user: string, + collection: string, + ) => Effect.Effect; + readonly storeTriples: ( + triples: ReadonlyArray, + user: string, + collection: string, + ) => Effect.Effect; + readonly deleteCollection: ( + user: string, + collection: string, + ) => Effect.Effect; +} + +const resolveFalkorDBStoreConfig = Effect.fn("FalkorDBTriplesStore.resolveConfig")(function* ( + config: FalkorDBConfig, +) { + const url = config.url ?? (yield* Config.string("FALKORDB_URL").pipe( + Config.withDefault("redis://localhost:6379"), + Effect.mapError((cause) => falkorDBTriplesStoreError("config", cause)), + )); + return { + url, + database: config.database ?? "falkordb", + }; +}); + +const connectFalkorDBTriplesStore = ( + config: FalkorDBConfig, +): Effect.Effect => + Effect.gen(function* () { + const { url, database } = yield* resolveFalkorDBStoreConfig(config); + const { client, graph } = yield* Effect.try({ + try: () => { + const client = createClient({ url }); + return { client, graph: new Graph(client, database) }; + }, + catch: (cause) => falkorDBTriplesStoreError("create-client", cause), + }); + + yield* Effect.tryPromise({ + try: () => client.connect(), + catch: (cause) => falkorDBTriplesStoreError("connect", cause), + }).pipe( + Effect.tapError((error) => + Effect.logError("[FalkorDBTriplesStore] Connection failed", { + error: error.message, + operation: error.operation, + }) + ), + ); + + yield* Effect.log(`[FalkorDBTriplesStore] Connected to ${url}, graph: ${database}`); + return { graph }; + }); + +const runGraphQuery = ( + graph: Graph, + operation: string, + query: string, + options?: FalkorDBQueryOptions, +): Effect.Effect => + Effect.tryPromise({ + try: () => graph.query(query, options), + catch: (cause) => falkorDBTriplesStoreError(operation, cause), + }).pipe( + Effect.asVoid, + ); + +const makeFalkorDBTriplesStoreEffect = ( + config: FalkorDBConfig = {}, +): FalkorDBTriplesStoreEffectShape => { + let cachedConnection: Effect.Effect | undefined; + + const getConnection = Effect.fn("FalkorDBTriplesStore.connection")(function* () { + if (cachedConnection === undefined) { + cachedConnection = yield* Effect.cached(connectFalkorDBTriplesStore(config)); + } + return yield* cachedConnection; + }); + + const createNode = Effect.fn("FalkorDBTriplesStore.createNode")(function* ( + uri: string, + user: string, + collection: string, + ) { + const { graph } = yield* getConnection(); + yield* runGraphQuery( + graph, + "create-node", + "MERGE (n:Node {uri: $uri, user: $user, collection: $collection})", + { params: { uri, user, collection } }, + ); + }); + + const createLiteral = Effect.fn("FalkorDBTriplesStore.createLiteral")(function* ( + value: string, + user: string, + collection: string, + ) { + const { graph } = yield* getConnection(); + yield* runGraphQuery( + graph, + "create-literal", + "MERGE (n:Literal {value: $value, user: $user, collection: $collection})", + { params: { value, user, collection } }, + ); + }); + + const relateNode = Effect.fn("FalkorDBTriplesStore.relateNode")(function* ( + src: string, + uri: string, + dest: string, + user: string, + collection: string, + ) { + const { graph } = yield* getConnection(); + yield* runGraphQuery( + graph, + "relate-node", + "MATCH (src:Node {uri: $src, user: $user, collection: $collection}) " + + "MATCH (dest:Node {uri: $dest, user: $user, collection: $collection}) " + + "MERGE (src)-[:Rel {uri: $uri, user: $user, collection: $collection}]->(dest)", + { params: { src, dest, uri, user, collection } }, + ); + }); + + const relateLiteral = Effect.fn("FalkorDBTriplesStore.relateLiteral")(function* ( + src: string, + uri: string, + dest: string, + user: string, + collection: string, + ) { + const { graph } = yield* getConnection(); + yield* runGraphQuery( + graph, + "relate-literal", + "MATCH (src:Node {uri: $src, user: $user, collection: $collection}) " + + "MATCH (dest:Literal {value: $dest, user: $user, collection: $collection}) " + + "MERGE (src)-[:Rel {uri: $uri, user: $user, collection: $collection}]->(dest)", + { params: { src, dest, uri, user, collection } }, + ); + }); + + const storeTriples = Effect.fn("FalkorDBTriplesStore.storeTriples")(function* ( + triples: ReadonlyArray, + user: string, + collection: string, + ) { + for (const triple of triples) { + const s = getTermValue(triple.s); + const p = getTermValue(triple.p); + const o = getTermValue(triple.o); + + yield* createNode(s, user, collection); + + if (triple.o.type === "IRI") { + yield* createNode(o, user, collection); + yield* relateNode(s, p, o, user, collection); + } else { + yield* createLiteral(o, user, collection); + yield* relateLiteral(s, p, o, user, collection); + } + } + }); + + const deleteCollection = Effect.fn("FalkorDBTriplesStore.deleteCollection")(function* ( + user: string, + collection: string, + ) { + const { graph } = yield* getConnection(); + yield* runGraphQuery( + graph, + "delete-collection-nodes", + "MATCH (n:Node {user: $user, collection: $collection}) DETACH DELETE n", + { params: { user, collection } }, + ); + yield* runGraphQuery( + graph, + "delete-collection-literals", + "MATCH (n:Literal {user: $user, collection: $collection}) DETACH DELETE n", + { params: { user, collection } }, + ); + yield* runGraphQuery( + graph, + "delete-collection-metadata", + "MATCH (c:CollectionMetadata {user: $user, collection: $collection}) DELETE c", + { params: { user, collection } }, + ); + }); + + return { + createNode, + createLiteral, + relateNode, + relateLiteral, + storeTriples, + deleteCollection, + }; +}; + +export function makeFalkorDBTriplesStore(config: FalkorDBConfig = {}): FalkorDBTriplesStore { + const store = makeFalkorDBTriplesStoreEffect(config); + return { + createNode: (uri, user, collection) => + Effect.runPromise(store.createNode(uri, user, collection)), + createLiteral: (value, user, collection) => + Effect.runPromise(store.createLiteral(value, user, collection)), + relateNode: (src, uri, dest, user, collection) => + Effect.runPromise(store.relateNode(src, uri, dest, user, collection)), + relateLiteral: (src, uri, dest, user, collection) => + Effect.runPromise(store.relateLiteral(src, uri, dest, user, collection)), + storeTriples: (triples, user = "default", collection = "default") => + Effect.runPromise(store.storeTriples(triples, user, collection)), + deleteCollection: (user, collection) => + Effect.runPromise(store.deleteCollection(user, collection)), + }; +} + export const makeFalkorDBTriplesStoreService = ( config: FalkorDBConfig = {}, ): FalkorDBTriplesStoreServiceShape => { - const store = makeFalkorDBTriplesStore(config); + const store = makeFalkorDBTriplesStoreEffect(config); return { - storeTriples: Effect.fn("FalkorDBTriplesStore.storeTriples")(( - triples: ReadonlyArray, - user: string, - collection: string, - ) => - Effect.tryPromise({ - try: () => store.storeTriples(Array.from(triples), user, collection), - catch: (cause) => falkorDBTriplesStoreError("store-triples", cause), - })), - deleteCollection: Effect.fn("FalkorDBTriplesStore.deleteCollection")(( - user: string, - collection: string, - ) => - Effect.tryPromise({ - try: () => store.deleteCollection(user, collection), - catch: (cause) => falkorDBTriplesStoreError("delete-collection", cause), - })), + storeTriples: store.storeTriples, + deleteCollection: store.deleteCollection, }; }; diff --git a/ts/packages/workbench/src/atoms/workbench.ts b/ts/packages/workbench/src/atoms/workbench.ts index 204f39c2..88f8175c 100644 --- a/ts/packages/workbench/src/atoms/workbench.ts +++ b/ts/packages/workbench/src/atoms/workbench.ts @@ -2,7 +2,7 @@ import { Clipboard as BrowserClipboard } from "@effect/platform-browser"; import * as BrowserHttpClient from "@effect/platform-browser/BrowserHttpClient"; import * as BrowserKeyValueStore from "@effect/platform-browser/BrowserKeyValueStore"; import { BaseApi, type ConnectionState, type DocumentMetadata, type ExplainEvent, type StreamingMetadata, type Term, type Triple } from "@trustgraph/client"; -import { Cause, Context, Data, Effect, Layer, Metric, Option, Schema as S } from "effect"; +import { Cause, Clock, Context, Effect, Layer, Metric, Option, Random, Schema as S } from "effect"; import * as Otlp from "effect/unstable/observability/Otlp"; import * as AsyncResult from "effect/unstable/reactivity/AsyncResult"; import * as Atom from "effect/unstable/reactivity/Atom"; @@ -29,23 +29,31 @@ const browserObservabilityLayer = Otlp.layerJson({ shutdownTimeout: "1 second", }); -class WorkbenchPromiseError extends Data.TaggedError("WorkbenchPromiseError")<{ - readonly cause: unknown; - readonly message: string; -}> {} +class WorkbenchPromiseError extends S.TaggedErrorClass()( + "WorkbenchPromiseError", + { + cause: S.Unknown, + message: S.String, + }, +) {} type WorkbenchError = WorkbenchPromiseError; +const isWorkbenchPromiseError = S.is(WorkbenchPromiseError); + function errorMessage(error: unknown): string { - if (error instanceof WorkbenchPromiseError) return error.message; - if (error instanceof Error) return error.message; + if (isWorkbenchPromiseError(error)) return error.message; + if (typeof error === "object" && error !== null && "message" in error) { + const message = (error as { message?: unknown }).message; + if (typeof message === "string") return message; + } return String(error); } function promiseBoundary(evaluate: () => Promise): Effect.Effect { return Effect.tryPromise({ try: evaluate, - catch: (cause) => new WorkbenchPromiseError({ cause, message: errorMessage(cause) }), + catch: (cause) => WorkbenchPromiseError.make({ cause, message: errorMessage(cause) }), }); } @@ -73,7 +81,7 @@ export class WorkbenchFiles extends Context.Service< Effect.flatMap((buffer) => Effect.try({ try: () => base64FromArrayBuffer(buffer), - catch: (cause) => new WorkbenchPromiseError({ cause, message: errorMessage(cause) }), + catch: (cause) => WorkbenchPromiseError.make({ cause, message: errorMessage(cause) }), }) ), ); @@ -480,12 +488,12 @@ export function resultError(result: AsyncResult.AsyncResult): string return resultErrorMessage(result); } -function nextMessageId(): string { - return `msg-${crypto.randomUUID()}`; -} - -function nextNotificationId(): string { - return `notif-${crypto.randomUUID()}`; +function randomId(prefix: string): Effect.Effect { + return Effect.gen(function*() { + const left = yield* Random.nextIntBetween(0, 36 ** 6, { halfOpen: true }); + const right = yield* Random.nextIntBetween(0, 36 ** 6, { halfOpen: true }); + return `${prefix}-${left.toString(36).padStart(6, "0")}${right.toString(36).padStart(6, "0")}`; + }); } function metadataFrom(metadata: StreamingMetadata | undefined): ChatMessage["metadata"] | undefined { @@ -520,7 +528,7 @@ function parseConfigEntries(raw: unknown, label: string): T[] { for (const item of mapConfigEntries(raw)) { const config = parseJsonUnknown(item.value); if (config === undefined) { - console.warn(`[workbench-atoms] Failed to parse ${label}: ${item.key}`); + Effect.runSync(Effect.logWarning(`[workbench-atoms] Failed to parse ${label}: ${item.key}`)); } else { entries.push({ key: item.key, config } as T); } @@ -743,7 +751,7 @@ export const clearMessagesAtom = Atom.writable( export const pushNotificationAtom = localCommandAtom, void>( "pushNotification", Effect.fn("trustgraph.workbench.pushNotification")(function*(input, get) { - const id = nextNotificationId(); + const id = yield* randomId("notif"); const notification: Notification = { id, ...input }; get.set(notificationsAtom, [...get(notificationsAtom), notification]); yield* Effect.sleep("5 seconds"); @@ -1307,9 +1315,11 @@ const uploadDocumentChunkedEffect = Effect.fn("trustgraph.workbench.uploadDocume bytesUploaded: 0, } }); const lib = api.librarian(); + const documentId = yield* randomId("upload"); + const timestamp = yield* Clock.currentTimeMillis; const beginResp = yield* promiseBoundary(() => lib.beginUpload({ - id: crypto.randomUUID(), - time: Math.floor(Date.now() / 1000), + id: documentId, + time: Math.floor(timestamp / 1000), kind: input.mimeType, title: input.title, comments: input.comments, @@ -1478,7 +1488,7 @@ export const deleteCollectionAtom = commandAtom("deleteCollection" export const submitMessageAtom = commandAtom<{ input: string }, void>( "submitMessage", - Effect.fn("trustgraph.workbench.submitMessage")(({ input }, get, api) => Effect.sync(() => { + Effect.fn("trustgraph.workbench.submitMessage")(function*({ input }, get, api) { const trimmed = input.trim(); if (trimmed.length === 0) return; @@ -1488,7 +1498,8 @@ export const submitMessageAtom = commandAtom<{ input: string }, void>( const chatMode = get(conversationAtom).chatMode; const flow = api.flow(flowId); const explainEvents: ExplainEvent[] = []; - const requestId = crypto.randomUUID(); + const requestId = yield* randomId("chat"); + const timestamp = yield* Clock.currentTimeMillis; const isActiveRequest = () => get(activeChatRequestAtom) === requestId; const finishRequest = () => { if (isActiveRequest()) { @@ -1498,16 +1509,16 @@ export const submitMessageAtom = commandAtom<{ input: string }, void>( }; const userMsg: ChatMessage = { - id: nextMessageId(), + id: yield* randomId("msg"), role: "user", content: input, - timestamp: Date.now(), + timestamp, }; const assistantMsg: ChatMessage = { - id: nextMessageId(), + id: yield* randomId("msg"), role: "assistant", content: "", - timestamp: Date.now(), + timestamp, isStreaming: true, ...(chatMode === "agent" ? { @@ -1625,7 +1636,7 @@ export const submitMessageAtom = commandAtom<{ input: string }, void>( ); break; } - })), + }), ); export const cancelChatAtom = Atom.writable( diff --git a/ts/packages/workbench/src/components/error-boundary.tsx b/ts/packages/workbench/src/components/error-boundary.tsx index c41e9c76..36e5d93f 100644 --- a/ts/packages/workbench/src/components/error-boundary.tsx +++ b/ts/packages/workbench/src/components/error-boundary.tsx @@ -4,6 +4,7 @@ import { type FallbackProps, } from "react-error-boundary"; import { AlertTriangle, RefreshCw } from "lucide-react"; +import { Effect } from "effect"; interface Props { children: ReactNode; @@ -12,7 +13,9 @@ interface Props { } const errorMessage = (error: unknown): string => - error instanceof Error ? error.message : "An unexpected error occurred."; + typeof error === "object" && error !== null && "message" in error && typeof error.message === "string" + ? error.message + : "An unexpected error occurred."; function DefaultFallback({ error, resetErrorBoundary }: FallbackProps) { return ( @@ -42,7 +45,7 @@ export function ErrorBoundary({ children, fallback }: Props) { fallback ?? } onError={(error, info) => { - console.error("[ErrorBoundary]", error, info.componentStack); + Effect.runSync(Effect.logError("[ErrorBoundary]", { error, componentStack: info.componentStack })); }} > {children} diff --git a/ts/packages/workbench/src/pages/library.tsx b/ts/packages/workbench/src/pages/library.tsx index c5ecec30..eaa0b69d 100644 --- a/ts/packages/workbench/src/pages/library.tsx +++ b/ts/packages/workbench/src/pages/library.tsx @@ -36,6 +36,7 @@ import { import { Dialog } from "@/components/ui/dialog"; import { Badge } from "@/components/ui/badge"; import type { DocumentMetadata } from "@trustgraph/client"; +import { DateTime } from "effect"; function formatBytes(bytes: number): string { if (bytes === 0) return "0 B"; @@ -53,6 +54,10 @@ function guessKind(doc: DocumentMetadata): string { return kind.length > 0 ? kind : "--"; } +function formatUnixTimestamp(seconds: number): string { + return DateTime.formatLocal(DateTime.makeUnsafe(seconds * 1000)); +} + function resetUploadForm(form: UploadForm): UploadForm { return { ...form, file: null, title: "", tags: "", comments: "", uploading: false, dragOver: false, progress: null }; } @@ -242,7 +247,7 @@ function DocumentDetailDialog() {

Created

-

{new Date(doc.time * 1000).toLocaleString()}

+

{formatUnixTimestamp(doc.time)}

)} {doc.metadata !== undefined && doc.metadata.length > 0 && ( diff --git a/ts/packages/workbench/src/qa/mock-api.ts b/ts/packages/workbench/src/qa/mock-api.ts index 3e5099f4..e283d879 100644 --- a/ts/packages/workbench/src/qa/mock-api.ts +++ b/ts/packages/workbench/src/qa/mock-api.ts @@ -1,5 +1,5 @@ import { makeBaseApiWithRpc, type BaseApi, type DocumentMetadata, type ProcessingMetadata, type StreamingMetadata, type Triple } from "@trustgraph/client"; -import { Option, Schema as S } from "effect"; +import { Clock, Effect, Option, Schema as S } from "effect"; type ConfigValues = Record>; @@ -268,12 +268,13 @@ function configValues(state: MockState, type: string) { function addDocument(state: MockState, metadata: DocumentMetadata): DocumentMetadata { const id = metadata.id ?? `qa-doc-${state.library.documents.length + 1}`; + const currentTimeSeconds = Math.floor(Effect.runSync(Clock.currentTimeMillis) / 1000); const document = { ...metadata, id, title: metadata.title ?? id, kind: metadata.kind ?? metadata["document-type"] ?? "text/plain", - time: metadata.time ?? Math.floor(Date.now() / 1000), + time: metadata.time ?? currentTimeSeconds, user: metadata.user ?? state.settings.user, tags: metadata.tags ?? [], }; @@ -526,16 +527,14 @@ export function makeMockBaseApi(fixture: MockWorkbenchFixture = {}): BaseApi { input.flow, ), ), - dispatchStream: async (input, receiver) => { - await dispatchStream(state, input.service, (message) => { + dispatchStream: (input, receiver) => + dispatchStream(state, input.service, (message) => { const chunk = message as { response?: unknown; complete?: boolean }; return receiver({ response: chunk.response, complete: chunk.complete === true, }); - }); - return undefined; - }, + }).then(() => undefined), subscribe: (listener) => { listener({ status: token === undefined ? "connected" : "connected" }); return () => {}; diff --git a/ts/tsconfig.base.json b/ts/tsconfig.base.json index 8265529a..06212d36 100644 --- a/ts/tsconfig.base.json +++ b/ts/tsconfig.base.json @@ -38,26 +38,13 @@ "plugins": [ { "name": "@effect/language-service", - "namespaceImportPackages": ["effect", "@effect/*", "@beep/*"], + "namespaceImportPackages": ["effect", "@effect/*", "@trustgraph/*"], "ignoreEffectSuggestionsInTscExitCode": false, "ignoreEffectWarningsInTscExitCode": false, "ignoreEffectErrorsInTscExitCode": false, "includeSuggestionsInTsc": true, "skipDisabledOptimization": false, "effectFn": ["span", "inferred-span", "suggested-span"], - "overrides": [ - { - "include": ["**/test/**/*.ts", "**/test/**/*.tsx"], - "options": { - "diagnosticSeverity": { - "missingEffectContext": "off", - "nodeBuiltinImport": "off", - "strictEffectProvide": "off" - } - } - } - ], - "importAliases": { "Array": "A", "Option": "O", @@ -68,39 +55,66 @@ }, "diagnosticSeverity": { "anyUnknownInErrorContext": "error", + "asyncFunction": "error", "catchAllToMapError": "error", - "classSelfMismatch": "error", - "floatingEffect": "error", + "catchToOrElseSucceed": "error", "catchUnfailableEffect": "error", + "classSelfMismatch": "error", + "cryptoRandomUUID": "error", + "cryptoRandomUUIDInEffect": "error", "deterministicKeys": "error", "duplicatePackage": "error", + "effectDoNotation": "error", "effectFnIife": "error", + "effectFnImplicitAny": "error", "effectFnOpportunity": "error", "effectGenUsesAdapter": "error", "effectInFailure": "error", "effectInVoidSuccess": "error", + "effectMapFlatten": "error", "effectMapVoid": "error", "effectSucceedWithVoid": "error", "extendsNativeError": "error", + "floatingEffect": "error", "genericEffectServices": "error", - "missingEffectContext": "error", - "missingEffectError": "error", + "globalConsole": "error", + "globalConsoleInEffect": "error", + "globalDate": "error", + "globalDateInEffect": "error", "globalErrorInEffectCatch": "error", "globalErrorInEffectFailure": "error", - "missingLayerContext": "error", + "globalFetch": "error", + "globalFetchInEffect": "error", + "globalRandom": "error", + "globalRandomInEffect": "error", + "globalTimers": "error", + "globalTimersInEffect": "error", "instanceOfSchema": "error", - "missingReturnYieldStar": "error", - "missingStarInYieldEffectGen": "error", "layerMergeAllWithDependencies": "error", - "overriddenSchemaConstructor": "error", - + "lazyEffect": "error", + "lazyPromiseInEffectSync": "error", "leakingRequirements": "error", "missedPipeableOpportunity": "error", + "missingEffectContext": "error", + "missingEffectError": "error", "missingEffectServiceDependency": "error", + "missingLayerContext": "error", + "missingReturnYieldStar": "error", + "missingStarInYieldEffectGen": "error", + "multipleCatchTag": "error", "multipleEffectProvide": "error", + "nestedEffectGenYield": "error", + "newPromise": "error", + "newSchemaClass": "error", "nodeBuiltinImport": "error", + "nonObjectEffectServiceType": "error", "outdatedApi": "error", + "overriddenSchemaConstructor": "error", "preferSchemaOverJson": "error", + "processEnv": "error", + "processEnvInEffect": "error", + "redundantOrDie": "error", + "redundantMapError": "error", "redundantSchemaTagIdentifier": "error", "returnEffectInGen": "error", "runEffectInsideEffect": "error", @@ -113,10 +127,13 @@ "strictEffectProvide": "error", "tryCatchInEffectGen": "error", "unknownInEffectCatch": "error", + "unnecessaryArrowBlock": "error", "unnecessaryEffectGen": "error", "unnecessaryFailYieldableError": "error", "unnecessaryPipe": "error", - "unnecessaryPipeChain": "error" + "unnecessaryPipeChain": "error", + "unnecessaryTypeofType": "error", + "unsafeEffectTypeAssertion": "error" } } ]