mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-01 09:29:38 +02:00
Advance TS port Effect workbench
This commit is contained in:
elpresidank
parent
92dae8c374
commit
3515106670
116 changed files with 12286 additions and 9584 deletions
|
|
@ -20,12 +20,19 @@
|
|||
"test": "bunx --bun vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"effect": "4.0.0-beta.65",
|
||||
"@effect/ai-anthropic": "4.0.0-beta.74",
|
||||
"@effect/ai-openai": "4.0.0-beta.74",
|
||||
"@effect/ai-openrouter": "4.0.0-beta.74",
|
||||
"@effect/atom-react": "4.0.0-beta.74",
|
||||
"@effect/openapi-generator": "4.0.0-beta.74",
|
||||
"@effect/opentelemetry": "4.0.0-beta.74",
|
||||
"@effect/platform-browser": "4.0.0-beta.74",
|
||||
"@effect/platform-bun": "4.0.0-beta.74",
|
||||
"nats": "^2.29.0",
|
||||
"prom-client": "^15.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@effect/vitest": "4.0.0-beta.65",
|
||||
"@effect/vitest": "4.0.0-beta.74",
|
||||
"@types/node": "^22.0.0",
|
||||
"typescript": "^5.8.0",
|
||||
"vitest": "^4.1.6"
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
MessagingRuntimeLive,
|
||||
ProducerSpec,
|
||||
PubSub,
|
||||
runFlowProcessorDefinitionScoped,
|
||||
runProcessorScoped,
|
||||
topics,
|
||||
type BackendConsumer,
|
||||
|
|
@ -212,4 +213,45 @@ describe("Effect-native FlowProcessor runtime", () => {
|
|||
expect(backend.closeCount).toBe(1);
|
||||
}),
|
||||
);
|
||||
|
||||
it.effect(
|
||||
"runs flow specs without a FlowProcessor subclass",
|
||||
Effect.fnUntraced(function* () {
|
||||
const backend = new FlowProcessorBackend();
|
||||
const events: Array<string> = [];
|
||||
|
||||
yield* Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const fiber = yield* runFlowProcessorDefinitionScoped({
|
||||
id: "functional-flow-processor-test",
|
||||
pubsub: backend,
|
||||
specifications: [new ProducerSpec<string>("output")],
|
||||
configHandlers: [
|
||||
(_config, version) => Effect.sync(() => {
|
||||
events.push(`handler:${version}`);
|
||||
}),
|
||||
],
|
||||
}).pipe(
|
||||
Effect.provide(MessagingRuntimeLive),
|
||||
Effect.provide(PubSub.layer(backend)),
|
||||
Effect.provide(fastMessagingConfig),
|
||||
Effect.forkChild,
|
||||
);
|
||||
|
||||
yield* waitFor(() => backend.consumerOptions.length === 1, "config subscription");
|
||||
|
||||
backend.pushConfig(1, { default: { topics: { output: "functional-output" } } });
|
||||
yield* waitFor(() => backend.producers.length === 1, "functional flow producer");
|
||||
yield* waitFor(() => backend.configConsumer.acknowledged.length === 1, "functional config ack");
|
||||
|
||||
yield* Fiber.interrupt(fiber);
|
||||
}),
|
||||
);
|
||||
|
||||
expect(backend.producerOptions.map((options) => options.topic)).toEqual(["functional-output"]);
|
||||
expect(events).toEqual(["handler:1"]);
|
||||
expect(backend.configConsumer.closeCount).toBeGreaterThanOrEqual(1);
|
||||
expect(backend.closeCount).toBe(1);
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,9 +7,13 @@
|
|||
* Python reference: trustgraph-base/trustgraph/base/flow_processor.py
|
||||
*/
|
||||
|
||||
import { AsyncProcessor, type ProcessorConfig } from "./async-processor.js";
|
||||
import {
|
||||
AsyncProcessor,
|
||||
type EffectConfigHandler,
|
||||
type ProcessorConfig,
|
||||
} from "./async-processor.js";
|
||||
import type { Spec } from "../spec/types.js";
|
||||
import type { BackendConsumer } from "../backend/types.js";
|
||||
import type { BackendConsumer, PubSubBackend } from "../backend/types.js";
|
||||
import { Flow, type FlowDefinition } from "./flow.js";
|
||||
import { topics } from "../schema/topics.js";
|
||||
import {
|
||||
|
|
@ -42,11 +46,241 @@ interface ActiveFlow {
|
|||
readonly scope: Scope.Closeable;
|
||||
}
|
||||
|
||||
export interface FlowProcessorRuntimeOptions<
|
||||
FlowRequirements = never,
|
||||
ConfigHandlerError = never,
|
||||
ConfigHandlerRequirements = never,
|
||||
> {
|
||||
readonly id: string;
|
||||
readonly pubsub: PubSubBackend;
|
||||
readonly specifications: ReadonlyArray<Spec<FlowRequirements>>;
|
||||
readonly configHandlers?: ReadonlyArray<
|
||||
EffectConfigHandler<ConfigHandlerError, ConfigHandlerRequirements>
|
||||
>;
|
||||
readonly isRunning?: () => boolean;
|
||||
}
|
||||
|
||||
const ConfigPushSchema = S.Struct({
|
||||
version: S.Number,
|
||||
config: S.Record(S.String, S.Unknown),
|
||||
});
|
||||
|
||||
export function runFlowProcessorDefinitionScoped<
|
||||
FlowRequirements = never,
|
||||
ConfigHandlerError = never,
|
||||
ConfigHandlerRequirements = never,
|
||||
>(
|
||||
options: FlowProcessorRuntimeOptions<
|
||||
FlowRequirements,
|
||||
ConfigHandlerError,
|
||||
ConfigHandlerRequirements
|
||||
>,
|
||||
): Effect.Effect<
|
||||
void,
|
||||
PubSubError | FlowRuntimeError | ConfigHandlerError,
|
||||
| PubSub
|
||||
| FlowRuntime
|
||||
| ProducerFactory
|
||||
| ConsumerFactory
|
||||
| RequestResponseFactory
|
||||
| Scope.Scope
|
||||
| FlowRequirements
|
||||
| ConfigHandlerRequirements
|
||||
> {
|
||||
const flows = new Map<string, ActiveFlow>();
|
||||
let configConsumer: BackendConsumer<ConfigPush> | null = null;
|
||||
let lastFlowsJson = "";
|
||||
const isRunning = options.isRunning ?? (() => true);
|
||||
|
||||
const closeFlowEffect = (name: string, activeFlow: ActiveFlow): Effect.Effect<void> =>
|
||||
Scope.close(activeFlow.scope, Exit.void).pipe(
|
||||
Effect.tap(() => Effect.log(`[${options.id}] Flow "${name}" stopped`)),
|
||||
);
|
||||
|
||||
const closeAllFlowsEffect = Effect.gen(function* () {
|
||||
const activeFlows = Array.from(flows.entries());
|
||||
for (const [name, activeFlow] of activeFlows) {
|
||||
yield* closeFlowEffect(name, activeFlow);
|
||||
}
|
||||
flows.clear();
|
||||
});
|
||||
|
||||
const closeConfigConsumerEffect = (): Effect.Effect<void> => {
|
||||
const consumer = configConsumer;
|
||||
configConsumer = null;
|
||||
if (consumer === null) {
|
||||
return Effect.void;
|
||||
}
|
||||
return Effect.tryPromise({
|
||||
try: () => consumer.close(),
|
||||
catch: (error) => pubSubError("close:config-push", error),
|
||||
}).pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logError(`[${options.id}] Failed to close config consumer`, {
|
||||
error: error.message,
|
||||
}),
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
const startFlowEffect = (
|
||||
name: string,
|
||||
definition: FlowDefinition,
|
||||
): Effect.Effect<
|
||||
ActiveFlow,
|
||||
FlowRuntimeError,
|
||||
FlowRuntime | ProducerFactory | ConsumerFactory | RequestResponseFactory | FlowRequirements
|
||||
> =>
|
||||
Effect.gen(function* () {
|
||||
const flowRuntime = yield* FlowRuntime;
|
||||
const scope = yield* Scope.make();
|
||||
const flow = new Flow<FlowRequirements>(
|
||||
name,
|
||||
options.id,
|
||||
options.pubsub,
|
||||
definition,
|
||||
options.specifications,
|
||||
);
|
||||
return yield* flowRuntime.run(flow).pipe(
|
||||
Scope.provide(scope),
|
||||
Effect.as({ scope } satisfies ActiveFlow),
|
||||
Effect.catch((error) =>
|
||||
Scope.close(scope, Exit.void).pipe(
|
||||
Effect.flatMap(() => Effect.fail(error)),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
const onConfigureFlowsEffect = (
|
||||
config: Record<string, unknown>,
|
||||
_version: number,
|
||||
): Effect.Effect<
|
||||
void,
|
||||
FlowRuntimeError,
|
||||
FlowRuntime | ProducerFactory | ConsumerFactory | RequestResponseFactory | FlowRequirements
|
||||
> =>
|
||||
Effect.gen(function* () {
|
||||
const flowDefs = config.flows as Record<string, FlowDefinition> | undefined;
|
||||
if (flowDefs === undefined) {
|
||||
yield* Effect.log(`[${options.id}] No flows in config push, skipping`);
|
||||
return;
|
||||
}
|
||||
|
||||
const flowsJson = yield* S.encodeUnknownEffect(S.UnknownFromJsonString)(flowDefs).pipe(
|
||||
Effect.catch((error) => Effect.succeed(String(error))),
|
||||
);
|
||||
if (lastFlowsJson.length > 0 && flowsJson === lastFlowsJson && flows.size > 0) {
|
||||
yield* Effect.log(`[${options.id}] Flow definitions unchanged, skipping restart`);
|
||||
return;
|
||||
}
|
||||
lastFlowsJson = flowsJson;
|
||||
|
||||
for (const [name, activeFlow] of flows) {
|
||||
if (!(name in flowDefs)) {
|
||||
yield* Effect.log(`[${options.id}] Stopping removed flow: ${name}`);
|
||||
yield* closeFlowEffect(name, activeFlow);
|
||||
flows.delete(name);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [name, defn] of Object.entries(flowDefs)) {
|
||||
if (typeof defn !== "object" || defn === null) {
|
||||
yield* Effect.logWarning(`[${options.id}] Skipping flow "${name}": definition is not an object`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const existing = flows.get(name);
|
||||
if (existing !== undefined) {
|
||||
yield* Effect.log(`[${options.id}] Restarting flow "${name}" with updated config`);
|
||||
yield* closeFlowEffect(name, existing);
|
||||
flows.delete(name);
|
||||
}
|
||||
|
||||
yield* Effect.log(`[${options.id}] Starting flow "${name}"`);
|
||||
const activeFlow = yield* startFlowEffect(name, defn);
|
||||
flows.set(name, activeFlow);
|
||||
yield* Effect.log(`[${options.id}] Flow "${name}" started`);
|
||||
}
|
||||
});
|
||||
|
||||
const processNextConfigPushEffect = (): Effect.Effect<
|
||||
void,
|
||||
never,
|
||||
| FlowRuntime
|
||||
| ProducerFactory
|
||||
| ConsumerFactory
|
||||
| RequestResponseFactory
|
||||
| FlowRequirements
|
||||
| ConfigHandlerRequirements
|
||||
> =>
|
||||
Effect.gen(function* () {
|
||||
const consumer = configConsumer;
|
||||
if (consumer === null) {
|
||||
yield* Effect.sleep(Duration.millis(1000));
|
||||
return;
|
||||
}
|
||||
|
||||
const msg = yield* Effect.tryPromise({
|
||||
try: () => consumer.receive(2000),
|
||||
catch: (error) => pubSubError("receive:config-push", error),
|
||||
});
|
||||
if (msg === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const push = msg.value();
|
||||
yield* Effect.log(`[${options.id}] Received config push version=${push.version}`);
|
||||
|
||||
yield* onConfigureFlowsEffect(push.config, push.version);
|
||||
|
||||
for (const handler of options.configHandlers ?? []) {
|
||||
yield* handler(push.config, push.version);
|
||||
}
|
||||
|
||||
yield* Effect.tryPromise({
|
||||
try: () => consumer.acknowledge(msg),
|
||||
catch: (error) => pubSubError("acknowledge:config-push", error),
|
||||
});
|
||||
}).pipe(
|
||||
Effect.catch((error) => {
|
||||
if (!isRunning()) {
|
||||
return Effect.void;
|
||||
}
|
||||
return Effect.logError(`[${options.id}] Config consumer error`, {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
}).pipe(
|
||||
Effect.flatMap(() => Effect.sleep(Duration.millis(1000))),
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
return Effect.gen(function* () {
|
||||
const pubsub = yield* PubSub;
|
||||
|
||||
configConsumer = yield* pubsub.createConsumer<ConfigPush>({
|
||||
topic: topics.configPush,
|
||||
subscription: `${options.id}-config-push`,
|
||||
initialPosition: "earliest",
|
||||
schema: ConfigPushSchema,
|
||||
});
|
||||
|
||||
yield* Effect.addFinalizer(() =>
|
||||
closeConfigConsumerEffect().pipe(
|
||||
Effect.flatMap(() => closeAllFlowsEffect),
|
||||
),
|
||||
);
|
||||
|
||||
yield* Effect.log(`[${options.id}] Listening for config pushes on ${topics.configPush}`);
|
||||
|
||||
yield* Effect.whileLoop({
|
||||
while: isRunning,
|
||||
body: processNextConfigPushEffect,
|
||||
step: () => undefined,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export abstract class FlowProcessor<FlowRequirements = never> extends AsyncProcessor<
|
||||
PubSubError | FlowRuntimeError | ProcessorLifecycleError,
|
||||
| PubSub
|
||||
|
|
@ -58,9 +292,6 @@ export abstract class FlowProcessor<FlowRequirements = never> extends AsyncProce
|
|||
| FlowRequirements
|
||||
> {
|
||||
private specifications: Array<Spec<FlowRequirements>> = [];
|
||||
private flows = new Map<string, ActiveFlow>();
|
||||
private configConsumer: BackendConsumer<ConfigPush> | null = null;
|
||||
private lastFlowsJson = "";
|
||||
|
||||
protected constructor(config: ProcessorConfig) {
|
||||
super(config);
|
||||
|
|
@ -104,216 +335,24 @@ export abstract class FlowProcessor<FlowRequirements = never> extends AsyncProce
|
|||
| FlowRequirements
|
||||
> {
|
||||
const processor = this;
|
||||
return Effect.gen(function* () {
|
||||
const pubsub = yield* PubSub;
|
||||
|
||||
// Subscribe to config-push topic to receive flow definitions.
|
||||
// Use "earliest" to replay any config pushes that arrived before this service started.
|
||||
processor.configConsumer = yield* pubsub.createConsumer<ConfigPush>({
|
||||
topic: topics.configPush,
|
||||
subscription: `${processor.config.id}-config-push`,
|
||||
initialPosition: "earliest",
|
||||
schema: ConfigPushSchema,
|
||||
});
|
||||
|
||||
yield* Effect.addFinalizer(() =>
|
||||
processor.closeConfigConsumerEffect().pipe(
|
||||
Effect.flatMap(() => processor.closeAllFlowsEffect()),
|
||||
),
|
||||
);
|
||||
|
||||
yield* Effect.log(`[${processor.config.id}] Listening for config pushes on ${topics.configPush}`);
|
||||
|
||||
yield* Effect.whileLoop({
|
||||
while: () => processor.running,
|
||||
body: () => processor.processNextConfigPushEffect(),
|
||||
step: () => undefined,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private onConfigureFlowsEffect(
|
||||
config: Record<string, unknown>,
|
||||
_version: number,
|
||||
): Effect.Effect<
|
||||
void,
|
||||
FlowRuntimeError,
|
||||
FlowRuntime | ProducerFactory | ConsumerFactory | RequestResponseFactory | FlowRequirements
|
||||
> {
|
||||
const processor = this;
|
||||
return Effect.gen(function* () {
|
||||
const flowDefs = config.flows as Record<string, FlowDefinition> | undefined;
|
||||
if (flowDefs === undefined) {
|
||||
yield* Effect.log(`[${processor.config.id}] No flows in config push, skipping`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip flow restart if the flow definitions haven't changed.
|
||||
// This prevents disrupting in-flight requests when non-flow config
|
||||
// sections (prompts, tools, mcp) are updated.
|
||||
const flowsJson = yield* S.encodeUnknownEffect(S.UnknownFromJsonString)(flowDefs).pipe(
|
||||
Effect.catch((error) => Effect.succeed(String(error))),
|
||||
);
|
||||
if (processor.lastFlowsJson.length > 0 && flowsJson === processor.lastFlowsJson && processor.flows.size > 0) {
|
||||
yield* Effect.log(`[${processor.config.id}] Flow definitions unchanged, skipping restart`);
|
||||
return;
|
||||
}
|
||||
processor.lastFlowsJson = flowsJson;
|
||||
|
||||
// Stop removed flows
|
||||
for (const [name, activeFlow] of processor.flows) {
|
||||
if (!(name in flowDefs)) {
|
||||
yield* Effect.log(`[${processor.config.id}] Stopping removed flow: ${name}`);
|
||||
yield* processor.closeFlowEffect(name, activeFlow);
|
||||
processor.flows.delete(name);
|
||||
}
|
||||
}
|
||||
|
||||
// Start or update flows
|
||||
for (const [name, defn] of Object.entries(flowDefs)) {
|
||||
// Skip invalid definitions (e.g., stringified JSON)
|
||||
if (typeof defn !== "object" || defn === null) {
|
||||
yield* Effect.logWarning(`[${processor.config.id}] Skipping flow "${name}": definition is not an object`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Stop existing flow before (re)starting with new config
|
||||
const existing = processor.flows.get(name);
|
||||
if (existing !== undefined) {
|
||||
yield* Effect.log(`[${processor.config.id}] Restarting flow "${name}" with updated config`);
|
||||
yield* processor.closeFlowEffect(name, existing);
|
||||
processor.flows.delete(name);
|
||||
}
|
||||
|
||||
yield* Effect.log(`[${processor.config.id}] Starting flow "${name}"`);
|
||||
const activeFlow = yield* processor.startFlowEffect(name, defn);
|
||||
processor.flows.set(name, activeFlow);
|
||||
yield* Effect.log(`[${processor.config.id}] Flow "${name}" started`);
|
||||
}
|
||||
const configHandlers = processor.configHandlers.map(
|
||||
(handler): EffectConfigHandler<PubSubError> =>
|
||||
(config, version) =>
|
||||
Effect.tryPromise({
|
||||
try: () => handler(config, version),
|
||||
catch: (error) => pubSubError("config-handler", error),
|
||||
}),
|
||||
);
|
||||
return runFlowProcessorDefinitionScoped({
|
||||
id: processor.config.id,
|
||||
pubsub: processor.pubsub,
|
||||
specifications: processor.specifications,
|
||||
configHandlers,
|
||||
isRunning: () => processor.running,
|
||||
});
|
||||
}
|
||||
|
||||
override stopEffect(): Effect.Effect<void, ProcessorLifecycleError> {
|
||||
return this.closeConfigConsumerEffect().pipe(
|
||||
Effect.flatMap(() => this.closeAllFlowsEffect()),
|
||||
Effect.flatMap(() => super.stopEffect()),
|
||||
);
|
||||
}
|
||||
|
||||
private processNextConfigPushEffect(): Effect.Effect<
|
||||
void,
|
||||
never,
|
||||
FlowRuntime | ProducerFactory | ConsumerFactory | RequestResponseFactory | FlowRequirements
|
||||
> {
|
||||
const processor = this;
|
||||
return Effect.gen(function* () {
|
||||
const consumer = processor.configConsumer;
|
||||
if (consumer === null) {
|
||||
yield* Effect.sleep(Duration.millis(1000));
|
||||
return;
|
||||
}
|
||||
|
||||
const msg = yield* Effect.tryPromise({
|
||||
try: () => consumer.receive(2000),
|
||||
catch: (error) => pubSubError("receive:config-push", error),
|
||||
});
|
||||
if (msg === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const push = msg.value();
|
||||
yield* Effect.log(`[${processor.config.id}] Received config push version=${push.version}`);
|
||||
|
||||
yield* processor.onConfigureFlowsEffect(push.config, push.version);
|
||||
|
||||
for (const handler of processor.configHandlers) {
|
||||
yield* Effect.tryPromise({
|
||||
try: () => handler(push.config, push.version),
|
||||
catch: (error) => pubSubError("config-handler", error),
|
||||
});
|
||||
}
|
||||
|
||||
yield* Effect.tryPromise({
|
||||
try: () => consumer.acknowledge(msg),
|
||||
catch: (error) => pubSubError("acknowledge:config-push", error),
|
||||
});
|
||||
}).pipe(
|
||||
Effect.catch((error) => {
|
||||
if (!processor.running) {
|
||||
return Effect.void;
|
||||
}
|
||||
return Effect.logError(`[${processor.config.id}] Config consumer error`, {
|
||||
error: error.message,
|
||||
}).pipe(
|
||||
Effect.flatMap(() => Effect.sleep(Duration.millis(1000))),
|
||||
);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
private startFlowEffect(
|
||||
name: string,
|
||||
definition: FlowDefinition,
|
||||
): Effect.Effect<
|
||||
ActiveFlow,
|
||||
FlowRuntimeError,
|
||||
FlowRuntime | ProducerFactory | ConsumerFactory | RequestResponseFactory | FlowRequirements
|
||||
> {
|
||||
const processor = this;
|
||||
return Effect.gen(function* () {
|
||||
const flowRuntime = yield* FlowRuntime;
|
||||
const scope = yield* Scope.make();
|
||||
const flow = new Flow<FlowRequirements>(
|
||||
name,
|
||||
processor.config.id,
|
||||
processor.pubsub,
|
||||
definition,
|
||||
processor.specifications,
|
||||
);
|
||||
return yield* flowRuntime.run(flow).pipe(
|
||||
Scope.provide(scope),
|
||||
Effect.as({ scope } satisfies ActiveFlow),
|
||||
Effect.catch((error) =>
|
||||
Scope.close(scope, Exit.void).pipe(
|
||||
Effect.flatMap(() => Effect.fail(error)),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private closeFlowEffect(name: string, activeFlow: ActiveFlow): Effect.Effect<void> {
|
||||
return Scope.close(activeFlow.scope, Exit.void).pipe(
|
||||
Effect.tap(() => Effect.log(`[${this.config.id}] Flow "${name}" stopped`)),
|
||||
);
|
||||
}
|
||||
|
||||
private closeAllFlowsEffect(): Effect.Effect<void> {
|
||||
const processor = this;
|
||||
return Effect.gen(function* () {
|
||||
const flows = Array.from(processor.flows.entries());
|
||||
for (const [name, activeFlow] of flows) {
|
||||
yield* processor.closeFlowEffect(name, activeFlow);
|
||||
}
|
||||
processor.flows.clear();
|
||||
});
|
||||
}
|
||||
|
||||
private closeConfigConsumerEffect(): Effect.Effect<void> {
|
||||
const consumer = this.configConsumer;
|
||||
this.configConsumer = null;
|
||||
if (consumer === null) {
|
||||
return Effect.void;
|
||||
}
|
||||
return Effect.tryPromise({
|
||||
try: () => consumer.close(),
|
||||
catch: (error) => pubSubError("close:config-push", error),
|
||||
}).pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logError(`[${this.config.id}] Failed to close config consumer`, {
|
||||
error: error.message,
|
||||
}),
|
||||
),
|
||||
);
|
||||
return super.stopEffect();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,11 @@ export {
|
|||
type EffectConfigHandler,
|
||||
type ProcessorConfig,
|
||||
} from "./async-processor.js";
|
||||
export { FlowProcessor } from "./flow-processor.js";
|
||||
export {
|
||||
FlowProcessor,
|
||||
runFlowProcessorDefinitionScoped,
|
||||
type FlowProcessorRuntimeOptions,
|
||||
} from "./flow-processor.js";
|
||||
export {
|
||||
Flow,
|
||||
type FlowConsumer,
|
||||
|
|
@ -18,5 +22,6 @@ export {
|
|||
makeFlowProcessorProgram,
|
||||
makeProcessorProgram,
|
||||
runProcessorScoped,
|
||||
type FlowProcessorProgramOptions,
|
||||
type ProcessorProgramOptions,
|
||||
} from "./program.js";
|
||||
|
|
|
|||
|
|
@ -5,8 +5,13 @@
|
|||
* executable path while the processor internals remain Promise-based.
|
||||
*/
|
||||
|
||||
import { Effect, Scope } from "effect";
|
||||
import { processorLifecycleError, type ProcessorLifecycleError } from "../errors.js";
|
||||
import { Config as EffectConfig, Effect, Layer, Scope } from "effect";
|
||||
import {
|
||||
processorLifecycleError,
|
||||
type FlowRuntimeError,
|
||||
type ProcessorLifecycleError,
|
||||
type PubSubError,
|
||||
} from "../errors.js";
|
||||
import { NatsBackend } from "../backend/nats.js";
|
||||
import { makePubSubService, PubSub } from "../backend/pubsub.js";
|
||||
import {
|
||||
|
|
@ -24,7 +29,13 @@ import {
|
|||
type ProcessorRuntimeConfigOptions,
|
||||
} from "../runtime/config.js";
|
||||
import { loadMessagingRuntimeConfig } from "../runtime/messaging-config.js";
|
||||
import type { AsyncProcessor, ProcessorConfig } from "./async-processor.js";
|
||||
import type {
|
||||
AsyncProcessor,
|
||||
EffectConfigHandler,
|
||||
ProcessorConfig,
|
||||
} from "./async-processor.js";
|
||||
import { runFlowProcessorDefinitionScoped } from "./flow-processor.js";
|
||||
import type { Spec } from "../spec/types.js";
|
||||
|
||||
type ProcessorRunError<Processor> = Processor extends AsyncProcessor<infer Error, unknown> ? Error : never;
|
||||
type ProcessorRunRequirements<Processor> = Processor extends AsyncProcessor<unknown, infer Requirements> ? Requirements : never;
|
||||
|
|
@ -40,6 +51,23 @@ export interface ProcessorProgramOptions<
|
|||
readonly loadConfig?: Effect.Effect<Config, Error, Requirements>;
|
||||
}
|
||||
|
||||
export interface FlowProcessorProgramOptions<
|
||||
Config extends ProcessorConfig,
|
||||
Error = never,
|
||||
FlowRequirements = never,
|
||||
LayerRequirements = never,
|
||||
> {
|
||||
readonly id: string;
|
||||
readonly loadConfig?: Effect.Effect<Config, Error, LayerRequirements>;
|
||||
readonly specs: (config: Config) => ReadonlyArray<Spec<FlowRequirements>>;
|
||||
readonly configHandlers?: (
|
||||
config: Config,
|
||||
) => ReadonlyArray<EffectConfigHandler<Error, FlowRequirements>>;
|
||||
readonly layer?: (
|
||||
config: Config,
|
||||
) => Layer.Layer<FlowRequirements, Error, LayerRequirements>;
|
||||
}
|
||||
|
||||
export function runProcessorScoped<
|
||||
Config extends ProcessorConfig,
|
||||
Processor extends AsyncProcessor<unknown, unknown>,
|
||||
|
|
@ -136,4 +164,74 @@ export function makeProcessorProgram<
|
|||
}
|
||||
|
||||
export const makeAsyncProcessorProgram = makeProcessorProgram;
|
||||
export const makeFlowProcessorProgram = makeProcessorProgram;
|
||||
|
||||
export function makeFlowProcessorProgram<
|
||||
Config extends ProcessorConfig,
|
||||
Error = never,
|
||||
FlowRequirements = never,
|
||||
LayerRequirements = never,
|
||||
>(
|
||||
options: FlowProcessorProgramOptions<Config, Error, FlowRequirements, LayerRequirements>,
|
||||
): Effect.Effect<
|
||||
void,
|
||||
Error | EffectConfig.ConfigError | PubSubError | FlowRuntimeError,
|
||||
LayerRequirements
|
||||
> {
|
||||
return Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const config = yield* (
|
||||
options.loadConfig ??
|
||||
loadProcessorRuntimeConfig(options.id, {
|
||||
manageProcessSignals: false,
|
||||
} satisfies ProcessorRuntimeConfigOptions)
|
||||
);
|
||||
|
||||
const runtimeConfig = {
|
||||
...config,
|
||||
manageProcessSignals: false,
|
||||
} as Config;
|
||||
|
||||
const pubsub = makePubSubService(new NatsBackend(runtimeConfig.pubsubUrl ?? "nats://localhost:4222"));
|
||||
const messagingConfig = yield* loadMessagingRuntimeConfig();
|
||||
yield* Effect.addFinalizer(() =>
|
||||
pubsub.close.pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logError("[PubSub] Failed to close processor backend", {
|
||||
error: error.message,
|
||||
operation: error.operation,
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const configHandlers = options.configHandlers?.(runtimeConfig);
|
||||
const processorOptions = {
|
||||
id: runtimeConfig.id,
|
||||
pubsub: pubsub.backend,
|
||||
specifications: options.specs(runtimeConfig),
|
||||
...(configHandlers === undefined ? {} : { configHandlers }),
|
||||
};
|
||||
const processorLayer = Layer.effectDiscard(
|
||||
runFlowProcessorDefinitionScoped<FlowRequirements, Error, FlowRequirements>(processorOptions),
|
||||
);
|
||||
const runtimeLayer = Layer.mergeAll(
|
||||
Layer.succeed(PubSub, PubSub.of(pubsub)),
|
||||
Layer.succeed(ProducerFactory, ProducerFactory.of(makeProducerFactoryService(pubsub))),
|
||||
Layer.succeed(ConsumerFactory, ConsumerFactory.of(makeConsumerFactoryService(pubsub, messagingConfig))),
|
||||
Layer.succeed(
|
||||
RequestResponseFactory,
|
||||
RequestResponseFactory.of(makeRequestResponseFactoryService(pubsub, messagingConfig)),
|
||||
),
|
||||
Layer.succeed(FlowRuntime, FlowRuntime.of({ run: runFlowRuntimeScoped })),
|
||||
);
|
||||
const dependencyLayer = options.layer?.(runtimeConfig) ??
|
||||
(Layer.empty as unknown as Layer.Layer<FlowRequirements, Error, LayerRequirements>);
|
||||
const providedProcessorLayer = processorLayer.pipe(
|
||||
Layer.provide(dependencyLayer),
|
||||
Layer.provide(runtimeLayer),
|
||||
);
|
||||
|
||||
return yield* Layer.launch(providedProcessorLayer);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -179,15 +179,16 @@ export const ConfigOperation = S.Union([
|
|||
S.Literal("put"),
|
||||
S.Literal("config"),
|
||||
S.Literal("getvalues"),
|
||||
S.Literal("getvalues-all-ws"),
|
||||
]);
|
||||
export type ConfigOperation = typeof ConfigOperation.Type;
|
||||
|
||||
export const ConfigRequest = S.Struct({
|
||||
operation: ConfigOperation,
|
||||
keys: S.optionalKey(StringArray),
|
||||
values: S.optionalKey(UnknownRecord),
|
||||
type: S.optionalKey(S.String),
|
||||
});
|
||||
export const ConfigRequest = S.StructWithRest(
|
||||
S.Struct({
|
||||
operation: ConfigOperation,
|
||||
}),
|
||||
[UnknownRecord],
|
||||
);
|
||||
export type ConfigRequest = typeof ConfigRequest.Type;
|
||||
|
||||
export const ConfigResponse = S.Struct({
|
||||
|
|
@ -271,7 +272,9 @@ export const DocumentMetadata = S.Struct({
|
|||
user: S.String,
|
||||
tags: StringArray,
|
||||
parentId: S.optionalKey(S.String),
|
||||
documentType: S.String,
|
||||
"parent-id": S.optionalKey(S.String),
|
||||
documentType: S.optionalKey(S.String),
|
||||
"document-type": S.optionalKey(S.String),
|
||||
metadata: OptionalMutableArray(Triple),
|
||||
});
|
||||
export type DocumentMetadata = typeof DocumentMetadata.Type;
|
||||
|
|
@ -279,6 +282,7 @@ export type DocumentMetadata = typeof DocumentMetadata.Type;
|
|||
export const ProcessingMetadata = S.Struct({
|
||||
id: S.String,
|
||||
documentId: S.String,
|
||||
"document-id": S.optionalKey(S.String),
|
||||
time: S.Number,
|
||||
flow: S.String,
|
||||
user: S.String,
|
||||
|
|
@ -291,6 +295,7 @@ export type ProcessingMetadata = typeof ProcessingMetadata.Type;
|
|||
export const LibrarianOperation = S.Literals([
|
||||
"add-document",
|
||||
"remove-document",
|
||||
"update-document",
|
||||
"list-documents",
|
||||
"get-document-metadata",
|
||||
"get-document-content",
|
||||
|
|
@ -299,15 +304,26 @@ export const LibrarianOperation = S.Literals([
|
|||
"add-processing",
|
||||
"remove-processing",
|
||||
"list-processing",
|
||||
"begin-upload",
|
||||
"upload-chunk",
|
||||
"complete-upload",
|
||||
"get-upload-status",
|
||||
"abort-upload",
|
||||
"list-uploads",
|
||||
"stream-document",
|
||||
]);
|
||||
export type LibrarianOperation = typeof LibrarianOperation.Type;
|
||||
|
||||
export const LibrarianRequest = S.Struct({
|
||||
operation: LibrarianOperation,
|
||||
documentId: S.optionalKey(S.String),
|
||||
"document-id": S.optionalKey(S.String),
|
||||
processingId: S.optionalKey(S.String),
|
||||
"processing-id": S.optionalKey(S.String),
|
||||
documentMetadata: S.optionalKey(DocumentMetadata),
|
||||
"document-metadata": S.optionalKey(DocumentMetadata),
|
||||
processingMetadata: S.optionalKey(ProcessingMetadata),
|
||||
"processing-metadata": S.optionalKey(ProcessingMetadata),
|
||||
content: S.optionalKey(S.String),
|
||||
user: S.optionalKey(S.String),
|
||||
collection: S.optionalKey(S.String),
|
||||
|
|
@ -317,9 +333,13 @@ export type LibrarianRequest = typeof LibrarianRequest.Type;
|
|||
export const LibrarianResponse = S.Struct({
|
||||
error: S.optionalKey(TgError),
|
||||
documentMetadata: S.optionalKey(DocumentMetadata),
|
||||
"document-metadata": S.optionalKey(DocumentMetadata),
|
||||
content: S.optionalKey(S.String),
|
||||
documents: OptionalMutableArray(DocumentMetadata),
|
||||
"document-metadatas": OptionalMutableArray(DocumentMetadata),
|
||||
processing: OptionalMutableArray(ProcessingMetadata),
|
||||
"processing-metadata": OptionalMutableArray(ProcessingMetadata),
|
||||
"processing-metadatas": OptionalMutableArray(ProcessingMetadata),
|
||||
});
|
||||
export type LibrarianResponse = typeof LibrarianResponse.Type;
|
||||
|
||||
|
|
@ -330,6 +350,12 @@ export const KnowledgeOperation = S.Literals([
|
|||
"delete-kg-core",
|
||||
"put-kg-core",
|
||||
"load-kg-core",
|
||||
"unload-kg-core",
|
||||
"list-de-cores",
|
||||
"get-de-core",
|
||||
"delete-de-core",
|
||||
"put-de-core",
|
||||
"load-de-core",
|
||||
]);
|
||||
export type KnowledgeOperation = typeof KnowledgeOperation.Type;
|
||||
|
||||
|
|
@ -338,6 +364,14 @@ const GraphEmbedding = S.Struct({
|
|||
vectors: NumberArrays,
|
||||
});
|
||||
|
||||
const DocumentEmbeddingsCore = S.StructWithRest(
|
||||
S.Struct({
|
||||
metadata: S.optionalKey(UnknownRecord),
|
||||
chunks: S.optionalKey(S.Unknown.pipe(S.Array, S.mutable)),
|
||||
}),
|
||||
[UnknownRecord],
|
||||
);
|
||||
|
||||
export const KnowledgeRequest = S.Struct({
|
||||
operation: KnowledgeOperation,
|
||||
user: S.optionalKey(S.String),
|
||||
|
|
@ -346,6 +380,7 @@ export const KnowledgeRequest = S.Struct({
|
|||
collection: S.optionalKey(S.String),
|
||||
triples: OptionalMutableArray(Triple),
|
||||
graphEmbeddings: OptionalMutableArray(GraphEmbedding),
|
||||
documentEmbeddings: S.optionalKey(DocumentEmbeddingsCore),
|
||||
});
|
||||
export type KnowledgeRequest = typeof KnowledgeRequest.Type;
|
||||
|
||||
|
|
@ -355,6 +390,7 @@ export const KnowledgeResponse = S.Struct({
|
|||
eos: S.optionalKey(S.Boolean),
|
||||
triples: OptionalMutableArray(Triple),
|
||||
graphEmbeddings: OptionalMutableArray(GraphEmbedding),
|
||||
documentEmbeddings: S.optionalKey(DocumentEmbeddingsCore),
|
||||
});
|
||||
export type KnowledgeResponse = typeof KnowledgeResponse.Type;
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import type { EmbeddingsRequest, EmbeddingsResponse } from "../schema/messages.j
|
|||
import { ConsumerSpec } from "../spec/consumer-spec.js";
|
||||
import { ParameterSpec } from "../spec/parameter-spec.js";
|
||||
import { ProducerSpec } from "../spec/producer-spec.js";
|
||||
import type { Spec } from "../spec/types.js";
|
||||
|
||||
export interface EmbeddingsServiceShape {
|
||||
readonly embed: (
|
||||
|
|
@ -30,53 +31,55 @@ export class Embeddings extends Context.Service<Embeddings, EmbeddingsServiceSha
|
|||
"@trustgraph/base/services/embeddings-service/Embeddings",
|
||||
) {}
|
||||
|
||||
const onEmbeddingsRequest = Effect.fn("EmbeddingsService.onRequest")(function* (
|
||||
msg: EmbeddingsRequest,
|
||||
properties: Record<string, string>,
|
||||
flowCtx: FlowContext<Embeddings>,
|
||||
): Effect.fn.Return<void, FlowResourceNotFoundError | MessagingDeliveryError, Embeddings> {
|
||||
const requestId = properties.id;
|
||||
if (requestId === undefined || requestId.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const responseProducer = yield* flowCtx.flow.producerEffect<EmbeddingsResponse>("embeddings-response");
|
||||
const embeddings = yield* Embeddings;
|
||||
const response = yield* embeddings.embed(msg.text, msg.model).pipe(
|
||||
Effect.map((vectors) => ({ vectors }) satisfies EmbeddingsResponse),
|
||||
Effect.catch((error) =>
|
||||
Effect.logError("[EmbeddingsService] Error processing request", {
|
||||
error: errorMessage(error),
|
||||
operation: error.operation,
|
||||
provider: error.provider ?? "unknown",
|
||||
}).pipe(
|
||||
Effect.as({
|
||||
vectors: [],
|
||||
error: {
|
||||
type: "embeddings-error",
|
||||
message: errorMessage(error),
|
||||
},
|
||||
} satisfies EmbeddingsResponse),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
yield* responseProducer.send(requestId, response);
|
||||
});
|
||||
|
||||
export const makeEmbeddingsSpecs = (): ReadonlyArray<Spec<Embeddings>> => [
|
||||
new ConsumerSpec<EmbeddingsRequest, FlowResourceNotFoundError | MessagingDeliveryError, Embeddings>(
|
||||
"embeddings-request",
|
||||
onEmbeddingsRequest,
|
||||
),
|
||||
new ProducerSpec<EmbeddingsResponse>("embeddings-response"),
|
||||
new ParameterSpec("model"),
|
||||
];
|
||||
|
||||
export class EmbeddingsService extends FlowProcessor<Embeddings> {
|
||||
constructor(config: ProcessorConfig) {
|
||||
super(config);
|
||||
|
||||
this.registerSpecification(
|
||||
new ConsumerSpec<EmbeddingsRequest, FlowResourceNotFoundError | MessagingDeliveryError, Embeddings>(
|
||||
"embeddings-request",
|
||||
this.onRequestEffect.bind(this),
|
||||
),
|
||||
);
|
||||
this.registerSpecification(new ProducerSpec<EmbeddingsResponse>("embeddings-response"));
|
||||
this.registerSpecification(new ParameterSpec("model"));
|
||||
}
|
||||
|
||||
private onRequestEffect(
|
||||
msg: EmbeddingsRequest,
|
||||
properties: Record<string, string>,
|
||||
flowCtx: FlowContext<Embeddings>,
|
||||
): Effect.Effect<void, FlowResourceNotFoundError | MessagingDeliveryError, Embeddings> {
|
||||
const requestId = properties.id;
|
||||
if (requestId === undefined || requestId.length === 0) {
|
||||
return Effect.void;
|
||||
for (const spec of makeEmbeddingsSpecs()) {
|
||||
this.registerSpecification(spec);
|
||||
}
|
||||
|
||||
return Effect.gen(function* () {
|
||||
const responseProducer = yield* flowCtx.flow.producerEffect<EmbeddingsResponse>("embeddings-response");
|
||||
const embeddings = yield* Embeddings;
|
||||
const response = yield* embeddings.embed(msg.text, msg.model).pipe(
|
||||
Effect.map((vectors) => ({ vectors }) satisfies EmbeddingsResponse),
|
||||
Effect.catch((error) =>
|
||||
Effect.logError("[EmbeddingsService] Error processing request", {
|
||||
error: errorMessage(error),
|
||||
operation: error.operation,
|
||||
provider: error.provider ?? "unknown",
|
||||
}).pipe(
|
||||
Effect.as({
|
||||
vectors: [],
|
||||
error: {
|
||||
type: "embeddings-error",
|
||||
message: errorMessage(error),
|
||||
},
|
||||
} satisfies EmbeddingsResponse),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
yield* responseProducer.send(requestId, response);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,15 @@
|
|||
export { LlmService } from "./llm-service.js";
|
||||
export {
|
||||
Llm,
|
||||
LlmService,
|
||||
LlmServiceError,
|
||||
makeLlmServiceShape,
|
||||
makeLlmSpecs,
|
||||
type LlmProvider,
|
||||
type LlmServiceShape,
|
||||
} from "./llm-service.js";
|
||||
export {
|
||||
Embeddings,
|
||||
EmbeddingsService,
|
||||
makeEmbeddingsSpecs,
|
||||
type EmbeddingsServiceShape,
|
||||
} from "./embeddings-service.js";
|
||||
|
|
|
|||
|
|
@ -1,125 +1,247 @@
|
|||
/**
|
||||
* Base LLM service — handles message plumbing, subclasses implement the LLM call.
|
||||
* Base LLM capability contract and message-bus adapter.
|
||||
*
|
||||
* Python reference: trustgraph-base/trustgraph/base/llm_service.py
|
||||
*/
|
||||
|
||||
import {FlowProcessor} from "../processor/index.js";
|
||||
import { Context, Effect } from "effect";
|
||||
import * as S from "effect/Schema";
|
||||
import {
|
||||
ConsumerSpec, ProducerSpec,
|
||||
ParameterSpec
|
||||
} from "../spec/index.js";
|
||||
import type {ProcessorConfig} from "../processor/index.js";
|
||||
import type {FlowContext} from "../messaging/consumer.js";
|
||||
errorMessage,
|
||||
type FlowResourceNotFoundError,
|
||||
type MessagingDeliveryError,
|
||||
} from "../errors.js";
|
||||
import type { FlowContext } from "../messaging/consumer.js";
|
||||
import { FlowProcessor } from "../processor/flow-processor.js";
|
||||
import type { ProcessorConfig } from "../processor/async-processor.js";
|
||||
import type {
|
||||
TextCompletionRequest,
|
||||
TextCompletionResponse,
|
||||
TextCompletionRequest,
|
||||
TextCompletionResponse,
|
||||
} from "../schema/messages.js";
|
||||
import type {LlmResult, LlmChunk} from "../schema/index.js";
|
||||
import type { LlmChunk, LlmResult } from "../schema/primitives.js";
|
||||
import { ConsumerSpec } from "../spec/consumer-spec.js";
|
||||
import { ParameterSpec } from "../spec/parameter-spec.js";
|
||||
import { ProducerSpec } from "../spec/producer-spec.js";
|
||||
import type { Spec } from "../spec/types.js";
|
||||
|
||||
export abstract class LlmService extends FlowProcessor {
|
||||
protected constructor(config: ProcessorConfig) {
|
||||
super(config);
|
||||
export class LlmServiceError extends S.TaggedErrorClass<LlmServiceError>()(
|
||||
"LlmServiceError",
|
||||
{
|
||||
message: S.String,
|
||||
operation: S.String,
|
||||
},
|
||||
) {}
|
||||
|
||||
this.registerSpecification(
|
||||
ConsumerSpec.fromPromise<TextCompletionRequest>(
|
||||
"text-completion-request",
|
||||
this.onRequest.bind(this),
|
||||
),
|
||||
);
|
||||
this.registerSpecification(new ProducerSpec<TextCompletionResponse>("text-completion-response"));
|
||||
this.registerSpecification(new ParameterSpec("model"));
|
||||
this.registerSpecification(new ParameterSpec("temperature"));
|
||||
}
|
||||
|
||||
private async onRequest(
|
||||
msg: TextCompletionRequest,
|
||||
properties: Record<string, string>,
|
||||
flowCtx: FlowContext,
|
||||
): Promise<void> {
|
||||
const requestId = properties.id;
|
||||
if (requestId === undefined || requestId.length === 0) return;
|
||||
|
||||
const responseProducer = flowCtx.flow.producer<TextCompletionResponse>("text-completion-response");
|
||||
|
||||
try {
|
||||
if (msg.streaming === true && this.supportsStreaming()) {
|
||||
for await (const chunk of this.generateContentStream(
|
||||
msg.system,
|
||||
msg.prompt,
|
||||
msg.model,
|
||||
msg.temperature,
|
||||
)) {
|
||||
const response = {
|
||||
response: chunk.text,
|
||||
...(chunk.model !== undefined ? { model: chunk.model } : {}),
|
||||
...(chunk.inToken !== null ? { inToken: chunk.inToken } : {}),
|
||||
...(chunk.outToken !== null ? { outToken: chunk.outToken } : {}),
|
||||
endOfStream: chunk.isFinal,
|
||||
};
|
||||
await responseProducer.send(
|
||||
requestId,
|
||||
response
|
||||
);
|
||||
}
|
||||
} else {
|
||||
const result = await this.generateContent(
|
||||
msg.system,
|
||||
msg.prompt,
|
||||
msg.model,
|
||||
msg.temperature,
|
||||
);
|
||||
const response = {
|
||||
response: result.text,
|
||||
...(result.model !== undefined ? { model: result.model } : {}),
|
||||
...(result.inToken !== undefined ? { inToken: result.inToken } : {}),
|
||||
...(result.outToken !== undefined ? { outToken: result.outToken } : {}),
|
||||
endOfStream: true,
|
||||
};
|
||||
|
||||
await responseProducer.send(
|
||||
requestId,
|
||||
response
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[LlmService] Error processing request:`,
|
||||
err
|
||||
);
|
||||
|
||||
const message = err instanceof Error
|
||||
? err.message
|
||||
: String(err);
|
||||
await responseProducer.send(
|
||||
requestId,
|
||||
{
|
||||
response: "",
|
||||
error: {
|
||||
type: "llm-error",
|
||||
message
|
||||
},
|
||||
endOfStream: true,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
abstract generateContent(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult>;
|
||||
|
||||
abstract generateContentStream(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk>;
|
||||
|
||||
supportsStreaming(): boolean {
|
||||
return false;
|
||||
}
|
||||
export interface LlmProvider {
|
||||
readonly generateContent: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
) => Promise<LlmResult>;
|
||||
readonly generateContentStream: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
) => AsyncGenerator<LlmChunk>;
|
||||
readonly supportsStreaming: () => boolean;
|
||||
}
|
||||
|
||||
export interface LlmServiceShape {
|
||||
readonly generateContent: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
) => Effect.Effect<LlmResult, LlmServiceError>;
|
||||
readonly generateContentStream: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
) => AsyncGenerator<LlmChunk>;
|
||||
readonly supportsStreaming: () => boolean;
|
||||
}
|
||||
|
||||
export class Llm extends Context.Service<Llm, LlmServiceShape>()(
|
||||
"@trustgraph/base/services/llm-service/Llm",
|
||||
) {}
|
||||
|
||||
const llmServiceError = (operation: string, cause: unknown) =>
|
||||
new LlmServiceError({
|
||||
operation,
|
||||
message: errorMessage(cause),
|
||||
});
|
||||
|
||||
export const makeLlmServiceShape = (provider: LlmProvider): LlmServiceShape => ({
|
||||
generateContent: Effect.fn("Llm.generateContent")((
|
||||
system,
|
||||
prompt,
|
||||
model,
|
||||
temperature,
|
||||
) =>
|
||||
Effect.tryPromise({
|
||||
try: () => provider.generateContent(system, prompt, model, temperature),
|
||||
catch: (cause) => llmServiceError("generate-content", cause),
|
||||
}),
|
||||
),
|
||||
generateContentStream: (
|
||||
system,
|
||||
prompt,
|
||||
model,
|
||||
temperature,
|
||||
) => provider.generateContentStream(system, prompt, model, temperature),
|
||||
supportsStreaming: () => provider.supportsStreaming(),
|
||||
});
|
||||
|
||||
type LlmHandlerError =
|
||||
| FlowResourceNotFoundError
|
||||
| MessagingDeliveryError;
|
||||
|
||||
const resultToResponse = (result: LlmResult): TextCompletionResponse => ({
|
||||
response: result.text,
|
||||
model: result.model,
|
||||
inToken: result.inToken,
|
||||
outToken: result.outToken,
|
||||
endOfStream: true,
|
||||
});
|
||||
|
||||
const chunkToResponse = (chunk: LlmChunk): TextCompletionResponse => ({
|
||||
response: chunk.text,
|
||||
model: chunk.model,
|
||||
...(chunk.inToken !== null ? { inToken: chunk.inToken } : {}),
|
||||
...(chunk.outToken !== null ? { outToken: chunk.outToken } : {}),
|
||||
endOfStream: chunk.isFinal,
|
||||
});
|
||||
|
||||
const llmErrorResponse = (error: LlmServiceError): TextCompletionResponse => ({
|
||||
response: "",
|
||||
error: {
|
||||
type: "llm-error",
|
||||
message: error.message,
|
||||
},
|
||||
endOfStream: true,
|
||||
});
|
||||
|
||||
const sendStreamingResponse = Effect.fn("LlmService.sendStreamingResponse")(function* (
|
||||
llm: LlmServiceShape,
|
||||
requestId: string,
|
||||
msg: TextCompletionRequest,
|
||||
responseProducer: {
|
||||
readonly send: (
|
||||
id: string,
|
||||
message: TextCompletionResponse,
|
||||
) => Effect.Effect<void, MessagingDeliveryError>;
|
||||
},
|
||||
) {
|
||||
const context = yield* Effect.context<never>();
|
||||
yield* Effect.tryPromise({
|
||||
try: async () => {
|
||||
for await (const chunk of llm.generateContentStream(
|
||||
msg.system,
|
||||
msg.prompt,
|
||||
msg.model,
|
||||
msg.temperature,
|
||||
)) {
|
||||
await Effect.runPromiseWith(context)(
|
||||
responseProducer.send(requestId, chunkToResponse(chunk)),
|
||||
);
|
||||
}
|
||||
},
|
||||
catch: (cause) => llmServiceError("generate-content-stream", cause),
|
||||
});
|
||||
});
|
||||
|
||||
const onLlmRequest = Effect.fn("LlmService.onRequest")(function* (
|
||||
msg: TextCompletionRequest,
|
||||
properties: Record<string, string>,
|
||||
flowCtx: FlowContext<Llm>,
|
||||
): Effect.fn.Return<void, LlmHandlerError, Llm> {
|
||||
const requestId = properties.id;
|
||||
if (requestId === undefined || requestId.length === 0) return;
|
||||
|
||||
const responseProducer = yield* flowCtx.flow.producerEffect<TextCompletionResponse>(
|
||||
"text-completion-response",
|
||||
);
|
||||
const llm = yield* Llm;
|
||||
|
||||
if (msg.streaming === true && llm.supportsStreaming()) {
|
||||
yield* sendStreamingResponse(llm, requestId, msg, responseProducer).pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logError("[LlmService] Error processing streaming request", {
|
||||
error: error.message,
|
||||
operation: error.operation,
|
||||
}).pipe(
|
||||
Effect.flatMap(() =>
|
||||
responseProducer.send(requestId, llmErrorResponse(error)),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const response = yield* llm.generateContent(
|
||||
msg.system,
|
||||
msg.prompt,
|
||||
msg.model,
|
||||
msg.temperature,
|
||||
).pipe(
|
||||
Effect.map(resultToResponse),
|
||||
Effect.catch((error) =>
|
||||
Effect.logError("[LlmService] Error processing request", {
|
||||
error: error.message,
|
||||
operation: error.operation,
|
||||
}).pipe(
|
||||
Effect.as(llmErrorResponse(error)),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
yield* responseProducer.send(requestId, response);
|
||||
});
|
||||
|
||||
export const makeLlmSpecs = (): ReadonlyArray<Spec<Llm>> => [
|
||||
new ConsumerSpec<TextCompletionRequest, LlmHandlerError, Llm>(
|
||||
"text-completion-request",
|
||||
onLlmRequest,
|
||||
),
|
||||
new ProducerSpec<TextCompletionResponse>("text-completion-response"),
|
||||
new ParameterSpec("model"),
|
||||
new ParameterSpec("temperature"),
|
||||
];
|
||||
|
||||
export abstract class LlmService extends FlowProcessor<Llm> implements LlmProvider {
|
||||
protected constructor(config: ProcessorConfig) {
|
||||
super(config);
|
||||
|
||||
for (const spec of makeLlmSpecs()) {
|
||||
this.registerSpecification(spec);
|
||||
}
|
||||
}
|
||||
|
||||
override startEffect() {
|
||||
return super.startEffect().pipe(
|
||||
Effect.provideService(Llm, Llm.of(makeLlmServiceShape(this))),
|
||||
);
|
||||
}
|
||||
|
||||
abstract generateContent(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult>;
|
||||
|
||||
abstract generateContentStream(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk>;
|
||||
|
||||
supportsStreaming(): boolean {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue