mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-01 17:39:39 +02:00
Enforce strict Effect tsgo migrations
This commit is contained in:
elpresidank
parent
64fb23e7d0
commit
f6878d4dd7
49 changed files with 5547 additions and 3250 deletions
|
|
@ -19,9 +19,15 @@ import {
|
|||
type ProcessorConfig,
|
||||
type LlmResult,
|
||||
type LlmChunk,
|
||||
tooManyRequestsError,
|
||||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
import { Effect, Layer, Stream } from "effect";
|
||||
import {
|
||||
optionalStringConfig,
|
||||
providerStatusError,
|
||||
requiredString,
|
||||
toAsyncGenerator,
|
||||
type TextCompletionRuntimeError,
|
||||
} from "./common.ts";
|
||||
|
||||
export type AzureOpenAIProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
|
|
@ -32,32 +38,65 @@ export type AzureOpenAIProcessorConfig = ProcessorConfig & {
|
|||
maxOutput?: number;
|
||||
};
|
||||
|
||||
export function makeAzureOpenAIProvider(config: AzureOpenAIProcessorConfig): LlmProvider {
|
||||
const defaultModel = config.model ?? process.env.AZURE_MODEL ?? "gpt-4o";
|
||||
const defaultTemperature = config.temperature ?? 0.0;
|
||||
const maxOutput = config.maxOutput ?? 4096;
|
||||
|
||||
const apiKey = config.apiKey ?? process.env.AZURE_TOKEN;
|
||||
if (apiKey === undefined || apiKey.length === 0) {
|
||||
throw new Error("Azure OpenAI API key not specified");
|
||||
}
|
||||
|
||||
const endpoint = config.endpoint ?? process.env.AZURE_ENDPOINT;
|
||||
if (endpoint === undefined || endpoint.length === 0) {
|
||||
throw new Error("Azure OpenAI endpoint not specified");
|
||||
}
|
||||
type ResolvedAzureOpenAIConfig = {
|
||||
readonly defaultModel: string;
|
||||
readonly defaultTemperature: number;
|
||||
readonly maxOutput: number;
|
||||
readonly apiKey: string;
|
||||
readonly endpoint: string;
|
||||
readonly apiVersion: string;
|
||||
};
|
||||
|
||||
const loadAzureOpenAIConfig = Effect.fn("loadAzureOpenAIConfig")(function* (
|
||||
config: AzureOpenAIProcessorConfig,
|
||||
) {
|
||||
const defaultModel =
|
||||
config.model ?? (yield* optionalStringConfig("AzureOpenAI", "AZURE_MODEL")) ?? "gpt-4o";
|
||||
const apiKey = yield* requiredString(
|
||||
config.apiKey ?? (yield* optionalStringConfig("AzureOpenAI", "AZURE_TOKEN")),
|
||||
"AzureOpenAI",
|
||||
"AZURE_TOKEN",
|
||||
"Azure OpenAI API key not specified",
|
||||
);
|
||||
const endpoint = yield* requiredString(
|
||||
config.endpoint ?? (yield* optionalStringConfig("AzureOpenAI", "AZURE_ENDPOINT")),
|
||||
"AzureOpenAI",
|
||||
"AZURE_ENDPOINT",
|
||||
"Azure OpenAI endpoint not specified",
|
||||
);
|
||||
const apiVersion =
|
||||
config.apiVersion ??
|
||||
process.env.AZURE_API_VERSION ??
|
||||
(yield* optionalStringConfig("AzureOpenAI", "AZURE_API_VERSION")) ??
|
||||
"2024-12-01-preview";
|
||||
|
||||
return {
|
||||
defaultModel,
|
||||
defaultTemperature: config.temperature ?? 0.0,
|
||||
maxOutput: config.maxOutput ?? 4096,
|
||||
apiKey,
|
||||
endpoint,
|
||||
apiVersion,
|
||||
};
|
||||
});
|
||||
|
||||
const mapAzureOpenAIError = (error: unknown): TextCompletionRuntimeError =>
|
||||
providerStatusError("AzureOpenAI", error);
|
||||
|
||||
export function makeAzureOpenAIProvider(config: AzureOpenAIProcessorConfig): LlmProvider {
|
||||
const {
|
||||
defaultModel,
|
||||
defaultTemperature,
|
||||
maxOutput,
|
||||
apiKey,
|
||||
endpoint,
|
||||
apiVersion,
|
||||
} = Effect.runSync(loadAzureOpenAIConfig(config)) satisfies ResolvedAzureOpenAIConfig;
|
||||
const client = new AzureOpenAI({ apiKey, apiVersion, endpoint });
|
||||
|
||||
console.log("[AzureOpenAI] LLM service initialized");
|
||||
Effect.runSync(Effect.log("[AzureOpenAI] LLM service initialized"));
|
||||
|
||||
return {
|
||||
generateContent: async (
|
||||
generateContent: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
|
|
@ -66,87 +105,106 @@ export function makeAzureOpenAIProvider(config: AzureOpenAIProcessorConfig): Llm
|
|||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const resp = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
});
|
||||
|
||||
return {
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
}),
|
||||
catch: mapAzureOpenAIError,
|
||||
}).pipe(
|
||||
Effect.map((resp): LlmResult => ({
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
})),
|
||||
),
|
||||
);
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
generateContentStream: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
): AsyncGenerator<LlmChunk> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const stream = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
});
|
||||
|
||||
const stream = Stream.fromEffect(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
}),
|
||||
catch: mapAzureOpenAIError,
|
||||
}),
|
||||
).pipe(
|
||||
Stream.flatMap((openAIStream) => {
|
||||
const iterator = openAIStream[Symbol.asyncIterator]();
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>(
|
||||
"pulling",
|
||||
(state) => {
|
||||
if (state === "done") return Effect.void as Effect.Effect<undefined>;
|
||||
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
}
|
||||
}
|
||||
return Effect.gen(function* () {
|
||||
while (true) {
|
||||
const next = yield* Effect.tryPromise({
|
||||
try: () => iterator.next(),
|
||||
catch: mapAzureOpenAIError,
|
||||
});
|
||||
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (next.done === true) {
|
||||
return [{
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
}, "done"] as const;
|
||||
}
|
||||
|
||||
const chunk = next.value;
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
}
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
return [{
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
}, "pulling"] as const;
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
return toAsyncGenerator(Stream.toAsyncIterable(stream), mapAzureOpenAIError);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -171,6 +229,6 @@ export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
|||
),
|
||||
});
|
||||
|
||||
export async function run(): Promise<void> {
|
||||
await Effect.runPromise(program);
|
||||
export function run(): Promise<void> {
|
||||
return Effect.runPromise(program);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,9 +15,15 @@ import {
|
|||
type ProcessorConfig,
|
||||
type LlmResult,
|
||||
type LlmChunk,
|
||||
tooManyRequestsError,
|
||||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
import { Effect, Layer, Stream } from "effect";
|
||||
import {
|
||||
optionalStringConfig,
|
||||
providerStatusError,
|
||||
requiredString,
|
||||
toAsyncGenerator,
|
||||
type TextCompletionRuntimeError,
|
||||
} from "./common.ts";
|
||||
|
||||
export type ClaudeProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
|
|
@ -26,21 +32,46 @@ export type ClaudeProcessorConfig = ProcessorConfig & {
|
|||
maxOutput?: number;
|
||||
};
|
||||
|
||||
type ResolvedClaudeConfig = {
|
||||
readonly defaultModel: string;
|
||||
readonly defaultTemperature: number;
|
||||
readonly maxOutput: number;
|
||||
readonly apiKey: string;
|
||||
};
|
||||
|
||||
const loadClaudeConfig = Effect.fn("loadClaudeConfig")(function*(config: ClaudeProcessorConfig) {
|
||||
const apiKey = yield* requiredString(
|
||||
config.apiKey ?? (yield* optionalStringConfig("Claude", "CLAUDE_KEY")),
|
||||
"Claude",
|
||||
"CLAUDE_KEY",
|
||||
"Claude API key not specified",
|
||||
);
|
||||
|
||||
return {
|
||||
defaultModel: config.model ?? "claude-sonnet-4-20250514",
|
||||
defaultTemperature: config.temperature ?? 0.0,
|
||||
maxOutput: config.maxOutput ?? 8192,
|
||||
apiKey,
|
||||
} satisfies ResolvedClaudeConfig;
|
||||
});
|
||||
|
||||
const mapClaudeError = (error: unknown): TextCompletionRuntimeError =>
|
||||
providerStatusError("Claude", error);
|
||||
|
||||
export function makeClaudeProvider(config: ClaudeProcessorConfig): LlmProvider {
|
||||
const defaultModel = config.model ?? "claude-sonnet-4-20250514";
|
||||
const defaultTemperature = config.temperature ?? 0.0;
|
||||
const maxOutput = config.maxOutput ?? 8192;
|
||||
const apiKey = config.apiKey ?? process.env.CLAUDE_KEY;
|
||||
if (apiKey === undefined || apiKey.length === 0) {
|
||||
throw new Error("Claude API key not specified");
|
||||
}
|
||||
const {
|
||||
defaultModel,
|
||||
defaultTemperature,
|
||||
maxOutput,
|
||||
apiKey,
|
||||
} = Effect.runSync(loadClaudeConfig(config)) satisfies ResolvedClaudeConfig;
|
||||
|
||||
const client = new Anthropic({ apiKey });
|
||||
|
||||
console.log("[Claude] LLM service initialized");
|
||||
Effect.runSync(Effect.log("[Claude] LLM service initialized"));
|
||||
|
||||
return {
|
||||
generateContent: async (
|
||||
generateContent: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
|
|
@ -49,88 +80,120 @@ export function makeClaudeProvider(config: ClaudeProcessorConfig): LlmProvider {
|
|||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const response = await client.messages.create({
|
||||
model: modelName,
|
||||
max_tokens: maxOutput,
|
||||
temperature: temp,
|
||||
system,
|
||||
messages: [
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
});
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
client.messages.create({
|
||||
model: modelName,
|
||||
max_tokens: maxOutput,
|
||||
temperature: temp,
|
||||
system,
|
||||
messages: [
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
}),
|
||||
catch: mapClaudeError,
|
||||
}).pipe(
|
||||
Effect.map((response): LlmResult => {
|
||||
const firstContent = response.content[0];
|
||||
const text = firstContent?.type === "text"
|
||||
? firstContent.text
|
||||
: "";
|
||||
|
||||
const text = response.content[0].type === "text"
|
||||
? response.content[0].text
|
||||
: "";
|
||||
|
||||
return {
|
||||
text,
|
||||
inToken: response.usage.input_tokens,
|
||||
outToken: response.usage.output_tokens,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof Anthropic.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return {
|
||||
text,
|
||||
inToken: response.usage.input_tokens,
|
||||
outToken: response.usage.output_tokens,
|
||||
model: modelName,
|
||||
};
|
||||
}),
|
||||
),
|
||||
);
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
generateContentStream: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
): AsyncGenerator<LlmChunk> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const stream = client.messages.stream({
|
||||
model: modelName,
|
||||
max_tokens: maxOutput,
|
||||
temperature: temp,
|
||||
system,
|
||||
messages: [
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
});
|
||||
|
||||
for await (const event of stream) {
|
||||
if (event.type === "content_block_delta" && event.delta.type === "text_delta") {
|
||||
yield {
|
||||
text: event.delta.text,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
const stream = Stream.fromEffect(
|
||||
Effect.try({
|
||||
try: () =>
|
||||
client.messages.stream({
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
max_tokens: maxOutput,
|
||||
temperature: temp,
|
||||
system,
|
||||
messages: [
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
}),
|
||||
catch: mapClaudeError,
|
||||
}),
|
||||
).pipe(
|
||||
Stream.flatMap((anthropicStream) => {
|
||||
const iterator = anthropicStream[Symbol.asyncIterator]();
|
||||
|
||||
const finalMessage = await stream.finalMessage();
|
||||
yield {
|
||||
text: "",
|
||||
inToken: finalMessage.usage.input_tokens,
|
||||
outToken: finalMessage.usage.output_tokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof Anthropic.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>(
|
||||
"pulling",
|
||||
(state) => {
|
||||
if (state === "done") return Effect.void as Effect.Effect<undefined>;
|
||||
|
||||
return Effect.gen(function* () {
|
||||
while (true) {
|
||||
const next = yield* Effect.tryPromise({
|
||||
try: () => iterator.next(),
|
||||
catch: mapClaudeError,
|
||||
});
|
||||
|
||||
if (next.done === true) {
|
||||
const finalMessage = yield* Effect.tryPromise({
|
||||
try: () => anthropicStream.finalMessage(),
|
||||
catch: mapClaudeError,
|
||||
});
|
||||
return [{
|
||||
text: "",
|
||||
inToken: finalMessage.usage.input_tokens,
|
||||
outToken: finalMessage.usage.output_tokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
}, "done"] as const;
|
||||
}
|
||||
|
||||
const event = next.value;
|
||||
if (
|
||||
event.type === "content_block_delta" &&
|
||||
event.delta.type === "text_delta"
|
||||
) {
|
||||
return [{
|
||||
text: event.delta.text,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
}, "pulling"] as const;
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
return toAsyncGenerator(Stream.toAsyncIterable(stream), mapClaudeError);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type ClaudeProcessor = ReturnType<typeof makeClaudeProcessor>;
|
||||
|
||||
export function makeClaudeProcessor(config: ClaudeProcessorConfig): ReturnType<typeof makeLlmService> {
|
||||
export function makeClaudeProcessor(
|
||||
config: ClaudeProcessorConfig,
|
||||
): ReturnType<typeof makeLlmService> {
|
||||
return makeLlmService(config, makeClaudeProvider(config));
|
||||
}
|
||||
|
||||
|
|
@ -146,6 +209,6 @@ export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
|||
),
|
||||
});
|
||||
|
||||
export async function run(): Promise<void> {
|
||||
await Effect.runPromise(program);
|
||||
export function run(): Promise<void> {
|
||||
return Effect.runPromise(program);
|
||||
}
|
||||
|
|
|
|||
101
ts/packages/flow/src/model/text-completion/common.ts
Normal file
101
ts/packages/flow/src/model/text-completion/common.ts
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
import {
|
||||
TooManyRequestsError,
|
||||
errorMessage,
|
||||
type LlmChunk,
|
||||
} from "@trustgraph/base";
|
||||
import { Config, Effect } from "effect";
|
||||
import * as O from "effect/Option";
|
||||
import * as S from "effect/Schema";
|
||||
|
||||
export class TextCompletionConfigError extends S.TaggedErrorClass<TextCompletionConfigError>()(
|
||||
"TextCompletionConfigError",
|
||||
{
|
||||
message: S.String,
|
||||
provider: S.String,
|
||||
key: S.String,
|
||||
},
|
||||
) {}
|
||||
|
||||
export class TextCompletionProviderError extends S.TaggedErrorClass<TextCompletionProviderError>()(
|
||||
"TextCompletionProviderError",
|
||||
{
|
||||
message: S.String,
|
||||
provider: S.String,
|
||||
},
|
||||
) {}
|
||||
|
||||
export type TextCompletionRuntimeError =
|
||||
| TextCompletionProviderError
|
||||
| TooManyRequestsError;
|
||||
|
||||
export const optionalStringConfig = Effect.fn("TextCompletion.optionalStringConfig")(function*(
|
||||
provider: string,
|
||||
name: string,
|
||||
) {
|
||||
const value = yield* Config.string(name).pipe(
|
||||
Config.option,
|
||||
Effect.mapError((cause) =>
|
||||
TextCompletionConfigError.make({
|
||||
provider,
|
||||
key: name,
|
||||
message: errorMessage(cause),
|
||||
})
|
||||
),
|
||||
);
|
||||
return O.getOrUndefined(value);
|
||||
});
|
||||
|
||||
export const requiredString = (
|
||||
value: string | undefined,
|
||||
provider: string,
|
||||
key: string,
|
||||
message: string,
|
||||
) =>
|
||||
value !== undefined && value.length > 0
|
||||
? Effect.succeed(value)
|
||||
: Effect.fail(TextCompletionConfigError.make({ provider, key, message }));
|
||||
|
||||
export const providerRuntimeError = (
|
||||
provider: string,
|
||||
error: unknown,
|
||||
): TextCompletionRuntimeError =>
|
||||
TextCompletionProviderError.make({
|
||||
provider,
|
||||
message: errorMessage(error),
|
||||
});
|
||||
|
||||
export const providerStatusError = (
|
||||
provider: string,
|
||||
error: unknown,
|
||||
): TextCompletionRuntimeError => {
|
||||
const status = typeof error === "object" && error !== null && "status" in error
|
||||
? (error as { readonly status?: unknown }).status
|
||||
: undefined;
|
||||
const statusCode = typeof error === "object" && error !== null && "statusCode" in error
|
||||
? (error as { readonly statusCode?: unknown }).statusCode
|
||||
: undefined;
|
||||
return status === 429 || statusCode === 429
|
||||
? TooManyRequestsError.make({ message: "Rate limit exceeded" })
|
||||
: providerRuntimeError(provider, error);
|
||||
};
|
||||
|
||||
export const toAsyncGenerator = (
|
||||
iterable: AsyncIterable<LlmChunk>,
|
||||
mapError: (error: unknown) => TextCompletionRuntimeError,
|
||||
): AsyncGenerator<LlmChunk> => {
|
||||
const iterator = iterable[Symbol.asyncIterator]();
|
||||
let generator: AsyncGenerator<LlmChunk>;
|
||||
generator = {
|
||||
next: (value?: unknown) => iterator.next(value as never),
|
||||
return: (value?: unknown) =>
|
||||
iterator.return === undefined
|
||||
? Promise.resolve({ done: true, value: value as LlmChunk })
|
||||
: iterator.return(value as never) as Promise<IteratorResult<LlmChunk>>,
|
||||
throw: (error?: unknown) =>
|
||||
iterator.throw === undefined
|
||||
? Effect.runPromise(Effect.fail(mapError(error))) as Promise<IteratorResult<LlmChunk>>
|
||||
: iterator.throw(error) as Promise<IteratorResult<LlmChunk>>,
|
||||
[Symbol.asyncIterator]: () => generator,
|
||||
} as AsyncGenerator<LlmChunk>;
|
||||
return generator;
|
||||
};
|
||||
|
|
@ -17,9 +17,15 @@ import {
|
|||
type ProcessorConfig,
|
||||
type LlmResult,
|
||||
type LlmChunk,
|
||||
tooManyRequestsError,
|
||||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
import { Effect, Layer, Stream } from "effect";
|
||||
import {
|
||||
optionalStringConfig,
|
||||
providerStatusError,
|
||||
requiredString,
|
||||
toAsyncGenerator,
|
||||
type TextCompletionRuntimeError,
|
||||
} from "./common.ts";
|
||||
|
||||
export type MistralProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
|
|
@ -28,22 +34,49 @@ export type MistralProcessorConfig = ProcessorConfig & {
|
|||
maxOutput?: number;
|
||||
};
|
||||
|
||||
type ResolvedMistralConfig = {
|
||||
readonly defaultModel: string;
|
||||
readonly defaultTemperature: number;
|
||||
readonly maxOutput: number;
|
||||
readonly apiKey: string;
|
||||
};
|
||||
|
||||
const loadMistralConfig = Effect.fn("loadMistralConfig")(function*(config: MistralProcessorConfig) {
|
||||
const apiKey = yield* requiredString(
|
||||
config.apiKey ?? (yield* optionalStringConfig("Mistral", "MISTRAL_TOKEN")),
|
||||
"Mistral",
|
||||
"MISTRAL_TOKEN",
|
||||
"Mistral API key not specified",
|
||||
);
|
||||
|
||||
return {
|
||||
defaultModel:
|
||||
config.model ??
|
||||
(yield* optionalStringConfig("Mistral", "MISTRAL_MODEL")) ??
|
||||
"ministral-8b-latest",
|
||||
defaultTemperature: config.temperature ?? 0.0,
|
||||
maxOutput: config.maxOutput ?? 4096,
|
||||
apiKey,
|
||||
} satisfies ResolvedMistralConfig;
|
||||
});
|
||||
|
||||
const mapMistralError = (error: unknown): TextCompletionRuntimeError =>
|
||||
providerStatusError("Mistral", error);
|
||||
|
||||
export function makeMistralProvider(config: MistralProcessorConfig): LlmProvider {
|
||||
const defaultModel =
|
||||
config.model ?? process.env.MISTRAL_MODEL ?? "ministral-8b-latest";
|
||||
const defaultTemperature = config.temperature ?? 0.0;
|
||||
const maxOutput = config.maxOutput ?? 4096;
|
||||
const apiKey = config.apiKey ?? process.env.MISTRAL_TOKEN;
|
||||
if (apiKey === undefined || apiKey.length === 0) {
|
||||
throw new Error("Mistral API key not specified");
|
||||
}
|
||||
const {
|
||||
defaultModel,
|
||||
defaultTemperature,
|
||||
maxOutput,
|
||||
apiKey,
|
||||
} = Effect.runSync(loadMistralConfig(config)) satisfies ResolvedMistralConfig;
|
||||
|
||||
const client = new Mistral({ apiKey });
|
||||
|
||||
console.log("[Mistral] LLM service initialized");
|
||||
Effect.runSync(Effect.log("[Mistral] LLM service initialized"));
|
||||
|
||||
return {
|
||||
generateContent: async (
|
||||
generateContent: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
|
|
@ -52,93 +85,114 @@ export function makeMistralProvider(config: MistralProcessorConfig): LlmProvider
|
|||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const resp = await client.chat.complete({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
maxTokens: maxOutput,
|
||||
});
|
||||
|
||||
return {
|
||||
text: (resp.choices?.[0]?.message?.content as string) ?? "",
|
||||
inToken: resp.usage?.promptTokens ?? 0,
|
||||
outToken: resp.usage?.completionTokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.statusCode === 429 || (err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
client.chat.complete({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
maxTokens: maxOutput,
|
||||
}),
|
||||
catch: mapMistralError,
|
||||
}).pipe(
|
||||
Effect.map((resp): LlmResult => ({
|
||||
text: (resp.choices?.[0]?.message?.content as string) ?? "",
|
||||
inToken: resp.usage?.promptTokens ?? 0,
|
||||
outToken: resp.usage?.completionTokens ?? 0,
|
||||
model: modelName,
|
||||
})),
|
||||
),
|
||||
);
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
generateContentStream: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
): AsyncGenerator<LlmChunk> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const stream = await client.chat.stream({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
maxTokens: maxOutput,
|
||||
});
|
||||
|
||||
const stream = Stream.fromEffect(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
client.chat.stream({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
maxTokens: maxOutput,
|
||||
}),
|
||||
catch: mapMistralError,
|
||||
}),
|
||||
).pipe(
|
||||
Stream.flatMap((mistralStream) => {
|
||||
const iterator = mistralStream[Symbol.asyncIterator]();
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
const delta = chunk.data?.choices?.[0]?.delta;
|
||||
const content = delta?.content;
|
||||
if (typeof content === "string" && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>(
|
||||
"pulling",
|
||||
(state) => {
|
||||
if (state === "done") return Effect.void as Effect.Effect<undefined>;
|
||||
|
||||
if (chunk.data?.usage !== undefined) {
|
||||
totalInputTokens = chunk.data.usage.promptTokens ?? 0;
|
||||
totalOutputTokens = chunk.data.usage.completionTokens ?? 0;
|
||||
}
|
||||
}
|
||||
return Effect.gen(function* () {
|
||||
while (true) {
|
||||
const next = yield* Effect.tryPromise({
|
||||
try: () => iterator.next(),
|
||||
catch: mapMistralError,
|
||||
});
|
||||
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.statusCode === 429 || (err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (next.done === true) {
|
||||
return [{
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
}, "done"] as const;
|
||||
}
|
||||
|
||||
const chunk = next.value;
|
||||
const delta = chunk.data?.choices?.[0]?.delta;
|
||||
const content = delta?.content;
|
||||
if (chunk.data?.usage !== undefined) {
|
||||
totalInputTokens = chunk.data.usage.promptTokens ?? 0;
|
||||
totalOutputTokens = chunk.data.usage.completionTokens ?? 0;
|
||||
}
|
||||
if (typeof content === "string" && content.length > 0) {
|
||||
return [{
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
}, "pulling"] as const;
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
return toAsyncGenerator(Stream.toAsyncIterable(stream), mapMistralError);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type MistralProcessor = ReturnType<typeof makeMistralProcessor>;
|
||||
|
||||
export function makeMistralProcessor(config: MistralProcessorConfig): ReturnType<typeof makeLlmService> {
|
||||
export function makeMistralProcessor(
|
||||
config: MistralProcessorConfig,
|
||||
): ReturnType<typeof makeLlmService> {
|
||||
return makeLlmService(config, makeMistralProvider(config));
|
||||
}
|
||||
|
||||
|
|
@ -154,6 +208,6 @@ export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
|||
),
|
||||
});
|
||||
|
||||
export async function run(): Promise<void> {
|
||||
await Effect.runPromise(program);
|
||||
export function run(): Promise<void> {
|
||||
return Effect.runPromise(program);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,32 +18,51 @@ import {
|
|||
type LlmResult,
|
||||
type LlmChunk,
|
||||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
import { Effect, Layer, Stream } from "effect";
|
||||
import {
|
||||
optionalStringConfig,
|
||||
providerRuntimeError,
|
||||
toAsyncGenerator,
|
||||
type TextCompletionRuntimeError,
|
||||
} from "./common.ts";
|
||||
|
||||
export type OllamaProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
ollamaUrl?: string;
|
||||
};
|
||||
|
||||
export function makeOllamaProvider(config: OllamaProcessorConfig): LlmProvider {
|
||||
const defaultModel =
|
||||
config.model ??
|
||||
process.env.OLLAMA_MODEL ??
|
||||
"qwen2.5:0.5b";
|
||||
type ResolvedOllamaConfig = {
|
||||
readonly defaultModel: string;
|
||||
readonly host: string;
|
||||
};
|
||||
|
||||
const host =
|
||||
config.ollamaUrl ??
|
||||
process.env.OLLAMA_URL ??
|
||||
"http://localhost:11434";
|
||||
const loadOllamaConfig = Effect.fn("loadOllamaConfig")(function*(config: OllamaProcessorConfig) {
|
||||
return {
|
||||
defaultModel:
|
||||
config.model ??
|
||||
(yield* optionalStringConfig("Ollama", "OLLAMA_MODEL")) ??
|
||||
"qwen2.5:0.5b",
|
||||
host:
|
||||
config.ollamaUrl ??
|
||||
(yield* optionalStringConfig("Ollama", "OLLAMA_URL")) ??
|
||||
"http://localhost:11434",
|
||||
} satisfies ResolvedOllamaConfig;
|
||||
});
|
||||
|
||||
const mapOllamaError = (error: unknown): TextCompletionRuntimeError =>
|
||||
providerRuntimeError("Ollama", error);
|
||||
|
||||
export function makeOllamaProvider(config: OllamaProcessorConfig): LlmProvider {
|
||||
const { defaultModel, host } = Effect.runSync(loadOllamaConfig(config)) satisfies ResolvedOllamaConfig;
|
||||
|
||||
const client = new Ollama({ host });
|
||||
|
||||
console.log(
|
||||
Effect.runSync(Effect.log(
|
||||
`[Ollama] LLM service initialized (host=${host}, model=${defaultModel})`,
|
||||
);
|
||||
));
|
||||
|
||||
return {
|
||||
generateContent: async (
|
||||
generateContent: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
|
|
@ -52,73 +71,107 @@ export function makeOllamaProvider(config: OllamaProcessorConfig): LlmProvider {
|
|||
const modelName = model ?? defaultModel;
|
||||
const fullPrompt = system + "\n\n" + prompt;
|
||||
|
||||
const resp = await client.generate({
|
||||
model: modelName,
|
||||
prompt: fullPrompt,
|
||||
stream: false,
|
||||
});
|
||||
|
||||
return {
|
||||
text: resp.response,
|
||||
inToken: resp.prompt_eval_count ?? 0,
|
||||
outToken: resp.eval_count ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
client.generate({
|
||||
model: modelName,
|
||||
prompt: fullPrompt,
|
||||
stream: false,
|
||||
}),
|
||||
catch: mapOllamaError,
|
||||
}).pipe(
|
||||
Effect.map((resp): LlmResult => ({
|
||||
text: resp.response,
|
||||
inToken: resp.prompt_eval_count ?? 0,
|
||||
outToken: resp.eval_count ?? 0,
|
||||
model: modelName,
|
||||
})),
|
||||
),
|
||||
);
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
generateContentStream: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
_temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
): AsyncGenerator<LlmChunk> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const fullPrompt = system + "\n\n" + prompt;
|
||||
|
||||
const stream = await client.generate({
|
||||
model: modelName,
|
||||
prompt: fullPrompt,
|
||||
stream: true,
|
||||
});
|
||||
|
||||
const stream = Stream.fromEffect(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
client.generate({
|
||||
model: modelName,
|
||||
prompt: fullPrompt,
|
||||
stream: true,
|
||||
}),
|
||||
catch: mapOllamaError,
|
||||
}),
|
||||
).pipe(
|
||||
Stream.flatMap((ollamaStream) => {
|
||||
const iterator = ollamaStream[Symbol.asyncIterator]();
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
// Token counts accumulate across chunks; keep the latest values
|
||||
if (chunk.prompt_eval_count !== undefined) {
|
||||
totalInputTokens = chunk.prompt_eval_count;
|
||||
}
|
||||
if (chunk.eval_count !== undefined) {
|
||||
totalOutputTokens = chunk.eval_count;
|
||||
}
|
||||
return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>(
|
||||
"pulling",
|
||||
(state) => {
|
||||
if (state === "done") return Effect.void as Effect.Effect<undefined>;
|
||||
|
||||
if (chunk.response.length > 0) {
|
||||
yield {
|
||||
text: chunk.response,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
return Effect.gen(function* () {
|
||||
while (true) {
|
||||
const next = yield* Effect.tryPromise({
|
||||
try: () => iterator.next(),
|
||||
catch: mapOllamaError,
|
||||
});
|
||||
|
||||
// Final chunk with accumulated token counts
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
if (next.done === true) {
|
||||
return [{
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
}, "done"] as const;
|
||||
}
|
||||
|
||||
const chunk = next.value;
|
||||
if (chunk.prompt_eval_count !== undefined) {
|
||||
totalInputTokens = chunk.prompt_eval_count;
|
||||
}
|
||||
if (chunk.eval_count !== undefined) {
|
||||
totalOutputTokens = chunk.eval_count;
|
||||
}
|
||||
|
||||
if (chunk.response.length > 0) {
|
||||
return [{
|
||||
text: chunk.response,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
}, "pulling"] as const;
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
return toAsyncGenerator(Stream.toAsyncIterable(stream), mapOllamaError);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type OllamaProcessor = ReturnType<typeof makeOllamaProcessor>;
|
||||
|
||||
export function makeOllamaProcessor(config: OllamaProcessorConfig): ReturnType<typeof makeLlmService> {
|
||||
export function makeOllamaProcessor(
|
||||
config: OllamaProcessorConfig,
|
||||
): ReturnType<typeof makeLlmService> {
|
||||
return makeLlmService(config, makeOllamaProvider(config));
|
||||
}
|
||||
|
||||
|
|
@ -134,6 +187,6 @@ export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
|||
),
|
||||
});
|
||||
|
||||
export async function run(): Promise<void> {
|
||||
await Effect.runPromise(program);
|
||||
export function run(): Promise<void> {
|
||||
return Effect.runPromise(program);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,14 @@ import {
|
|||
type LlmResult,
|
||||
type LlmChunk,
|
||||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
import { Effect, Layer, Stream } from "effect";
|
||||
import {
|
||||
optionalStringConfig,
|
||||
providerStatusError,
|
||||
requiredString,
|
||||
toAsyncGenerator,
|
||||
type TextCompletionRuntimeError,
|
||||
} from "./common.ts";
|
||||
|
||||
export type OpenAICompatibleProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
|
|
@ -31,30 +38,57 @@ export type OpenAICompatibleProcessorConfig = ProcessorConfig & {
|
|||
maxOutput?: number;
|
||||
};
|
||||
|
||||
type ResolvedOpenAICompatibleConfig = {
|
||||
readonly defaultModel: string;
|
||||
readonly defaultTemperature: number;
|
||||
readonly maxOutput: number;
|
||||
readonly apiKey: string;
|
||||
readonly baseURL: string;
|
||||
};
|
||||
|
||||
const loadOpenAICompatibleConfig = Effect.fn("loadOpenAICompatibleConfig")(function*(
|
||||
config: OpenAICompatibleProcessorConfig,
|
||||
) {
|
||||
const defaultModel =
|
||||
config.model ?? (yield* optionalStringConfig("OpenAI-Compatible", "OPENAI_COMPAT_MODEL")) ?? "default";
|
||||
const baseURL = yield* requiredString(
|
||||
config.baseUrl ?? (yield* optionalStringConfig("OpenAI-Compatible", "OPENAI_COMPAT_URL")),
|
||||
"OpenAI-Compatible",
|
||||
"OPENAI_COMPAT_URL",
|
||||
"OpenAI-compatible server URL not specified (set OPENAI_COMPAT_URL)",
|
||||
);
|
||||
const apiKey =
|
||||
config.apiKey ?? (yield* optionalStringConfig("OpenAI-Compatible", "OPENAI_COMPAT_KEY")) ?? "sk-no-key-required";
|
||||
|
||||
return {
|
||||
defaultModel,
|
||||
defaultTemperature: config.temperature ?? 0.0,
|
||||
maxOutput: config.maxOutput ?? 4096,
|
||||
apiKey,
|
||||
baseURL,
|
||||
} satisfies ResolvedOpenAICompatibleConfig;
|
||||
});
|
||||
|
||||
const mapOpenAICompatibleError = (error: unknown): TextCompletionRuntimeError =>
|
||||
providerStatusError("OpenAI-Compatible", error);
|
||||
|
||||
export function makeOpenAICompatibleProvider(
|
||||
config: OpenAICompatibleProcessorConfig,
|
||||
): LlmProvider {
|
||||
const defaultModel =
|
||||
config.model ?? process.env.OPENAI_COMPAT_MODEL ?? "default";
|
||||
const defaultTemperature = config.temperature ?? 0.0;
|
||||
const maxOutput = config.maxOutput ?? 4096;
|
||||
|
||||
const baseURL = config.baseUrl ?? process.env.OPENAI_COMPAT_URL;
|
||||
if (baseURL === undefined || baseURL.length === 0) {
|
||||
throw new Error(
|
||||
"OpenAI-compatible server URL not specified (set OPENAI_COMPAT_URL)",
|
||||
);
|
||||
}
|
||||
|
||||
const apiKey =
|
||||
config.apiKey ?? process.env.OPENAI_COMPAT_KEY ?? "sk-no-key-required";
|
||||
const {
|
||||
defaultModel,
|
||||
defaultTemperature,
|
||||
maxOutput,
|
||||
apiKey,
|
||||
baseURL,
|
||||
} = Effect.runSync(loadOpenAICompatibleConfig(config)) satisfies ResolvedOpenAICompatibleConfig;
|
||||
|
||||
const client = new OpenAI({ baseURL, apiKey });
|
||||
|
||||
console.log("[OpenAI-Compatible] LLM service initialized");
|
||||
Effect.runSync(Effect.log("[OpenAI-Compatible] LLM service initialized"));
|
||||
|
||||
return {
|
||||
generateContent: async (
|
||||
generateContent: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
|
|
@ -63,72 +97,105 @@ export function makeOpenAICompatibleProvider(
|
|||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
const resp = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_tokens: maxOutput,
|
||||
});
|
||||
|
||||
return {
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_tokens: maxOutput,
|
||||
}),
|
||||
catch: mapOpenAICompatibleError,
|
||||
}).pipe(
|
||||
Effect.map((resp): LlmResult => ({
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
})),
|
||||
),
|
||||
);
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
generateContentStream: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
): AsyncGenerator<LlmChunk> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
const stream = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_tokens: maxOutput,
|
||||
stream: true,
|
||||
});
|
||||
|
||||
const stream = Stream.fromEffect(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_tokens: maxOutput,
|
||||
stream: true,
|
||||
}),
|
||||
catch: mapOpenAICompatibleError,
|
||||
}),
|
||||
).pipe(
|
||||
Stream.flatMap((openAIStream) => {
|
||||
const iterator = openAIStream[Symbol.asyncIterator]();
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>(
|
||||
"pulling",
|
||||
(state) => {
|
||||
if (state === "done") return Effect.void as Effect.Effect<undefined>;
|
||||
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
}
|
||||
}
|
||||
return Effect.gen(function* () {
|
||||
while (true) {
|
||||
const next = yield* Effect.tryPromise({
|
||||
try: () => iterator.next(),
|
||||
catch: mapOpenAICompatibleError,
|
||||
});
|
||||
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
if (next.done === true) {
|
||||
return [{
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
}, "done"] as const;
|
||||
}
|
||||
|
||||
const chunk = next.value;
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
}
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
return [{
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
}, "pulling"] as const;
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
return toAsyncGenerator(Stream.toAsyncIterable(stream), mapOpenAICompatibleError);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -153,6 +220,6 @@ export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
|||
),
|
||||
});
|
||||
|
||||
export async function run(): Promise<void> {
|
||||
await Effect.runPromise(program);
|
||||
export function run(): Promise<void> {
|
||||
return Effect.runPromise(program);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,9 +15,15 @@ import {
|
|||
type ProcessorConfig,
|
||||
type LlmResult,
|
||||
type LlmChunk,
|
||||
tooManyRequestsError,
|
||||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
import { Effect, Layer, Stream } from "effect";
|
||||
import {
|
||||
optionalStringConfig,
|
||||
providerStatusError,
|
||||
requiredString,
|
||||
toAsyncGenerator,
|
||||
type TextCompletionRuntimeError,
|
||||
} from "./common.ts";
|
||||
|
||||
export type OpenAIProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
|
|
@ -27,24 +33,52 @@ export type OpenAIProcessorConfig = ProcessorConfig & {
|
|||
maxOutput?: number;
|
||||
};
|
||||
|
||||
type ResolvedOpenAIConfig = {
|
||||
readonly defaultModel: string;
|
||||
readonly defaultTemperature: number;
|
||||
readonly maxOutput: number;
|
||||
readonly apiKey: string;
|
||||
readonly baseURL: string | undefined;
|
||||
};
|
||||
|
||||
const loadOpenAIConfig = Effect.fn("loadOpenAIConfig")(function*(config: OpenAIProcessorConfig) {
|
||||
const apiKey = yield* requiredString(
|
||||
config.apiKey ?? (yield* optionalStringConfig("OpenAI", "OPENAI_TOKEN")),
|
||||
"OpenAI",
|
||||
"OPENAI_TOKEN",
|
||||
"OpenAI API key not specified",
|
||||
);
|
||||
|
||||
return {
|
||||
defaultModel: config.model ?? "gpt-4o",
|
||||
defaultTemperature: config.temperature ?? 0.0,
|
||||
maxOutput: config.maxOutput ?? 4096,
|
||||
apiKey,
|
||||
baseURL: config.baseUrl ?? (yield* optionalStringConfig("OpenAI", "OPENAI_BASE_URL")),
|
||||
} satisfies ResolvedOpenAIConfig;
|
||||
});
|
||||
|
||||
const mapOpenAIError = (error: unknown): TextCompletionRuntimeError =>
|
||||
providerStatusError("OpenAI", error);
|
||||
|
||||
export function makeOpenAIProvider(config: OpenAIProcessorConfig): LlmProvider {
|
||||
const defaultModel = config.model ?? "gpt-4o";
|
||||
const defaultTemperature = config.temperature ?? 0.0;
|
||||
const maxOutput = config.maxOutput ?? 4096;
|
||||
const apiKey = config.apiKey ?? process.env.OPENAI_TOKEN;
|
||||
if (apiKey === undefined || apiKey.length === 0) {
|
||||
throw new Error("OpenAI API key not specified");
|
||||
}
|
||||
const {
|
||||
defaultModel,
|
||||
defaultTemperature,
|
||||
maxOutput,
|
||||
apiKey,
|
||||
baseURL,
|
||||
} = Effect.runSync(loadOpenAIConfig(config)) satisfies ResolvedOpenAIConfig;
|
||||
|
||||
const client = new OpenAI({
|
||||
apiKey,
|
||||
baseURL: config.baseUrl ?? process.env.OPENAI_BASE_URL,
|
||||
});
|
||||
apiKey,
|
||||
baseURL,
|
||||
});
|
||||
|
||||
console.log("[OpenAI] LLM service initialized");
|
||||
Effect.runSync(Effect.log("[OpenAI] LLM service initialized"));
|
||||
|
||||
return {
|
||||
generateContent: async (
|
||||
generateContent: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
|
|
@ -53,94 +87,115 @@ export function makeOpenAIProvider(config: OpenAIProcessorConfig): LlmProvider {
|
|||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const resp = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
});
|
||||
|
||||
return {
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof OpenAI.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
}),
|
||||
catch: mapOpenAIError,
|
||||
}).pipe(
|
||||
Effect.map((resp): LlmResult => ({
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
})),
|
||||
),
|
||||
);
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
generateContentStream: (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
): AsyncGenerator<LlmChunk> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const stream = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
});
|
||||
|
||||
const stream = Stream.fromEffect(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
}),
|
||||
catch: mapOpenAIError,
|
||||
}),
|
||||
).pipe(
|
||||
Stream.flatMap((openAIStream) => {
|
||||
const iterator = openAIStream[Symbol.asyncIterator]();
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
return Stream.unfold<"pulling" | "done", LlmChunk, TextCompletionRuntimeError, never>(
|
||||
"pulling",
|
||||
(state) => {
|
||||
if (state === "done") return Effect.void as Effect.Effect<undefined>;
|
||||
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
}
|
||||
}
|
||||
return Effect.gen(function* () {
|
||||
while (true) {
|
||||
const next = yield* Effect.tryPromise({
|
||||
try: () => iterator.next(),
|
||||
catch: mapOpenAIError,
|
||||
});
|
||||
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof OpenAI.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (next.done === true) {
|
||||
return [{
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
}, "done"] as const;
|
||||
}
|
||||
|
||||
const chunk = next.value;
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
}
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
return [{
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
}, "pulling"] as const;
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
return toAsyncGenerator(Stream.toAsyncIterable(stream), mapOpenAIError);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type OpenAIProcessor = ReturnType<typeof makeOpenAIProcessor>;
|
||||
|
||||
export function makeOpenAIProcessor(config: OpenAIProcessorConfig): ReturnType<typeof makeLlmService> {
|
||||
export function makeOpenAIProcessor(
|
||||
config: OpenAIProcessorConfig,
|
||||
): ReturnType<typeof makeLlmService> {
|
||||
return makeLlmService(config, makeOpenAIProvider(config));
|
||||
}
|
||||
|
||||
|
|
@ -156,6 +211,6 @@ export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
|||
),
|
||||
});
|
||||
|
||||
export async function run(): Promise<void> {
|
||||
await Effect.runPromise(program);
|
||||
export function run(): Promise<void> {
|
||||
return Effect.runPromise(program);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue