mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-02 02:58:10 +02:00
Remove native classes from TS runtime
This commit is contained in:
elpresidank
parent
952daf325d
commit
dca2786828
79 changed files with 7622 additions and 6703 deletions
|
|
@ -11,10 +11,11 @@
|
|||
import { AzureOpenAI } from "openai";
|
||||
import {
|
||||
Llm,
|
||||
LlmService,
|
||||
makeLlmService,
|
||||
makeFlowProcessorProgram,
|
||||
makeLlmServiceShape,
|
||||
makeLlmSpecs,
|
||||
type LlmProvider,
|
||||
type ProcessorConfig,
|
||||
type LlmResult,
|
||||
type LlmChunk,
|
||||
|
|
@ -22,27 +23,19 @@ import {
|
|||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
|
||||
export class AzureOpenAIProcessor extends LlmService {
|
||||
private client: AzureOpenAI;
|
||||
private readonly defaultModel: string;
|
||||
private readonly defaultTemperature: number;
|
||||
private readonly maxOutput: number;
|
||||
export type AzureOpenAIProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
endpoint?: string;
|
||||
apiVersion?: string;
|
||||
temperature?: number;
|
||||
maxOutput?: number;
|
||||
};
|
||||
|
||||
constructor(
|
||||
config: ProcessorConfig & {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
endpoint?: string;
|
||||
apiVersion?: string;
|
||||
temperature?: number;
|
||||
maxOutput?: number;
|
||||
},
|
||||
) {
|
||||
super(config);
|
||||
|
||||
this.defaultModel = config.model ?? process.env.AZURE_MODEL ?? "gpt-4o";
|
||||
this.defaultTemperature = config.temperature ?? 0.0;
|
||||
this.maxOutput = config.maxOutput ?? 4096;
|
||||
export function makeAzureOpenAIProvider(config: AzureOpenAIProcessorConfig): LlmProvider {
|
||||
const defaultModel = config.model ?? process.env.AZURE_MODEL ?? "gpt-4o";
|
||||
const defaultTemperature = config.temperature ?? 0.0;
|
||||
const maxOutput = config.maxOutput ?? 4096;
|
||||
|
||||
const apiKey = config.apiKey ?? process.env.AZURE_TOKEN;
|
||||
if (apiKey === undefined || apiKey.length === 0) {
|
||||
|
|
@ -59,115 +52,122 @@ export class AzureOpenAIProcessor extends LlmService {
|
|||
process.env.AZURE_API_VERSION ??
|
||||
"2024-12-01-preview";
|
||||
|
||||
this.client = new AzureOpenAI({ apiKey, apiVersion, endpoint });
|
||||
const client = new AzureOpenAI({ apiKey, apiVersion, endpoint });
|
||||
|
||||
console.log("[AzureOpenAI] LLM service initialized");
|
||||
}
|
||||
|
||||
async generateContent(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const temp = temperature ?? this.defaultTemperature;
|
||||
return {
|
||||
generateContent: async (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const resp = await this.client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: this.maxOutput,
|
||||
});
|
||||
try {
|
||||
const resp = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
});
|
||||
|
||||
return {
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
return {
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
override supportsStreaming(): boolean {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const stream = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
});
|
||||
|
||||
async *generateContentStream(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const temp = temperature ?? this.defaultTemperature;
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
try {
|
||||
const stream = await this.client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: this.maxOutput,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
});
|
||||
for await (const chunk of stream) {
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
}
|
||||
}
|
||||
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type AzureOpenAIProcessor = ReturnType<typeof makeAzureOpenAIProcessor>;
|
||||
|
||||
export function makeAzureOpenAIProcessor(
|
||||
config: AzureOpenAIProcessorConfig,
|
||||
): ReturnType<typeof makeLlmService> {
|
||||
return makeLlmService(config, makeAzureOpenAIProvider(config));
|
||||
}
|
||||
|
||||
export const AzureOpenAIProcessor = makeAzureOpenAIProcessor;
|
||||
|
||||
export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
||||
id: "text-completion",
|
||||
specs: () => makeLlmSpecs(),
|
||||
layer: (config) =>
|
||||
Layer.succeed(
|
||||
Llm,
|
||||
Llm.of(makeLlmServiceShape(new AzureOpenAIProcessor(config))),
|
||||
Llm.of(makeLlmServiceShape(makeAzureOpenAIProvider(config))),
|
||||
),
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -7,10 +7,11 @@
|
|||
import Anthropic from "@anthropic-ai/sdk";
|
||||
import {
|
||||
Llm,
|
||||
LlmService,
|
||||
makeLlmService,
|
||||
makeFlowProcessorProgram,
|
||||
makeLlmServiceShape,
|
||||
makeLlmSpecs,
|
||||
type LlmProvider,
|
||||
type ProcessorConfig,
|
||||
type LlmResult,
|
||||
type LlmChunk,
|
||||
|
|
@ -18,132 +19,130 @@ import {
|
|||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
|
||||
export class ClaudeProcessor extends LlmService {
|
||||
private client: Anthropic;
|
||||
private readonly defaultModel: string;
|
||||
private readonly defaultTemperature: number;
|
||||
private readonly maxOutput: number;
|
||||
|
||||
constructor(config: ProcessorConfig & {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
temperature?: number;
|
||||
maxOutput?: number;
|
||||
}) {
|
||||
super(config);
|
||||
|
||||
this.defaultModel = config.model ?? "claude-sonnet-4-20250514";
|
||||
this.defaultTemperature = config.temperature ?? 0.0;
|
||||
this.maxOutput = config.maxOutput ?? 8192;
|
||||
export type ClaudeProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
temperature?: number;
|
||||
maxOutput?: number;
|
||||
};
|
||||
|
||||
export function makeClaudeProvider(config: ClaudeProcessorConfig): LlmProvider {
|
||||
const defaultModel = config.model ?? "claude-sonnet-4-20250514";
|
||||
const defaultTemperature = config.temperature ?? 0.0;
|
||||
const maxOutput = config.maxOutput ?? 8192;
|
||||
const apiKey = config.apiKey ?? process.env.CLAUDE_KEY;
|
||||
if (apiKey === undefined || apiKey.length === 0) {
|
||||
throw new Error("Claude API key not specified");
|
||||
}
|
||||
|
||||
this.client = new Anthropic({ apiKey });
|
||||
const client = new Anthropic({ apiKey });
|
||||
|
||||
console.log("[Claude] LLM service initialized");
|
||||
}
|
||||
|
||||
async generateContent(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const temp = temperature ?? this.defaultTemperature;
|
||||
return {
|
||||
generateContent: async (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const response = await this.client.messages.create({
|
||||
model: modelName,
|
||||
max_tokens: this.maxOutput,
|
||||
temperature: temp,
|
||||
system,
|
||||
messages: [
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
});
|
||||
try {
|
||||
const response = await client.messages.create({
|
||||
model: modelName,
|
||||
max_tokens: maxOutput,
|
||||
temperature: temp,
|
||||
system,
|
||||
messages: [
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
});
|
||||
|
||||
const text = response.content[0].type === "text"
|
||||
? response.content[0].text
|
||||
: "";
|
||||
const text = response.content[0].type === "text"
|
||||
? response.content[0].text
|
||||
: "";
|
||||
|
||||
return {
|
||||
text,
|
||||
inToken: response.usage.input_tokens,
|
||||
outToken: response.usage.output_tokens,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof Anthropic.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
override supportsStreaming(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
async *generateContentStream(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const temp = temperature ?? this.defaultTemperature;
|
||||
|
||||
try {
|
||||
const stream = this.client.messages.stream({
|
||||
model: modelName,
|
||||
max_tokens: this.maxOutput,
|
||||
temperature: temp,
|
||||
system,
|
||||
messages: [
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
});
|
||||
|
||||
for await (const event of stream) {
|
||||
if (event.type === "content_block_delta" && event.delta.type === "text_delta") {
|
||||
yield {
|
||||
text: event.delta.text,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
return {
|
||||
text,
|
||||
inToken: response.usage.input_tokens,
|
||||
outToken: response.usage.output_tokens,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof Anthropic.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
const finalMessage = await stream.finalMessage();
|
||||
yield {
|
||||
text: "",
|
||||
inToken: finalMessage.usage.input_tokens,
|
||||
outToken: finalMessage.usage.output_tokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof Anthropic.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
try {
|
||||
const stream = client.messages.stream({
|
||||
model: modelName,
|
||||
max_tokens: maxOutput,
|
||||
temperature: temp,
|
||||
system,
|
||||
messages: [
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
});
|
||||
|
||||
for await (const event of stream) {
|
||||
if (event.type === "content_block_delta" && event.delta.type === "text_delta") {
|
||||
yield {
|
||||
text: event.delta.text,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const finalMessage = await stream.finalMessage();
|
||||
yield {
|
||||
text: "",
|
||||
inToken: finalMessage.usage.input_tokens,
|
||||
outToken: finalMessage.usage.output_tokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof Anthropic.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type ClaudeProcessor = ReturnType<typeof makeClaudeProcessor>;
|
||||
|
||||
export function makeClaudeProcessor(config: ClaudeProcessorConfig): ReturnType<typeof makeLlmService> {
|
||||
return makeLlmService(config, makeClaudeProvider(config));
|
||||
}
|
||||
|
||||
export const ClaudeProcessor = makeClaudeProcessor;
|
||||
|
||||
export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
||||
id: "text-completion",
|
||||
specs: () => makeLlmSpecs(),
|
||||
layer: (config) =>
|
||||
Layer.succeed(
|
||||
Llm,
|
||||
Llm.of(makeLlmServiceShape(new ClaudeProcessor(config))),
|
||||
Llm.of(makeLlmServiceShape(makeClaudeProvider(config))),
|
||||
),
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -9,10 +9,11 @@
|
|||
import { Mistral } from "@mistralai/mistralai";
|
||||
import {
|
||||
Llm,
|
||||
LlmService,
|
||||
makeLlmService,
|
||||
makeFlowProcessorProgram,
|
||||
makeLlmServiceShape,
|
||||
makeLlmSpecs,
|
||||
type LlmProvider,
|
||||
type ProcessorConfig,
|
||||
type LlmResult,
|
||||
type LlmChunk,
|
||||
|
|
@ -20,140 +21,136 @@ import {
|
|||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
|
||||
export class MistralProcessor extends LlmService {
|
||||
private client: Mistral;
|
||||
private readonly defaultModel: string;
|
||||
private readonly defaultTemperature: number;
|
||||
private readonly maxOutput: number;
|
||||
|
||||
constructor(
|
||||
config: ProcessorConfig & {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
temperature?: number;
|
||||
maxOutput?: number;
|
||||
},
|
||||
) {
|
||||
super(config);
|
||||
|
||||
this.defaultModel =
|
||||
config.model ?? process.env.MISTRAL_MODEL ?? "ministral-8b-latest";
|
||||
this.defaultTemperature = config.temperature ?? 0.0;
|
||||
this.maxOutput = config.maxOutput ?? 4096;
|
||||
export type MistralProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
temperature?: number;
|
||||
maxOutput?: number;
|
||||
};
|
||||
|
||||
export function makeMistralProvider(config: MistralProcessorConfig): LlmProvider {
|
||||
const defaultModel =
|
||||
config.model ?? process.env.MISTRAL_MODEL ?? "ministral-8b-latest";
|
||||
const defaultTemperature = config.temperature ?? 0.0;
|
||||
const maxOutput = config.maxOutput ?? 4096;
|
||||
const apiKey = config.apiKey ?? process.env.MISTRAL_TOKEN;
|
||||
if (apiKey === undefined || apiKey.length === 0) {
|
||||
throw new Error("Mistral API key not specified");
|
||||
}
|
||||
|
||||
this.client = new Mistral({ apiKey });
|
||||
const client = new Mistral({ apiKey });
|
||||
|
||||
console.log("[Mistral] LLM service initialized");
|
||||
}
|
||||
|
||||
async generateContent(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const temp = temperature ?? this.defaultTemperature;
|
||||
return {
|
||||
generateContent: async (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const resp = await this.client.chat.complete({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
maxTokens: this.maxOutput,
|
||||
});
|
||||
try {
|
||||
const resp = await client.chat.complete({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
maxTokens: maxOutput,
|
||||
});
|
||||
|
||||
return {
|
||||
text: (resp.choices?.[0]?.message?.content as string) ?? "",
|
||||
inToken: resp.usage?.promptTokens ?? 0,
|
||||
outToken: resp.usage?.completionTokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.statusCode === 429 || (err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
return {
|
||||
text: (resp.choices?.[0]?.message?.content as string) ?? "",
|
||||
inToken: resp.usage?.promptTokens ?? 0,
|
||||
outToken: resp.usage?.completionTokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.statusCode === 429 || (err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
override supportsStreaming(): boolean {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const stream = await client.chat.stream({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
maxTokens: maxOutput,
|
||||
});
|
||||
|
||||
async *generateContentStream(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const temp = temperature ?? this.defaultTemperature;
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
try {
|
||||
const stream = await this.client.chat.stream({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
maxTokens: this.maxOutput,
|
||||
});
|
||||
for await (const chunk of stream) {
|
||||
const delta = chunk.data?.choices?.[0]?.delta;
|
||||
const content = delta?.content;
|
||||
if (typeof content === "string" && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
const delta = chunk.data?.choices?.[0]?.delta;
|
||||
const content = delta?.content;
|
||||
if (typeof content === "string" && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
if (chunk.data?.usage !== undefined) {
|
||||
totalInputTokens = chunk.data.usage.promptTokens ?? 0;
|
||||
totalOutputTokens = chunk.data.usage.completionTokens ?? 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (chunk.data?.usage !== undefined) {
|
||||
totalInputTokens = chunk.data.usage.promptTokens ?? 0;
|
||||
totalOutputTokens = chunk.data.usage.completionTokens ?? 0;
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.statusCode === 429 || (err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if ((err as any)?.statusCode === 429 || (err as any)?.status === 429) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type MistralProcessor = ReturnType<typeof makeMistralProcessor>;
|
||||
|
||||
export function makeMistralProcessor(config: MistralProcessorConfig): ReturnType<typeof makeLlmService> {
|
||||
return makeLlmService(config, makeMistralProvider(config));
|
||||
}
|
||||
|
||||
export const MistralProcessor = makeMistralProcessor;
|
||||
|
||||
export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
||||
id: "text-completion",
|
||||
specs: () => makeLlmSpecs(),
|
||||
layer: (config) =>
|
||||
Layer.succeed(
|
||||
Llm,
|
||||
Llm.of(makeLlmServiceShape(new MistralProcessor(config))),
|
||||
Llm.of(makeLlmServiceShape(makeMistralProvider(config))),
|
||||
),
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -9,27 +9,24 @@
|
|||
import { Ollama } from "ollama";
|
||||
import {
|
||||
Llm,
|
||||
LlmService,
|
||||
makeLlmService,
|
||||
makeFlowProcessorProgram,
|
||||
makeLlmServiceShape,
|
||||
makeLlmSpecs,
|
||||
type LlmProvider,
|
||||
type ProcessorConfig,
|
||||
type LlmResult,
|
||||
type LlmChunk,
|
||||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
|
||||
export class OllamaProcessor extends LlmService {
|
||||
private client: Ollama;
|
||||
private readonly defaultModel: string;
|
||||
export type OllamaProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
ollamaUrl?: string;
|
||||
};
|
||||
|
||||
constructor(config: ProcessorConfig & {
|
||||
model?: string;
|
||||
ollamaUrl?: string;
|
||||
}) {
|
||||
super(config);
|
||||
|
||||
this.defaultModel =
|
||||
export function makeOllamaProvider(config: OllamaProcessorConfig): LlmProvider {
|
||||
const defaultModel =
|
||||
config.model ??
|
||||
process.env.OLLAMA_MODEL ??
|
||||
"qwen2.5:0.5b";
|
||||
|
|
@ -39,96 +36,101 @@ export class OllamaProcessor extends LlmService {
|
|||
process.env.OLLAMA_URL ??
|
||||
"http://localhost:11434";
|
||||
|
||||
this.client = new Ollama({ host });
|
||||
const client = new Ollama({ host });
|
||||
|
||||
console.log(
|
||||
`[Ollama] LLM service initialized (host=${host}, model=${this.defaultModel})`,
|
||||
`[Ollama] LLM service initialized (host=${host}, model=${defaultModel})`,
|
||||
);
|
||||
}
|
||||
|
||||
async generateContent(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
_temperature?: number,
|
||||
): Promise<LlmResult> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const fullPrompt = system + "\n\n" + prompt;
|
||||
return {
|
||||
generateContent: async (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
_temperature?: number,
|
||||
): Promise<LlmResult> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const fullPrompt = system + "\n\n" + prompt;
|
||||
|
||||
const resp = await this.client.generate({
|
||||
model: modelName,
|
||||
prompt: fullPrompt,
|
||||
stream: false,
|
||||
});
|
||||
const resp = await client.generate({
|
||||
model: modelName,
|
||||
prompt: fullPrompt,
|
||||
stream: false,
|
||||
});
|
||||
|
||||
return {
|
||||
text: resp.response,
|
||||
inToken: resp.prompt_eval_count ?? 0,
|
||||
outToken: resp.eval_count ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
}
|
||||
return {
|
||||
text: resp.response,
|
||||
inToken: resp.prompt_eval_count ?? 0,
|
||||
outToken: resp.eval_count ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
_temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? defaultModel;
|
||||
const fullPrompt = system + "\n\n" + prompt;
|
||||
|
||||
override supportsStreaming(): boolean {
|
||||
return true;
|
||||
}
|
||||
const stream = await client.generate({
|
||||
model: modelName,
|
||||
prompt: fullPrompt,
|
||||
stream: true,
|
||||
});
|
||||
|
||||
async *generateContentStream(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
_temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const fullPrompt = system + "\n\n" + prompt;
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
const stream = await this.client.generate({
|
||||
model: modelName,
|
||||
prompt: fullPrompt,
|
||||
stream: true,
|
||||
});
|
||||
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
for await (const chunk of stream) {
|
||||
// Token counts accumulate across chunks; keep the latest values
|
||||
if (chunk.prompt_eval_count !== undefined) {
|
||||
totalInputTokens = chunk.prompt_eval_count;
|
||||
}
|
||||
if (chunk.eval_count !== undefined) {
|
||||
totalOutputTokens = chunk.eval_count;
|
||||
}
|
||||
if (chunk.prompt_eval_count !== undefined) {
|
||||
totalInputTokens = chunk.prompt_eval_count;
|
||||
}
|
||||
if (chunk.eval_count !== undefined) {
|
||||
totalOutputTokens = chunk.eval_count;
|
||||
}
|
||||
|
||||
if (chunk.response.length > 0) {
|
||||
yield {
|
||||
text: chunk.response,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
if (chunk.response.length > 0) {
|
||||
yield {
|
||||
text: chunk.response,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Final chunk with accumulated token counts
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
}
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type OllamaProcessor = ReturnType<typeof makeOllamaProcessor>;
|
||||
|
||||
export function makeOllamaProcessor(config: OllamaProcessorConfig): ReturnType<typeof makeLlmService> {
|
||||
return makeLlmService(config, makeOllamaProvider(config));
|
||||
}
|
||||
|
||||
export const OllamaProcessor = makeOllamaProcessor;
|
||||
|
||||
export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
||||
id: "text-completion",
|
||||
specs: () => makeLlmSpecs(),
|
||||
layer: (config) =>
|
||||
Layer.succeed(
|
||||
Llm,
|
||||
Llm.of(makeLlmServiceShape(new OllamaProcessor(config))),
|
||||
Llm.of(makeLlmServiceShape(makeOllamaProvider(config))),
|
||||
),
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -12,37 +12,32 @@
|
|||
import OpenAI from "openai";
|
||||
import {
|
||||
Llm,
|
||||
LlmService,
|
||||
makeLlmService,
|
||||
makeFlowProcessorProgram,
|
||||
makeLlmServiceShape,
|
||||
makeLlmSpecs,
|
||||
type LlmProvider,
|
||||
type ProcessorConfig,
|
||||
type LlmResult,
|
||||
type LlmChunk,
|
||||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
|
||||
export class OpenAICompatibleProcessor extends LlmService {
|
||||
private client: OpenAI;
|
||||
private readonly defaultModel: string;
|
||||
private readonly defaultTemperature: number;
|
||||
private readonly maxOutput: number;
|
||||
export type OpenAICompatibleProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
temperature?: number;
|
||||
maxOutput?: number;
|
||||
};
|
||||
|
||||
constructor(
|
||||
config: ProcessorConfig & {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
temperature?: number;
|
||||
maxOutput?: number;
|
||||
},
|
||||
) {
|
||||
super(config);
|
||||
|
||||
this.defaultModel =
|
||||
config.model ?? process.env.OPENAI_COMPAT_MODEL ?? "default";
|
||||
this.defaultTemperature = config.temperature ?? 0.0;
|
||||
this.maxOutput = config.maxOutput ?? 4096;
|
||||
export function makeOpenAICompatibleProvider(
|
||||
config: OpenAICompatibleProcessorConfig,
|
||||
): LlmProvider {
|
||||
const defaultModel =
|
||||
config.model ?? process.env.OPENAI_COMPAT_MODEL ?? "default";
|
||||
const defaultTemperature = config.temperature ?? 0.0;
|
||||
const maxOutput = config.maxOutput ?? 4096;
|
||||
|
||||
const baseURL = config.baseUrl ?? process.env.OPENAI_COMPAT_URL;
|
||||
if (baseURL === undefined || baseURL.length === 0) {
|
||||
|
|
@ -54,100 +49,107 @@ export class OpenAICompatibleProcessor extends LlmService {
|
|||
const apiKey =
|
||||
config.apiKey ?? process.env.OPENAI_COMPAT_KEY ?? "sk-no-key-required";
|
||||
|
||||
this.client = new OpenAI({ baseURL, apiKey });
|
||||
const client = new OpenAI({ baseURL, apiKey });
|
||||
|
||||
console.log("[OpenAI-Compatible] LLM service initialized");
|
||||
}
|
||||
|
||||
async generateContent(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const temp = temperature ?? this.defaultTemperature;
|
||||
return {
|
||||
generateContent: async (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
const resp = await this.client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_tokens: this.maxOutput,
|
||||
});
|
||||
const resp = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_tokens: maxOutput,
|
||||
});
|
||||
|
||||
return {
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
}
|
||||
return {
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
override supportsStreaming(): boolean {
|
||||
return true;
|
||||
}
|
||||
const stream = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_tokens: maxOutput,
|
||||
stream: true,
|
||||
});
|
||||
|
||||
async *generateContentStream(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const temp = temperature ?? this.defaultTemperature;
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
const stream = await this.client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_tokens: this.maxOutput,
|
||||
stream: true,
|
||||
});
|
||||
for await (const chunk of stream) {
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
}
|
||||
}
|
||||
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
}
|
||||
}
|
||||
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
}
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type OpenAICompatibleProcessor = ReturnType<typeof makeOpenAICompatibleProcessor>;
|
||||
|
||||
export function makeOpenAICompatibleProcessor(
|
||||
config: OpenAICompatibleProcessorConfig,
|
||||
): ReturnType<typeof makeLlmService> {
|
||||
return makeLlmService(config, makeOpenAICompatibleProvider(config));
|
||||
}
|
||||
|
||||
export const OpenAICompatibleProcessor = makeOpenAICompatibleProcessor;
|
||||
|
||||
export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
||||
id: "text-completion",
|
||||
specs: () => makeLlmSpecs(),
|
||||
layer: (config) =>
|
||||
Layer.succeed(
|
||||
Llm,
|
||||
Llm.of(makeLlmServiceShape(new OpenAICompatibleProcessor(config))),
|
||||
Llm.of(makeLlmServiceShape(makeOpenAICompatibleProvider(config))),
|
||||
),
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -7,10 +7,11 @@
|
|||
import OpenAI from "openai";
|
||||
import {
|
||||
Llm,
|
||||
LlmService,
|
||||
makeLlmService,
|
||||
makeFlowProcessorProgram,
|
||||
makeLlmServiceShape,
|
||||
makeLlmSpecs,
|
||||
type LlmProvider,
|
||||
type ProcessorConfig,
|
||||
type LlmResult,
|
||||
type LlmChunk,
|
||||
|
|
@ -18,142 +19,140 @@ import {
|
|||
} from "@trustgraph/base";
|
||||
import { Effect, Layer } from "effect";
|
||||
|
||||
export class OpenAIProcessor extends LlmService {
|
||||
private client: OpenAI;
|
||||
private readonly defaultModel: string;
|
||||
private readonly defaultTemperature: number;
|
||||
private readonly maxOutput: number;
|
||||
|
||||
constructor(config: ProcessorConfig & {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
temperature?: number;
|
||||
maxOutput?: number;
|
||||
}) {
|
||||
super(config);
|
||||
|
||||
this.defaultModel = config.model ?? "gpt-4o";
|
||||
this.defaultTemperature = config.temperature ?? 0.0;
|
||||
this.maxOutput = config.maxOutput ?? 4096;
|
||||
export type OpenAIProcessorConfig = ProcessorConfig & {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
temperature?: number;
|
||||
maxOutput?: number;
|
||||
};
|
||||
|
||||
export function makeOpenAIProvider(config: OpenAIProcessorConfig): LlmProvider {
|
||||
const defaultModel = config.model ?? "gpt-4o";
|
||||
const defaultTemperature = config.temperature ?? 0.0;
|
||||
const maxOutput = config.maxOutput ?? 4096;
|
||||
const apiKey = config.apiKey ?? process.env.OPENAI_TOKEN;
|
||||
if (apiKey === undefined || apiKey.length === 0) {
|
||||
throw new Error("OpenAI API key not specified");
|
||||
}
|
||||
|
||||
this.client = new OpenAI({
|
||||
const client = new OpenAI({
|
||||
apiKey,
|
||||
baseURL: config.baseUrl ?? process.env.OPENAI_BASE_URL,
|
||||
});
|
||||
|
||||
console.log("[OpenAI] LLM service initialized");
|
||||
}
|
||||
|
||||
async generateContent(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const temp = temperature ?? this.defaultTemperature;
|
||||
return {
|
||||
generateContent: async (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): Promise<LlmResult> => {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
try {
|
||||
const resp = await this.client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: this.maxOutput,
|
||||
});
|
||||
try {
|
||||
const resp = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
});
|
||||
|
||||
return {
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof OpenAI.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
return {
|
||||
text: resp.choices[0].message.content ?? "",
|
||||
inToken: resp.usage?.prompt_tokens ?? 0,
|
||||
outToken: resp.usage?.completion_tokens ?? 0,
|
||||
model: modelName,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof OpenAI.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
},
|
||||
supportsStreaming: () => true,
|
||||
generateContentStream: async function* (
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? defaultModel;
|
||||
const temp = temperature ?? defaultTemperature;
|
||||
|
||||
override supportsStreaming(): boolean {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const stream = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: maxOutput,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
});
|
||||
|
||||
async *generateContentStream(
|
||||
system: string,
|
||||
prompt: string,
|
||||
model?: string,
|
||||
temperature?: number,
|
||||
): AsyncGenerator<LlmChunk> {
|
||||
const modelName = model ?? this.defaultModel;
|
||||
const temp = temperature ?? this.defaultTemperature;
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
try {
|
||||
const stream = await this.client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
temperature: temp,
|
||||
max_completion_tokens: this.maxOutput,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
});
|
||||
for await (const chunk of stream) {
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
}
|
||||
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (content !== null && content !== undefined && content.length > 0) {
|
||||
yield {
|
||||
text: content,
|
||||
inToken: null,
|
||||
outToken: null,
|
||||
model: modelName,
|
||||
isFinal: false,
|
||||
};
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
}
|
||||
}
|
||||
|
||||
if (chunk.usage !== null && chunk.usage !== undefined) {
|
||||
totalInputTokens = chunk.usage.prompt_tokens;
|
||||
totalOutputTokens = chunk.usage.completion_tokens;
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof OpenAI.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
yield {
|
||||
text: "",
|
||||
inToken: totalInputTokens,
|
||||
outToken: totalOutputTokens,
|
||||
model: modelName,
|
||||
isFinal: true,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof OpenAI.RateLimitError) {
|
||||
throw tooManyRequestsError();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type OpenAIProcessor = ReturnType<typeof makeOpenAIProcessor>;
|
||||
|
||||
export function makeOpenAIProcessor(config: OpenAIProcessorConfig): ReturnType<typeof makeLlmService> {
|
||||
return makeLlmService(config, makeOpenAIProvider(config));
|
||||
}
|
||||
|
||||
export const OpenAIProcessor = makeOpenAIProcessor;
|
||||
|
||||
export const program = makeFlowProcessorProgram<ProcessorConfig, never, Llm>({
|
||||
id: "text-completion",
|
||||
specs: () => makeLlmSpecs(),
|
||||
layer: (config) =>
|
||||
Layer.succeed(
|
||||
Llm,
|
||||
Llm.of(makeLlmServiceShape(new OpenAIProcessor(config))),
|
||||
Llm.of(makeLlmServiceShape(makeOpenAIProvider(config))),
|
||||
),
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue