mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-04-28 09:56:23 +02:00
minor improvements
This commit is contained in:
parent
1dabb971c4
commit
6a065d3332
3 changed files with 37 additions and 44 deletions
|
|
@ -1068,35 +1068,34 @@ export async function* streamAgent({
|
||||||
// Local-provider prioritization: background agents yield to active chat
|
// Local-provider prioritization: background agents yield to active chat
|
||||||
if (isBackgroundAgent) {
|
if (isBackgroundAgent) {
|
||||||
await waitIfChatActive(providerFlavor, signal);
|
await waitIfChatActive(providerFlavor, signal);
|
||||||
} else {
|
|
||||||
markChatActive();
|
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
for await (const event of streamLlm(
|
if (!isBackgroundAgent) markChatActive();
|
||||||
model,
|
for await (const event of streamLlm(
|
||||||
state.messages,
|
model,
|
||||||
instructionsWithDateTime,
|
state.messages,
|
||||||
tools,
|
instructionsWithDateTime,
|
||||||
signal,
|
tools,
|
||||||
)) {
|
signal,
|
||||||
messageBuilder.ingest(event);
|
)) {
|
||||||
yield* processEvent({
|
messageBuilder.ingest(event);
|
||||||
runId,
|
|
||||||
type: "llm-stream-event",
|
|
||||||
event: event,
|
|
||||||
subflow: [],
|
|
||||||
});
|
|
||||||
if (event.type === "error") {
|
|
||||||
streamError = event.error;
|
|
||||||
yield* processEvent({
|
yield* processEvent({
|
||||||
runId,
|
runId,
|
||||||
type: "error",
|
type: "llm-stream-event",
|
||||||
error: streamError,
|
event: event,
|
||||||
subflow: [],
|
subflow: [],
|
||||||
});
|
});
|
||||||
break;
|
if (event.type === "error") {
|
||||||
|
streamError = event.error;
|
||||||
|
yield* processEvent({
|
||||||
|
runId,
|
||||||
|
type: "error",
|
||||||
|
error: streamError,
|
||||||
|
subflow: [],
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
} finally {
|
} finally {
|
||||||
if (!isBackgroundAgent) {
|
if (!isBackgroundAgent) {
|
||||||
markChatIdle();
|
markChatIdle();
|
||||||
|
|
|
||||||
|
|
@ -9,23 +9,24 @@
|
||||||
* Cloud providers bypass this entirely — they handle concurrency fine.
|
* Cloud providers bypass this entirely — they handle concurrency fine.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import type { z } from "zod";
|
||||||
|
import type { LlmProvider } from "@x/shared/dist/models.js";
|
||||||
|
|
||||||
|
type ProviderFlavor = z.infer<typeof LlmProvider>["flavor"];
|
||||||
|
|
||||||
let chatActiveCount = 0;
|
let chatActiveCount = 0;
|
||||||
let chatIdleResolvers: Array<() => void> = [];
|
let chatIdleResolvers: Array<() => void> = [];
|
||||||
|
|
||||||
/**
|
|
||||||
* Call when an interactive chat LLM stream starts.
|
|
||||||
* Nestable — supports concurrent interactive streams.
|
|
||||||
*/
|
|
||||||
export function markChatActive(): void {
|
export function markChatActive(): void {
|
||||||
chatActiveCount++;
|
chatActiveCount++;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Call when an interactive chat LLM stream ends.
|
|
||||||
* When all interactive streams finish, waiting background tasks resume.
|
|
||||||
*/
|
|
||||||
export function markChatIdle(): void {
|
export function markChatIdle(): void {
|
||||||
chatActiveCount = Math.max(0, chatActiveCount - 1);
|
if (chatActiveCount <= 0) {
|
||||||
|
console.warn("[llm-queue] markChatIdle called with no active chat — possible mismatched calls");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
chatActiveCount--;
|
||||||
if (chatActiveCount === 0) {
|
if (chatActiveCount === 0) {
|
||||||
const resolvers = chatIdleResolvers;
|
const resolvers = chatIdleResolvers;
|
||||||
chatIdleResolvers = [];
|
chatIdleResolvers = [];
|
||||||
|
|
@ -35,23 +36,16 @@ export function markChatIdle(): void {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
export function isLocalProvider(flavor: ProviderFlavor): boolean {
|
||||||
* Returns true if the provider flavor represents a local inference server.
|
|
||||||
*/
|
|
||||||
export function isLocalProvider(flavor: string): boolean {
|
|
||||||
return flavor === "ollama" || flavor === "openai-compatible";
|
return flavor === "ollama" || flavor === "openai-compatible";
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Background services call this before each LLM request.
|
* Background services call this before each LLM request.
|
||||||
* - If the provider is cloud-based, returns immediately.
|
* Returns immediately for cloud providers or when no chat is active.
|
||||||
* - If the provider is local and no chat is active, returns immediately.
|
* For local providers with active chat, waits until chat finishes.
|
||||||
* - If the provider is local and chat IS active, waits until chat finishes.
|
|
||||||
*
|
|
||||||
* @param providerFlavor - The provider flavor string (e.g. "ollama", "openai", "anthropic")
|
|
||||||
* @param signal - Optional AbortSignal to cancel the wait
|
|
||||||
*/
|
*/
|
||||||
export function waitIfChatActive(providerFlavor: string, signal?: AbortSignal): Promise<void> {
|
export function waitIfChatActive(providerFlavor: ProviderFlavor, signal?: AbortSignal): Promise<void> {
|
||||||
if (!isLocalProvider(providerFlavor)) {
|
if (!isLocalProvider(providerFlavor)) {
|
||||||
return Promise.resolve();
|
return Promise.resolve();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ import { createOllama } from "ollama-ai-provider-v2";
|
||||||
import { createOpenRouter } from '@openrouter/ai-sdk-provider';
|
import { createOpenRouter } from '@openrouter/ai-sdk-provider';
|
||||||
import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
|
import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
|
||||||
import { LlmModelConfig, LlmProvider } from "@x/shared/dist/models.js";
|
import { LlmModelConfig, LlmProvider } from "@x/shared/dist/models.js";
|
||||||
|
import { isLocalProvider } from "./llm-queue.js";
|
||||||
import z from "zod";
|
import z from "zod";
|
||||||
import { isSignedIn } from "../account/account.js";
|
import { isSignedIn } from "../account/account.js";
|
||||||
import { getGatewayProvider } from "./gateway.js";
|
import { getGatewayProvider } from "./gateway.js";
|
||||||
|
|
@ -75,8 +76,7 @@ export async function testModelConnection(
|
||||||
model: string,
|
model: string,
|
||||||
timeoutMs?: number,
|
timeoutMs?: number,
|
||||||
): Promise<{ success: boolean; error?: string }> {
|
): Promise<{ success: boolean; error?: string }> {
|
||||||
const isLocal = providerConfig.flavor === "ollama" || providerConfig.flavor === "openai-compatible";
|
const effectiveTimeout = timeoutMs ?? (isLocalProvider(providerConfig.flavor) ? 60000 : 8000);
|
||||||
const effectiveTimeout = timeoutMs ?? (isLocal ? 60000 : 8000);
|
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
const timeout = setTimeout(() => controller.abort(), effectiveTimeout);
|
const timeout = setTimeout(() => controller.abort(), effectiveTimeout);
|
||||||
try {
|
try {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue