From 799d7584b8b62c11a2a2403aed54b364ae26be31 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Sun, 5 Jul 2026 11:45:22 +0530 Subject: [PATCH 1/5] make local models work well --- .../src/components/settings-dialog.tsx | 12 +- apps/x/packages/core/src/agents/runtime.ts | 6 +- .../core/src/application/lib/builtin-tools.ts | 5 +- .../src/background-tasks/event-consumer.ts | 4 +- apps/x/packages/core/src/events/routing.ts | 5 +- .../core/src/knowledge/classify_thread.ts | 9 +- .../core/src/knowledge/inline_tasks.ts | 5 +- .../src/knowledge/live-note/event-consumer.ts | 4 +- .../core/src/knowledge/meeting_prep_brief.ts | 4 +- .../core/src/knowledge/summarize_meeting.ts | 4 +- apps/x/packages/core/src/models/defaults.ts | 1 + apps/x/packages/core/src/models/local.test.ts | 118 ++++++++ apps/x/packages/core/src/models/local.ts | 252 ++++++++++++++++++ apps/x/packages/core/src/models/models.ts | 143 +++++++++- apps/x/packages/core/src/models/repo.ts | 4 + .../core/src/models/structured.test.ts | 87 ++++++ apps/x/packages/core/src/models/structured.ts | 135 ++++++++++ .../security/auto-permission-classifier.ts | 10 +- .../src/turns/bridges/real-model-registry.ts | 240 ++++++++++------- .../packages/core/src/turns/model-registry.ts | 4 + apps/x/packages/core/src/turns/runtime.ts | 5 + apps/x/packages/shared/src/ipc.ts | 7 + apps/x/packages/shared/src/models.ts | 5 + 23 files changed, 938 insertions(+), 131 deletions(-) create mode 100644 apps/x/packages/core/src/models/local.test.ts create mode 100644 apps/x/packages/core/src/models/local.ts create mode 100644 apps/x/packages/core/src/models/structured.test.ts create mode 100644 apps/x/packages/core/src/models/structured.ts diff --git a/apps/x/apps/renderer/src/components/settings-dialog.tsx b/apps/x/apps/renderer/src/components/settings-dialog.tsx index 259a7253..c358c816 100644 --- a/apps/x/apps/renderer/src/components/settings-dialog.tsx +++ b/apps/x/apps/renderer/src/components/settings-dialog.tsx @@ -528,7 +528,17 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { setDefaultProvider(provider) setTestState({ status: "success" }) window.dispatchEvent(new Event('models-config-changed')) - toast.success("Model configuration saved") + // Capability probe caveats (local models): saved, but the user should + // know when the model can't do tools or has a too-small context. + const warnings: string[] = result.warnings ?? [] + if (warnings.length > 0) { + for (const warning of warnings) { + toast.warning(warning, { duration: 12000 }) + } + toast.success("Model configuration saved (with warnings)") + } else { + toast.success("Model configuration saved") + } } else { setTestState({ status: "error", error: result.error }) toast.error(result.error || "Connection test failed") diff --git a/apps/x/packages/core/src/agents/runtime.ts b/apps/x/packages/core/src/agents/runtime.ts index b0297aaa..357e0ff3 100644 --- a/apps/x/packages/core/src/agents/runtime.ts +++ b/apps/x/packages/core/src/agents/runtime.ts @@ -19,7 +19,7 @@ import { resolveFilePathForPermission } from "../filesystem/files.js"; import container from "../di/container.js"; import { notifyIfEnabled } from "../application/notification/notifier.js"; import { IModelConfigRepo } from "../models/repo.js"; -import { createProvider } from "../models/models.js"; +import { createLanguageModel } from "../models/models.js"; import { resolveProviderConfig } from "../models/defaults.js"; import { IAgentsRepo } from "./repo.js"; import { IMonotonicallyIncreasingIdGenerator } from "../application/lib/id-gen.js"; @@ -1365,8 +1365,8 @@ export async function* streamAgent({ } const modelId = state.runModel; const providerConfig = await resolveProviderConfig(state.runProvider); - const provider = createProvider(providerConfig); - const model = provider.languageModel(modelId); + // Legacy runs are user-facing chats: interactive priority on local models. + const model = createLanguageModel(providerConfig, modelId, { priority: "interactive" }); logger.log(`using model: ${modelId} (provider: ${state.runProvider})`); // Install use-case context for tool-internal LLM calls (e.g. parseFile) diff --git a/apps/x/packages/core/src/application/lib/builtin-tools.ts b/apps/x/packages/core/src/application/lib/builtin-tools.ts index 2f74a9c5..8d4faad4 100644 --- a/apps/x/packages/core/src/application/lib/builtin-tools.ts +++ b/apps/x/packages/core/src/application/lib/builtin-tools.ts @@ -93,7 +93,7 @@ async function resolveCodeProject(dirPath: string): Promise< import { ensureLoaded as ensureBrowserSkillsLoaded, readSkillContent as readBrowserSkillContent, refreshFromRemote as refreshBrowserSkills } from "../browser-skills/index.js"; import type { ToolContext } from "./exec-tool.js"; import { generateText } from "ai"; -import { createProvider } from "../../models/models.js"; +import { createLanguageModel } from "../../models/models.js"; import { getDefaultModelAndProvider, resolveProviderConfig } from "../../models/defaults.js"; import { captureLlmUsage } from "../../analytics/usage.js"; import { getCurrentUseCase, withUseCase } from "../../analytics/use_case.js"; @@ -584,7 +584,8 @@ export const BuiltinTools: z.infer = { const { model: modelId, provider: providerName } = await getDefaultModelAndProvider(); const providerConfig = await resolveProviderConfig(providerName); - const model = createProvider(providerConfig).languageModel(modelId); + // Runs as a tool inside a chat turn more often than not. + const model = createLanguageModel(providerConfig, modelId, { priority: 'interactive' }); const userPrompt = prompt || 'Convert this file to well-structured markdown.'; diff --git a/apps/x/packages/core/src/background-tasks/event-consumer.ts b/apps/x/packages/core/src/background-tasks/event-consumer.ts index 4d3e5f81..c80c640d 100644 --- a/apps/x/packages/core/src/background-tasks/event-consumer.ts +++ b/apps/x/packages/core/src/background-tasks/event-consumer.ts @@ -1,6 +1,6 @@ import type { EventConsumer, EventConsumerTarget } from '../events/consumer.js'; import { routeBatch } from '../events/routing.js'; -import { createProvider } from '../models/models.js'; +import { createLanguageModel } from '../models/models.js'; import { getDefaultModelAndProvider, getBackgroundTaskAgentModel, @@ -14,7 +14,7 @@ async function resolveRoutingModel() { const { provider } = await getDefaultModelAndProvider(); const config = await resolveProviderConfig(provider); return { - model: createProvider(config).languageModel(modelId), + model: createLanguageModel(config, modelId, { priority: 'classifier' }), modelId, providerName: provider, }; diff --git a/apps/x/packages/core/src/events/routing.ts b/apps/x/packages/core/src/events/routing.ts index 5f9e4222..e7a73cc1 100644 --- a/apps/x/packages/core/src/events/routing.ts +++ b/apps/x/packages/core/src/events/routing.ts @@ -1,6 +1,6 @@ -import { generateObject } from 'ai'; import type { LanguageModel } from 'ai'; import { events, PrefixLogger } from '@x/shared'; +import { generateObjectSafe } from '../models/structured.js'; import type { RowboatEvent } from '@x/shared/dist/events.js'; import { captureLlmUsage } from '../analytics/usage.js'; import { withUseCase, type UseCase } from '../analytics/use_case.js'; @@ -89,11 +89,12 @@ export async function routeBatch( for (let i = 0; i < targets.length; i += BATCH_SIZE) { const batch = targets.slice(i, i + BATCH_SIZE); try { - const result = await withUseCase({ useCase: opts.useCase, subUseCase: 'routing' }, () => generateObject({ + const result = await withUseCase({ useCase: opts.useCase, subUseCase: 'routing' }, () => generateObjectSafe({ model, system: systemPrompt, prompt: buildPrompt(event, batch, opts.entityPlural), schema: events.Pass1OutputSchema, + retry: true, })); captureLlmUsage({ useCase: opts.useCase, diff --git a/apps/x/packages/core/src/knowledge/classify_thread.ts b/apps/x/packages/core/src/knowledge/classify_thread.ts index 25580c65..daa0851d 100644 --- a/apps/x/packages/core/src/knowledge/classify_thread.ts +++ b/apps/x/packages/core/src/knowledge/classify_thread.ts @@ -1,11 +1,11 @@ import fs from 'fs'; import path from 'path'; import { z } from 'zod'; -import { generateObject } from 'ai'; import { google } from 'googleapis'; import type { OAuth2Client } from 'google-auth-library'; import { WorkDir } from '../config/config.js'; -import { createProvider } from '../models/models.js'; +import { createLanguageModel } from '../models/models.js'; +import { generateObjectSafe } from '../models/structured.js'; import { getDefaultModelAndProvider, getKgModel, @@ -249,7 +249,7 @@ export async function classifyThread( const modelId = await getKgModel(); const { provider } = await getDefaultModelAndProvider(); const config = await resolveProviderConfig(provider); - const model = createProvider(config).languageModel(modelId); + const model = createLanguageModel(config, modelId, { priority: 'background' }); let systemPrompt = options.skipDraft ? `${SYSTEM_PROMPT}\n\n# Skip the draft\n\nThe user already has their own draft in progress for this thread — DO NOT generate a draftResponse. Always omit the draftResponse field.` @@ -262,11 +262,12 @@ export async function classifyThread( systemPrompt = `${systemPrompt}\n\n${feedback}`; } - const result = await withUseCase({ useCase: 'knowledge_sync', subUseCase: 'email_classifier' }, () => generateObject({ + const result = await withUseCase({ useCase: 'knowledge_sync', subUseCase: 'email_classifier' }, () => generateObjectSafe({ model, system: systemPrompt, prompt: buildPrompt(snapshot, userEmail, styleGuide, calendar), schema: ClassificationSchema, + retry: true, })); captureLlmUsage({ diff --git a/apps/x/packages/core/src/knowledge/inline_tasks.ts b/apps/x/packages/core/src/knowledge/inline_tasks.ts index f24972ed..e1002b58 100644 --- a/apps/x/packages/core/src/knowledge/inline_tasks.ts +++ b/apps/x/packages/core/src/knowledge/inline_tasks.ts @@ -7,7 +7,7 @@ import { runHeadlessAgent } from '../agents/headless-app.js'; import { getKgModel } from '../models/defaults.js'; import container from '../di/container.js'; import type { IModelConfigRepo } from '../models/repo.js'; -import { createProvider } from '../models/models.js'; +import { createLanguageModel } from '../models/models.js'; import { inlineTask } from '@x/shared'; import { captureLlmUsage } from '../analytics/usage.js'; import { withUseCase } from '../analytics/use_case.js'; @@ -613,8 +613,7 @@ export async function processRowboatInstruction( export async function classifySchedule(instruction: string): Promise { const repo = container.resolve('modelConfigRepo'); const config = await repo.getConfig(); - const provider = createProvider(config.provider); - const model = provider.languageModel(config.model); + const model = createLanguageModel(config.provider, config.model, { priority: 'classifier' }); const now = new Date(); const defaultEnd = new Date(now.getTime() + 7 * 24 * 60 * 60 * 1000); diff --git a/apps/x/packages/core/src/knowledge/live-note/event-consumer.ts b/apps/x/packages/core/src/knowledge/live-note/event-consumer.ts index 8000aac5..f314c66c 100644 --- a/apps/x/packages/core/src/knowledge/live-note/event-consumer.ts +++ b/apps/x/packages/core/src/knowledge/live-note/event-consumer.ts @@ -3,7 +3,7 @@ import { fetchLiveNote } from './fileops.js'; import { runLiveNoteAgent } from './runner.js'; import type { EventConsumer, EventConsumerTarget } from '../../events/consumer.js'; import { routeBatch } from '../../events/routing.js'; -import { createProvider } from '../../models/models.js'; +import { createLanguageModel } from '../../models/models.js'; import { getDefaultModelAndProvider, getLiveNoteAgentModel, resolveProviderConfig } from '../../models/defaults.js'; async function resolveRoutingModel() { @@ -11,7 +11,7 @@ async function resolveRoutingModel() { const { provider } = await getDefaultModelAndProvider(); const config = await resolveProviderConfig(provider); return { - model: createProvider(config).languageModel(modelId), + model: createLanguageModel(config, modelId, { priority: 'classifier' }), modelId, providerName: provider, }; diff --git a/apps/x/packages/core/src/knowledge/meeting_prep_brief.ts b/apps/x/packages/core/src/knowledge/meeting_prep_brief.ts index 45cb26e0..ed41895f 100644 --- a/apps/x/packages/core/src/knowledge/meeting_prep_brief.ts +++ b/apps/x/packages/core/src/knowledge/meeting_prep_brief.ts @@ -2,7 +2,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { generateText } from 'ai'; import { WorkDir } from '../config/config.js'; -import { createProvider } from '../models/models.js'; +import { createLanguageModel } from '../models/models.js'; import { getDefaultModelAndProvider, getMeetingNotesModel, resolveProviderConfig } from '../models/defaults.js'; import { captureLlmUsage } from '../analytics/usage.js'; import { withUseCase } from '../analytics/use_case.js'; @@ -178,7 +178,7 @@ async function generateBrief(event: CalendarEvent, ctx: Awaited generateText({ model, diff --git a/apps/x/packages/core/src/knowledge/summarize_meeting.ts b/apps/x/packages/core/src/knowledge/summarize_meeting.ts index dde7acb5..6bca2d24 100644 --- a/apps/x/packages/core/src/knowledge/summarize_meeting.ts +++ b/apps/x/packages/core/src/knowledge/summarize_meeting.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; import { generateText } from 'ai'; -import { createProvider } from '../models/models.js'; +import { createLanguageModel } from '../models/models.js'; import { getDefaultModelAndProvider, getMeetingNotesModel, resolveProviderConfig } from '../models/defaults.js'; import { WorkDir } from '../config/config.js'; import { captureLlmUsage } from '../analytics/usage.js'; @@ -140,7 +140,7 @@ export async function summarizeMeeting(transcript: string, meetingStartTime?: st const modelId = await getMeetingNotesModel(); const { provider: providerName } = await getDefaultModelAndProvider(); const providerConfig = await resolveProviderConfig(providerName); - const model = createProvider(providerConfig).languageModel(modelId); + const model = createLanguageModel(providerConfig, modelId, { priority: 'background' }); // If a specific calendar event was linked, use it directly. // Otherwise fall back to scanning events within ±3 hours. diff --git a/apps/x/packages/core/src/models/defaults.ts b/apps/x/packages/core/src/models/defaults.ts index 3f84a62a..9bd4eaf1 100644 --- a/apps/x/packages/core/src/models/defaults.ts +++ b/apps/x/packages/core/src/models/defaults.ts @@ -52,6 +52,7 @@ export async function resolveProviderConfig(name: string): Promise void; + const promise = new Promise((r) => { + resolve = r; + }); + return { promise, resolve }; +} + +const tick = () => new Promise((r) => setTimeout(r, 0)); + +describe("isLocalProvider", () => { + it("treats ollama as local", () => { + expect(isLocalProvider({ flavor: "ollama" })).toBe(true); + }); + + it("treats loopback openai-compatible endpoints as local", () => { + expect(isLocalProvider({ flavor: "openai-compatible", baseURL: "http://localhost:1234/v1" })).toBe(true); + expect(isLocalProvider({ flavor: "openai-compatible", baseURL: "http://127.0.0.1:8080/v1" })).toBe(true); + }); + + it("treats remote openai-compatible endpoints and cloud flavors as non-local", () => { + expect(isLocalProvider({ flavor: "openai-compatible", baseURL: "https://api.together.xyz/v1" })).toBe(false); + expect(isLocalProvider({ flavor: "openai" })).toBe(false); + expect(isLocalProvider({ flavor: "rowboat" })).toBe(false); + }); +}); + +describe("LocalLlmScheduler", () => { + it("serves waiters by priority, FIFO within a priority", async () => { + const scheduler = new LocalLlmScheduler(1); + const order: string[] = []; + const first = deferred(); + + const running = scheduler.run("background", undefined, async () => { + await first.promise; + order.push("initial"); + }); + await tick(); + + const bg1 = scheduler.run("background", undefined, async () => { + order.push("bg1"); + }); + const bg2 = scheduler.run("background", undefined, async () => { + order.push("bg2"); + }); + const chat = scheduler.run("interactive", undefined, async () => { + order.push("chat"); + }); + const classifier = scheduler.run("classifier", undefined, async () => { + order.push("classifier"); + }); + await tick(); + + first.resolve(); + await Promise.all([running, bg1, bg2, chat, classifier]); + expect(order).toEqual(["initial", "chat", "classifier", "bg1", "bg2"]); + }); + + it("releases the slot when a task throws", async () => { + const scheduler = new LocalLlmScheduler(1); + await expect( + scheduler.run("interactive", undefined, async () => { + throw new Error("boom"); + }), + ).rejects.toThrow("boom"); + // Slot must be free again. + await scheduler.run("interactive", undefined, async () => undefined); + }); + + it("rejects queued waiters whose signal aborts, without leaking the slot", async () => { + const scheduler = new LocalLlmScheduler(1); + const gate = deferred(); + const running = scheduler.run("background", undefined, () => gate.promise); + await tick(); + + const controller = new AbortController(); + const waiting = scheduler.acquire("interactive", controller.signal); + controller.abort(); + await expect(waiting).rejects.toThrow(/abort/i); + + gate.resolve(); + await running; + // Queue is clean: a fresh acquire succeeds immediately. + const release = await scheduler.acquire("background"); + release(); + }); + + it("rejects immediately when acquiring with an already-aborted signal", async () => { + const scheduler = new LocalLlmScheduler(1); + const controller = new AbortController(); + controller.abort(); + await expect(scheduler.acquire("interactive", controller.signal)).rejects.toThrow(/abort/i); + }); + + it("allows up to maxConcurrent tasks at once", async () => { + const scheduler = new LocalLlmScheduler(2); + const gateA = deferred(); + const gateB = deferred(); + let active = 0; + let peak = 0; + const track = async (gate: Promise) => { + active++; + peak = Math.max(peak, active); + await gate; + active--; + }; + const a = scheduler.run("background", undefined, () => track(gateA.promise)); + const b = scheduler.run("background", undefined, () => track(gateB.promise)); + await tick(); + expect(peak).toBe(2); + gateA.resolve(); + gateB.resolve(); + await Promise.all([a, b]); + }); +}); diff --git a/apps/x/packages/core/src/models/local.ts b/apps/x/packages/core/src/models/local.ts new file mode 100644 index 00000000..16dc7759 --- /dev/null +++ b/apps/x/packages/core/src/models/local.ts @@ -0,0 +1,252 @@ +import { wrapLanguageModel, type LanguageModel } from "ai"; +import type { z } from "zod"; +import type { LlmProvider } from "@x/shared/dist/models.js"; +import { PrefixLogger } from "@x/shared"; + +const log = new PrefixLogger("LocalLlm"); + +// Ollama's server-side default context window (~4k tokens) is far below what +// Rowboat's agents need (the copilot's system prompt + tool schemas alone are +// ~15-20k tokens) and Ollama silently truncates the prompt from the top when +// it overflows — the model loses its own instructions. We therefore always +// request an explicit window for Ollama models. Overridable per provider via +// `contextLength` in models.json. +export const DEFAULT_OLLAMA_CONTEXT_LENGTH = 32768; + +const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "0.0.0.0", "::1", "[::1]"]); + +/** + * Whether requests to this provider are served by a model running on the + * user's own machine (and therefore need scheduling: local runtimes process + * requests mostly serially, so background work must not starve chat). + */ +export function isLocalProvider(config: z.infer): boolean { + if (config.flavor === "ollama") { + return true; + } + if (config.flavor === "openai-compatible") { + // LM Studio, llama.cpp server, vLLM on the same machine, etc. + try { + const host = new URL(config.baseURL ?? "").hostname; + return LOOPBACK_HOSTS.has(host) || host.endsWith(".local"); + } catch { + return false; + } + } + return false; +} + +export type LlmPriority = "interactive" | "classifier" | "background"; + +const PRIORITY_ORDER: Record = { + interactive: 0, + classifier: 1, + background: 2, +}; + +interface Waiter { + priority: LlmPriority; + seq: number; + resolve: (release: () => void) => void; + reject: (error: Error) => void; + signal?: AbortSignal; + onAbort?: () => void; +} + +function abortError(): Error { + const error = new Error("local LLM slot acquisition aborted"); + error.name = "AbortError"; + return error; +} + +/** + * Serializes access to local LLM runtimes. One slot by default: local servers + * effectively process one request at a time, and interleaving requests + * destroys their KV-cache reuse. Waiters are served strictly by priority + * (interactive chat > lightweight classifiers > background knowledge + * pipeline), FIFO within a priority — so a queued email-labeling job can + * never delay the user's chat by more than the one request already running. + */ +export class LocalLlmScheduler { + private active = 0; + private seq = 0; + private readonly waiting: Waiter[] = []; + + constructor(private readonly maxConcurrent = 1) {} + + async acquire(priority: LlmPriority, signal?: AbortSignal): Promise<() => void> { + if (signal?.aborted) { + throw abortError(); + } + if (this.active < this.maxConcurrent) { + this.active++; + return this.makeRelease(); + } + return new Promise<() => void>((resolve, reject) => { + const waiter: Waiter = { priority, seq: this.seq++, resolve, reject, signal }; + if (signal) { + waiter.onAbort = () => { + const index = this.waiting.indexOf(waiter); + if (index >= 0) { + this.waiting.splice(index, 1); + reject(abortError()); + } + }; + signal.addEventListener("abort", waiter.onAbort, { once: true }); + } + this.waiting.push(waiter); + if (this.waiting.length === 1 || this.waiting.length % 10 === 0) { + log.log(`queueing ${priority} request (${this.waiting.length} waiting, ${this.active} active)`); + } + }); + } + + /** Run `fn` while holding a slot. */ + async run(priority: LlmPriority, signal: AbortSignal | undefined, fn: () => PromiseLike): Promise { + const release = await this.acquire(priority, signal); + try { + return await fn(); + } finally { + release(); + } + } + + get queueDepth(): number { + return this.waiting.length; + } + + private makeRelease(): () => void { + let released = false; + return () => { + if (released) { + return; + } + released = true; + this.active--; + this.dequeue(); + }; + } + + private dequeue(): void { + while (this.active < this.maxConcurrent && this.waiting.length > 0) { + let best = 0; + for (let i = 1; i < this.waiting.length; i++) { + const a = this.waiting[i]; + const b = this.waiting[best]; + if ( + PRIORITY_ORDER[a.priority] < PRIORITY_ORDER[b.priority] || + (PRIORITY_ORDER[a.priority] === PRIORITY_ORDER[b.priority] && a.seq < b.seq) + ) { + best = i; + } + } + const [waiter] = this.waiting.splice(best, 1); + if (waiter.signal && waiter.onAbort) { + waiter.signal.removeEventListener("abort", waiter.onAbort); + } + this.active++; + waiter.resolve(this.makeRelease()); + } + } +} + +function envConcurrency(): number { + const raw = process.env.ROWBOAT_LOCAL_LLM_CONCURRENCY; + const parsed = raw ? Number.parseInt(raw, 10) : NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : 1; +} + +// One queue for all local providers: users run one local server, and even +// with several, the machine's compute is the shared resource. +export const localLlmScheduler = new LocalLlmScheduler(envConcurrency()); + +/** + * Wrap a language model so every call requests an explicit context window + * from Ollama (merged under the caller's providerOptions — an explicit + * caller value wins) and, when `priority` is set and the provider is local, + * goes through the shared scheduler. + */ +export function applyLocalModelSettings( + model: LanguageModel, + providerConfig: z.infer, + priority: LlmPriority | null, +): LanguageModel { + if (typeof model === "string") { + // Bare model-id strings resolve through the global registry; local + // providers never take this path. + return model; + } + const local = isLocalProvider(providerConfig); + const wantsNumCtx = providerConfig.flavor === "ollama"; + if (!wantsNumCtx && !(local && priority)) { + return model; + } + const numCtx = providerConfig.contextLength ?? DEFAULT_OLLAMA_CONTEXT_LENGTH; + const schedule = local && priority ? priority : null; + return wrapLanguageModel({ + model, + middleware: { + ...(wantsNumCtx + ? { + transformParams: async ({ params }) => { + const providerOptions = (params.providerOptions ?? {}) as Record>; + const ollama = (providerOptions.ollama ?? {}) as Record; + const options = (ollama.options ?? {}) as Record; + return { + ...params, + providerOptions: { + ...providerOptions, + ollama: { + ...ollama, + options: { num_ctx: numCtx, ...options }, + }, + }, + }; + }, + } + : {}), + ...(schedule + ? { + wrapGenerate: async ({ doGenerate, params }) => + localLlmScheduler.run(schedule, params.abortSignal, () => doGenerate()), + wrapStream: async ({ doStream, params }) => { + const release = await localLlmScheduler.acquire(schedule, params.abortSignal); + try { + const { stream, ...rest } = await doStream(); + return { ...rest, stream: releaseOnSettled(stream, release) }; + } catch (error) { + release(); + throw error; + } + }, + } + : {}), + }, + }); +} + +// Hold the slot until the provider stream drains, errors, or is cancelled — +// a streaming response occupies the local server for its full duration. +function releaseOnSettled(stream: ReadableStream, release: () => void): ReadableStream { + const reader = stream.getReader(); + return new ReadableStream({ + async pull(controller) { + try { + const { done, value } = await reader.read(); + if (done) { + release(); + controller.close(); + } else { + controller.enqueue(value); + } + } catch (error) { + release(); + controller.error(error); + } + }, + cancel(reason) { + release(); + return reader.cancel(reason); + }, + }); +} diff --git a/apps/x/packages/core/src/models/models.ts b/apps/x/packages/core/src/models/models.ts index c50393c6..b05df695 100644 --- a/apps/x/packages/core/src/models/models.ts +++ b/apps/x/packages/core/src/models/models.ts @@ -1,5 +1,5 @@ import { ProviderV2 } from "@ai-sdk/provider"; -import { createGateway, generateText } from "ai"; +import { createGateway, generateText, type LanguageModel } from "ai"; import { createOpenAI } from "@ai-sdk/openai"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { createAnthropic } from "@ai-sdk/anthropic"; @@ -12,6 +12,12 @@ import { getGatewayProvider } from "./gateway.js"; import { getDefaultModelAndProvider, resolveProviderConfig } from "./defaults.js"; import { getChatModelIds } from "./models-dev.js"; import { withUseCase } from "../analytics/use_case.js"; +import { + applyLocalModelSettings, + isLocalProvider, + DEFAULT_OLLAMA_CONTEXT_LENGTH, + type LlmPriority, +} from "./local.js"; export const Provider = LlmProvider; export const ModelConfig = LlmModelConfig; @@ -74,24 +80,149 @@ export function createProvider(config: z.infer): ProviderV2 { } } +/** + * The one place model instances are created. Applies local-runtime settings + * (explicit Ollama context window) and, when `priority` is given, routes the + * call through the shared local scheduler so background work cannot starve + * interactive chat. Pass `priority: null` when the caller does its own + * scheduling (the turn runtime's model registry). + */ +export function createLanguageModel( + providerConfig: z.infer, + modelId: string, + opts: { priority?: LlmPriority | null } = {}, +): LanguageModel { + const model = createProvider(providerConfig).languageModel(modelId); + return applyLocalModelSettings(model, providerConfig, opts.priority ?? null); +} + +export interface ModelCapabilities { + /** undefined = could not be determined (endpoint missing, non-local provider). */ + supportsTools?: boolean; + maxContextLength?: number; +} + +/** + * Best-effort capability probe for local runtimes. Ollama reports a + * `capabilities` list and the model's trained context window via /api/show; + * LM Studio exposes the same through its /api/v0/models REST endpoint. + * Failures are swallowed — an unknown capability is not an error. + */ +export async function probeModelCapabilities( + providerConfig: z.infer, + model: string, + timeoutMs = 5000, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + if (providerConfig.flavor === "ollama") { + const base = (providerConfig.baseURL ?? "http://localhost:11434") + .replace(/\/+$/, "") + .replace(/\/api$/, ""); + const res = await fetch(`${base}/api/show`, { + method: "POST", + headers: { "Content-Type": "application/json", ...(providerConfig.headers ?? {}) }, + body: JSON.stringify({ model }), + signal: controller.signal, + }); + if (!res.ok) return {}; + const data = await res.json() as { + capabilities?: string[]; + model_info?: Record; + }; + const result: ModelCapabilities = {}; + if (Array.isArray(data.capabilities)) { + result.supportsTools = data.capabilities.includes("tools"); + } + for (const [key, value] of Object.entries(data.model_info ?? {})) { + if (key.endsWith(".context_length") && typeof value === "number") { + result.maxContextLength = value; + break; + } + } + return result; + } + if (providerConfig.flavor === "openai-compatible" && isLocalProvider(providerConfig)) { + // LM Studio's enhanced REST API lives at /api/v0 on the same + // origin as the OpenAI-compatible /v1 endpoint. + const origin = new URL(providerConfig.baseURL ?? "").origin; + const res = await fetch(`${origin}/api/v0/models`, { + headers: providerConfig.headers ?? {}, + signal: controller.signal, + }); + if (!res.ok) return {}; + const data = await res.json() as { data?: Array> }; + const entry = (data.data ?? []).find((m) => m.id === model); + if (!entry) return {}; + const result: ModelCapabilities = {}; + if (Array.isArray(entry.capabilities)) { + result.supportsTools = (entry.capabilities as string[]).includes("tool_use"); + } + const max = entry.loaded_context_length ?? entry.max_context_length; + if (typeof max === "number") { + result.maxContextLength = max; + } + return result; + } + return {}; + } catch { + return {}; + } finally { + clearTimeout(timeout); + } +} + +function capabilityWarnings( + providerConfig: z.infer, + model: string, + capabilities: ModelCapabilities, +): string[] { + const warnings: string[] = []; + if (capabilities.supportsTools === false) { + warnings.push( + `${model} does not support tool calling. Rowboat's assistant and background agents rely on tools; pick a tool-capable model (e.g. qwen3, gpt-oss, llama3.3).`, + ); + } + const configured = providerConfig.contextLength + ?? (providerConfig.flavor === "ollama" ? DEFAULT_OLLAMA_CONTEXT_LENGTH : undefined); + if (capabilities.maxContextLength !== undefined) { + if (capabilities.maxContextLength < 16384) { + warnings.push( + `${model} has a ${capabilities.maxContextLength}-token context window. Rowboat's assistant needs ~16k+ tokens; expect truncated or confused responses.`, + ); + } else if (configured !== undefined && capabilities.maxContextLength < configured) { + warnings.push( + `${model} supports at most ${capabilities.maxContextLength} context tokens, below the configured ${configured}. Set "contextLength" for this provider in models.json to ${capabilities.maxContextLength} or less.`, + ); + } + } + return warnings; +} + export async function testModelConnection( providerConfig: z.infer, model: string, timeoutMs?: number, -): Promise<{ success: boolean; error?: string }> { +): Promise<{ success: boolean; error?: string; warnings?: string[]; capabilities?: ModelCapabilities }> { const isLocal = providerConfig.flavor === "ollama" || providerConfig.flavor === "openai-compatible"; const effectiveTimeout = timeoutMs ?? (isLocal ? 60000 : 8000); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), effectiveTimeout); try { - const provider = createProvider(providerConfig); - const languageModel = provider.languageModel(model); + const languageModel = createLanguageModel(providerConfig, model); await generateText({ model: languageModel, prompt: "ping", abortSignal: controller.signal, }); - return { success: true }; + const capabilities = await probeModelCapabilities(providerConfig, model); + const warnings = capabilityWarnings(providerConfig, model, capabilities); + return { + success: true, + ...(warnings.length > 0 ? { warnings } : {}), + capabilities, + }; } catch (error) { const message = error instanceof Error ? error.message : "Connection test failed"; return { success: false, error: message }; @@ -203,7 +334,7 @@ export async function generateOneShot(opts: GenerateTextOptions): Promise generateText({ diff --git a/apps/x/packages/core/src/models/repo.ts b/apps/x/packages/core/src/models/repo.ts index 29febf4b..b36cb34a 100644 --- a/apps/x/packages/core/src/models/repo.ts +++ b/apps/x/packages/core/src/models/repo.ts @@ -48,6 +48,10 @@ export class FSModelConfigRepo implements IModelConfigRepo { apiKey: config.provider.apiKey, baseURL: config.provider.baseURL, headers: config.provider.headers, + // Preserve a hand-edited contextLength unless the caller sets one. + ...(config.provider.contextLength !== undefined + ? { contextLength: config.provider.contextLength } + : {}), model: config.model, models: config.models, knowledgeGraphModel: config.knowledgeGraphModel, diff --git a/apps/x/packages/core/src/models/structured.test.ts b/apps/x/packages/core/src/models/structured.test.ts new file mode 100644 index 00000000..848188cd --- /dev/null +++ b/apps/x/packages/core/src/models/structured.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { NoObjectGeneratedError } from "ai"; +import type { LanguageModel } from "ai"; +import { generateObjectSafe } from "./structured.js"; + +const Schema = z.object({ ids: z.array(z.string()) }); + +// A minimal LanguageModelV2 double whose doGenerate returns the given texts +// in sequence. generateObject parses the text against the schema itself, so +// malformed text surfaces as NoObjectGeneratedError — exactly the local-model +// failure mode generateObjectSafe exists to absorb. +function fakeModel(responses: string[]): LanguageModel { + let call = 0; + return { + specificationVersion: "v2", + provider: "fake", + modelId: "fake-model", + supportedUrls: {}, + doGenerate: async () => { + const text = responses[Math.min(call++, responses.length - 1)]; + return { + content: [{ type: "text" as const, text }], + finishReason: "stop" as const, + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + warnings: [], + }; + }, + doStream: async () => { + throw new Error("not used"); + }, + } as unknown as LanguageModel; +} + +describe("generateObjectSafe", () => { + it("passes through a clean structured response", async () => { + const result = await generateObjectSafe({ + model: fakeModel(['{"ids":["a","b"]}']), + prompt: "p", + schema: Schema, + }); + expect(result.object).toEqual({ ids: ["a", "b"] }); + expect(result.usage?.totalTokens).toBe(2); + }); + + it("salvages JSON wrapped in prose and think blocks", async () => { + const result = await generateObjectSafe({ + model: fakeModel([ + 'hmm {"ids":["x"]} maybeSure! Here you go:\n```json\n{"ids":["note-1","note-2"]}\n```\nLet me know!', + ]), + prompt: "p", + schema: Schema, + }); + expect(result.object).toEqual({ ids: ["note-1", "note-2"] }); + }); + + it("retries once with a reinforced instruction when enabled", async () => { + const result = await generateObjectSafe({ + model: fakeModel(["I cannot answer in JSON, sorry!", '{"ids":[]}']), + prompt: "p", + schema: Schema, + retry: true, + }); + expect(result.object).toEqual({ ids: [] }); + }); + + it("throws the original error when salvage and retry both fail", async () => { + await expect( + generateObjectSafe({ + model: fakeModel(["not json at all"]), + prompt: "p", + schema: Schema, + retry: true, + }), + ).rejects.toSatisfy((error: unknown) => NoObjectGeneratedError.isInstance(error)); + }); + + it("does not retry when retry is disabled", async () => { + await expect( + generateObjectSafe({ + model: fakeModel(["nope", '{"ids":[]}']), + prompt: "p", + schema: Schema, + }), + ).rejects.toSatisfy((error: unknown) => NoObjectGeneratedError.isInstance(error)); + }); +}); diff --git a/apps/x/packages/core/src/models/structured.ts b/apps/x/packages/core/src/models/structured.ts new file mode 100644 index 00000000..75097fbb --- /dev/null +++ b/apps/x/packages/core/src/models/structured.ts @@ -0,0 +1,135 @@ +import { + generateObject, + NoObjectGeneratedError, + type LanguageModel, + type LanguageModelUsage, +} from "ai"; +import type { z } from "zod"; +import { PrefixLogger } from "@x/shared"; + +const log = new PrefixLogger("StructuredOutput"); + +const NO_JSON = Symbol("no-json"); + +export interface GenerateObjectSafeOptions { + model: LanguageModel; + system?: string; + prompt: string; + schema: z.ZodType; + /** + * Retry once with a reinforced JSON-only instruction when the first + * attempt produces unparseable output. Local/small models miss strict + * schema output far more often than frontier models, so callers that may + * run on a local model should enable this. + */ + retry?: boolean; +} + +export interface GenerateObjectSafeResult { + object: T; + usage?: LanguageModelUsage; +} + +/** + * generateObject with degradation paths for models that can't reliably emit + * strict JSON: (1) salvage a schema-valid JSON value out of the raw response + * text (small models wrap JSON in prose, fences, or blocks), then + * (2) optionally retry once with a reinforced instruction. Throws the + * original error when nothing works, so callers' failure handling is + * unchanged. + */ +export async function generateObjectSafe( + options: GenerateObjectSafeOptions, +): Promise> { + try { + const result = await generateObject({ + model: options.model, + ...(options.system ? { system: options.system } : {}), + prompt: options.prompt, + schema: options.schema, + }); + return { object: result.object, usage: result.usage }; + } catch (error) { + const salvaged = salvage(error, options.schema); + if (salvaged) { + log.log("salvaged schema-valid JSON from a malformed response"); + return salvaged; + } + if (!options.retry) { + throw error; + } + log.log( + `first attempt failed (${error instanceof Error ? error.message : String(error)}); retrying with reinforced JSON instruction`, + ); + try { + const system = [ + options.system ?? "", + "Return ONLY a single valid JSON value that matches the requested schema. No prose, no markdown fences, no explanations.", + ].join("\n\n").trim(); + const result = await generateObject({ + model: options.model, + system, + prompt: options.prompt, + schema: options.schema, + }); + return { object: result.object, usage: result.usage }; + } catch (retryError) { + const retrySalvaged = salvage(retryError, options.schema); + if (retrySalvaged) { + log.log("salvaged schema-valid JSON from the retry response"); + return retrySalvaged; + } + throw error; + } + } +} + +function salvage( + error: unknown, + schema: z.ZodType, +): GenerateObjectSafeResult | null { + if (!NoObjectGeneratedError.isInstance(error) || typeof error.text !== "string") { + return null; + } + const candidate = extractJson(error.text); + if (candidate === NO_JSON) { + return null; + } + const parsed = schema.safeParse(candidate); + if (!parsed.success) { + return null; + } + return { object: parsed.data, usage: error.usage }; +} + +// Pull a JSON value out of chatty model output: drop blocks, prefer +// fenced content, then fall back to the widest parseable {...}/[...] span. +function extractJson(raw: string): unknown { + let text = raw.replace(/[\s\S]*?<\/think>/gi, "").trim(); + const fence = text.match(/```(?:json)?\s*([\s\S]*?)```/i); + if (fence) { + text = fence[1].trim(); + } + try { + return JSON.parse(text); + } catch { + // fall through to span scan + } + const starts = [text.indexOf("{"), text.indexOf("[")].filter((i) => i >= 0); + if (starts.length === 0) { + return NO_JSON; + } + const start = Math.min(...starts); + for (let end = text.length; end > start; end--) { + const tail = text[end - 1]; + if (tail !== "}" && tail !== "]") { + continue; + } + try { + return JSON.parse(text.slice(start, end)); + } catch { + // keep shrinking + } + } + return NO_JSON; +} diff --git a/apps/x/packages/core/src/security/auto-permission-classifier.ts b/apps/x/packages/core/src/security/auto-permission-classifier.ts index 352512be..fdd6df14 100644 --- a/apps/x/packages/core/src/security/auto-permission-classifier.ts +++ b/apps/x/packages/core/src/security/auto-permission-classifier.ts @@ -1,11 +1,12 @@ -import { generateObject, type ModelMessage } from "ai"; +import type { ModelMessage } from "ai"; import z from "zod"; import { ToolPermissionMetadata } from "@x/shared/dist/runs.js"; import { ToolCallPart } from "@x/shared/dist/message.js"; import { captureLlmUsage } from "../analytics/usage.js"; import { withUseCase, type UseCase } from "../analytics/use_case.js"; import { getAutoPermissionDecisionModel, getDefaultModelAndProvider, resolveProviderConfig } from "../models/defaults.js"; -import { createProvider } from "../models/models.js"; +import { createLanguageModel } from "../models/models.js"; +import { generateObjectSafe } from "../models/structured.js"; const DecisionSchema = z.object({ decisions: z.array(z.object({ @@ -83,7 +84,7 @@ export async function classifyToolPermissions(input: { const modelId = await getAutoPermissionDecisionModel(); const { provider: providerName } = await getDefaultModelAndProvider(); const providerConfig = await resolveProviderConfig(providerName); - const model = createProvider(providerConfig).languageModel(modelId); + const model = createLanguageModel(providerConfig, modelId, { priority: "classifier" }); const result = await withUseCase( { @@ -91,11 +92,12 @@ export async function classifyToolPermissions(input: { subUseCase: "auto_permission_classifier", ...(input.agentName ? { agentName: input.agentName } : {}), }, - () => generateObject({ + () => generateObjectSafe({ model, system: SYSTEM_PROMPT, prompt: buildPrompt(input), schema: DecisionSchema, + retry: true, }), ); diff --git a/apps/x/packages/core/src/turns/bridges/real-model-registry.ts b/apps/x/packages/core/src/turns/bridges/real-model-registry.ts index f79ef32d..dbbda950 100644 --- a/apps/x/packages/core/src/turns/bridges/real-model-registry.ts +++ b/apps/x/packages/core/src/turns/bridges/real-model-registry.ts @@ -14,6 +14,11 @@ import type { JsonValue, ModelDescriptor, TurnUsage } from "@x/shared/dist/turns import { convertFromMessages } from "../../agents/runtime.js"; import { resolveProviderConfig } from "../../models/defaults.js"; import { createProvider } from "../../models/models.js"; +import { + applyLocalModelSettings, + isLocalProvider, + localLlmScheduler, +} from "../../models/local.js"; import type { IModelRegistry, LlmStreamEvent, @@ -29,6 +34,10 @@ export type StreamTextInvoker = (options: { messages: ModelMessage[]; tools: ToolSet; abortSignal: AbortSignal; + temperature?: number; + topP?: number; + maxOutputTokens?: number; + providerOptions?: Record>; }) => { fullStream: AsyncIterable }; const defaultInvoker: StreamTextInvoker = (options) => @@ -61,7 +70,15 @@ export class RealModelRegistry implements IModelRegistry { ): Promise { const providerConfig = await this.resolveProvider(descriptor.provider); const provider = this.createProviderImpl(providerConfig); - const model = provider.languageModel(descriptor.model); + // Local settings (Ollama context window) are applied here, but + // scheduling happens per-step in run() where the turn's priority is + // known — so priority stays null. + const model = applyLocalModelSettings( + provider.languageModel(descriptor.model), + providerConfig, + null, + ); + const local = isLocalProvider(providerConfig); return { descriptor, // The structural -> wire conversion the app uses today: weaves @@ -70,13 +87,14 @@ export class RealModelRegistry implements IModelRegistry { // per-message, so composed requests are byte-stable. encodeMessages: (messages) => convertFromMessages(messages) as unknown as JsonValue[], - stream: (request) => this.run(model, request), + stream: (request) => this.run(model, request, local), }; } private async *run( model: LanguageModel, request: ModelStreamRequest, + local: boolean, ): AsyncGenerator { const tools: ToolSet = {}; for (const descriptor of request.tools) { @@ -93,13 +111,26 @@ export class RealModelRegistry implements IModelRegistry { }); } - const result = this.invoke({ - model, - system: request.systemPrompt, - messages: request.messages as ModelMessage[], - tools, - abortSignal: request.signal, - }); + // Persisted per-call parameters (turn-runtime-design.md §8.3): only + // the whitelisted generation knobs are forwarded to the provider. + const params = request.parameters ?? {}; + const generationParams = { + ...(typeof params.temperature === "number" ? { temperature: params.temperature } : {}), + ...(typeof params.topP === "number" ? { topP: params.topP } : {}), + ...(typeof params.maxOutputTokens === "number" ? { maxOutputTokens: params.maxOutputTokens } : {}), + ...(params.providerOptions && typeof params.providerOptions === "object" && !Array.isArray(params.providerOptions) + ? { providerOptions: params.providerOptions as Record> } + : {}), + }; + + // One scheduler slot per model step: local runtimes serve requests + // serially, and headless turns must not starve interactive chat. + const release = local + ? await localLlmScheduler.acquire( + request.priority ?? "interactive", + request.signal, + ) + : null; const parts: Array> = []; let textBuffer = ""; @@ -108,99 +139,112 @@ export class RealModelRegistry implements IModelRegistry { let usage: z.infer = {}; let providerMetadata: JsonValue | undefined; - for await (const raw of result.fullStream) { - request.signal.throwIfAborted(); - const event = raw as { - type: string; - text?: string; - toolCallId?: string; - toolName?: string; - input?: unknown; - finishReason?: string; - usage?: Record; - providerMetadata?: unknown; - error?: unknown; - }; - switch (event.type) { - case "text-start": - textBuffer = ""; - yield { type: "step_event", event: { type: "text_start" } }; - break; - case "text-delta": { - const delta = event.text ?? ""; - textBuffer += delta; - const last = parts[parts.length - 1]; - if (last?.type === "text") { - last.text += delta; - } else { - parts.push({ type: "text", text: delta }); + try { + const result = this.invoke({ + model, + system: request.systemPrompt, + messages: request.messages as ModelMessage[], + tools, + abortSignal: request.signal, + ...generationParams, + }); + + for await (const raw of result.fullStream) { + request.signal.throwIfAborted(); + const event = raw as { + type: string; + text?: string; + toolCallId?: string; + toolName?: string; + input?: unknown; + finishReason?: string; + usage?: Record; + providerMetadata?: unknown; + error?: unknown; + }; + switch (event.type) { + case "text-start": + textBuffer = ""; + yield { type: "step_event", event: { type: "text_start" } }; + break; + case "text-delta": { + const delta = event.text ?? ""; + textBuffer += delta; + const last = parts[parts.length - 1]; + if (last?.type === "text") { + last.text += delta; + } else { + parts.push({ type: "text", text: delta }); + } + yield { type: "text_delta", delta }; + break; } - yield { type: "text_delta", delta }; - break; - } - case "text-end": - yield { - type: "step_event", - event: { type: "text_end", text: textBuffer }, - }; - break; - case "reasoning-start": - reasoningBuffer = ""; - yield { type: "step_event", event: { type: "reasoning_start" } }; - break; - case "reasoning-delta": { - const delta = event.text ?? ""; - reasoningBuffer += delta; - const last = parts[parts.length - 1]; - if (last?.type === "reasoning") { - last.text += delta; - } else { - parts.push({ type: "reasoning", text: delta }); + case "text-end": + yield { + type: "step_event", + event: { type: "text_end", text: textBuffer }, + }; + break; + case "reasoning-start": + reasoningBuffer = ""; + yield { type: "step_event", event: { type: "reasoning_start" } }; + break; + case "reasoning-delta": { + const delta = event.text ?? ""; + reasoningBuffer += delta; + const last = parts[parts.length - 1]; + if (last?.type === "reasoning") { + last.text += delta; + } else { + parts.push({ type: "reasoning", text: delta }); + } + yield { type: "reasoning_delta", delta }; + break; } - yield { type: "reasoning_delta", delta }; - break; + case "reasoning-end": + yield { + type: "step_event", + event: { type: "reasoning_end", text: reasoningBuffer }, + }; + break; + case "tool-call": { + const toolCall = { + type: "tool-call" as const, + toolCallId: String(event.toolCallId), + toolName: String(event.toolName), + arguments: event.input, + }; + parts.push(toolCall); + yield { type: "step_event", event: { type: "tool_call", toolCall } }; + break; + } + case "finish-step": { + finishReason = event.finishReason ?? "unknown"; + usage = mapUsage(event.usage); + providerMetadata = toJsonValue(event.providerMetadata); + yield { + type: "step_event", + event: { + type: "finish_step", + finishReason, + usage, + ...(providerMetadata === undefined + ? {} + : { providerMetadata }), + }, + }; + break; + } + case "error": + throw event.error instanceof Error + ? event.error + : new Error(formatStreamError(event.error)); + default: + break; } - case "reasoning-end": - yield { - type: "step_event", - event: { type: "reasoning_end", text: reasoningBuffer }, - }; - break; - case "tool-call": { - const toolCall = { - type: "tool-call" as const, - toolCallId: String(event.toolCallId), - toolName: String(event.toolName), - arguments: event.input, - }; - parts.push(toolCall); - yield { type: "step_event", event: { type: "tool_call", toolCall } }; - break; - } - case "finish-step": { - finishReason = event.finishReason ?? "unknown"; - usage = mapUsage(event.usage); - providerMetadata = toJsonValue(event.providerMetadata); - yield { - type: "step_event", - event: { - type: "finish_step", - finishReason, - usage, - ...(providerMetadata === undefined - ? {} - : { providerMetadata }), - }, - }; - break; - } - case "error": - throw event.error instanceof Error - ? event.error - : new Error(formatStreamError(event.error)); - default: - break; } + } finally { + release?.(); } yield { diff --git a/apps/x/packages/core/src/turns/model-registry.ts b/apps/x/packages/core/src/turns/model-registry.ts index d4c83a76..65631d87 100644 --- a/apps/x/packages/core/src/turns/model-registry.ts +++ b/apps/x/packages/core/src/turns/model-registry.ts @@ -34,6 +34,10 @@ export interface ModelStreamRequest { tools: Array>; parameters: Record; signal: AbortSignal; + // Scheduling class for local-model queueing: interactive turns (a human + // is watching) preempt queued headless/background work. Defaults to + // "interactive" so an unset priority never slows a user down. + priority?: "interactive" | "background"; } export interface ResolvedModel { diff --git a/apps/x/packages/core/src/turns/runtime.ts b/apps/x/packages/core/src/turns/runtime.ts index da2362a4..90b72d7b 100644 --- a/apps/x/packages/core/src/turns/runtime.ts +++ b/apps/x/packages/core/src/turns/runtime.ts @@ -951,6 +951,11 @@ class TurnAdvance { tools: composed.tools, parameters: composed.parameters, signal: this.signal, + // Headless turns (no human waiting) yield the local-model + // queue to interactive chat. + priority: this.definition.config.humanAvailable + ? "interactive" + : "background", })) { switch (event.type) { case "text_delta": diff --git a/apps/x/packages/shared/src/ipc.ts b/apps/x/packages/shared/src/ipc.ts index 04651dc8..1d8b7cf3 100644 --- a/apps/x/packages/shared/src/ipc.ts +++ b/apps/x/packages/shared/src/ipc.ts @@ -566,6 +566,13 @@ const ipcSchemas = { res: z.object({ success: z.boolean(), error: z.string().optional(), + // Capability caveats from the local-model probe (tool support, context + // window) — the connection still succeeded. + warnings: z.array(z.string()).optional(), + capabilities: z.object({ + supportsTools: z.boolean().optional(), + maxContextLength: z.number().optional(), + }).optional(), }), }, 'models:listForProvider': { diff --git a/apps/x/packages/shared/src/models.ts b/apps/x/packages/shared/src/models.ts index 4571de2c..d8e48060 100644 --- a/apps/x/packages/shared/src/models.ts +++ b/apps/x/packages/shared/src/models.ts @@ -5,6 +5,10 @@ export const LlmProvider = z.object({ apiKey: z.string().optional(), baseURL: z.string().optional(), headers: z.record(z.string(), z.string()).optional(), + // Context window (in tokens) to request from local runtimes. Ollama defaults + // to a ~4k window that silently truncates Rowboat's prompts; when unset, + // local providers get a larger default (see core/models/local.ts). + contextLength: z.number().int().positive().optional(), }); export const LlmModelConfig = z.object({ @@ -15,6 +19,7 @@ export const LlmModelConfig = z.object({ apiKey: z.string().optional(), baseURL: z.string().optional(), headers: z.record(z.string(), z.string()).optional(), + contextLength: z.number().int().positive().optional(), model: z.string().optional(), models: z.array(z.string()).optional(), knowledgeGraphModel: z.string().optional(), From db7df256ca9d5571910681165a97c4e8ad0807a4 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Sun, 5 Jul 2026 13:48:27 +0530 Subject: [PATCH 2/5] fix ollama model selection issue --- .../onboarding/use-onboarding-state.ts | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/apps/x/apps/renderer/src/components/onboarding/use-onboarding-state.ts b/apps/x/apps/renderer/src/components/onboarding/use-onboarding-state.ts index d548ef32..f5e6f213 100644 --- a/apps/x/apps/renderer/src/components/onboarding/use-onboarding-state.ts +++ b/apps/x/apps/renderer/src/components/onboarding/use-onboarding-state.ts @@ -434,10 +434,29 @@ export function useOnboardingState(open: boolean, onComplete: () => void) { } const catalog: string[] = result.models ?? [] + const typed = activeConfig.model.trim() + // Hosted providers hide the model field (it holds an auto-seeded + // default), so only treat it as user intent where the field is shown — + // mirrors showModelInput in llm-setup-step. + const hostedProviders: LlmProviderFlavor[] = ["openai", "anthropic", "google"] + const modelInputShown = !hostedProviders.includes(llmProvider) + + if (modelInputShown && typed && llmProvider === "ollama" && catalog.length > 0 && !catalog.includes(typed)) { + // Ollama's tag list is authoritative: an unlisted model isn't pulled, + // so saving it would break chat at runtime with no obvious cause. + const error = `Model '${typed}' is not available on this Ollama server. Pull it first (ollama pull ${typed}) or pick one of: ${catalog.slice(0, 5).join(", ")}${catalog.length > 5 ? ", …" : ""}` + setTestState({ status: "error", error }) + toast.error(error) + return false + } + const preferred = preferredDefaults[llmProvider] - const model = - (preferred && catalog.includes(preferred) && preferred) || - catalog[0] || activeConfig.model.trim() || "" + // A model the user explicitly entered always wins — this used to prefer + // catalog[0], which silently replaced the user's Ollama model with + // whatever model the local server happened to list first. + const model = modelInputShown + ? (typed || catalog[0] || "") + : ((preferred && catalog.includes(preferred) && preferred) || catalog[0] || typed || "") // `models` is the user's curated assistant-model list (shown in Settings), // NOT the full provider catalog. Onboarding seeds it with just the selected From 0b55dc3300de43127f85396a57dd653b63c0ee4a Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Sun, 5 Jul 2026 14:50:15 +0530 Subject: [PATCH 3/5] fix ollama model effort level --- apps/x/packages/core/src/models/defaults.ts | 1 + apps/x/packages/core/src/models/local.test.ts | 88 +++++++++++++++++- apps/x/packages/core/src/models/local.ts | 92 +++++++++++++++++++ apps/x/packages/core/src/models/models.ts | 8 ++ apps/x/packages/core/src/models/repo.ts | 5 +- apps/x/packages/shared/src/models.ts | 6 ++ 6 files changed, 197 insertions(+), 3 deletions(-) diff --git a/apps/x/packages/core/src/models/defaults.ts b/apps/x/packages/core/src/models/defaults.ts index 9bd4eaf1..35399396 100644 --- a/apps/x/packages/core/src/models/defaults.ts +++ b/apps/x/packages/core/src/models/defaults.ts @@ -53,6 +53,7 @@ export async function resolveProviderConfig(name: string): Promise void; @@ -28,6 +33,85 @@ describe("isLocalProvider", () => { }); }); +describe("resolveThinkValue", () => { + it("passes effort levels straight through for gpt-oss variants", () => { + expect(resolveThinkValue("gpt-oss:20b", "low", true)).toBe("low"); + expect(resolveThinkValue("gpt-oss:120b", "high", true)).toBe("high"); + // gpt-oss can't disable thinking, so levels apply even if the + // capability probe failed. + expect(resolveThinkValue("gpt-oss:latest", "medium", false)).toBe("medium"); + }); + + it("maps effort to a boolean toggle for other thinking models", () => { + expect(resolveThinkValue("qwen3.5:27b", "low", true)).toBe(false); + expect(resolveThinkValue("qwen3.5:27b", "medium", true)).toBeUndefined(); + expect(resolveThinkValue("deepseek-r1:8b", "high", true)).toBe(true); + }); + + it("strips think for models without the thinking capability", () => { + expect(resolveThinkValue("llama3.2:3b", "low", false)).toBeUndefined(); + expect(resolveThinkValue("llama3.2:3b", "high", false)).toBeUndefined(); + }); +}); + +describe("makeOllamaThinkFetch", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + function stubFetch(capabilities: string[]) { + const calls: Array<{ url: string; body: unknown }> = []; + vi.stubGlobal("fetch", vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + calls.push({ url, body: init?.body ? JSON.parse(init.body as string) : undefined }); + if (url.endsWith("/api/show")) { + return new Response(JSON.stringify({ capabilities }), { status: 200 }); + } + return new Response("{}", { status: 200 }); + })); + return calls; + } + + it("rewrites think on /api/chat for gpt-oss", async () => { + const calls = stubFetch(["completion", "tools", "thinking"]); + const wrapped = makeOllamaThinkFetch("low"); + await wrapped("http://localhost:11434/api/chat", { + method: "POST", + body: JSON.stringify({ model: "gpt-oss:20b", think: false, messages: [] }), + }); + const chat = calls.find((c) => c.url.endsWith("/api/chat")); + expect((chat?.body as { think?: unknown }).think).toBe("low"); + }); + + it("strips think for non-thinking models", async () => { + const calls = stubFetch(["completion", "tools"]); + const wrapped = makeOllamaThinkFetch("high"); + await wrapped("http://localhost:11434/api/chat", { + method: "POST", + body: JSON.stringify({ model: "llama3.2:3b", think: false, messages: [] }), + }); + const chat = calls.find((c) => c.url.endsWith("/api/chat")); + expect(chat?.body as Record).not.toHaveProperty("think"); + }); + + it("probes /api/show once per model and leaves other endpoints untouched", async () => { + const calls = stubFetch(["completion", "thinking"]); + const wrapped = makeOllamaThinkFetch("high"); + const chatBody = JSON.stringify({ model: "qwen3.5:27b", think: false, messages: [] }); + await wrapped("http://localhost:11434/api/chat", { method: "POST", body: chatBody }); + await wrapped("http://localhost:11434/api/chat", { method: "POST", body: chatBody }); + await wrapped("http://localhost:11434/api/tags", { method: "GET" }); + expect(calls.filter((c) => c.url.endsWith("/api/show")).length).toBe(1); + const chats = calls.filter((c) => c.url.endsWith("/api/chat")); + expect(chats).toHaveLength(2); + for (const chat of chats) { + expect((chat.body as { think?: unknown }).think).toBe(true); + } + // Non-chat request passed through with no body rewrite. + expect(calls.some((c) => c.url.endsWith("/api/tags"))).toBe(true); + }); +}); + describe("LocalLlmScheduler", () => { it("serves waiters by priority, FIFO within a priority", async () => { const scheduler = new LocalLlmScheduler(1); diff --git a/apps/x/packages/core/src/models/local.ts b/apps/x/packages/core/src/models/local.ts index 16dc7759..43a574f9 100644 --- a/apps/x/packages/core/src/models/local.ts +++ b/apps/x/packages/core/src/models/local.ts @@ -225,6 +225,98 @@ export function applyLocalModelSettings( }); } +export type ReasoningEffort = "low" | "medium" | "high"; + +// Local models default to snappy: gpt-oss at medium effort spends ~3x the +// tokens of low on the same answer, and the AI SDK Ollama provider can't +// express effort at all (its `think` option is boolean-only and hardcoded to +// false, which thinking models like gpt-oss simply ignore). +export const DEFAULT_OLLAMA_REASONING_EFFORT: ReasoningEffort = "low"; + +/** + * Map a configured effort to the Ollama `think` value for a given model. + * - gpt-oss accepts effort levels directly ("low" | "medium" | "high"). + * - Other thinking models (qwen3, deepseek-r1, …) only toggle: low turns + * thinking off, high forces it on, medium leaves the model default. + * - Models without the thinking capability must not receive `think` at all. + * Returns undefined when `think` should be stripped from the request. + */ +export function resolveThinkValue( + modelName: string, + effort: ReasoningEffort, + supportsThinking: boolean, +): boolean | string | undefined { + if (/gpt-oss/i.test(modelName)) { + return effort; + } + if (!supportsThinking) { + return undefined; + } + switch (effort) { + case "low": + return false; + case "medium": + return undefined; + case "high": + return true; + } +} + +// The ollama-ai-provider-v2 request builder always writes `think: false` +// (its providerOptions schema is boolean-only), so effort can only be set by +// rewriting the wire request. This wraps fetch for createOllama: /api/chat +// bodies get `think` set per resolveThinkValue; every other request passes +// through untouched. Thinking capability is probed once per model via +// /api/show and cached for the process lifetime. +export function makeOllamaThinkFetch( + effort: ReasoningEffort, +): typeof fetch { + const thinkingSupport = new Map>(); + + const supportsThinking = (chatUrl: string, model: string): Promise => { + const showUrl = chatUrl.replace(/\/chat(\?.*)?$/, "/show"); + const key = `${showUrl}|${model}`; + let cached = thinkingSupport.get(key); + if (!cached) { + cached = fetch(showUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model }), + }) + .then(async (res) => { + if (!res.ok) return false; + const data = await res.json() as { capabilities?: string[] }; + return Array.isArray(data.capabilities) && data.capabilities.includes("thinking"); + }) + .catch(() => false); + thinkingSupport.set(key, cached); + } + return cached; + }; + + return async (input, init) => { + try { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + const isChat = /\/api\/chat(\?.*)?$/.test(url); + const body = init?.body; + if (isChat && typeof body === "string") { + const parsed = JSON.parse(body) as Record; + const model = typeof parsed.model === "string" ? parsed.model : ""; + const think = resolveThinkValue(model, effort, await supportsThinking(url, model)); + if (think === undefined) { + delete parsed.think; + } else { + parsed.think = think; + } + return fetch(input, { ...init, body: JSON.stringify(parsed) }); + } + } catch { + // Malformed body or URL — send the original request unchanged. + } + return fetch(input, init); + }; +} + // Hold the slot until the provider stream drains, errors, or is cancelled — // a streaming response occupies the local server for its full duration. function releaseOnSettled(stream: ReadableStream, release: () => void): ReadableStream { diff --git a/apps/x/packages/core/src/models/models.ts b/apps/x/packages/core/src/models/models.ts index b05df695..624d08a9 100644 --- a/apps/x/packages/core/src/models/models.ts +++ b/apps/x/packages/core/src/models/models.ts @@ -15,7 +15,9 @@ import { withUseCase } from "../analytics/use_case.js"; import { applyLocalModelSettings, isLocalProvider, + makeOllamaThinkFetch, DEFAULT_OLLAMA_CONTEXT_LENGTH, + DEFAULT_OLLAMA_REASONING_EFFORT, type LlmPriority, } from "./local.js"; @@ -58,6 +60,12 @@ export function createProvider(config: z.infer): ProviderV2 { return createOllama({ baseURL: ollamaURL, headers, + // Rewrites `think` on the wire: the provider itself can only + // send think:false, which thinking models ignore — leaving + // e.g. gpt-oss at medium effort. See makeOllamaThinkFetch. + fetch: makeOllamaThinkFetch( + config.reasoningEffort ?? DEFAULT_OLLAMA_REASONING_EFFORT, + ), }); } case "openai-compatible": diff --git a/apps/x/packages/core/src/models/repo.ts b/apps/x/packages/core/src/models/repo.ts index b36cb34a..e0c60aed 100644 --- a/apps/x/packages/core/src/models/repo.ts +++ b/apps/x/packages/core/src/models/repo.ts @@ -48,10 +48,13 @@ export class FSModelConfigRepo implements IModelConfigRepo { apiKey: config.provider.apiKey, baseURL: config.provider.baseURL, headers: config.provider.headers, - // Preserve a hand-edited contextLength unless the caller sets one. + // Preserve hand-edited local-model tuning unless the caller sets it. ...(config.provider.contextLength !== undefined ? { contextLength: config.provider.contextLength } : {}), + ...(config.provider.reasoningEffort !== undefined + ? { reasoningEffort: config.provider.reasoningEffort } + : {}), model: config.model, models: config.models, knowledgeGraphModel: config.knowledgeGraphModel, diff --git a/apps/x/packages/shared/src/models.ts b/apps/x/packages/shared/src/models.ts index d8e48060..320492b2 100644 --- a/apps/x/packages/shared/src/models.ts +++ b/apps/x/packages/shared/src/models.ts @@ -9,6 +9,11 @@ export const LlmProvider = z.object({ // to a ~4k window that silently truncates Rowboat's prompts; when unset, // local providers get a larger default (see core/models/local.ts). contextLength: z.number().int().positive().optional(), + // Reasoning effort for local thinking models (Ollama `think` parameter). + // gpt-oss supports the levels directly; other thinking models map + // low → thinking off, high → thinking on. Defaults to "low" for Ollama — + // background agents and chat both want snappy responses on local hardware. + reasoningEffort: z.enum(["low", "medium", "high"]).optional(), }); export const LlmModelConfig = z.object({ @@ -20,6 +25,7 @@ export const LlmModelConfig = z.object({ baseURL: z.string().optional(), headers: z.record(z.string(), z.string()).optional(), contextLength: z.number().int().positive().optional(), + reasoningEffort: z.enum(["low", "medium", "high"]).optional(), model: z.string().optional(), models: z.array(z.string()).optional(), knowledgeGraphModel: z.string().optional(), From c1fd4e6221fd415285bc04b3e9fa8b0a0457e0c2 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Sun, 5 Jul 2026 16:46:32 +0530 Subject: [PATCH 4/5] allow byok even when logged in --- apps/x/apps/main/src/ipc.ts | 5 + .../components/chat-input-with-mentions.tsx | 85 ++--- .../src/components/settings-dialog.tsx | 292 ++++++++++++------ .../src/background-tasks/event-consumer.ts | 4 +- .../core/src/background-tasks/runner.ts | 9 +- .../core/src/knowledge/agent_notes.ts | 2 +- .../core/src/knowledge/build_graph.ts | 4 +- .../core/src/knowledge/classify_thread.ts | 4 +- .../knowledge/email_importance_feedback.ts | 14 +- .../core/src/knowledge/inline_tasks.ts | 4 +- .../core/src/knowledge/label_emails.ts | 2 +- .../src/knowledge/live-note/event-consumer.ts | 5 +- .../core/src/knowledge/live-note/runner.ts | 9 +- .../core/src/knowledge/meeting_prep_brief.ts | 5 +- .../core/src/knowledge/summarize_meeting.ts | 5 +- .../packages/core/src/knowledge/tag_notes.ts | 2 +- apps/x/packages/core/src/models/defaults.ts | 121 +++++--- apps/x/packages/core/src/models/repo.ts | 46 ++- apps/x/packages/core/src/pre_built/runner.ts | 2 +- .../security/auto-permission-classifier.ts | 5 +- apps/x/packages/shared/src/ipc.ts | 18 +- apps/x/packages/shared/src/models.ts | 30 +- 22 files changed, 452 insertions(+), 221 deletions(-) diff --git a/apps/x/apps/main/src/ipc.ts b/apps/x/apps/main/src/ipc.ts index a558f4ae..cb6e98b1 100644 --- a/apps/x/apps/main/src/ipc.ts +++ b/apps/x/apps/main/src/ipc.ts @@ -1080,6 +1080,11 @@ export function setupIpcHandlers() { await repo.setConfig(args); return { success: true }; }, + 'models:updateConfig': async (_event, args) => { + const repo = container.resolve('modelConfigRepo'); + await repo.updateConfig(args); + return { success: true }; + }, 'oauth:connect': async (_event, args) => { const credentials = args.clientId && args.clientSecret ? { clientId: args.clientId.trim(), clientSecret: args.clientSecret.trim() } diff --git a/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx b/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx index 13540e6f..223bfac8 100644 --- a/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx +++ b/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx @@ -400,8 +400,10 @@ function ChatInputInner({ return cleanup }, []) - // Load the list of models the user can choose from. - // Signed-in: gateway model list. Signed-out: providers configured in models.json. + // Load the list of models the user can choose from. Hybrid mode: signed-in + // users get the gateway list AND every BYOK provider configured in + // models.json (selecting a BYOK model routes that message through the + // user's own key / local server). Signed-out users get BYOK only. const loadModelConfig = useCallback(async () => { // Concurrent runs race (mount fires one before the sign-in state resolves, // which fires another) — only the newest run may write state, else a slow @@ -415,44 +417,37 @@ function ChatInputInner({ if (loadModelConfigEpoch.current === epoch) setDefaultModel(null) } try { - if (isRowboatConnected) { + const models: ConfiguredModel[] = [] + const seen = new Set() + const push = (provider: string, model: string) => { + if (!model) return + const key = `${provider}/${model}` + if (seen.has(key)) return + seen.add(key) + models.push({ provider: provider as ProviderName, model }) + } + + // Full catalog per provider (gateway + cloud). Providers with no + // catalog (Ollama, OpenAI-compatible) fall back to the models saved in + // config below. + const catalog: Record = {} + try { const listResult = await window.ipc.invoke('models:list', null) - const rowboatProvider = listResult.providers?.find( - (p: { id: string }) => p.id === 'rowboat' - ) - const models: ConfiguredModel[] = (rowboatProvider?.models || []).map( - (m: { id: string }) => ({ provider: 'rowboat', model: m.id }) - ) - if (loadModelConfigEpoch.current !== epoch) return - setConfiguredModels(models) - } else { + for (const p of listResult.providers || []) { + catalog[p.id] = (p.models || []).map((m: { id: string }) => m.id) + } + } catch { /* offline / no catalog — fall back to saved config below */ } + + if (isRowboatConnected) { + for (const m of catalog['rowboat'] || []) push('rowboat', m) + } + + try { const result = await window.ipc.invoke('workspace:readFile', { path: 'config/models.json' }) const parsed = JSON.parse(result.data) - // Offer every model the configured key supports — the same full catalog - // Settings uses for its dropdowns — so BYOK chat matches the signed-in - // gateway picker. The picker is no longer limited to a hand-curated - // config.models list. Providers with no catalog (Ollama, OpenAI-compatible) - // fall back to the model saved in config. - const catalog: Record = {} - try { - const listResult = await window.ipc.invoke('models:list', null) - for (const p of listResult.providers || []) { - catalog[p.id] = (p.models || []).map((m: { id: string }) => m.id) - } - } catch { /* offline / no catalog — fall back to saved config below */ } - - const models: ConfiguredModel[] = [] - const seen = new Set() - const push = (provider: string, model: string) => { - if (!model) return - const key = `${provider}/${model}` - if (seen.has(key)) return - seen.add(key) - models.push({ provider: provider as ProviderName, model }) - } - - // List the default provider first so its default model leads the picker. + // List the default provider first so its default model leads the + // BYOK section of the picker. const defaultFlavor = typeof parsed?.provider?.flavor === 'string' ? parsed.provider.flavor : '' const flavors = Object.keys(parsed?.providers || {}) .sort((a, b) => (a === defaultFlavor ? -1 : b === defaultFlavor ? 1 : 0)) @@ -474,9 +469,21 @@ function ChatInputInner({ for (const m of saved) push(flavor, m) } } - if (loadModelConfigEpoch.current !== epoch) return - setConfiguredModels(models) - } + + // The user's explicit default selection leads the whole picker. + const sel = parsed?.defaultSelection + if (sel && typeof sel.provider === 'string' && typeof sel.model === 'string') { + const selKey = `${sel.provider}/${sel.model}` + const index = models.findIndex((m) => `${m.provider}/${m.model}` === selKey) + if (index > 0) { + const [entry] = models.splice(index, 1) + models.unshift(entry) + } + } + } catch { /* no BYOK config yet */ } + + if (loadModelConfigEpoch.current !== epoch) return + setConfiguredModels(models) } catch (err) { // No config yet — but surface unexpected failures for diagnosis. console.error('[chat-input] failed to load model list', err) diff --git a/apps/x/apps/renderer/src/components/settings-dialog.tsx b/apps/x/apps/renderer/src/components/settings-dialog.tsx index c358c816..657b66b9 100644 --- a/apps/x/apps/renderer/src/components/settings-dialog.tsx +++ b/apps/x/apps/renderer/src/components/settings-dialog.tsx @@ -346,7 +346,7 @@ type ProviderModelConfig = { autoPermissionDecisionModel: string } -function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { +function ModelSettings({ dialogOpen, rowboatConnected = false }: { dialogOpen: boolean; rowboatConnected?: boolean }) { const [provider, setProvider] = useState("openai") const [defaultProvider, setDefaultProvider] = useState(null) const [providerConfigs, setProviderConfigs] = useState>({ @@ -397,6 +397,8 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { useEffect(() => { if (!dialogOpen) return + const asString = (v: unknown): string => (typeof v === "string" ? v : "") + async function loadCurrentConfig() { try { setConfigLoading(true) @@ -422,10 +424,10 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { apiKey: e.apiKey || "", baseURL: e.baseURL || (defaultBaseURLs[key as LlmProviderFlavor] || ""), models: savedModels, - knowledgeGraphModel: e.knowledgeGraphModel || "", - meetingNotesModel: e.meetingNotesModel || "", - liveNoteAgentModel: e.liveNoteAgentModel || "", - autoPermissionDecisionModel: e.autoPermissionDecisionModel || "", + knowledgeGraphModel: asString(e.knowledgeGraphModel), + meetingNotesModel: asString(e.meetingNotesModel), + liveNoteAgentModel: asString(e.liveNoteAgentModel), + autoPermissionDecisionModel: asString(e.autoPermissionDecisionModel), }; } } @@ -441,10 +443,10 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { apiKey: parsed.provider.apiKey || "", baseURL: parsed.provider.baseURL || (defaultBaseURLs[flavor] || ""), models: activeModels.length > 0 ? activeModels : [""], - knowledgeGraphModel: parsed.knowledgeGraphModel || "", - meetingNotesModel: parsed.meetingNotesModel || "", - liveNoteAgentModel: parsed.liveNoteAgentModel || "", - autoPermissionDecisionModel: parsed.autoPermissionDecisionModel || "", + knowledgeGraphModel: asString(parsed.knowledgeGraphModel), + meetingNotesModel: asString(parsed.meetingNotesModel), + liveNoteAgentModel: asString(parsed.liveNoteAgentModel), + autoPermissionDecisionModel: asString(parsed.autoPermissionDecisionModel), }; } return next; @@ -517,10 +519,12 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { }, model: allModels[0] || "", models: allModels, - knowledgeGraphModel: activeConfig.knowledgeGraphModel.trim() || undefined, - meetingNotesModel: activeConfig.meetingNotesModel.trim() || undefined, - liveNoteAgentModel: activeConfig.liveNoteAgentModel.trim() || undefined, - autoPermissionDecisionModel: activeConfig.autoPermissionDecisionModel.trim() || undefined, + ...(rowboatConnected ? {} : { + knowledgeGraphModel: activeConfig.knowledgeGraphModel.trim() || undefined, + meetingNotesModel: activeConfig.meetingNotesModel.trim() || undefined, + liveNoteAgentModel: activeConfig.liveNoteAgentModel.trim() || undefined, + autoPermissionDecisionModel: activeConfig.autoPermissionDecisionModel.trim() || undefined, + }), } const result = await window.ipc.invoke("models:test", providerConfig) if (result.success) { @@ -547,7 +551,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { setTestState({ status: "error", error: "Connection test failed" }) toast.error("Connection test failed") } - }, [canTest, provider, activeConfig]) + }, [canTest, provider, activeConfig, rowboatConnected]) const handleSetDefault = useCallback(async (prov: LlmProviderFlavor) => { const config = providerConfigs[prov] @@ -562,10 +566,12 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { }, model: allModels[0], models: allModels, - knowledgeGraphModel: config.knowledgeGraphModel.trim() || undefined, - meetingNotesModel: config.meetingNotesModel.trim() || undefined, - liveNoteAgentModel: config.liveNoteAgentModel.trim() || undefined, - autoPermissionDecisionModel: config.autoPermissionDecisionModel.trim() || undefined, + ...(rowboatConnected ? {} : { + knowledgeGraphModel: config.knowledgeGraphModel.trim() || undefined, + meetingNotesModel: config.meetingNotesModel.trim() || undefined, + liveNoteAgentModel: config.liveNoteAgentModel.trim() || undefined, + autoPermissionDecisionModel: config.autoPermissionDecisionModel.trim() || undefined, + }), }) setDefaultProvider(prov) window.dispatchEvent(new Event('models-config-changed')) @@ -573,7 +579,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { } catch { toast.error("Failed to set default provider") } - }, [providerConfigs]) + }, [providerConfigs, rowboatConnected]) const handleDeleteProvider = useCallback(async (prov: LlmProviderFlavor) => { try { @@ -594,10 +600,12 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { } parsed.model = defModels[0] || "" parsed.models = defModels - parsed.knowledgeGraphModel = defConfig.knowledgeGraphModel.trim() || undefined - parsed.meetingNotesModel = defConfig.meetingNotesModel.trim() || undefined - parsed.liveNoteAgentModel = defConfig.liveNoteAgentModel.trim() || undefined - parsed.autoPermissionDecisionModel = defConfig.autoPermissionDecisionModel.trim() || undefined + if (!rowboatConnected) { + parsed.knowledgeGraphModel = defConfig.knowledgeGraphModel.trim() || undefined + parsed.meetingNotesModel = defConfig.meetingNotesModel.trim() || undefined + parsed.liveNoteAgentModel = defConfig.liveNoteAgentModel.trim() || undefined + parsed.autoPermissionDecisionModel = defConfig.autoPermissionDecisionModel.trim() || undefined + } } await window.ipc.invoke("workspace:writeFile", { path: "config/models.json", @@ -613,7 +621,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { } catch { toast.error("Failed to remove provider") } - }, [defaultProvider, providerConfigs]) + }, [defaultProvider, providerConfigs, rowboatConnected]) const renderProviderCard = (p: { id: LlmProviderFlavor; name: string; description: string }) => { const isDefault = defaultProvider === p.id @@ -635,7 +643,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { >
{p.name} - {isDefault && ( + {isDefault && !rowboatConnected && ( Default @@ -644,16 +652,18 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
{p.description}
{!isDefault && hasModel && isSelected && (
- { - e.stopPropagation() - handleSetDefault(p.id) - }} - className="inline-flex text-[11px] text-muted-foreground hover:text-primary transition-colors cursor-pointer" - > - Set as default - + {!rowboatConnected && ( + { + e.stopPropagation() + handleSetDefault(p.id) + }} + className="inline-flex text-[11px] text-muted-foreground hover:text-primary transition-colors cursor-pointer" + > + Set as default + + )} { @@ -705,7 +715,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
{/* Assistant models (left column) */}
- Assistant model + {rowboatConnected ? "Model" : "Assistant model"} {modelsLoading ? (
@@ -743,6 +753,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { )}
+ {!rowboatConnected && (<> {/* Knowledge graph model (right column) */}
Knowledge graph model @@ -878,6 +889,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) { )}
+ )}
{/* API Key */} @@ -1262,11 +1274,45 @@ function ToolsLibrarySettings({ dialogOpen, rowboatConnected }: { dialogOpen: bo } // --- Rowboat Model Settings (when signed in via Rowboat) --- +// +// Hybrid mode: every dropdown lists the gateway catalog PLUS any models from +// BYOK providers configured below. Values are provider-qualified +// ("provider::model") and saved via models:updateConfig as {provider, model} +// refs, so a signed-in user can e.g. keep the gateway assistant while +// running background agents on a local Ollama model. + +interface HybridModelOption { + provider: string + model: string + label: string +} + +const providerDisplayNames: Record = { + openai: 'OpenAI', + anthropic: 'Anthropic', + google: 'Gemini', + ollama: 'Ollama', + openrouter: 'OpenRouter', + aigateway: 'AI Gateway', + 'openai-compatible': 'OpenAI-Compatible', + rowboat: 'Rowboat', +} + +const HYBRID_SEP = "::" +const hybridKey = (provider: string, model: string) => `${provider}${HYBRID_SEP}${model}` + +function parseHybridKey(key: string): { provider: string; model: string } | null { + const index = key.indexOf(HYBRID_SEP) + if (index <= 0) return null + return { provider: key.slice(0, index), model: key.slice(index + HYBRID_SEP.length) } +} function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) { - const [gatewayModels, setGatewayModels] = useState([]) - const [selectedModel, setSelectedModel] = useState("") - const [selectedKgModel, setSelectedKgModel] = useState("") + const [options, setOptions] = useState([]) + const [selectedDefault, setSelectedDefault] = useState("") + const [selectedKg, setSelectedKg] = useState("") + const [selectedLiveNote, setSelectedLiveNote] = useState("") + const [selectedAutoPermission, setSelectedAutoPermission] = useState("") const [loading, setLoading] = useState(true) const [saving, setSaving] = useState(false) @@ -1276,22 +1322,63 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) { async function load() { setLoading(true) try { - // Fetch gateway models - const listResult = await window.ipc.invoke("models:list", null) - const rowboatProvider = listResult.providers?.find((p: { id: string }) => p.id === "rowboat") - const models = rowboatProvider?.models || [] - setGatewayModels(models) + const collected: HybridModelOption[] = [] + const seen = new Set() + const push = (provider: string, model: string, label?: string) => { + if (!model) return + const key = hybridKey(provider, model) + if (seen.has(key)) return + seen.add(key) + collected.push({ provider, model, label: label || model }) + } - // Read current selection from config + const catalog: Record = {} + try { + const listResult = await window.ipc.invoke("models:list", null) + for (const p of listResult.providers || []) { + catalog[p.id] = p.models || [] + } + } catch { /* offline — BYOK entries below still load */ } + for (const m of catalog["rowboat"] || []) push("rowboat", m.id, m.name || m.id) + + let parsed: Record = {} try { const configResult = await window.ipc.invoke("workspace:readFile", { path: "config/models.json" }) - const parsed = JSON.parse(configResult.data) - if (parsed?.model) setSelectedModel(parsed.model) - if (parsed?.knowledgeGraphModel) setSelectedKgModel(parsed.knowledgeGraphModel) - } catch { - // No config yet — pick first model as default - if (models.length > 0) setSelectedModel(models[0].id) + parsed = JSON.parse(configResult.data) + } catch { /* no BYOK config yet */ } + + const providersMap = (parsed.providers ?? {}) as Record> + for (const [flavor, entry] of Object.entries(providersMap)) { + const hasKey = typeof entry.apiKey === "string" && (entry.apiKey as string).trim().length > 0 + const hasBaseURL = typeof entry.baseURL === "string" && (entry.baseURL as string).trim().length > 0 + if (!hasKey && !hasBaseURL) continue + push(flavor, typeof entry.model === "string" ? entry.model : "") + const catalogModels = catalog[flavor] || [] + if (catalogModels.length > 0) { + for (const m of catalogModels) push(flavor, m.id, m.name || m.id) + } else { + for (const m of Array.isArray(entry.models) ? entry.models as string[] : []) push(flavor, m) + } } + setOptions(collected) + + // Current selections. Legacy string overrides pair with the BYOK + // top-level flavor (mirrors core/models/defaults.ts). + const legacyFlavor = (parsed.provider as Record | undefined)?.flavor + const toKey = (value: unknown): string => { + if (!value) return "" + if (typeof value === "string") { + return typeof legacyFlavor === "string" ? hybridKey(legacyFlavor, value) : "" + } + const ref = value as { provider?: unknown; model?: unknown } + return typeof ref.provider === "string" && typeof ref.model === "string" + ? hybridKey(ref.provider, ref.model) + : "" + } + setSelectedDefault(toKey(parsed.defaultSelection)) + setSelectedKg(toKey(parsed.knowledgeGraphModel)) + setSelectedLiveNote(toKey(parsed.liveNoteAgentModel)) + setSelectedAutoPermission(toKey(parsed.autoPermissionDecisionModel)) } catch { toast.error("Failed to load models") } finally { @@ -1303,13 +1390,14 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) { }, [dialogOpen]) const handleSave = useCallback(async () => { - if (!selectedModel) return setSaving(true) try { - await window.ipc.invoke("models:saveConfig", { - provider: { flavor: "openrouter" as const }, - model: selectedModel, - knowledgeGraphModel: selectedKgModel || undefined, + const toRef = (key: string) => (key ? parseHybridKey(key) : null) + await window.ipc.invoke("models:updateConfig", { + defaultSelection: toRef(selectedDefault), + knowledgeGraphModel: toRef(selectedKg), + liveNoteAgentModel: toRef(selectedLiveNote), + autoPermissionDecisionModel: toRef(selectedAutoPermission), }) window.dispatchEvent(new Event("models-config-changed")) toast.success("Model configuration saved") @@ -1318,7 +1406,37 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) { } finally { setSaving(false) } - }, [selectedModel, selectedKgModel]) + }, [selectedDefault, selectedKg, selectedLiveNote, selectedAutoPermission]) + + const renderSelect = ( + label: string, + value: string, + onChange: (v: string) => void, + defaultLabel: string, + ) => ( +
+ + +
+ ) if (loading) { return ( @@ -1331,46 +1449,16 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) { return (

- Select the models Rowboat uses. These are provided through your Rowboat account. + Select the models Rowboat uses. Rowboat models are provided through your account; models from your own providers route through your keys or local runtimes.

- {/* Assistant model */} -
- - -
- - {/* Knowledge graph model */} -
- - -
+ {renderSelect("Assistant model", selectedDefault, setSelectedDefault, "Rowboat default")} + {renderSelect("Knowledge graph model", selectedKg, setSelectedKg, "Rowboat default")} + {renderSelect("Background agents model", selectedLiveNote, setSelectedLiveNote, "Rowboat default")} + {renderSelect("Permission checks model", selectedAutoPermission, setSelectedAutoPermission, "Rowboat default")} {/* Save */} -
)} + {/* Defer background tasks while chatting */} +
+
+
Defer background tasks while chatting
+
+ Background agents (knowledge sync, live notes, tasks) wait until your chat finishes. Recommended for local models. +
+
+ +
+ {/* Test & Save button */}