diff --git a/apps/x/packages/core/src/models/defaults.ts b/apps/x/packages/core/src/models/defaults.ts index 9bd4eaf1..35399396 100644 --- a/apps/x/packages/core/src/models/defaults.ts +++ b/apps/x/packages/core/src/models/defaults.ts @@ -53,6 +53,7 @@ export async function resolveProviderConfig(name: string): Promise void; @@ -28,6 +33,85 @@ describe("isLocalProvider", () => { }); }); +describe("resolveThinkValue", () => { + it("passes effort levels straight through for gpt-oss variants", () => { + expect(resolveThinkValue("gpt-oss:20b", "low", true)).toBe("low"); + expect(resolveThinkValue("gpt-oss:120b", "high", true)).toBe("high"); + // gpt-oss can't disable thinking, so levels apply even if the + // capability probe failed. + expect(resolveThinkValue("gpt-oss:latest", "medium", false)).toBe("medium"); + }); + + it("maps effort to a boolean toggle for other thinking models", () => { + expect(resolveThinkValue("qwen3.5:27b", "low", true)).toBe(false); + expect(resolveThinkValue("qwen3.5:27b", "medium", true)).toBeUndefined(); + expect(resolveThinkValue("deepseek-r1:8b", "high", true)).toBe(true); + }); + + it("strips think for models without the thinking capability", () => { + expect(resolveThinkValue("llama3.2:3b", "low", false)).toBeUndefined(); + expect(resolveThinkValue("llama3.2:3b", "high", false)).toBeUndefined(); + }); +}); + +describe("makeOllamaThinkFetch", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + function stubFetch(capabilities: string[]) { + const calls: Array<{ url: string; body: unknown }> = []; + vi.stubGlobal("fetch", vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + calls.push({ url, body: init?.body ? JSON.parse(init.body as string) : undefined }); + if (url.endsWith("/api/show")) { + return new Response(JSON.stringify({ capabilities }), { status: 200 }); + } + return new Response("{}", { status: 200 }); + })); + return calls; + } + + it("rewrites think on /api/chat for gpt-oss", async () => { + const calls = stubFetch(["completion", "tools", "thinking"]); + const wrapped = makeOllamaThinkFetch("low"); + await wrapped("http://localhost:11434/api/chat", { + method: "POST", + body: JSON.stringify({ model: "gpt-oss:20b", think: false, messages: [] }), + }); + const chat = calls.find((c) => c.url.endsWith("/api/chat")); + expect((chat?.body as { think?: unknown }).think).toBe("low"); + }); + + it("strips think for non-thinking models", async () => { + const calls = stubFetch(["completion", "tools"]); + const wrapped = makeOllamaThinkFetch("high"); + await wrapped("http://localhost:11434/api/chat", { + method: "POST", + body: JSON.stringify({ model: "llama3.2:3b", think: false, messages: [] }), + }); + const chat = calls.find((c) => c.url.endsWith("/api/chat")); + expect(chat?.body as Record).not.toHaveProperty("think"); + }); + + it("probes /api/show once per model and leaves other endpoints untouched", async () => { + const calls = stubFetch(["completion", "thinking"]); + const wrapped = makeOllamaThinkFetch("high"); + const chatBody = JSON.stringify({ model: "qwen3.5:27b", think: false, messages: [] }); + await wrapped("http://localhost:11434/api/chat", { method: "POST", body: chatBody }); + await wrapped("http://localhost:11434/api/chat", { method: "POST", body: chatBody }); + await wrapped("http://localhost:11434/api/tags", { method: "GET" }); + expect(calls.filter((c) => c.url.endsWith("/api/show")).length).toBe(1); + const chats = calls.filter((c) => c.url.endsWith("/api/chat")); + expect(chats).toHaveLength(2); + for (const chat of chats) { + expect((chat.body as { think?: unknown }).think).toBe(true); + } + // Non-chat request passed through with no body rewrite. + expect(calls.some((c) => c.url.endsWith("/api/tags"))).toBe(true); + }); +}); + describe("LocalLlmScheduler", () => { it("serves waiters by priority, FIFO within a priority", async () => { const scheduler = new LocalLlmScheduler(1); diff --git a/apps/x/packages/core/src/models/local.ts b/apps/x/packages/core/src/models/local.ts index 16dc7759..43a574f9 100644 --- a/apps/x/packages/core/src/models/local.ts +++ b/apps/x/packages/core/src/models/local.ts @@ -225,6 +225,98 @@ export function applyLocalModelSettings( }); } +export type ReasoningEffort = "low" | "medium" | "high"; + +// Local models default to snappy: gpt-oss at medium effort spends ~3x the +// tokens of low on the same answer, and the AI SDK Ollama provider can't +// express effort at all (its `think` option is boolean-only and hardcoded to +// false, which thinking models like gpt-oss simply ignore). +export const DEFAULT_OLLAMA_REASONING_EFFORT: ReasoningEffort = "low"; + +/** + * Map a configured effort to the Ollama `think` value for a given model. + * - gpt-oss accepts effort levels directly ("low" | "medium" | "high"). + * - Other thinking models (qwen3, deepseek-r1, …) only toggle: low turns + * thinking off, high forces it on, medium leaves the model default. + * - Models without the thinking capability must not receive `think` at all. + * Returns undefined when `think` should be stripped from the request. + */ +export function resolveThinkValue( + modelName: string, + effort: ReasoningEffort, + supportsThinking: boolean, +): boolean | string | undefined { + if (/gpt-oss/i.test(modelName)) { + return effort; + } + if (!supportsThinking) { + return undefined; + } + switch (effort) { + case "low": + return false; + case "medium": + return undefined; + case "high": + return true; + } +} + +// The ollama-ai-provider-v2 request builder always writes `think: false` +// (its providerOptions schema is boolean-only), so effort can only be set by +// rewriting the wire request. This wraps fetch for createOllama: /api/chat +// bodies get `think` set per resolveThinkValue; every other request passes +// through untouched. Thinking capability is probed once per model via +// /api/show and cached for the process lifetime. +export function makeOllamaThinkFetch( + effort: ReasoningEffort, +): typeof fetch { + const thinkingSupport = new Map>(); + + const supportsThinking = (chatUrl: string, model: string): Promise => { + const showUrl = chatUrl.replace(/\/chat(\?.*)?$/, "/show"); + const key = `${showUrl}|${model}`; + let cached = thinkingSupport.get(key); + if (!cached) { + cached = fetch(showUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model }), + }) + .then(async (res) => { + if (!res.ok) return false; + const data = await res.json() as { capabilities?: string[] }; + return Array.isArray(data.capabilities) && data.capabilities.includes("thinking"); + }) + .catch(() => false); + thinkingSupport.set(key, cached); + } + return cached; + }; + + return async (input, init) => { + try { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + const isChat = /\/api\/chat(\?.*)?$/.test(url); + const body = init?.body; + if (isChat && typeof body === "string") { + const parsed = JSON.parse(body) as Record; + const model = typeof parsed.model === "string" ? parsed.model : ""; + const think = resolveThinkValue(model, effort, await supportsThinking(url, model)); + if (think === undefined) { + delete parsed.think; + } else { + parsed.think = think; + } + return fetch(input, { ...init, body: JSON.stringify(parsed) }); + } + } catch { + // Malformed body or URL — send the original request unchanged. + } + return fetch(input, init); + }; +} + // Hold the slot until the provider stream drains, errors, or is cancelled — // a streaming response occupies the local server for its full duration. function releaseOnSettled(stream: ReadableStream, release: () => void): ReadableStream { diff --git a/apps/x/packages/core/src/models/models.ts b/apps/x/packages/core/src/models/models.ts index b05df695..624d08a9 100644 --- a/apps/x/packages/core/src/models/models.ts +++ b/apps/x/packages/core/src/models/models.ts @@ -15,7 +15,9 @@ import { withUseCase } from "../analytics/use_case.js"; import { applyLocalModelSettings, isLocalProvider, + makeOllamaThinkFetch, DEFAULT_OLLAMA_CONTEXT_LENGTH, + DEFAULT_OLLAMA_REASONING_EFFORT, type LlmPriority, } from "./local.js"; @@ -58,6 +60,12 @@ export function createProvider(config: z.infer): ProviderV2 { return createOllama({ baseURL: ollamaURL, headers, + // Rewrites `think` on the wire: the provider itself can only + // send think:false, which thinking models ignore — leaving + // e.g. gpt-oss at medium effort. See makeOllamaThinkFetch. + fetch: makeOllamaThinkFetch( + config.reasoningEffort ?? DEFAULT_OLLAMA_REASONING_EFFORT, + ), }); } case "openai-compatible": diff --git a/apps/x/packages/core/src/models/repo.ts b/apps/x/packages/core/src/models/repo.ts index b36cb34a..e0c60aed 100644 --- a/apps/x/packages/core/src/models/repo.ts +++ b/apps/x/packages/core/src/models/repo.ts @@ -48,10 +48,13 @@ export class FSModelConfigRepo implements IModelConfigRepo { apiKey: config.provider.apiKey, baseURL: config.provider.baseURL, headers: config.provider.headers, - // Preserve a hand-edited contextLength unless the caller sets one. + // Preserve hand-edited local-model tuning unless the caller sets it. ...(config.provider.contextLength !== undefined ? { contextLength: config.provider.contextLength } : {}), + ...(config.provider.reasoningEffort !== undefined + ? { reasoningEffort: config.provider.reasoningEffort } + : {}), model: config.model, models: config.models, knowledgeGraphModel: config.knowledgeGraphModel, diff --git a/apps/x/packages/shared/src/models.ts b/apps/x/packages/shared/src/models.ts index d8e48060..320492b2 100644 --- a/apps/x/packages/shared/src/models.ts +++ b/apps/x/packages/shared/src/models.ts @@ -9,6 +9,11 @@ export const LlmProvider = z.object({ // to a ~4k window that silently truncates Rowboat's prompts; when unset, // local providers get a larger default (see core/models/local.ts). contextLength: z.number().int().positive().optional(), + // Reasoning effort for local thinking models (Ollama `think` parameter). + // gpt-oss supports the levels directly; other thinking models map + // low → thinking off, high → thinking on. Defaults to "low" for Ollama — + // background agents and chat both want snappy responses on local hardware. + reasoningEffort: z.enum(["low", "medium", "high"]).optional(), }); export const LlmModelConfig = z.object({ @@ -20,6 +25,7 @@ export const LlmModelConfig = z.object({ baseURL: z.string().optional(), headers: z.record(z.string(), z.string()).optional(), contextLength: z.number().int().positive().optional(), + reasoningEffort: z.enum(["low", "medium", "high"]).optional(), model: z.string().optional(), models: z.array(z.string()).optional(), knowledgeGraphModel: z.string().optional(),