diff --git a/apps/x/ANALYTICS.md b/apps/x/ANALYTICS.md index cf3a0790..f016c066 100644 --- a/apps/x/ANALYTICS.md +++ b/apps/x/ANALYTICS.md @@ -89,7 +89,7 @@ Emit points: `apps/main/src/oauth-handler.ts:369` and `apps/renderer/src/hooks/u Privacy rules (enforced in `packages/core/src/analytics/model-providers.ts`): only provider **flavors** are captured — never instance ids (future-proofing for user-named instances), never `apiKey`/`headers`, and never `baseURL` (local endpoints can carry internal hostnames). Model ids are allowed. - `llm_provider_connected` / `llm_provider_disconnected` — `{ flavor }` — one event family across every surface. BYOK fires from `FSModelConfigRepo.setProvider` (new entries only — key rotation is not a connect) / `removeProvider`; `rowboat` from sign-in/out (`apps/main/src/oauth-handler.ts`); `codex` from ChatGPT sign-in/out (`apps/main/src/ipc.ts`). -- `llm_initial_model_selected` — `{ flavor, model, recommended, source: 'connect' | 'onboarding' | 'sign_in' }` — a connect seeded the assistant model (only when none was configured). `recommended: false` = first-listed fallback; the hit rate measures backend recommendation quality. Emit points: `apps/renderer/src/components/settings/providers-section.tsx` (connect/onboarding) and `packages/core/src/models/rowboat-selection.ts` (sign-in). +- `llm_initial_model_selected` — `{ flavor, model, recommended, task_overrides_seeded, source: 'connect' | 'onboarding' | 'sign_in' }` — a connect seeded the assistant model (only when none was configured). `recommended: false` = first-listed fallback; the hit rate measures backend recommendation quality. `task_overrides_seeded` counts the per-task recommendations written alongside (the server-controlled lite-tier task models — 0 when the provider has none). Emit points: `apps/renderer/src/components/settings/providers-section.tsx` (connect/onboarding) and `packages/core/src/models/rowboat-selection.ts` / `chatgpt-selection.ts` (sign-in). - `models_config_migrated` — `{ had_assistant, materialized_overrides, provider_count }` — one-shot per install at the models.json v1 → v2 boot migration (`FSModelConfigRepo.ensureConfig`); rollout health for the schema change. ### Other events (pre-existing, not added by the LLM-usage work) diff --git a/apps/x/apps/renderer/src/components/settings/providers-section.tsx b/apps/x/apps/renderer/src/components/settings/providers-section.tsx index 0eb15424..b46f140e 100644 --- a/apps/x/apps/renderer/src/components/settings/providers-section.tsx +++ b/apps/x/apps/renderer/src/components/settings/providers-section.tsx @@ -8,6 +8,8 @@ import { Input } from "@/components/ui/input" import { Switch } from "@/components/ui/switch" import { cn } from "@/lib/utils" import { providerDisplayNames, type ModelRef } from "@/components/model-selector" +import { selectInitialModel, selectInitialTaskModels } from "@x/shared/dist/initial-selection.js" +import { normalizeModelRecommendation, type ModelRecommendations } from "@x/shared/dist/rowboat-account.js" import { useModels } from "@/hooks/use-models" import { useRowboatConfig } from "@/hooks/use-rowboat-config" import { useChatGPT } from "@/hooks/useChatGPT" @@ -259,7 +261,7 @@ function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnecte chatgptSignedIn: boolean onChatGPTSignIn: () => Promise | void hadAssistant: boolean - modelRecommendations: Record | undefined + modelRecommendations: ModelRecommendations | undefined analyticsSource: 'connect' | 'onboarding' }) { const [step, setStep] = useState({ kind: "choose" }) @@ -368,11 +370,7 @@ function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnecte const listRes = await window.ipc.invoke("models:listForProvider", { provider: providerEntry }) const list = listRes.success ? listRes.models ?? [] : [] const typed = manualModel.trim() - let model = typed - if (!model) { - const recommended = modelRecommendations?.[flavor] - model = recommended && list.includes(recommended) ? recommended : (list[0] ?? "") - } + const model = typed || (selectInitialModel(flavor, list, modelRecommendations) ?? "") if (!listRes.success && !model) { setStep({ kind: "error", flavor, message: listRes.error || "Could not load the provider's model list." }) return @@ -394,11 +392,18 @@ function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnecte const cfgNow = await window.ipc.invoke("models:getConfig", null).catch(() => null) const hasAssistantNow = cfgNow ? cfgNow.assistantModel !== null : hadAssistant if (!hasAssistantNow) { - await window.ipc.invoke("models:updateConfig", { assistantModel: { provider: flavor, model } }) + // Task recommendations ride along the seeding moment as visible + // overrides (validated against the live list; only differences). + const taskModels = selectInitialTaskModels(flavor, flavor, list, modelRecommendations, model) + await window.ipc.invoke("models:updateConfig", { + assistantModel: { provider: flavor, model }, + ...(Object.keys(taskModels).length > 0 ? { taskModels } : {}), + }) analytics.llmInitialModelSelected({ flavor, model, - recommended: model === modelRecommendations?.[flavor], + recommended: model === normalizeModelRecommendation(modelRecommendations, flavor)?.assistantModel, + taskOverridesSeeded: Object.keys(taskModels).length, source: analyticsSource, }) } diff --git a/apps/x/apps/renderer/src/lib/analytics.ts b/apps/x/apps/renderer/src/lib/analytics.ts index 621c637b..d0c2fa56 100644 --- a/apps/x/apps/renderer/src/lib/analytics.ts +++ b/apps/x/apps/renderer/src/lib/analytics.ts @@ -345,7 +345,9 @@ export function llmInitialModelSelected(props: { flavor: string model: string recommended: boolean + taskOverridesSeeded: number source: 'connect' | 'onboarding' }) { - posthog.capture('llm_initial_model_selected', { ...props }) + const { taskOverridesSeeded, ...rest } = props + posthog.capture('llm_initial_model_selected', { ...rest, task_overrides_seeded: taskOverridesSeeded }) } diff --git a/apps/x/packages/core/src/models/chatgpt-selection.ts b/apps/x/packages/core/src/models/chatgpt-selection.ts index 1f1a57c4..b78788ea 100644 --- a/apps/x/packages/core/src/models/chatgpt-selection.ts +++ b/apps/x/packages/core/src/models/chatgpt-selection.ts @@ -2,7 +2,8 @@ import container from "../di/container.js"; import { IModelConfigRepo } from "./repo.js"; import { listCodexModels } from "./codex.js"; import { getRowboatConfig } from "../config/rowboat.js"; -import { selectInitialModel } from "./initial-selection.js"; +import { selectInitialModel, selectInitialTaskModels } from "./initial-selection.js"; +import { normalizeModelRecommendation } from "@x/shared/dist/rowboat-account.js"; import { capture } from "../analytics/posthog.js"; /** @@ -27,11 +28,18 @@ export async function applyCodexInitialSelection(): Promise { const recommendations = (await getRowboatConfig().catch(() => null))?.modelRecommendations; const model = selectInitialModel("codex", ids, recommendations); if (model) { - await repo.updateConfig({ assistantModel: { provider: "codex", model } }); + // Task recommendations ride along the seeding moment (codex has + // none today; the path is uniform across providers). + const taskModels = selectInitialTaskModels("codex", "codex", ids, recommendations, model); + await repo.updateConfig({ + assistantModel: { provider: "codex", model }, + ...(Object.keys(taskModels).length > 0 ? { taskModels } : {}), + }); capture("llm_initial_model_selected", { flavor: "codex", model, - recommended: model === recommendations?.["codex"], + recommended: model === normalizeModelRecommendation(recommendations, "codex")?.assistantModel, + task_overrides_seeded: Object.keys(taskModels).length, source: "sign_in", }); } diff --git a/apps/x/packages/core/src/models/initial-selection.test.ts b/apps/x/packages/core/src/models/initial-selection.test.ts index 44ca5fc0..26ebad1c 100644 --- a/apps/x/packages/core/src/models/initial-selection.test.ts +++ b/apps/x/packages/core/src/models/initial-selection.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { selectInitialModel } from './initial-selection.js'; +import { selectInitialModel, selectInitialTaskModels } from './initial-selection.js'; describe('selectInitialModel', () => { const recommendations = { @@ -29,4 +29,47 @@ describe('selectInitialModel', () => { it('returns null when the provider listed nothing', () => { expect(selectInitialModel('openai', [], recommendations)).toBeNull(); }); + + it('accepts the nested { assistantModel, taskModels } wire shape', () => { + const nested = { rowboat: { assistantModel: 'google/gemini-3.5-flash', taskModels: {} } }; + expect(selectInitialModel('rowboat', ['a', 'google/gemini-3.5-flash'], nested)) + .toBe('google/gemini-3.5-flash'); + }); +}); + +describe('selectInitialTaskModels', () => { + const gatewayList = [ + 'google/gemini-3.5-flash', + 'google/gemini-3.1-flash-lite', + 'google/gemini-3.5-flash-lite', + ]; + const nested = { + rowboat: { + assistantModel: 'google/gemini-3.5-flash', + taskModels: { + knowledgeGraph: 'google/gemini-3.1-flash-lite', + chatTitle: 'google/gemini-3.5-flash-lite', + // Equal to the assistant → redundant, inherit produces it. + meetingNotes: 'google/gemini-3.5-flash', + // Not in the provider's list → stale hint, skipped. + liveNoteAgent: 'google/gemini-9-experimental', + // Unknown key → ignored. + somethingNew: 'google/gemini-3.1-flash-lite', + }, + }, + }; + + it('writes overrides only for listed recs that differ from the assistant', () => { + expect(selectInitialTaskModels('rowboat', 'rowboat', gatewayList, nested, 'google/gemini-3.5-flash')) + .toEqual({ + knowledgeGraph: { provider: 'rowboat', model: 'google/gemini-3.1-flash-lite' }, + chatTitle: { provider: 'rowboat', model: 'google/gemini-3.5-flash-lite' }, + }); + }); + + it('returns nothing for legacy flat recommendations or absent maps', () => { + expect(selectInitialTaskModels('rowboat', 'rowboat', gatewayList, { rowboat: 'google/gemini-3.5-flash' }, 'x')) + .toEqual({}); + expect(selectInitialTaskModels('rowboat', 'rowboat', gatewayList, undefined, 'x')).toEqual({}); + }); }); diff --git a/apps/x/packages/core/src/models/initial-selection.ts b/apps/x/packages/core/src/models/initial-selection.ts index de6ad1d0..d894390e 100644 --- a/apps/x/packages/core/src/models/initial-selection.ts +++ b/apps/x/packages/core/src/models/initial-selection.ts @@ -1,34 +1,3 @@ -/** - * Initial model selection for a provider being connected for the first time. - * - * Implements the selection order from the provider/model-selection spec: - * 1. If Rowboat's recommended model for this flavor appears in the - * provider's available list, pick it. - * 2. Otherwise pick the first model the provider returned. - * 3. With no list at all, return null — the caller offers retry or manual - * entry. - * - * This runs ONLY when a provider is first connected and has no saved - * selection. It must never run over an existing choice: after initial setup - * the saved model configuration is the source of truth, and changes to the - * recommendations or to the provider's list order must not silently replace - * what the user picked. - * - * Pure function by design: the caller supplies the provider's available - * models (from the unified catalog / a live probe) and the recommendations - * map (from /v1/config via rowboat:getConfig, keyed by provider FLAVOR in - * each provider's native id format). Recommendations are best-effort — an - * absent map, an unknown flavor, or a recommendation the provider doesn't - * serve all degrade to "first available model". - */ -export function selectInitialModel( - flavor: string, - availableModelIds: string[], - recommendations: Record | undefined, -): string | null { - const recommended = recommendations?.[flavor]; - if (recommended && availableModelIds.includes(recommended)) { - return recommended; - } - return availableModelIds[0] ?? null; -} +// Pure selection logic lives in @x/shared (the renderer's connect flow uses +// the same implementation); re-exported here for core call sites. +export { selectInitialModel, selectInitialTaskModels } from "@x/shared/dist/initial-selection.js"; diff --git a/apps/x/packages/core/src/models/rowboat-selection.ts b/apps/x/packages/core/src/models/rowboat-selection.ts index 7a6bc1dc..23d4aa40 100644 --- a/apps/x/packages/core/src/models/rowboat-selection.ts +++ b/apps/x/packages/core/src/models/rowboat-selection.ts @@ -2,7 +2,8 @@ import container from "../di/container.js"; import { IModelConfigRepo } from "./repo.js"; import { listGatewayModels } from "./gateway.js"; import { getRowboatConfig } from "../config/rowboat.js"; -import { selectInitialModel } from "./initial-selection.js"; +import { selectInitialModel, selectInitialTaskModels } from "./initial-selection.js"; +import { normalizeModelRecommendation } from "@x/shared/dist/rowboat-account.js"; import { capture } from "../analytics/posthog.js"; /** @@ -28,13 +29,22 @@ export async function applyRowboatInitialSelection(): Promise { const recommendations = (await getRowboatConfig().catch(() => null))?.modelRecommendations; const model = selectInitialModel("rowboat", ids, recommendations); if (model) { - await repo.updateConfig({ assistantModel: { provider: "rowboat", model } }); + // Task recommendations ride along the seeding moment: the + // gateway's lite-tier task models become visible overrides so + // always-on background work doesn't run on assistant-class + // models (plan-credit economics). + const taskModels = selectInitialTaskModels("rowboat", "rowboat", ids, recommendations, model); + await repo.updateConfig({ + assistantModel: { provider: "rowboat", model }, + ...(Object.keys(taskModels).length > 0 ? { taskModels } : {}), + }); // Measures recommendation quality: hit = the backend's pick was // in the gateway list; miss = first-listed fallback. capture("llm_initial_model_selected", { flavor: "rowboat", model, - recommended: model === recommendations?.["rowboat"], + recommended: model === normalizeModelRecommendation(recommendations, "rowboat")?.assistantModel, + task_overrides_seeded: Object.keys(taskModels).length, source: "sign_in", }); } diff --git a/apps/x/packages/shared/src/initial-selection.ts b/apps/x/packages/shared/src/initial-selection.ts new file mode 100644 index 00000000..11bb17c3 --- /dev/null +++ b/apps/x/packages/shared/src/initial-selection.ts @@ -0,0 +1,67 @@ +import { z } from "zod"; +import { ModelRef, TaskModels } from "./models.js"; +import { normalizeModelRecommendation, type ModelRecommendations } from "./rowboat-account.js"; + +/** + * Initial model selection for a provider being connected for the first time. + * + * Implements the selection order from the provider/model-selection spec: + * 1. If Rowboat's recommended model for this flavor appears in the + * provider's available list, pick it. + * 2. Otherwise pick the first model the provider returned. + * 3. With no list at all, return null — the caller offers retry or manual + * entry. + * + * Task-model recommendations ride along the same moment: when (and only + * when) a connect seeds the assistant, the provider's per-task + * recommendations become visible taskModels overrides — each validated + * against the live list and skipped when it equals the chosen assistant + * (inheritance already produces it; only differences are written). + * + * This runs ONLY when a provider is first connected and has no saved + * selection. It must never run over an existing choice: after initial setup + * the saved model configuration is the source of truth, and changes to the + * recommendations or to the provider's list order must not silently replace + * what the user picked. + * + * Pure functions by design: callers supply the provider's available models + * (from the unified catalog / a live probe) and the recommendations map + * (from /v1/config via rowboat:getConfig, keyed by provider FLAVOR in each + * provider's native id format). Everything is best-effort — an absent map, + * an unknown flavor, or a recommendation the provider doesn't serve all + * degrade gracefully. + */ + +const TASK_MODEL_KEYS = Object.keys(TaskModels.shape) as Array>; + +export function selectInitialModel( + flavor: string, + availableModelIds: string[], + recommendations: ModelRecommendations | undefined, +): string | null { + const recommended = normalizeModelRecommendation(recommendations, flavor)?.assistantModel; + if (recommended && availableModelIds.includes(recommended)) { + return recommended; + } + return availableModelIds[0] ?? null; +} + +export function selectInitialTaskModels( + providerId: string, + flavor: string, + availableModelIds: string[], + recommendations: ModelRecommendations | undefined, + assistantModel: string, +): Partial, z.infer>> { + const taskRecommendations = normalizeModelRecommendation(recommendations, flavor)?.taskModels; + if (!taskRecommendations) return {}; + const overrides: Partial, z.infer>> = {}; + for (const key of TASK_MODEL_KEYS) { + const model = taskRecommendations[key]; + // Unknown keys are ignored; a rec equal to the assistant is redundant + // (inherit produces it); an unlisted rec is a stale hint. + if (!model || model === assistantModel || !availableModelIds.includes(model)) continue; + overrides[key] = { provider: providerId, model }; + } + return overrides; +} diff --git a/apps/x/packages/shared/src/rowboat-account.ts b/apps/x/packages/shared/src/rowboat-account.ts index 526adf30..9c4d457e 100644 --- a/apps/x/packages/shared/src/rowboat-account.ts +++ b/apps/x/packages/shared/src/rowboat-account.ts @@ -11,13 +11,43 @@ export const RowboatApiConfig = z.object({ // app keeps working against API deployments that predate it — the rewards // UI just stays empty until the backend serves the catalog creditActivations: z.array(CreditActivationCatalogEntrySchema).optional(), - // One recommended model id per provider FLAVOR (e.g. { openai: "gpt-5.4", - // openrouter: "anthropic/claude-opus-4.8" }), in each provider's native id - // format. A hint for the INITIAL selection when a provider is first + // Recommended models per provider FLAVOR, in each provider's native id + // format: one assistantModel (the primary) plus optional per-task + // taskModels overrides mirroring models.json v2 vocabulary (a missing + // task key = inherit the assistant — task recs exist only where the + // intended model differs; for rowboat they reproduce the pre-v2 curated + // lite-tier task models so plan credits aren't burned by background + // services). Hints for the INITIAL selection when a provider is first // connected — never a catalog, and never applied over a saved choice - // (see core/models/initial-selection.ts). Local/custom flavors are - // intentionally absent: the API can't know which models exist in a user's - // environment. Optional so older API deployments and failed fetches never - // break parsing — recommendations are best-effort by design. - modelRecommendations: z.record(z.string(), z.string()).optional(), + // (see shared/initial-selection.ts). The bare-string form is the legacy + // wire shape, accepted so backend deploy order and rollback are + // non-events. Local/custom flavors are intentionally absent: the API + // can't know which models exist in a user's environment. Optional so + // older API deployments and failed fetches never break parsing — + // recommendations are best-effort by design. + modelRecommendations: z.record(z.string(), z.union([ + z.string(), + z.object({ + assistantModel: z.string(), + taskModels: z.record(z.string(), z.string()).optional(), + }), + ])).optional(), }); + +export type ModelRecommendations = NonNullable['modelRecommendations']>; + +export interface NormalizedModelRecommendation { + assistantModel: string; + taskModels: Record; +} + +/** One provider's recommendation in canonical form; null when absent. */ +export function normalizeModelRecommendation( + recommendations: ModelRecommendations | undefined, + flavor: string, +): NormalizedModelRecommendation | null { + const raw = recommendations?.[flavor]; + if (!raw) return null; + if (typeof raw === 'string') return { assistantModel: raw, taskModels: {} }; + return { assistantModel: raw.assistantModel, taskModels: raw.taskModels ?? {} }; +}