feat(x): apply per-task model recommendations at initial selection

The backend's modelRecommendations entries are now { assistantModel,
taskModels? } (rowboatx-backend#18), mirroring models.json v2 vocabulary.
When a provider connect/sign-in seeds the assistant (and ONLY then), its
per-task recommendations become visible taskModels overrides — each
validated against the live list and skipped when equal to the chosen
assistant (only differences are written, same principle as the migration).

This restores pre-v2 economics for signed-in task models without hidden
defaults: fresh Rowboat sign-ins, new devices, and sign-out→sign-in round
trips all land with the lite-tier task models the app used to hardcode —
server-controlled, updateable without a release. Saved choices are still
never touched: no seeding happens when an assistant already exists.

- selection logic moves to @x/shared/initial-selection (core re-exports;
  the renderer connect flow uses the same implementation)
- RowboatApiConfig accepts both the legacy flat shape and the nested one
  (normalizeModelRecommendation), so backend deploy order and rollback
  are non-events
- llm_initial_model_selected gains task_overrides_seeded

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Ramnique Singh 2026-07-24 17:33:59 +05:30
parent d9317f3ca1
commit 72e71e0ab5
9 changed files with 193 additions and 59 deletions

View file

@ -89,7 +89,7 @@ Emit points: `apps/main/src/oauth-handler.ts:369` and `apps/renderer/src/hooks/u
Privacy rules (enforced in `packages/core/src/analytics/model-providers.ts`): only provider **flavors** are captured — never instance ids (future-proofing for user-named instances), never `apiKey`/`headers`, and never `baseURL` (local endpoints can carry internal hostnames). Model ids are allowed.
- `llm_provider_connected` / `llm_provider_disconnected``{ flavor }` — one event family across every surface. BYOK fires from `FSModelConfigRepo.setProvider` (new entries only — key rotation is not a connect) / `removeProvider`; `rowboat` from sign-in/out (`apps/main/src/oauth-handler.ts`); `codex` from ChatGPT sign-in/out (`apps/main/src/ipc.ts`).
- `llm_initial_model_selected``{ flavor, model, recommended, source: 'connect' | 'onboarding' | 'sign_in' }` — a connect seeded the assistant model (only when none was configured). `recommended: false` = first-listed fallback; the hit rate measures backend recommendation quality. Emit points: `apps/renderer/src/components/settings/providers-section.tsx` (connect/onboarding) and `packages/core/src/models/rowboat-selection.ts` (sign-in).
- `llm_initial_model_selected``{ flavor, model, recommended, task_overrides_seeded, source: 'connect' | 'onboarding' | 'sign_in' }` — a connect seeded the assistant model (only when none was configured). `recommended: false` = first-listed fallback; the hit rate measures backend recommendation quality. `task_overrides_seeded` counts the per-task recommendations written alongside (the server-controlled lite-tier task models — 0 when the provider has none). Emit points: `apps/renderer/src/components/settings/providers-section.tsx` (connect/onboarding) and `packages/core/src/models/rowboat-selection.ts` / `chatgpt-selection.ts` (sign-in).
- `models_config_migrated``{ had_assistant, materialized_overrides, provider_count }` — one-shot per install at the models.json v1 → v2 boot migration (`FSModelConfigRepo.ensureConfig`); rollout health for the schema change.
### Other events (pre-existing, not added by the LLM-usage work)

View file

@ -8,6 +8,8 @@ import { Input } from "@/components/ui/input"
import { Switch } from "@/components/ui/switch"
import { cn } from "@/lib/utils"
import { providerDisplayNames, type ModelRef } from "@/components/model-selector"
import { selectInitialModel, selectInitialTaskModels } from "@x/shared/dist/initial-selection.js"
import { normalizeModelRecommendation, type ModelRecommendations } from "@x/shared/dist/rowboat-account.js"
import { useModels } from "@/hooks/use-models"
import { useRowboatConfig } from "@/hooks/use-rowboat-config"
import { useChatGPT } from "@/hooks/useChatGPT"
@ -259,7 +261,7 @@ function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnecte
chatgptSignedIn: boolean
onChatGPTSignIn: () => Promise<unknown> | void
hadAssistant: boolean
modelRecommendations: Record<string, string> | undefined
modelRecommendations: ModelRecommendations | undefined
analyticsSource: 'connect' | 'onboarding'
}) {
const [step, setStep] = useState<AddStep>({ kind: "choose" })
@ -368,11 +370,7 @@ function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnecte
const listRes = await window.ipc.invoke("models:listForProvider", { provider: providerEntry })
const list = listRes.success ? listRes.models ?? [] : []
const typed = manualModel.trim()
let model = typed
if (!model) {
const recommended = modelRecommendations?.[flavor]
model = recommended && list.includes(recommended) ? recommended : (list[0] ?? "")
}
const model = typed || (selectInitialModel(flavor, list, modelRecommendations) ?? "")
if (!listRes.success && !model) {
setStep({ kind: "error", flavor, message: listRes.error || "Could not load the provider's model list." })
return
@ -394,11 +392,18 @@ function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnecte
const cfgNow = await window.ipc.invoke("models:getConfig", null).catch(() => null)
const hasAssistantNow = cfgNow ? cfgNow.assistantModel !== null : hadAssistant
if (!hasAssistantNow) {
await window.ipc.invoke("models:updateConfig", { assistantModel: { provider: flavor, model } })
// Task recommendations ride along the seeding moment as visible
// overrides (validated against the live list; only differences).
const taskModels = selectInitialTaskModels(flavor, flavor, list, modelRecommendations, model)
await window.ipc.invoke("models:updateConfig", {
assistantModel: { provider: flavor, model },
...(Object.keys(taskModels).length > 0 ? { taskModels } : {}),
})
analytics.llmInitialModelSelected({
flavor,
model,
recommended: model === modelRecommendations?.[flavor],
recommended: model === normalizeModelRecommendation(modelRecommendations, flavor)?.assistantModel,
taskOverridesSeeded: Object.keys(taskModels).length,
source: analyticsSource,
})
}

View file

@ -345,7 +345,9 @@ export function llmInitialModelSelected(props: {
flavor: string
model: string
recommended: boolean
taskOverridesSeeded: number
source: 'connect' | 'onboarding'
}) {
posthog.capture('llm_initial_model_selected', { ...props })
const { taskOverridesSeeded, ...rest } = props
posthog.capture('llm_initial_model_selected', { ...rest, task_overrides_seeded: taskOverridesSeeded })
}

View file

@ -2,7 +2,8 @@ import container from "../di/container.js";
import { IModelConfigRepo } from "./repo.js";
import { listCodexModels } from "./codex.js";
import { getRowboatConfig } from "../config/rowboat.js";
import { selectInitialModel } from "./initial-selection.js";
import { selectInitialModel, selectInitialTaskModels } from "./initial-selection.js";
import { normalizeModelRecommendation } from "@x/shared/dist/rowboat-account.js";
import { capture } from "../analytics/posthog.js";
/**
@ -27,11 +28,18 @@ export async function applyCodexInitialSelection(): Promise<void> {
const recommendations = (await getRowboatConfig().catch(() => null))?.modelRecommendations;
const model = selectInitialModel("codex", ids, recommendations);
if (model) {
await repo.updateConfig({ assistantModel: { provider: "codex", model } });
// Task recommendations ride along the seeding moment (codex has
// none today; the path is uniform across providers).
const taskModels = selectInitialTaskModels("codex", "codex", ids, recommendations, model);
await repo.updateConfig({
assistantModel: { provider: "codex", model },
...(Object.keys(taskModels).length > 0 ? { taskModels } : {}),
});
capture("llm_initial_model_selected", {
flavor: "codex",
model,
recommended: model === recommendations?.["codex"],
recommended: model === normalizeModelRecommendation(recommendations, "codex")?.assistantModel,
task_overrides_seeded: Object.keys(taskModels).length,
source: "sign_in",
});
}

View file

@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
import { selectInitialModel } from './initial-selection.js';
import { selectInitialModel, selectInitialTaskModels } from './initial-selection.js';
describe('selectInitialModel', () => {
const recommendations = {
@ -29,4 +29,47 @@ describe('selectInitialModel', () => {
it('returns null when the provider listed nothing', () => {
expect(selectInitialModel('openai', [], recommendations)).toBeNull();
});
it('accepts the nested { assistantModel, taskModels } wire shape', () => {
const nested = { rowboat: { assistantModel: 'google/gemini-3.5-flash', taskModels: {} } };
expect(selectInitialModel('rowboat', ['a', 'google/gemini-3.5-flash'], nested))
.toBe('google/gemini-3.5-flash');
});
});
describe('selectInitialTaskModels', () => {
const gatewayList = [
'google/gemini-3.5-flash',
'google/gemini-3.1-flash-lite',
'google/gemini-3.5-flash-lite',
];
const nested = {
rowboat: {
assistantModel: 'google/gemini-3.5-flash',
taskModels: {
knowledgeGraph: 'google/gemini-3.1-flash-lite',
chatTitle: 'google/gemini-3.5-flash-lite',
// Equal to the assistant → redundant, inherit produces it.
meetingNotes: 'google/gemini-3.5-flash',
// Not in the provider's list → stale hint, skipped.
liveNoteAgent: 'google/gemini-9-experimental',
// Unknown key → ignored.
somethingNew: 'google/gemini-3.1-flash-lite',
},
},
};
it('writes overrides only for listed recs that differ from the assistant', () => {
expect(selectInitialTaskModels('rowboat', 'rowboat', gatewayList, nested, 'google/gemini-3.5-flash'))
.toEqual({
knowledgeGraph: { provider: 'rowboat', model: 'google/gemini-3.1-flash-lite' },
chatTitle: { provider: 'rowboat', model: 'google/gemini-3.5-flash-lite' },
});
});
it('returns nothing for legacy flat recommendations or absent maps', () => {
expect(selectInitialTaskModels('rowboat', 'rowboat', gatewayList, { rowboat: 'google/gemini-3.5-flash' }, 'x'))
.toEqual({});
expect(selectInitialTaskModels('rowboat', 'rowboat', gatewayList, undefined, 'x')).toEqual({});
});
});

View file

@ -1,34 +1,3 @@
/**
* Initial model selection for a provider being connected for the first time.
*
* Implements the selection order from the provider/model-selection spec:
* 1. If Rowboat's recommended model for this flavor appears in the
* provider's available list, pick it.
* 2. Otherwise pick the first model the provider returned.
* 3. With no list at all, return null the caller offers retry or manual
* entry.
*
* This runs ONLY when a provider is first connected and has no saved
* selection. It must never run over an existing choice: after initial setup
* the saved model configuration is the source of truth, and changes to the
* recommendations or to the provider's list order must not silently replace
* what the user picked.
*
* Pure function by design: the caller supplies the provider's available
* models (from the unified catalog / a live probe) and the recommendations
* map (from /v1/config via rowboat:getConfig, keyed by provider FLAVOR in
* each provider's native id format). Recommendations are best-effort an
* absent map, an unknown flavor, or a recommendation the provider doesn't
* serve all degrade to "first available model".
*/
export function selectInitialModel(
flavor: string,
availableModelIds: string[],
recommendations: Record<string, string> | undefined,
): string | null {
const recommended = recommendations?.[flavor];
if (recommended && availableModelIds.includes(recommended)) {
return recommended;
}
return availableModelIds[0] ?? null;
}
// Pure selection logic lives in @x/shared (the renderer's connect flow uses
// the same implementation); re-exported here for core call sites.
export { selectInitialModel, selectInitialTaskModels } from "@x/shared/dist/initial-selection.js";

View file

@ -2,7 +2,8 @@ import container from "../di/container.js";
import { IModelConfigRepo } from "./repo.js";
import { listGatewayModels } from "./gateway.js";
import { getRowboatConfig } from "../config/rowboat.js";
import { selectInitialModel } from "./initial-selection.js";
import { selectInitialModel, selectInitialTaskModels } from "./initial-selection.js";
import { normalizeModelRecommendation } from "@x/shared/dist/rowboat-account.js";
import { capture } from "../analytics/posthog.js";
/**
@ -28,13 +29,22 @@ export async function applyRowboatInitialSelection(): Promise<void> {
const recommendations = (await getRowboatConfig().catch(() => null))?.modelRecommendations;
const model = selectInitialModel("rowboat", ids, recommendations);
if (model) {
await repo.updateConfig({ assistantModel: { provider: "rowboat", model } });
// Task recommendations ride along the seeding moment: the
// gateway's lite-tier task models become visible overrides so
// always-on background work doesn't run on assistant-class
// models (plan-credit economics).
const taskModels = selectInitialTaskModels("rowboat", "rowboat", ids, recommendations, model);
await repo.updateConfig({
assistantModel: { provider: "rowboat", model },
...(Object.keys(taskModels).length > 0 ? { taskModels } : {}),
});
// Measures recommendation quality: hit = the backend's pick was
// in the gateway list; miss = first-listed fallback.
capture("llm_initial_model_selected", {
flavor: "rowboat",
model,
recommended: model === recommendations?.["rowboat"],
recommended: model === normalizeModelRecommendation(recommendations, "rowboat")?.assistantModel,
task_overrides_seeded: Object.keys(taskModels).length,
source: "sign_in",
});
}

View file

@ -0,0 +1,67 @@
import { z } from "zod";
import { ModelRef, TaskModels } from "./models.js";
import { normalizeModelRecommendation, type ModelRecommendations } from "./rowboat-account.js";
/**
* Initial model selection for a provider being connected for the first time.
*
* Implements the selection order from the provider/model-selection spec:
* 1. If Rowboat's recommended model for this flavor appears in the
* provider's available list, pick it.
* 2. Otherwise pick the first model the provider returned.
* 3. With no list at all, return null the caller offers retry or manual
* entry.
*
* Task-model recommendations ride along the same moment: when (and only
* when) a connect seeds the assistant, the provider's per-task
* recommendations become visible taskModels overrides each validated
* against the live list and skipped when it equals the chosen assistant
* (inheritance already produces it; only differences are written).
*
* This runs ONLY when a provider is first connected and has no saved
* selection. It must never run over an existing choice: after initial setup
* the saved model configuration is the source of truth, and changes to the
* recommendations or to the provider's list order must not silently replace
* what the user picked.
*
* Pure functions by design: callers supply the provider's available models
* (from the unified catalog / a live probe) and the recommendations map
* (from /v1/config via rowboat:getConfig, keyed by provider FLAVOR in each
* provider's native id format). Everything is best-effort an absent map,
* an unknown flavor, or a recommendation the provider doesn't serve all
* degrade gracefully.
*/
const TASK_MODEL_KEYS = Object.keys(TaskModels.shape) as Array<keyof z.infer<typeof TaskModels>>;
export function selectInitialModel(
flavor: string,
availableModelIds: string[],
recommendations: ModelRecommendations | undefined,
): string | null {
const recommended = normalizeModelRecommendation(recommendations, flavor)?.assistantModel;
if (recommended && availableModelIds.includes(recommended)) {
return recommended;
}
return availableModelIds[0] ?? null;
}
export function selectInitialTaskModels(
providerId: string,
flavor: string,
availableModelIds: string[],
recommendations: ModelRecommendations | undefined,
assistantModel: string,
): Partial<Record<keyof z.infer<typeof TaskModels>, z.infer<typeof ModelRef>>> {
const taskRecommendations = normalizeModelRecommendation(recommendations, flavor)?.taskModels;
if (!taskRecommendations) return {};
const overrides: Partial<Record<keyof z.infer<typeof TaskModels>, z.infer<typeof ModelRef>>> = {};
for (const key of TASK_MODEL_KEYS) {
const model = taskRecommendations[key];
// Unknown keys are ignored; a rec equal to the assistant is redundant
// (inherit produces it); an unlisted rec is a stale hint.
if (!model || model === assistantModel || !availableModelIds.includes(model)) continue;
overrides[key] = { provider: providerId, model };
}
return overrides;
}

View file

@ -11,13 +11,43 @@ export const RowboatApiConfig = z.object({
// app keeps working against API deployments that predate it — the rewards
// UI just stays empty until the backend serves the catalog
creditActivations: z.array(CreditActivationCatalogEntrySchema).optional(),
// One recommended model id per provider FLAVOR (e.g. { openai: "gpt-5.4",
// openrouter: "anthropic/claude-opus-4.8" }), in each provider's native id
// format. A hint for the INITIAL selection when a provider is first
// Recommended models per provider FLAVOR, in each provider's native id
// format: one assistantModel (the primary) plus optional per-task
// taskModels overrides mirroring models.json v2 vocabulary (a missing
// task key = inherit the assistant — task recs exist only where the
// intended model differs; for rowboat they reproduce the pre-v2 curated
// lite-tier task models so plan credits aren't burned by background
// services). Hints for the INITIAL selection when a provider is first
// connected — never a catalog, and never applied over a saved choice
// (see core/models/initial-selection.ts). Local/custom flavors are
// intentionally absent: the API can't know which models exist in a user's
// environment. Optional so older API deployments and failed fetches never
// break parsing — recommendations are best-effort by design.
modelRecommendations: z.record(z.string(), z.string()).optional(),
// (see shared/initial-selection.ts). The bare-string form is the legacy
// wire shape, accepted so backend deploy order and rollback are
// non-events. Local/custom flavors are intentionally absent: the API
// can't know which models exist in a user's environment. Optional so
// older API deployments and failed fetches never break parsing —
// recommendations are best-effort by design.
modelRecommendations: z.record(z.string(), z.union([
z.string(),
z.object({
assistantModel: z.string(),
taskModels: z.record(z.string(), z.string()).optional(),
}),
])).optional(),
});
export type ModelRecommendations = NonNullable<z.infer<typeof RowboatApiConfig>['modelRecommendations']>;
export interface NormalizedModelRecommendation {
assistantModel: string;
taskModels: Record<string, string>;
}
/** One provider's recommendation in canonical form; null when absent. */
export function normalizeModelRecommendation(
recommendations: ModelRecommendations | undefined,
flavor: string,
): NormalizedModelRecommendation | null {
const raw = recommendations?.[flavor];
if (!raw) return null;
if (typeof raw === 'string') return { assistantModel: raw, taskModels: {} };
return { assistantModel: raw.assistantModel, taskModels: raw.taskModels ?? {} };
}