mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-24 21:41:08 +02:00
feat(x): apply per-task model recommendations at initial selection
The backend's modelRecommendations entries are now { assistantModel,
taskModels? } (rowboatx-backend#18), mirroring models.json v2 vocabulary.
When a provider connect/sign-in seeds the assistant (and ONLY then), its
per-task recommendations become visible taskModels overrides — each
validated against the live list and skipped when equal to the chosen
assistant (only differences are written, same principle as the migration).
This restores pre-v2 economics for signed-in task models without hidden
defaults: fresh Rowboat sign-ins, new devices, and sign-out→sign-in round
trips all land with the lite-tier task models the app used to hardcode —
server-controlled, updateable without a release. Saved choices are still
never touched: no seeding happens when an assistant already exists.
- selection logic moves to @x/shared/initial-selection (core re-exports;
the renderer connect flow uses the same implementation)
- RowboatApiConfig accepts both the legacy flat shape and the nested one
(normalizeModelRecommendation), so backend deploy order and rollback
are non-events
- llm_initial_model_selected gains task_overrides_seeded
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
d9317f3ca1
commit
72e71e0ab5
9 changed files with 193 additions and 59 deletions
|
|
@ -89,7 +89,7 @@ Emit points: `apps/main/src/oauth-handler.ts:369` and `apps/renderer/src/hooks/u
|
|||
Privacy rules (enforced in `packages/core/src/analytics/model-providers.ts`): only provider **flavors** are captured — never instance ids (future-proofing for user-named instances), never `apiKey`/`headers`, and never `baseURL` (local endpoints can carry internal hostnames). Model ids are allowed.
|
||||
|
||||
- `llm_provider_connected` / `llm_provider_disconnected` — `{ flavor }` — one event family across every surface. BYOK fires from `FSModelConfigRepo.setProvider` (new entries only — key rotation is not a connect) / `removeProvider`; `rowboat` from sign-in/out (`apps/main/src/oauth-handler.ts`); `codex` from ChatGPT sign-in/out (`apps/main/src/ipc.ts`).
|
||||
- `llm_initial_model_selected` — `{ flavor, model, recommended, source: 'connect' | 'onboarding' | 'sign_in' }` — a connect seeded the assistant model (only when none was configured). `recommended: false` = first-listed fallback; the hit rate measures backend recommendation quality. Emit points: `apps/renderer/src/components/settings/providers-section.tsx` (connect/onboarding) and `packages/core/src/models/rowboat-selection.ts` (sign-in).
|
||||
- `llm_initial_model_selected` — `{ flavor, model, recommended, task_overrides_seeded, source: 'connect' | 'onboarding' | 'sign_in' }` — a connect seeded the assistant model (only when none was configured). `recommended: false` = first-listed fallback; the hit rate measures backend recommendation quality. `task_overrides_seeded` counts the per-task recommendations written alongside (the server-controlled lite-tier task models — 0 when the provider has none). Emit points: `apps/renderer/src/components/settings/providers-section.tsx` (connect/onboarding) and `packages/core/src/models/rowboat-selection.ts` / `chatgpt-selection.ts` (sign-in).
|
||||
- `models_config_migrated` — `{ had_assistant, materialized_overrides, provider_count }` — one-shot per install at the models.json v1 → v2 boot migration (`FSModelConfigRepo.ensureConfig`); rollout health for the schema change.
|
||||
|
||||
### Other events (pre-existing, not added by the LLM-usage work)
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ import { Input } from "@/components/ui/input"
|
|||
import { Switch } from "@/components/ui/switch"
|
||||
import { cn } from "@/lib/utils"
|
||||
import { providerDisplayNames, type ModelRef } from "@/components/model-selector"
|
||||
import { selectInitialModel, selectInitialTaskModels } from "@x/shared/dist/initial-selection.js"
|
||||
import { normalizeModelRecommendation, type ModelRecommendations } from "@x/shared/dist/rowboat-account.js"
|
||||
import { useModels } from "@/hooks/use-models"
|
||||
import { useRowboatConfig } from "@/hooks/use-rowboat-config"
|
||||
import { useChatGPT } from "@/hooks/useChatGPT"
|
||||
|
|
@ -259,7 +261,7 @@ function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnecte
|
|||
chatgptSignedIn: boolean
|
||||
onChatGPTSignIn: () => Promise<unknown> | void
|
||||
hadAssistant: boolean
|
||||
modelRecommendations: Record<string, string> | undefined
|
||||
modelRecommendations: ModelRecommendations | undefined
|
||||
analyticsSource: 'connect' | 'onboarding'
|
||||
}) {
|
||||
const [step, setStep] = useState<AddStep>({ kind: "choose" })
|
||||
|
|
@ -368,11 +370,7 @@ function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnecte
|
|||
const listRes = await window.ipc.invoke("models:listForProvider", { provider: providerEntry })
|
||||
const list = listRes.success ? listRes.models ?? [] : []
|
||||
const typed = manualModel.trim()
|
||||
let model = typed
|
||||
if (!model) {
|
||||
const recommended = modelRecommendations?.[flavor]
|
||||
model = recommended && list.includes(recommended) ? recommended : (list[0] ?? "")
|
||||
}
|
||||
const model = typed || (selectInitialModel(flavor, list, modelRecommendations) ?? "")
|
||||
if (!listRes.success && !model) {
|
||||
setStep({ kind: "error", flavor, message: listRes.error || "Could not load the provider's model list." })
|
||||
return
|
||||
|
|
@ -394,11 +392,18 @@ function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnecte
|
|||
const cfgNow = await window.ipc.invoke("models:getConfig", null).catch(() => null)
|
||||
const hasAssistantNow = cfgNow ? cfgNow.assistantModel !== null : hadAssistant
|
||||
if (!hasAssistantNow) {
|
||||
await window.ipc.invoke("models:updateConfig", { assistantModel: { provider: flavor, model } })
|
||||
// Task recommendations ride along the seeding moment as visible
|
||||
// overrides (validated against the live list; only differences).
|
||||
const taskModels = selectInitialTaskModels(flavor, flavor, list, modelRecommendations, model)
|
||||
await window.ipc.invoke("models:updateConfig", {
|
||||
assistantModel: { provider: flavor, model },
|
||||
...(Object.keys(taskModels).length > 0 ? { taskModels } : {}),
|
||||
})
|
||||
analytics.llmInitialModelSelected({
|
||||
flavor,
|
||||
model,
|
||||
recommended: model === modelRecommendations?.[flavor],
|
||||
recommended: model === normalizeModelRecommendation(modelRecommendations, flavor)?.assistantModel,
|
||||
taskOverridesSeeded: Object.keys(taskModels).length,
|
||||
source: analyticsSource,
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -345,7 +345,9 @@ export function llmInitialModelSelected(props: {
|
|||
flavor: string
|
||||
model: string
|
||||
recommended: boolean
|
||||
taskOverridesSeeded: number
|
||||
source: 'connect' | 'onboarding'
|
||||
}) {
|
||||
posthog.capture('llm_initial_model_selected', { ...props })
|
||||
const { taskOverridesSeeded, ...rest } = props
|
||||
posthog.capture('llm_initial_model_selected', { ...rest, task_overrides_seeded: taskOverridesSeeded })
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ import container from "../di/container.js";
|
|||
import { IModelConfigRepo } from "./repo.js";
|
||||
import { listCodexModels } from "./codex.js";
|
||||
import { getRowboatConfig } from "../config/rowboat.js";
|
||||
import { selectInitialModel } from "./initial-selection.js";
|
||||
import { selectInitialModel, selectInitialTaskModels } from "./initial-selection.js";
|
||||
import { normalizeModelRecommendation } from "@x/shared/dist/rowboat-account.js";
|
||||
import { capture } from "../analytics/posthog.js";
|
||||
|
||||
/**
|
||||
|
|
@ -27,11 +28,18 @@ export async function applyCodexInitialSelection(): Promise<void> {
|
|||
const recommendations = (await getRowboatConfig().catch(() => null))?.modelRecommendations;
|
||||
const model = selectInitialModel("codex", ids, recommendations);
|
||||
if (model) {
|
||||
await repo.updateConfig({ assistantModel: { provider: "codex", model } });
|
||||
// Task recommendations ride along the seeding moment (codex has
|
||||
// none today; the path is uniform across providers).
|
||||
const taskModels = selectInitialTaskModels("codex", "codex", ids, recommendations, model);
|
||||
await repo.updateConfig({
|
||||
assistantModel: { provider: "codex", model },
|
||||
...(Object.keys(taskModels).length > 0 ? { taskModels } : {}),
|
||||
});
|
||||
capture("llm_initial_model_selected", {
|
||||
flavor: "codex",
|
||||
model,
|
||||
recommended: model === recommendations?.["codex"],
|
||||
recommended: model === normalizeModelRecommendation(recommendations, "codex")?.assistantModel,
|
||||
task_overrides_seeded: Object.keys(taskModels).length,
|
||||
source: "sign_in",
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { selectInitialModel } from './initial-selection.js';
|
||||
import { selectInitialModel, selectInitialTaskModels } from './initial-selection.js';
|
||||
|
||||
describe('selectInitialModel', () => {
|
||||
const recommendations = {
|
||||
|
|
@ -29,4 +29,47 @@ describe('selectInitialModel', () => {
|
|||
it('returns null when the provider listed nothing', () => {
|
||||
expect(selectInitialModel('openai', [], recommendations)).toBeNull();
|
||||
});
|
||||
|
||||
it('accepts the nested { assistantModel, taskModels } wire shape', () => {
|
||||
const nested = { rowboat: { assistantModel: 'google/gemini-3.5-flash', taskModels: {} } };
|
||||
expect(selectInitialModel('rowboat', ['a', 'google/gemini-3.5-flash'], nested))
|
||||
.toBe('google/gemini-3.5-flash');
|
||||
});
|
||||
});
|
||||
|
||||
describe('selectInitialTaskModels', () => {
|
||||
const gatewayList = [
|
||||
'google/gemini-3.5-flash',
|
||||
'google/gemini-3.1-flash-lite',
|
||||
'google/gemini-3.5-flash-lite',
|
||||
];
|
||||
const nested = {
|
||||
rowboat: {
|
||||
assistantModel: 'google/gemini-3.5-flash',
|
||||
taskModels: {
|
||||
knowledgeGraph: 'google/gemini-3.1-flash-lite',
|
||||
chatTitle: 'google/gemini-3.5-flash-lite',
|
||||
// Equal to the assistant → redundant, inherit produces it.
|
||||
meetingNotes: 'google/gemini-3.5-flash',
|
||||
// Not in the provider's list → stale hint, skipped.
|
||||
liveNoteAgent: 'google/gemini-9-experimental',
|
||||
// Unknown key → ignored.
|
||||
somethingNew: 'google/gemini-3.1-flash-lite',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
it('writes overrides only for listed recs that differ from the assistant', () => {
|
||||
expect(selectInitialTaskModels('rowboat', 'rowboat', gatewayList, nested, 'google/gemini-3.5-flash'))
|
||||
.toEqual({
|
||||
knowledgeGraph: { provider: 'rowboat', model: 'google/gemini-3.1-flash-lite' },
|
||||
chatTitle: { provider: 'rowboat', model: 'google/gemini-3.5-flash-lite' },
|
||||
});
|
||||
});
|
||||
|
||||
it('returns nothing for legacy flat recommendations or absent maps', () => {
|
||||
expect(selectInitialTaskModels('rowboat', 'rowboat', gatewayList, { rowboat: 'google/gemini-3.5-flash' }, 'x'))
|
||||
.toEqual({});
|
||||
expect(selectInitialTaskModels('rowboat', 'rowboat', gatewayList, undefined, 'x')).toEqual({});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,34 +1,3 @@
|
|||
/**
|
||||
* Initial model selection for a provider being connected for the first time.
|
||||
*
|
||||
* Implements the selection order from the provider/model-selection spec:
|
||||
* 1. If Rowboat's recommended model for this flavor appears in the
|
||||
* provider's available list, pick it.
|
||||
* 2. Otherwise pick the first model the provider returned.
|
||||
* 3. With no list at all, return null — the caller offers retry or manual
|
||||
* entry.
|
||||
*
|
||||
* This runs ONLY when a provider is first connected and has no saved
|
||||
* selection. It must never run over an existing choice: after initial setup
|
||||
* the saved model configuration is the source of truth, and changes to the
|
||||
* recommendations or to the provider's list order must not silently replace
|
||||
* what the user picked.
|
||||
*
|
||||
* Pure function by design: the caller supplies the provider's available
|
||||
* models (from the unified catalog / a live probe) and the recommendations
|
||||
* map (from /v1/config via rowboat:getConfig, keyed by provider FLAVOR in
|
||||
* each provider's native id format). Recommendations are best-effort — an
|
||||
* absent map, an unknown flavor, or a recommendation the provider doesn't
|
||||
* serve all degrade to "first available model".
|
||||
*/
|
||||
export function selectInitialModel(
|
||||
flavor: string,
|
||||
availableModelIds: string[],
|
||||
recommendations: Record<string, string> | undefined,
|
||||
): string | null {
|
||||
const recommended = recommendations?.[flavor];
|
||||
if (recommended && availableModelIds.includes(recommended)) {
|
||||
return recommended;
|
||||
}
|
||||
return availableModelIds[0] ?? null;
|
||||
}
|
||||
// Pure selection logic lives in @x/shared (the renderer's connect flow uses
|
||||
// the same implementation); re-exported here for core call sites.
|
||||
export { selectInitialModel, selectInitialTaskModels } from "@x/shared/dist/initial-selection.js";
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ import container from "../di/container.js";
|
|||
import { IModelConfigRepo } from "./repo.js";
|
||||
import { listGatewayModels } from "./gateway.js";
|
||||
import { getRowboatConfig } from "../config/rowboat.js";
|
||||
import { selectInitialModel } from "./initial-selection.js";
|
||||
import { selectInitialModel, selectInitialTaskModels } from "./initial-selection.js";
|
||||
import { normalizeModelRecommendation } from "@x/shared/dist/rowboat-account.js";
|
||||
import { capture } from "../analytics/posthog.js";
|
||||
|
||||
/**
|
||||
|
|
@ -28,13 +29,22 @@ export async function applyRowboatInitialSelection(): Promise<void> {
|
|||
const recommendations = (await getRowboatConfig().catch(() => null))?.modelRecommendations;
|
||||
const model = selectInitialModel("rowboat", ids, recommendations);
|
||||
if (model) {
|
||||
await repo.updateConfig({ assistantModel: { provider: "rowboat", model } });
|
||||
// Task recommendations ride along the seeding moment: the
|
||||
// gateway's lite-tier task models become visible overrides so
|
||||
// always-on background work doesn't run on assistant-class
|
||||
// models (plan-credit economics).
|
||||
const taskModels = selectInitialTaskModels("rowboat", "rowboat", ids, recommendations, model);
|
||||
await repo.updateConfig({
|
||||
assistantModel: { provider: "rowboat", model },
|
||||
...(Object.keys(taskModels).length > 0 ? { taskModels } : {}),
|
||||
});
|
||||
// Measures recommendation quality: hit = the backend's pick was
|
||||
// in the gateway list; miss = first-listed fallback.
|
||||
capture("llm_initial_model_selected", {
|
||||
flavor: "rowboat",
|
||||
model,
|
||||
recommended: model === recommendations?.["rowboat"],
|
||||
recommended: model === normalizeModelRecommendation(recommendations, "rowboat")?.assistantModel,
|
||||
task_overrides_seeded: Object.keys(taskModels).length,
|
||||
source: "sign_in",
|
||||
});
|
||||
}
|
||||
|
|
|
|||
67
apps/x/packages/shared/src/initial-selection.ts
Normal file
67
apps/x/packages/shared/src/initial-selection.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
import { z } from "zod";
|
||||
import { ModelRef, TaskModels } from "./models.js";
|
||||
import { normalizeModelRecommendation, type ModelRecommendations } from "./rowboat-account.js";
|
||||
|
||||
/**
|
||||
* Initial model selection for a provider being connected for the first time.
|
||||
*
|
||||
* Implements the selection order from the provider/model-selection spec:
|
||||
* 1. If Rowboat's recommended model for this flavor appears in the
|
||||
* provider's available list, pick it.
|
||||
* 2. Otherwise pick the first model the provider returned.
|
||||
* 3. With no list at all, return null — the caller offers retry or manual
|
||||
* entry.
|
||||
*
|
||||
* Task-model recommendations ride along the same moment: when (and only
|
||||
* when) a connect seeds the assistant, the provider's per-task
|
||||
* recommendations become visible taskModels overrides — each validated
|
||||
* against the live list and skipped when it equals the chosen assistant
|
||||
* (inheritance already produces it; only differences are written).
|
||||
*
|
||||
* This runs ONLY when a provider is first connected and has no saved
|
||||
* selection. It must never run over an existing choice: after initial setup
|
||||
* the saved model configuration is the source of truth, and changes to the
|
||||
* recommendations or to the provider's list order must not silently replace
|
||||
* what the user picked.
|
||||
*
|
||||
* Pure functions by design: callers supply the provider's available models
|
||||
* (from the unified catalog / a live probe) and the recommendations map
|
||||
* (from /v1/config via rowboat:getConfig, keyed by provider FLAVOR in each
|
||||
* provider's native id format). Everything is best-effort — an absent map,
|
||||
* an unknown flavor, or a recommendation the provider doesn't serve all
|
||||
* degrade gracefully.
|
||||
*/
|
||||
|
||||
const TASK_MODEL_KEYS = Object.keys(TaskModels.shape) as Array<keyof z.infer<typeof TaskModels>>;
|
||||
|
||||
export function selectInitialModel(
|
||||
flavor: string,
|
||||
availableModelIds: string[],
|
||||
recommendations: ModelRecommendations | undefined,
|
||||
): string | null {
|
||||
const recommended = normalizeModelRecommendation(recommendations, flavor)?.assistantModel;
|
||||
if (recommended && availableModelIds.includes(recommended)) {
|
||||
return recommended;
|
||||
}
|
||||
return availableModelIds[0] ?? null;
|
||||
}
|
||||
|
||||
export function selectInitialTaskModels(
|
||||
providerId: string,
|
||||
flavor: string,
|
||||
availableModelIds: string[],
|
||||
recommendations: ModelRecommendations | undefined,
|
||||
assistantModel: string,
|
||||
): Partial<Record<keyof z.infer<typeof TaskModels>, z.infer<typeof ModelRef>>> {
|
||||
const taskRecommendations = normalizeModelRecommendation(recommendations, flavor)?.taskModels;
|
||||
if (!taskRecommendations) return {};
|
||||
const overrides: Partial<Record<keyof z.infer<typeof TaskModels>, z.infer<typeof ModelRef>>> = {};
|
||||
for (const key of TASK_MODEL_KEYS) {
|
||||
const model = taskRecommendations[key];
|
||||
// Unknown keys are ignored; a rec equal to the assistant is redundant
|
||||
// (inherit produces it); an unlisted rec is a stale hint.
|
||||
if (!model || model === assistantModel || !availableModelIds.includes(model)) continue;
|
||||
overrides[key] = { provider: providerId, model };
|
||||
}
|
||||
return overrides;
|
||||
}
|
||||
|
|
@ -11,13 +11,43 @@ export const RowboatApiConfig = z.object({
|
|||
// app keeps working against API deployments that predate it — the rewards
|
||||
// UI just stays empty until the backend serves the catalog
|
||||
creditActivations: z.array(CreditActivationCatalogEntrySchema).optional(),
|
||||
// One recommended model id per provider FLAVOR (e.g. { openai: "gpt-5.4",
|
||||
// openrouter: "anthropic/claude-opus-4.8" }), in each provider's native id
|
||||
// format. A hint for the INITIAL selection when a provider is first
|
||||
// Recommended models per provider FLAVOR, in each provider's native id
|
||||
// format: one assistantModel (the primary) plus optional per-task
|
||||
// taskModels overrides mirroring models.json v2 vocabulary (a missing
|
||||
// task key = inherit the assistant — task recs exist only where the
|
||||
// intended model differs; for rowboat they reproduce the pre-v2 curated
|
||||
// lite-tier task models so plan credits aren't burned by background
|
||||
// services). Hints for the INITIAL selection when a provider is first
|
||||
// connected — never a catalog, and never applied over a saved choice
|
||||
// (see core/models/initial-selection.ts). Local/custom flavors are
|
||||
// intentionally absent: the API can't know which models exist in a user's
|
||||
// environment. Optional so older API deployments and failed fetches never
|
||||
// break parsing — recommendations are best-effort by design.
|
||||
modelRecommendations: z.record(z.string(), z.string()).optional(),
|
||||
// (see shared/initial-selection.ts). The bare-string form is the legacy
|
||||
// wire shape, accepted so backend deploy order and rollback are
|
||||
// non-events. Local/custom flavors are intentionally absent: the API
|
||||
// can't know which models exist in a user's environment. Optional so
|
||||
// older API deployments and failed fetches never break parsing —
|
||||
// recommendations are best-effort by design.
|
||||
modelRecommendations: z.record(z.string(), z.union([
|
||||
z.string(),
|
||||
z.object({
|
||||
assistantModel: z.string(),
|
||||
taskModels: z.record(z.string(), z.string()).optional(),
|
||||
}),
|
||||
])).optional(),
|
||||
});
|
||||
|
||||
export type ModelRecommendations = NonNullable<z.infer<typeof RowboatApiConfig>['modelRecommendations']>;
|
||||
|
||||
export interface NormalizedModelRecommendation {
|
||||
assistantModel: string;
|
||||
taskModels: Record<string, string>;
|
||||
}
|
||||
|
||||
/** One provider's recommendation in canonical form; null when absent. */
|
||||
export function normalizeModelRecommendation(
|
||||
recommendations: ModelRecommendations | undefined,
|
||||
flavor: string,
|
||||
): NormalizedModelRecommendation | null {
|
||||
const raw = recommendations?.[flavor];
|
||||
if (!raw) return null;
|
||||
if (typeof raw === 'string') return { assistantModel: raw, taskModels: {} };
|
||||
return { assistantModel: raw.assistantModel, taskModels: raw.taskModels ?? {} };
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue