From 412166f106ec9b35793858579c4f37fc91d9f805 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Sat, 6 Jun 2026 22:37:07 +0200 Subject: [PATCH] feat(setup): write per-role llm model presets --- packages/cli/src/setup-models.ts | 571 +++++++------------------ packages/cli/test/setup-models.test.ts | 533 ++++++++--------------- 2 files changed, 330 insertions(+), 774 deletions(-) diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index 8e8cf30b..aafcc880 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -10,7 +10,7 @@ import { resolveKtxConfigReference } from './context/core/config-reference.js'; import { type KtxProjectConfig, type KtxProjectLlmConfig, serializeKtxProjectConfig } from './context/project/config.js'; import { loadKtxProject } from './context/project/project.js'; import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; -import type { KtxLlmConfig } from './llm/types.js'; +import { type KtxModelRole, KTX_MODEL_ROLES, type KtxLlmConfig } from './llm/types.js'; import { type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from './llm/model-health.js'; import { formatClaudeCodePromptCachingWarning, @@ -37,7 +37,6 @@ export interface KtxSetupModelArgs { llmBackend?: KtxSetupLlmBackend; anthropicApiKeyEnv?: string; anthropicApiKeyFile?: string; - llmModel?: string; vertexProject?: string; vertexLocation?: string; forcePrompt?: boolean; @@ -52,13 +51,6 @@ export type KtxSetupModelResult = | { status: 'missing-input'; projectDir: string } | { status: 'failed'; projectDir: string }; -/** @internal */ -export interface AnthropicModelChoice { - id: string; - label: string; - recommended: boolean; -} - export type KtxSetupLlmBackend = 'anthropic' | 'vertex' | 'claude-code' | 'codex'; /** @internal */ @@ -76,9 +68,7 @@ export interface KtxSetupModelPromptAdapter { export interface KtxSetupModelDeps { env?: NodeJS.ProcessEnv; - fetch?: typeof fetch; prompts?: KtxSetupModelPromptAdapter; - listModels?: (apiKey: string) => Promise; healthCheck?: (config: KtxLlmConfig) => Promise; claudeCodeAuthProbe?: (input: { projectDir: string; @@ -91,91 +81,58 @@ export interface KtxSetupModelDeps { spinner?: () => KtxCliSpinner; } -/** @internal */ -export const BUNDLED_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ - { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }, - { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, - { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, -]; - -const VERTEX_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ - { id: 'claude-opus-4-7', label: 'Claude Opus 4.7', recommended: false }, - { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: false }, - { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, - { id: 'claude-opus-4-5', label: 'Claude Opus 4.5', recommended: false }, - { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, - { id: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5', recommended: false }, - { id: 'claude-opus-4-1', label: 'Claude Opus 4.1', recommended: false }, -]; - -const CLAUDE_CODE_MODELS: AnthropicModelChoice[] = [ - { id: 'sonnet', label: 'Claude Sonnet', recommended: true }, - { id: 'opus', label: 'Claude Opus', recommended: false }, - { id: 'haiku', label: 'Claude Haiku', recommended: false }, -]; - -// Curated Codex models from OpenAI's current lineup that work under both -// ChatGPT-account (subscription) and API-key auth. Intentionally omitted: -// the `*-codex` ids (e.g. gpt-5.3-codex, gpt-5.2-codex) are API-key-only and -// fail on ChatGPT-account auth, and gpt-5.3-codex-spark is a ChatGPT-Pro-only -// research preview. Codex resolves real availability per account at runtime -// (its binary remote-fetches the model list), so this is a convenience -// shortlist only — the manual-entry option accepts any id your account's -// `codex` picker exposes, and the auth probe reports an unsupported choice. -const CODEX_MODELS: AnthropicModelChoice[] = [ - { id: 'gpt-5.5', label: 'GPT-5.5', recommended: true }, - { id: 'gpt-5.4', label: 'GPT-5.4', recommended: false }, - { id: 'gpt-5.4-mini', label: 'GPT-5.4 mini', recommended: false }, -]; - -const HIDDEN_ANTHROPIC_MODEL_PATTERNS = [ - /^claude-sonnet-4$/i, - /^claude-opus-4$/i, - /^Claude Sonnet 4$/i, - /^Claude Opus 4$/i, -]; - const ANTHROPIC_CREDENTIAL_PROMPT_CONTEXT = 'KTX uses the key to verify Anthropic model access now and to run ingest agents that turn schemas, SQL, ' + 'BI metadata, and docs into semantic-layer sources and wiki context. ktx.yaml stores an env: or file: ' + 'reference, not the raw key.'; -const ANTHROPIC_MODEL_PROMPT_CONTEXT = - 'KTX uses this as the default model for ingest agents that turn schemas, SQL, BI metadata, and docs ' + - 'into semantic-layer sources and wiki context.'; - const VERTEX_PROJECT_PROMPT_CONTEXT = 'KTX stores the selected Google Cloud project ID in ktx.yaml and uses Application Default Credentials for ' + 'access. Project visibility depends on the signed-in Google account and organization permissions.'; const DEFAULT_VERTEX_LOCATION = 'us-east5'; +type KtxSetupModelPreset = Record; + +const ANTHROPIC_PRESET = { + default: 'claude-sonnet-4-6', + triage: 'claude-haiku-4-5', + candidateExtraction: 'claude-sonnet-4-6', + curator: 'claude-opus-4-7', + reconcile: 'claude-opus-4-7', + repair: 'claude-haiku-4-5', +} satisfies KtxSetupModelPreset; + +const CLAUDE_CODE_PRESET = { + default: 'sonnet', + triage: 'haiku', + candidateExtraction: 'sonnet', + curator: 'opus', + reconcile: 'opus', + repair: 'haiku', +} satisfies KtxSetupModelPreset; + +const CODEX_PRESET = { + default: DEFAULT_CODEX_MODEL, + triage: DEFAULT_CODEX_MODEL, + candidateExtraction: DEFAULT_CODEX_MODEL, + curator: DEFAULT_CODEX_MODEL, + reconcile: DEFAULT_CODEX_MODEL, + repair: DEFAULT_CODEX_MODEL, +} satisfies KtxSetupModelPreset; + +const MODEL_PRESETS = { + anthropic: ANTHROPIC_PRESET, + vertex: ANTHROPIC_PRESET, + 'claude-code': CLAUDE_CODE_PRESET, + codex: CODEX_PRESET, +} satisfies Record; + +function presetForBackend(backend: KtxSetupLlmBackend): KtxSetupModelPreset { + return MODEL_PRESETS[backend]; +} + const execFileAsync = promisify(execFile); -type AnthropicModelDiscoveryErrorReason = 'authentication' | 'http' | 'empty-response'; - -class AnthropicModelDiscoveryError extends Error { - constructor( - message: string, - public readonly reason: AnthropicModelDiscoveryErrorReason, - public readonly status?: number, - ) { - super(message); - this.name = 'AnthropicModelDiscoveryError'; - } -} - -function isAnthropicModelAuthenticationError(error: unknown): error is AnthropicModelDiscoveryError { - return error instanceof AnthropicModelDiscoveryError && error.reason === 'authentication'; -} - -function isSelectableAnthropicModel(model: AnthropicModelChoice): boolean { - return !HIDDEN_ANTHROPIC_MODEL_PATTERNS.some((pattern) => pattern.test(model.id) || pattern.test(model.label)); -} - -type ChooseModelResult = - | { status: 'ready'; model: string } - | { status: 'back' | 'missing-input' | 'invalid-credential' }; - type ChooseBackendResult = | { status: 'ready'; backend: KtxSetupLlmBackend; prompted: boolean } | { status: 'back' }; @@ -234,47 +191,6 @@ async function defaultListGcloudProjects(): Promise { .filter((project): project is GcloudProjectChoice => Boolean(project)); } -/** @internal */ -export async function fetchAnthropicModels( - apiKey: string, - fetchFn: typeof fetch = fetch, -): Promise { - const response = await fetchFn('https://api.anthropic.com/v1/models?limit=1000', { - headers: { - 'anthropic-version': '2023-06-01', - 'x-api-key': apiKey, - }, - }); - if (!response.ok) { - if (response.status === 401 || response.status === 403) { - throw new AnthropicModelDiscoveryError( - `Anthropic model discovery failed with HTTP ${response.status}`, - 'authentication', - response.status, - ); - } - throw new AnthropicModelDiscoveryError( - `Anthropic model discovery failed with HTTP ${response.status}`, - 'http', - response.status, - ); - } - const body = (await response.json()) as { data?: Array<{ id?: unknown; display_name?: unknown; type?: unknown }> }; - const models = (body.data ?? []) - .map((item) => ({ - id: typeof item.id === 'string' ? item.id : '', - label: typeof item.display_name === 'string' ? item.display_name : typeof item.id === 'string' ? item.id : '', - recommended: false, - })) - .filter((item) => item.id.startsWith('claude-')) - .filter(isSelectableAnthropicModel); - if (models.length === 0) { - throw new AnthropicModelDiscoveryError('Anthropic model discovery returned no Claude models', 'empty-response'); - } - const recommendedIndex = models.findIndex((item) => item.id.includes('sonnet')); - return models.map((item, index) => ({ ...item, recommended: index === Math.max(recommendedIndex, 0) })); -} - export function isKtxSetupLlmConfigReady(config: KtxProjectLlmConfig): boolean { let resolved: KtxLlmConfig | null; try { @@ -309,12 +225,12 @@ function buildProjectLlmConfig( | { backend: 'vertex'; vertex: { project?: string; location: string } } | { backend: 'claude-code' } | { backend: 'codex' }, - model: string, + models: KtxSetupModelPreset, ): KtxProjectLlmConfig { if (provider.backend === 'claude-code') { return { provider: { backend: 'claude-code' }, - models: { ...existing.models, default: model }, + models, promptCaching: existing.promptCaching, }; } @@ -322,7 +238,7 @@ function buildProjectLlmConfig( if (provider.backend === 'codex') { return { provider: { backend: 'codex' }, - models: { ...existing.models, default: model }, + models, promptCaching: existing.promptCaching, }; } @@ -333,7 +249,7 @@ function buildProjectLlmConfig( backend: 'vertex', vertex: provider.vertex, }, - models: { ...existing.models, default: model }, + models, promptCaching: { ...(existing.promptCaching ?? {}), enabled: true, vertexFallbackTo5m: true }, }; } @@ -343,7 +259,7 @@ function buildProjectLlmConfig( backend: 'anthropic', anthropic: { api_key: provider.credentialRef }, }, - models: { ...existing.models, default: model }, + models, promptCaching: { ...(existing.promptCaching ?? {}), enabled: true }, }; } @@ -514,16 +430,12 @@ function requestedBackend(args: KtxSetupModelArgs): KtxSetupLlmBackend | undefin if (args.vertexProject || args.vertexLocation) { return 'vertex'; } - if (args.anthropicApiKeyEnv || args.anthropicApiKeyFile || args.llmModel) { + if (args.anthropicApiKeyEnv || args.anthropicApiKeyFile) { return 'anthropic'; } return undefined; } -function requestedModel(args: KtxSetupModelArgs): string | undefined { - return args.llmModel; -} - async function chooseBackend( args: KtxSetupModelArgs, io: KtxCliIo, @@ -774,187 +686,6 @@ async function chooseVertexConfig( }; } -async function chooseModel( - args: KtxSetupModelArgs, - credentialValue: string, - io: KtxCliIo, - deps: KtxSetupModelDeps, -): Promise { - const providedModel = requestedModel(args); - if (providedModel) { - return { status: 'ready', model: providedModel }; - } - if (args.inputMode === 'disabled') { - io.stderr.write('Missing LLM model: pass --llm-model.\n'); - return { status: 'missing-input' }; - } - - let models: AnthropicModelChoice[]; - try { - models = deps.listModels - ? await deps.listModels(credentialValue) - : await fetchAnthropicModels(credentialValue, deps.fetch); - } catch (error) { - if (isAnthropicModelAuthenticationError(error)) { - const statusSuffix = error.status ? ` (HTTP ${error.status})` : ''; - io.stderr.write(`Anthropic API key is invalid or unauthorized${statusSuffix}. Check the key and try again.\n`); - return { status: 'invalid-credential' }; - } - io.stderr.write( - 'Could not fetch live Anthropic models. Showing bundled defaults. Setup will still test the selected model before saving it.\n', - ); - models = BUNDLED_ANTHROPIC_MODELS; - } - - const selectableModels = models.filter(isSelectableAnthropicModel); - const prompts = deps.prompts ?? createPromptAdapter(); - const modelOptions = [ - ...selectableModels.map((model) => ({ - value: model.id, - label: model.label || model.id, - ...(model.recommended ? { hint: 'recommended' } : {}), - })), - { value: 'manual', label: 'Enter a model ID manually' }, - { value: 'back', label: 'Back' }, - ]; - const choice = await prompts.autocomplete({ - message: `Which Anthropic model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, - placeholder: 'Type to search models', - options: modelOptions, - }); - if (choice === 'back') { - return { status: 'back' }; - } - if (choice === 'manual') { - const manual = await prompts.text({ - message: withTextInputNavigation('Anthropic model ID'), - placeholder: selectableModels.find((model) => model.recommended)?.id ?? selectableModels[0]?.id, - }); - if (manual === undefined) { - return { status: 'back' }; - } - return manual.trim() ? { status: 'ready', model: manual.trim() } : { status: 'missing-input' }; - } - return { status: 'ready', model: choice }; -} - -async function chooseVertexModel(args: KtxSetupModelArgs, io: KtxCliIo, deps: KtxSetupModelDeps): Promise { - const providedModel = requestedModel(args); - if (providedModel) { - return { status: 'ready', model: providedModel }; - } - if (args.inputMode === 'disabled') { - io.stderr.write('Missing LLM model: pass --llm-model.\n'); - return { status: 'missing-input' }; - } - - const selectableModels = VERTEX_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel); - const prompts = deps.prompts ?? createPromptAdapter(); - const choice = await prompts.autocomplete({ - message: `Which Anthropic model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, - placeholder: 'Type to search models', - options: [ - ...selectableModels.map((model) => ({ - value: model.id, - label: model.label || model.id, - ...(model.recommended ? { hint: 'recommended' } : {}), - })), - { value: 'manual', label: 'Enter a model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }); - if (choice === 'back') { - return { status: 'back' }; - } - if (choice === 'manual') { - const manual = await prompts.text({ - message: withTextInputNavigation('Anthropic model ID'), - placeholder: selectableModels.find((model) => model.recommended)?.id ?? selectableModels[0]?.id, - }); - if (manual === undefined) { - return { status: 'back' }; - } - return manual.trim() ? { status: 'ready', model: manual.trim() } : { status: 'missing-input' }; - } - return { status: 'ready', model: choice }; -} - -async function chooseClaudeCodeModel(args: KtxSetupModelArgs, deps: KtxSetupModelDeps): Promise { - const providedModel = requestedModel(args); - if (providedModel) { - return { status: 'ready', model: providedModel }; - } - if (args.inputMode === 'disabled') { - return { status: 'ready', model: 'sonnet' }; - } - - const prompts = deps.prompts ?? createPromptAdapter(); - const choice = await prompts.select({ - message: `Which Claude Code model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, - options: [ - ...CLAUDE_CODE_MODELS.map((model) => ({ - value: model.id, - label: model.label, - ...(model.recommended ? { hint: 'recommended' } : {}), - })), - { value: 'manual', label: 'Enter a Claude Code model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }); - if (choice === 'back') { - return { status: 'back' }; - } - if (choice === 'manual') { - const manual = await prompts.text({ - message: withTextInputNavigation('Claude Code model ID'), - placeholder: CLAUDE_CODE_MODELS.find((model) => model.recommended)?.id ?? CLAUDE_CODE_MODELS[0]?.id, - }); - if (manual === undefined) { - return { status: 'back' }; - } - return manual.trim() ? { status: 'ready', model: manual.trim() } : { status: 'missing-input' }; - } - return { status: 'ready', model: choice }; -} - -async function chooseCodexModel(args: KtxSetupModelArgs, deps: KtxSetupModelDeps): Promise { - const providedModel = requestedModel(args); - if (providedModel) { - return { status: 'ready', model: providedModel }; - } - if (args.inputMode === 'disabled') { - return { status: 'ready', model: DEFAULT_CODEX_MODEL }; - } - - const prompts = deps.prompts ?? createPromptAdapter(); - const choice = await prompts.select({ - message: `Which Codex model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, - options: [ - ...CODEX_MODELS.map((model) => ({ - value: model.id, - label: model.label, - ...(model.recommended ? { hint: 'recommended' } : {}), - })), - { value: 'manual', label: 'Enter a Codex model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }); - if (choice === 'back') { - return { status: 'back' }; - } - if (choice === 'manual') { - const manual = await prompts.text({ - message: withTextInputNavigation('Codex model ID'), - placeholder: CODEX_MODELS.find((model) => model.recommended)?.id ?? CODEX_MODELS[0]?.id, - }); - if (manual === undefined) { - return { status: 'back' }; - } - return manual.trim() ? { status: 'ready', model: manual.trim() } : { status: 'missing-input' }; - } - return { status: 'ready', model: choice }; -} - async function persistLlmConfig( projectDir: string, provider: @@ -962,12 +693,12 @@ async function persistLlmConfig( | { backend: 'vertex'; vertex: { project?: string; location: string } } | { backend: 'claude-code' } | { backend: 'codex' }, - model: string, + models: KtxSetupModelPreset, ): Promise { const project = await loadKtxProject({ projectDir }); const config = { ...project.config, - llm: buildProjectLlmConfig(project.config.llm, provider, model), + llm: buildProjectLlmConfig(project.config.llm, provider, models), scan: { ...project.config.scan, enrichment: { @@ -990,6 +721,61 @@ function buildInteractiveRetryArgs(args: KtxSetupModelArgs, backend?: KtxSetupLl }; } +type PresetModelValidationResult = { ok: true } | { ok: false; message: string }; + +function distinctPresetModels(preset: KtxSetupModelPreset): string[] { + const models: string[] = []; + const seen = new Set(); + for (const role of KTX_MODEL_ROLES) { + const model = preset[role]; + if (!seen.has(model)) { + seen.add(model); + models.push(model); + } + } + return models; +} + +function rolesUsingModel(preset: KtxSetupModelPreset, model: string): KtxModelRole[] { + return KTX_MODEL_ROLES.filter((role) => preset[role] === model); +} + +function formatPresetFallbackWarning(roles: KtxModelRole[], unavailableModel: string, anchorModel: string): string { + return `LLM model ${unavailableModel} is unavailable for ${roles.join(', ')}; using ${anchorModel} for those roles.`; +} + +async function validatePresetModels( + preset: KtxSetupModelPreset, + validateModel: (model: string) => Promise, + io: KtxCliIo, +): Promise<{ status: 'ready'; models: KtxSetupModelPreset } | { status: 'failed'; message: string }> { + const anchorModel = preset.default; + const degraded = { ...preset }; + const models = distinctPresetModels(preset); + + const anchorResult = await validateModel(anchorModel); + if (!anchorResult.ok) { + return { status: 'failed', message: anchorResult.message }; + } + + for (const model of models) { + if (model === anchorModel) { + continue; + } + const result = await validateModel(model); + if (result.ok) { + continue; + } + const affectedRoles = rolesUsingModel(degraded, model); + for (const role of affectedRoles) { + degraded[role] = anchorModel; + } + io.stderr.write(`${formatPresetFallbackWarning(affectedRoles, model, anchorModel)}\n`); + } + + return { status: 'ready', models: degraded }; +} + export async function runKtxSetupAnthropicModelStep( args: KtxSetupModelArgs, io: KtxCliIo, @@ -1007,7 +793,6 @@ export async function runKtxSetupAnthropicModelStep( !args.llmBackend && !args.anthropicApiKeyEnv && !args.anthropicApiKeyFile && - !args.llmModel && !args.vertexProject && !args.vertexLocation ) { @@ -1038,94 +823,74 @@ export async function runKtxSetupAnthropicModelStep( return { status: vertex.status, projectDir: args.projectDir }; } - const model = await chooseVertexModel(backendArgs, io, deps); - if (model.status === 'back' && !backendArgs.vertexLocation) { + const preset = presetForBackend('vertex'); + const validation = await validatePresetModels( + preset, + async (model) => + runLlmHealthCheckWithProgress( + buildVertexHealthConfig(vertex.values, model), + 'Vertex AI', + model, + healthCheck, + deps, + ), + io, + ); + if (validation.status !== 'ready') { + io.stderr.write( + `Vertex AI Anthropic model health check failed: ${formatVertexHealthFailure(validation.message, vertex.values)}\n`, + ); + if (args.inputMode === 'disabled') { + return { status: 'failed', projectDir: args.projectDir }; + } + io.stderr.write('Choose a different Vertex AI project or location, or Back.\n'); attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend); continue; } - if (model.status === 'invalid-credential') { - return { status: 'failed', projectDir: args.projectDir }; - } - if (model.status !== 'ready') { - return { status: model.status, projectDir: args.projectDir }; - } - const health = await runLlmHealthCheckWithProgress( - buildVertexHealthConfig(vertex.values, model.model), - 'Vertex AI', - model.model, - healthCheck, - deps, - ); - if (health.ok) { - await persistLlmConfig(args.projectDir, { backend: 'vertex', vertex: vertex.refs }, model.model); - io.stdout.write(`│ LLM ready: yes (${model.model})\n`); - return { status: 'ready', projectDir: args.projectDir }; - } - - io.stderr.write(`Vertex AI Anthropic model health check failed: ${formatVertexHealthFailure(health.message, vertex.values)}\n`); - if (args.inputMode === 'disabled') { - return { status: 'failed', projectDir: args.projectDir }; - } - io.stderr.write('Choose a different Vertex AI project, location, or model, or Back.\n'); - attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend); - continue; + await persistLlmConfig(args.projectDir, { backend: 'vertex', vertex: vertex.refs }, validation.models); + io.stdout.write(`│ LLM ready: yes (${validation.models.default})\n`); + return { status: 'ready', projectDir: args.projectDir }; } if (backendChoice.backend === 'claude-code') { - const model = await chooseClaudeCodeModel(backendArgs, deps); - if (model.status === 'back' && backendChoice.prompted) { - attemptArgs = buildInteractiveRetryArgs(args); - continue; - } - if (model.status === 'invalid-credential') { - return { status: 'failed', projectDir: args.projectDir }; - } - if (model.status !== 'ready') { - return { status: model.status, projectDir: args.projectDir }; - } + const preset = presetForBackend('claude-code'); const probe = deps.claudeCodeAuthProbe ?? runClaudeCodeAuthProbe; - const health = await probe({ projectDir: args.projectDir, model: model.model, env: deps.env ?? process.env }); - if (!health.ok) { - io.stderr.write(`${health.message}\n`); + const validation = await validatePresetModels( + preset, + async (model) => probe({ projectDir: args.projectDir, model, env: deps.env ?? process.env }), + io, + ); + if (validation.status !== 'ready') { + io.stderr.write(`${validation.message}\n`); return { status: 'failed', projectDir: args.projectDir }; } const warning = formatClaudeCodePromptCachingWarning( ignoredClaudeCodePromptCachingFields( - buildProjectLlmConfig(project.config.llm, { backend: 'claude-code' }, model.model), + buildProjectLlmConfig(project.config.llm, { backend: 'claude-code' }, validation.models), ), ); if (warning) { io.stderr.write(`${warning}\n`); } - await persistLlmConfig(args.projectDir, { backend: 'claude-code' }, model.model); - io.stdout.write(`│ LLM ready: yes (${model.model})\n`); + await persistLlmConfig(args.projectDir, { backend: 'claude-code' }, validation.models); + io.stdout.write(`│ LLM ready: yes (${validation.models.default})\n`); return { status: 'ready', projectDir: args.projectDir }; } if (backendChoice.backend === 'codex') { - const model = await chooseCodexModel(backendArgs, deps); - if (model.status === 'back' && backendChoice.prompted) { - attemptArgs = buildInteractiveRetryArgs(args); - continue; - } - if (model.status === 'invalid-credential') { - return { status: 'failed', projectDir: args.projectDir }; - } - if (model.status !== 'ready') { - return { status: model.status, projectDir: args.projectDir }; - } + const preset = presetForBackend('codex'); const probe = deps.codexAuthProbe ?? runCodexAuthProbe; - const health = await probe({ projectDir: args.projectDir, model: model.model }); - if (!health.ok) { - io.stderr.write(`${health.message}\n`); + const validation = await validatePresetModels(preset, async (model) => probe({ projectDir: args.projectDir, model }), io); + if (validation.status !== 'ready') { + io.stderr.write(`${validation.message}\n`); return { status: 'failed', projectDir: args.projectDir }; } // Prefix the clack gutter so the warning sits inside the setup frame // instead of breaking out of it; kept on stderr for scripted runs. io.stderr.write(`│ ${formatCodexIsolationWarning()}\n`); - await persistLlmConfig(args.projectDir, { backend: 'codex' }, model.model); - io.stdout.write(`│ LLM ready: yes (codex, ${model.model})\n`); + await persistLlmConfig(args.projectDir, { backend: 'codex' }, validation.models); + io.stdout.write(`│ LLM ready: yes (codex, ${validation.models.default})\n`); return { status: 'ready', projectDir: args.projectDir }; } @@ -1138,8 +903,21 @@ export async function runKtxSetupAnthropicModelStep( return { status: credential.status, projectDir: args.projectDir }; } - const model = await chooseModel(backendArgs, credential.value, io, deps); - if (model.status === 'invalid-credential') { + const preset = presetForBackend('anthropic'); + const validation = await validatePresetModels( + preset, + async (model) => + runLlmHealthCheckWithProgress( + buildAnthropicHealthConfig(credential.value, model), + 'Anthropic API', + model, + healthCheck, + deps, + ), + io, + ); + if (validation.status !== 'ready') { + io.stderr.write(`Anthropic model health check failed: ${validation.message}\n`); if (args.inputMode === 'disabled') { return { status: 'failed', projectDir: args.projectDir }; } @@ -1147,32 +925,9 @@ export async function runKtxSetupAnthropicModelStep( attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend); continue; } - if (model.status === 'back' && !backendArgs.anthropicApiKeyEnv && !backendArgs.anthropicApiKeyFile) { - attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend); - continue; - } - if (model.status !== 'ready') { - return { status: model.status, projectDir: args.projectDir }; - } - const health = await runLlmHealthCheckWithProgress( - buildAnthropicHealthConfig(credential.value, model.model), - 'Anthropic API', - model.model, - healthCheck, - deps, - ); - if (health.ok) { - await persistLlmConfig(args.projectDir, { backend: 'anthropic', credentialRef: credential.ref }, model.model); - io.stdout.write(`│ LLM ready: yes (${model.model})\n`); - return { status: 'ready', projectDir: args.projectDir }; - } - - io.stderr.write(`Anthropic model health check failed: ${health.message}\n`); - if (args.inputMode === 'disabled') { - return { status: 'failed', projectDir: args.projectDir }; - } - io.stderr.write('Choose a different credential source or model, or Back.\n'); - attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend); + await persistLlmConfig(args.projectDir, { backend: 'anthropic', credentialRef: credential.ref }, validation.models); + io.stdout.write(`│ LLM ready: yes (${validation.models.default})\n`); + return { status: 'ready', projectDir: args.projectDir }; } } diff --git a/packages/cli/test/setup-models.test.ts b/packages/cli/test/setup-models.test.ts index dedf03bd..f09691e0 100644 --- a/packages/cli/test/setup-models.test.ts +++ b/packages/cli/test/setup-models.test.ts @@ -6,8 +6,6 @@ import { parseKtxProjectConfig } from '../src/context/project/config.js'; import { readKtxSetupState, writeKtxSetupState } from '../src/context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { - BUNDLED_ANTHROPIC_MODELS, - fetchAnthropicModels, type KtxSetupModelPromptAdapter, runKtxSetupAnthropicModelStep, } from '../src/setup-models.js'; @@ -97,6 +95,33 @@ function makePromptAdapter(options: { }; } +const anthropicPreset = { + default: 'claude-sonnet-4-6', + triage: 'claude-haiku-4-5', + candidateExtraction: 'claude-sonnet-4-6', + curator: 'claude-opus-4-7', + reconcile: 'claude-opus-4-7', + repair: 'claude-haiku-4-5', +}; + +const claudeCodePreset = { + default: 'sonnet', + triage: 'haiku', + candidateExtraction: 'sonnet', + curator: 'opus', + reconcile: 'opus', + repair: 'haiku', +}; + +const codexPreset = { + default: 'gpt-5.5', + triage: 'gpt-5.5', + candidateExtraction: 'gpt-5.5', + curator: 'gpt-5.5', + reconcile: 'gpt-5.5', + repair: 'gpt-5.5', +}; + describe('setup Anthropic model step', () => { let tempDir: string; @@ -109,66 +134,6 @@ describe('setup Anthropic model step', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('does not expose Claude Sonnet 4 or Claude Opus 4 as selectable Anthropic models', async () => { - const fetchModels = vi.fn( - async () => - new Response( - JSON.stringify({ - data: [ - { id: 'claude-sonnet-4', display_name: 'Claude Sonnet 4' }, - { id: 'claude-opus-4', display_name: 'Claude Opus 4' }, - { id: 'claude-sonnet-4-6', display_name: 'Claude Sonnet 4.6' }, - { id: 'claude-opus-4-6', display_name: 'Claude Opus 4.6' }, - { id: 'claude-haiku-4-5', display_name: 'Claude Haiku 4.5' }, - ], - }), - { status: 200 }, - ), - ); - - await expect(fetchAnthropicModels('sk-ant-test', fetchModels)).resolves.toEqual([ // pragma: allowlist secret - { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }, - { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, - { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, - ]); - expect(BUNDLED_ANTHROPIC_MODELS.map((model) => model.id)).not.toEqual( - expect.arrayContaining(['claude-sonnet-4', 'claude-opus-4']), - ); - }); - - it('filters Claude Sonnet 4 and Claude Opus 4 from Anthropic model prompt choices', async () => { - const prompts = makePromptAdapter({ selectValues: ['env', 'back', 'back'] }); - - await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - makeIo().io, - { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => [ - { id: 'claude-sonnet-4', label: 'Claude Sonnet 4', recommended: true }, - { id: 'claude-opus-4', label: 'Claude Opus 4', recommended: false }, - { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }, - { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, - { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, - ]), - }, - ); - - expect(prompts.autocomplete).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Anthropic model should KTX use?'), - options: [ - { value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', hint: 'recommended' }, - { value: 'claude-opus-4-6', label: 'Claude Opus 4.6' }, - { value: 'claude-haiku-4-5', label: 'Claude Haiku 4.5' }, - { value: 'manual', label: 'Enter a model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }), - ); - }); - it('offers Anthropic provider paths in the preferred order', async () => { const prompts = makePromptAdapter({ providerChoice: 'back' }); @@ -212,9 +177,38 @@ describe('setup Anthropic model step', () => { const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm).toMatchObject({ provider: { backend: 'claude-code' }, - models: { default: 'sonnet' }, + models: claudeCodePreset, }); - expect(authProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'sonnet' })); + expect(authProbe).toHaveBeenCalledTimes(3); + expect(authProbe).toHaveBeenNthCalledWith(1, expect.objectContaining({ projectDir: tempDir, model: 'sonnet' })); + expect(authProbe).toHaveBeenNthCalledWith(2, expect.objectContaining({ projectDir: tempDir, model: 'haiku' })); + expect(authProbe).toHaveBeenNthCalledWith(3, expect.objectContaining({ projectDir: tempDir, model: 'opus' })); + }); + + it('does not prompt for a Claude Code model during interactive setup', async () => { + const io = makeIo(); + const prompts = makePromptAdapter({ selectValues: ['claude-code'] }); + const authProbe = vi.fn(async () => ({ ok: true as const })); + + const result = await runKtxSetupAnthropicModelStep( + { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, + io.io, + { prompts, claudeCodeAuthProbe: authProbe }, + ); + + expect(result.status).toBe('ready'); + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Which LLM provider should KTX use?'), + }), + ); + expect(prompts.select).not.toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Which Claude Code model should KTX use?'), + }), + ); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.llm.models).toMatchObject(claudeCodePreset); }); it('configures Codex backend and validates local auth', async () => { @@ -226,7 +220,6 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', llmBackend: 'codex', - llmModel: 'gpt-5.5', skipLlm: false, }, io.io, @@ -237,8 +230,9 @@ describe('setup Anthropic model step', () => { const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm).toMatchObject({ provider: { backend: 'codex' }, - models: { default: 'gpt-5.5' }, + models: codexPreset, }); + expect(codexAuthProbe).toHaveBeenCalledTimes(1); expect(codexAuthProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'gpt-5.5' })); // The warning carries the clack gutter so it renders inside the setup frame. expect(io.stderr()).toContain('│ Codex backend isolation is limited'); @@ -264,70 +258,12 @@ describe('setup Anthropic model step', () => { const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm).toMatchObject({ provider: { backend: 'codex' }, - models: { default: 'gpt-5.5' }, + models: codexPreset, }); + expect(codexAuthProbe).toHaveBeenCalledTimes(1); expect(codexAuthProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'gpt-5.5' })); }); - it('offers the curated Codex models during interactive setup', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['codex', 'gpt-5.5'] }); - const codexAuthProbe = vi.fn(async () => ({ ok: true as const })); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - io.io, - { prompts, codexAuthProbe }, - ); - - expect(result.status).toBe('ready'); - expect(prompts.select).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Codex model should KTX use?'), - options: [ - { value: 'gpt-5.5', label: 'GPT-5.5', hint: 'recommended' }, - { value: 'gpt-5.4', label: 'GPT-5.4' }, - { value: 'gpt-5.4-mini', label: 'GPT-5.4 mini' }, - { value: 'manual', label: 'Enter a Codex model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }), - ); - expect(codexAuthProbe).toHaveBeenCalledWith(expect.objectContaining({ model: 'gpt-5.5' })); - }); - - it('prompts for the Claude Code model during interactive setup', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['claude-code', 'opus'] }); - const authProbe = vi.fn(async () => ({ ok: true as const })); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - io.io, - { prompts, claudeCodeAuthProbe: authProbe }, - ); - - expect(result.status).toBe('ready'); - expect(prompts.select).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Claude Code model should KTX use?'), - options: [ - { value: 'sonnet', label: 'Claude Sonnet', hint: 'recommended' }, - { value: 'opus', label: 'Claude Opus' }, - { value: 'haiku', label: 'Claude Haiku' }, - { value: 'manual', label: 'Enter a Claude Code model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }), - ); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.llm).toMatchObject({ - provider: { backend: 'claude-code' }, - models: { default: 'opus' }, - }); - expect(authProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'opus' })); - }); - it('warns during Claude Code setup when existing prompt-caching fields will be ignored', async () => { await writeFile( join(tempDir, 'ktx.yaml'), @@ -392,7 +328,6 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -410,7 +345,7 @@ describe('setup Anthropic model step', () => { backend: 'anthropic', anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret }, - models: { default: 'claude-sonnet-4-6' }, + models: anthropicPreset, promptCaching: { enabled: true }, }); expect(config.scan.enrichment.mode).toBe('llm'); @@ -419,11 +354,62 @@ describe('setup Anthropic model step', () => { expect(spinnerEvents).toEqual([ 'start:Checking Anthropic API LLM (claude-sonnet-4-6).', 'stop:LLM test passed (Anthropic API, claude-sonnet-4-6)', + 'start:Checking Anthropic API LLM (claude-haiku-4-5).', + 'stop:LLM test passed (Anthropic API, claude-haiku-4-5)', + 'start:Checking Anthropic API LLM (claude-opus-4-7).', + 'stop:LLM test passed (Anthropic API, claude-opus-4-7)', ]); expect(io.stdout()).toContain('LLM ready: yes'); expect(io.stdout()).not.toContain('sk-ant-test'); }); + it('degrades unavailable Anthropic non-anchor models to the anchor before persisting', async () => { + const io = makeIo(); + const { events: spinnerEvents, spinner } = makeSpinnerEvents(); + const healthCheck = vi + .fn() + .mockResolvedValueOnce({ ok: true as const }) + .mockResolvedValueOnce({ ok: false as const, message: 'model not enabled' }) + .mockResolvedValueOnce({ ok: true as const }); + + const result = await runKtxSetupAnthropicModelStep( + { + projectDir: tempDir, + inputMode: 'disabled', + anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret + skipLlm: false, + }, + io.io, + { + env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret + healthCheck, + spinner, + }, + ); + + expect(result.status).toBe('ready'); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.llm.models).toMatchObject({ + default: 'claude-sonnet-4-6', + triage: 'claude-sonnet-4-6', + candidateExtraction: 'claude-sonnet-4-6', + curator: 'claude-opus-4-7', + reconcile: 'claude-opus-4-7', + repair: 'claude-sonnet-4-6', + }); + expect(io.stderr()).toContain( + 'LLM model claude-haiku-4-5 is unavailable for triage, repair; using claude-sonnet-4-6 for those roles.', + ); + expect(spinnerEvents).toEqual([ + 'start:Checking Anthropic API LLM (claude-sonnet-4-6).', + 'stop:LLM test passed (Anthropic API, claude-sonnet-4-6)', + 'start:Checking Anthropic API LLM (claude-haiku-4-5).', + 'error:LLM test failed', + 'start:Checking Anthropic API LLM (claude-opus-4-7).', + 'stop:LLM test passed (Anthropic API, claude-opus-4-7)', + ]); + }); + it('configures Vertex AI provider, selected model, prompt caching, and llm completion state', async () => { const io = makeIo(); const healthCheck = vi.fn(async () => ({ ok: true as const })); @@ -436,7 +422,6 @@ describe('setup Anthropic model step', () => { llmBackend: 'vertex', vertexProject: 'local-gcp-project', vertexLocation: 'us-east5', - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -444,19 +429,31 @@ describe('setup Anthropic model step', () => { ); expect(result.status).toBe('ready'); - expect(healthCheck).toHaveBeenCalledWith({ + expect(healthCheck).toHaveBeenNthCalledWith(1, { backend: 'vertex', vertex: { project: 'local-gcp-project', location: 'us-east5' }, modelSlots: { default: 'claude-sonnet-4-6' }, promptCaching: { enabled: true, vertexFallbackTo5m: true }, }); + expect(healthCheck).toHaveBeenNthCalledWith(2, { + backend: 'vertex', + vertex: { project: 'local-gcp-project', location: 'us-east5' }, + modelSlots: { default: 'claude-haiku-4-5' }, + promptCaching: { enabled: true, vertexFallbackTo5m: true }, + }); + expect(healthCheck).toHaveBeenNthCalledWith(3, { + backend: 'vertex', + vertex: { project: 'local-gcp-project', location: 'us-east5' }, + modelSlots: { default: 'claude-opus-4-7' }, + promptCaching: { enabled: true, vertexFallbackTo5m: true }, + }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm).toMatchObject({ provider: { backend: 'vertex', vertex: { project: 'local-gcp-project', location: 'us-east5' }, }, - models: { default: 'claude-sonnet-4-6' }, + models: anthropicPreset, promptCaching: { enabled: true, vertexFallbackTo5m: true }, }); expect(config.scan.enrichment.mode).toBe('llm'); @@ -465,13 +462,17 @@ describe('setup Anthropic model step', () => { expect(spinnerEvents).toEqual([ 'start:Checking Vertex AI LLM (claude-sonnet-4-6).', 'stop:LLM test passed (Vertex AI, claude-sonnet-4-6)', + 'start:Checking Vertex AI LLM (claude-haiku-4-5).', + 'stop:LLM test passed (Vertex AI, claude-haiku-4-5)', + 'start:Checking Vertex AI LLM (claude-opus-4-7).', + 'stop:LLM test passed (Vertex AI, claude-opus-4-7)', ]); expect(io.stdout()).toContain('LLM ready: yes (claude-sonnet-4-6)'); }); it('uses existing Vertex AI credentials without an extra auth choice', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'local-gcp-project', 'claude-sonnet-4-6'] }); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'local-gcp-project'] }); const readGcloudProject = vi.fn(async () => 'local-gcp-project'); const listGcloudProjects = vi.fn(async () => [ { projectId: 'local-gcp-project', name: 'Local project' }, @@ -511,22 +512,6 @@ describe('setup Anthropic model step', () => { ], }), ); - expect(prompts.autocomplete).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Anthropic model should KTX use?'), - options: [ - { value: 'claude-opus-4-7', label: 'Claude Opus 4.7' }, - { value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6' }, - { value: 'claude-opus-4-6', label: 'Claude Opus 4.6' }, - { value: 'claude-opus-4-5', label: 'Claude Opus 4.5' }, - { value: 'claude-haiku-4-5', label: 'Claude Haiku 4.5' }, - { value: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5' }, - { value: 'claude-opus-4-1', label: 'Claude Opus 4.1' }, - { value: 'manual', label: 'Enter a model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }), - ); expect(healthCheck).toHaveBeenCalledWith({ backend: 'vertex', vertex: { project: 'local-gcp-project', location: 'us-east5' }, @@ -542,7 +527,7 @@ describe('setup Anthropic model step', () => { it('skips the Vertex AI auth choice when Application Default Credentials are the only option', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'local-gcp-project', 'claude-sonnet-4-6'] }); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'local-gcp-project'] }); const healthCheck = vi.fn(async () => ({ ok: true as const })); const result = await runKtxSetupAnthropicModelStep( @@ -578,7 +563,7 @@ describe('setup Anthropic model step', () => { it('lets users choose a different visible gcloud project for Vertex AI', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'other-gcp-project', 'claude-sonnet-4-6'] }); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'other-gcp-project'] }); const healthCheck = vi.fn(async () => ({ ok: true as const })); const result = await runKtxSetupAnthropicModelStep( @@ -612,10 +597,7 @@ describe('setup Anthropic model step', () => { it('allows manual Vertex AI project entry when gcloud project listing is empty', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ - selectValues: ['vertex', 'manual', 'claude-sonnet-4-6'], - textValues: ['manual-gcp-project'], - }); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'manual'], textValues: ['manual-gcp-project'] }); const healthCheck = vi.fn(async () => ({ ok: true as const })); const result = await runKtxSetupAnthropicModelStep( @@ -654,7 +636,7 @@ describe('setup Anthropic model step', () => { it('lets users retry Vertex AI project listing after gcloud auth fails', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'retry', 'other-gcp-project', 'claude-sonnet-4-6'] }); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'retry', 'other-gcp-project'] }); const listGcloudProjects = vi .fn() .mockRejectedValueOnce(new Error('Reauthentication failed. cannot prompt during non-interactive execution.')) @@ -743,7 +725,6 @@ describe('setup Anthropic model step', () => { llmBackend: 'vertex', vertexProject: 'kaelio-orbit-looker-20260430', vertexLocation: 'us-east5', - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -771,7 +752,6 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyFile: secretPath, - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -779,19 +759,34 @@ describe('setup Anthropic model step', () => { ); expect(result.status).toBe('ready'); - expect(healthCheck).toHaveBeenCalledWith( + expect(healthCheck).toHaveBeenNthCalledWith( + 1, expect.objectContaining({ anthropic: { apiKey: 'sk-ant-file' }, // pragma: allowlist secret modelSlots: { default: 'claude-sonnet-4-6' }, }), ); + expect(healthCheck).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + anthropic: { apiKey: 'sk-ant-file' }, // pragma: allowlist secret + modelSlots: { default: 'claude-haiku-4-5' }, + }), + ); + expect(healthCheck).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ + anthropic: { apiKey: 'sk-ant-file' }, // pragma: allowlist secret + modelSlots: { default: 'claude-opus-4-7' }, + }), + ); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm).toMatchObject({ provider: { backend: 'anthropic', anthropic: { api_key: `file:${secretPath}` }, // pragma: allowlist secret }, - models: { default: 'claude-sonnet-4-6' }, + models: anthropicPreset, }); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); @@ -808,7 +803,6 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyFile: missingSecretPath, - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -835,32 +829,10 @@ describe('setup Anthropic model step', () => { expect(io.stderr()).not.toContain('--skip-llm'); }); - it('does not recommend skipping when non-interactive setup is missing an LLM model', async () => { - const io = makeIo(); - const healthCheck = vi.fn(async () => ({ ok: true as const })); - - const result = await runKtxSetupAnthropicModelStep( - { - projectDir: tempDir, - inputMode: 'disabled', - anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - skipLlm: false, - }, - io.io, - { env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, healthCheck }, // pragma: allowlist secret - ); - - expect(result.status).toBe('missing-input'); - expect(healthCheck).not.toHaveBeenCalled(); - expect(io.stderr()).toContain('Missing LLM model: pass --llm-model.'); - expect(io.stderr()).not.toContain('--skip-llm'); - }); - it('writes pasted keys to .ktx/secrets and never prints the key', async () => { const io = makeIo(); const prompts = makePromptAdapter({ credentialChoice: 'paste', - modelChoice: 'claude-sonnet-4-6', passwordValue: 'sk-ant-pasted', // pragma: allowlist secret }); @@ -870,7 +842,6 @@ describe('setup Anthropic model step', () => { { prompts, env: {}, - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), healthCheck: vi.fn(async () => ({ ok: true as const })), }, ); @@ -888,7 +859,7 @@ describe('setup Anthropic model step', () => { it('opens pasted key entry directly and tells users Escape goes back', async () => { const prompts = makePromptAdapter({ - selectValues: ['paste', 'claude-sonnet-4-6'], + selectValues: ['paste'], passwordValue: 'sk-ant-pasted', // pragma: allowlist secret }); @@ -898,7 +869,6 @@ describe('setup Anthropic model step', () => { { prompts, env: {}, - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), healthCheck: vi.fn(async () => ({ ok: true as const })), }, ); @@ -956,142 +926,6 @@ describe('setup Anthropic model step', () => { expect(io.stdout()).not.toContain('KTX uses the key'); }); - it('does not offer skipping while choosing an Anthropic model', async () => { - const prompts = makePromptAdapter({ selectValues: ['env', 'back', 'back'] }); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - makeIo().io, - { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), - }, - ); - - expect(result.status).toBe('back'); - expect(prompts.autocomplete).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Anthropic model should KTX use?'), - options: expect.not.arrayContaining([expect.objectContaining({ value: 'skip' })]), - }), - ); - }); - - it('explains why KTX asks for an Anthropic model', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ credentialChoice: 'env', modelChoice: 'claude-sonnet-4-6' }); - const expectedPromptMessage = [ - 'Which Anthropic model should KTX use?', - '', - [ - 'KTX uses this as the default model for ingest agents that turn schemas, SQL, BI metadata, and docs', - 'into semantic-layer sources and wiki context.', - ].join(' '), - ].join('\n'); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - io.io, - { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), - healthCheck: vi.fn(async () => ({ ok: true as const })), - }, - ); - - expect(result.status).toBe('ready'); - expect(prompts.autocomplete).toHaveBeenCalledWith( - expect.objectContaining({ - message: expectedPromptMessage, - }), - ); - expect(io.stdout()).not.toContain('KTX uses this as the default model'); - expect(io.stdout()).not.toContain('Setup verifies the selected model now'); - }); - - it('uses the bundled fallback registry when live discovery fails', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ credentialChoice: 'env', modelChoice: 'claude-sonnet-4-6' }); - - await expect( - runKtxSetupAnthropicModelStep({ projectDir: tempDir, inputMode: 'auto', skipLlm: false }, io.io, { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => { - throw new Error('network unavailable'); - }), - healthCheck: vi.fn(async () => ({ ok: true as const })), - }), - ).resolves.toMatchObject({ status: 'ready' }); - - expect(io.stderr()).toContain('Could not fetch live Anthropic models. Showing bundled defaults.'); - }); - - it('shows bundled model choices when live discovery fails', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['env', 'manual'], textValues: [''] }); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - io.io, - { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => { - throw new Error('network unavailable'); - }), - healthCheck: vi.fn(async () => ({ ok: true as const })), - }, - ); - - expect(result.status).toBe('missing-input'); - expect(BUNDLED_ANTHROPIC_MODELS.length).toBeGreaterThan(0); - expect(prompts.autocomplete).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Anthropic model should KTX use?'), - options: expect.arrayContaining([ - { value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', hint: 'recommended' }, - ]), - }), - ); - expect(prompts.text).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'Anthropic model ID\n│ Press Escape to go back.\n│', - placeholder: 'claude-sonnet-4-6', - }), - ); - }); - - it('reports invalid Anthropic API keys during live discovery instead of showing bundled defaults', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['env', 'back'] }); - const fetchModels = vi.fn( - async () => new Response(JSON.stringify({ error: { message: 'invalid x-api-key' } }), { status: 401 }), - ); - const healthCheck = vi.fn(async () => ({ ok: true as const })); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - io.io, - { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-invalid' }, // pragma: allowlist secret - fetch: fetchModels, - healthCheck, - }, - ); - - expect(result.status).toBe('back'); - expect(fetchModels).toHaveBeenCalledTimes(1); - expect(healthCheck).not.toHaveBeenCalled(); - expect(io.stderr()).toContain('Anthropic API key is invalid or unauthorized'); - expect(io.stderr()).toContain('Choose a different credential source or Back.'); - expect(io.stderr()).not.toContain('Could not fetch live Anthropic models. Showing bundled defaults.'); - expect(io.stderr()).not.toContain('sk-ant-invalid'); - }); - it('does not persist llm completion when the health check fails', async () => { const io = makeIo(); const result = await runKtxSetupAnthropicModelStep( @@ -1099,7 +933,6 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -1117,12 +950,12 @@ describe('setup Anthropic model step', () => { it('re-prompts after an interactive health-check failure and saves after retry success', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ - selectValues: ['env', 'claude-haiku-3-5', 'env', 'claude-sonnet-4-6'], - }); + const prompts = makePromptAdapter({ selectValues: ['env', 'env'] }); const healthCheck = vi .fn() .mockResolvedValueOnce({ ok: false as const, message: 'model not found' }) + .mockResolvedValueOnce({ ok: true as const }) + .mockResolvedValueOnce({ ok: true as const }) .mockResolvedValueOnce({ ok: true as const }); const result = await runKtxSetupAnthropicModelStep( @@ -1131,22 +964,22 @@ describe('setup Anthropic model step', () => { { prompts, env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => [ - { id: 'claude-haiku-3-5', label: 'Claude Haiku 3.5', recommended: false }, - { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }, - ]), healthCheck, }, ); expect(result.status).toBe('ready'); - expect(healthCheck).toHaveBeenCalledTimes(2); + expect(healthCheck).toHaveBeenCalledTimes(4); expect(prompts.select).toHaveBeenCalledTimes(3); - expect(prompts.autocomplete).toHaveBeenCalledTimes(2); + expect(prompts.autocomplete).not.toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Which Anthropic model should KTX use?'), + }), + ); expect(io.stderr()).toContain('Anthropic model health check failed: model not found'); - expect(io.stderr()).toContain('Choose a different credential source or model, or Back.'); + expect(io.stderr()).toContain('Choose a different credential source or Back.'); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.llm.models.default).toBe('claude-sonnet-4-6'); + expect(config.llm.models).toMatchObject(anthropicPreset); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); expect(io.stderr()).not.toContain('sk-ant-test'); @@ -1175,39 +1008,8 @@ describe('setup Anthropic model step', () => { expect(config.llm.provider.backend).toBe('none'); }); - it('returns from model selection Back to credential selection instead of exiting setup', async () => { - const prompts = makePromptAdapter({ - selectValues: ['paste', 'back', 'back'], - passwordValue: 'sk-ant-pasted', // pragma: allowlist secret - }); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - makeIo().io, - { - prompts, - env: {}, - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), - healthCheck: vi.fn(async () => ({ ok: true as const })), - }, - ); - - expect(result.status).toBe('back'); - expect(prompts.select).toHaveBeenNthCalledWith( - 3, - expect.objectContaining({ - message: expect.stringContaining('How should KTX find your Anthropic API key?'), - }), - ); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.llm.provider.backend).toBe('none'); - }); - it('returns from pasted key entry Escape to credential selection and can use env credentials', async () => { - const prompts = makePromptAdapter({ - selectValues: ['paste', 'env', 'claude-sonnet-4-6'], - passwordValues: [undefined], - }); + const prompts = makePromptAdapter({ selectValues: ['paste', 'env'], passwordValues: [undefined] }); const result = await runKtxSetupAnthropicModelStep( { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, @@ -1215,7 +1017,6 @@ describe('setup Anthropic model step', () => { { prompts, env: { ANTHROPIC_API_KEY: 'sk-ant-env' }, // pragma: allowlist secret - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), healthCheck: vi.fn(async () => ({ ok: true as const })), }, );