From 2c18a62de447143e7d67169ea20aaeaf436c5567 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Mon, 8 Jun 2026 15:30:48 +0200 Subject: [PATCH] feat(setup): apply per-role LLM model presets, remove --llm-model (#268) * feat(setup): write per-role llm model presets * feat(setup): remove llm model setup flag * chore(setup): update llm preset guidance * docs(setup): document llm model presets * chore(release): sync uv.lock to 0.9.0 * fix(cli): make sl query --execute work on secret-backed connections sl query --execute used a parallel SQL executor (createDefaultLocalQueryExecutor) that passed connection.url verbatim into pg, so file:/env: secret references failed with "SASL: SCRAM-SERVER-FIRST-MESSAGE: client password must be a string". Collapse onto the connector-based executor already used by MCP and ingest (createKtxCliIngestQueryExecutor), which resolves secret references and supports every driver. Delete the now-dead local/postgres/sqlite query executors, their tests, and the orphaned hasLocalQueryExecutor driver flag. * docs(agents): require one implementation per capability Add a design-reasoning default and a matching self-check question telling agents to route callers through a single shared implementation of a capability rather than forking a parallel one, and to fix the shared layer rather than patch one branch. Encodes the lesson from a divergent SQL-execution-path bug, stated generally. CLAUDE.md is a symlink to AGENTS.md, so both agent-instruction files are covered. --- AGENTS.md | 17 + .../content/docs/cli-reference/ktx-setup.mdx | 18 +- .../content/docs/configuration/ktx-yaml.mdx | 14 + .../content/docs/guides/building-context.mdx | 2 +- .../content/docs/guides/llm-configuration.mdx | 21 +- packages/cli/src/commands/setup-commands.ts | 4 - .../cli/src/context/connections/drivers.ts | 8 - .../connections/local-query-executor.ts | 59 -- .../connections/postgres-query-executor.ts | 78 --- .../src/context/connections/query-executor.ts | 2 +- .../connections/sqlite-query-executor.ts | 92 --- .../context/ingest/local-bundle-runtime.ts | 4 +- packages/cli/src/setup-models.ts | 571 +++++------------- packages/cli/src/setup.ts | 2 - packages/cli/src/sl.ts | 6 +- .../test/context/connections/drivers.test.ts | 2 - .../connections/local-query-executor.test.ts | 59 -- .../postgres-query-executor.test.ts | 103 ---- .../connections/sqlite-query-executor.test.ts | 139 ----- .../ingest/local-bundle-runtime.test.ts | 4 +- packages/cli/test/index.test.ts | 24 +- packages/cli/test/ingest.test.ts | 6 +- packages/cli/test/setup-models.test.ts | 533 +++++----------- packages/cli/test/setup.test.ts | 7 - scripts/codex-backend-live-smoke.mjs | 13 +- 25 files changed, 404 insertions(+), 1384 deletions(-) delete mode 100644 packages/cli/src/context/connections/local-query-executor.ts delete mode 100644 packages/cli/src/context/connections/postgres-query-executor.ts delete mode 100644 packages/cli/src/context/connections/sqlite-query-executor.ts delete mode 100644 packages/cli/test/context/connections/local-query-executor.test.ts delete mode 100644 packages/cli/test/context/connections/postgres-query-executor.test.ts delete mode 100644 packages/cli/test/context/connections/sqlite-query-executor.test.ts diff --git a/AGENTS.md b/AGENTS.md index ec715364..2c61a7e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -192,6 +192,19 @@ autonomously — without being asked the leading question — is the bar. next stack. The only acceptable static patterns are genuinely universal invariants (e.g. DB-engine system catalogs) and ktx's own self-emitted signatures. +- **MUST**: Give each capability one implementation and route every caller + through it. When some behavior — running a query, resolving a credential or + config reference, authenticating, selecting a dialect, loading config — + already has a working implementation that some call sites use, make new or + divergent call sites depend on that path instead of standing up a second one. + Parallel implementations of one capability drift apart silently: a fix, a + newly supported input, or an added case lands on one path and not the other, + so one entry point (a CLI command, an MCP tool, an ingest stage) succeeds + while another fails on the same input. When two paths already do the same + job, collapse onto the shared one and delete the duplicate instead of + keeping both. When fixing a defect that lives on one path, fix the shared + implementation; do not patch the symptom on a forked branch, which preserves + the divergence you set out to remove. - **SHOULD**: Before inventing an abstraction or hand-rolling structural logic, search for what already exists and reuse it — the codebase's canonical representation (a structured ref/key type) instead of a parallel string scheme, @@ -212,6 +225,10 @@ Before presenting a design, answer these explicitly: instead of building or parsing my own? 5. Am I discarding the better option on a weak or misapplied constraint (one-time vs recurring cost, "more surface area", "more work now")? +6. Does another entry point already perform this operation through a shared + implementation? If so, am I routing through that path instead of forking a + parallel one — and if I'm fixing a bug, am I fixing the shared layer rather + than one branch? A user question that nudges toward an alternative ("would X help?", "should I always do Y?", "will you hardcode Z?") is a signal that a better option exists. diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index 0e6cb57c..8ae4469d 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -54,7 +54,6 @@ prompts. | `--llm-backend ` | LLM backend: `anthropic`, `vertex`, `claude-code`, or `codex` | | `--llm-backend claude-code` | Use the local Claude Code session for **ktx** LLM calls | | `--llm-backend codex` | Use local Codex authentication for **ktx** LLM calls | -| `--llm-model ` | LLM model ID or backend model alias to validate and save | | `--anthropic-api-key-env ` | Environment variable containing the Anthropic API key | | `--anthropic-api-key-file ` | File containing the Anthropic API key | | `--vertex-project ` | Vertex AI project ID, `env:NAME`, or `file:/path` reference | @@ -64,13 +63,9 @@ prompts. Choose only one Anthropic credential source. Anthropic credential flags are only valid with the Anthropic backend; Vertex flags are only valid with the Vertex backend. The `claude-code` and `codex` backends use local authentication instead -of Anthropic API key or Vertex flags. For Claude Code, `--llm-model` accepts -`sonnet`, `opus`, `haiku`, or a full Claude model ID. For Codex, `--llm-model` -accepts `codex`, `default`, or a `gpt-*` / `codex-*` model ID such as -`gpt-5.5`; any other value is rejected before the auth probe. Run `codex` to -see the models available to your login, and pick a `gpt-*` / `codex-*` id from -that list. Note that `*-codex` API-billing model IDs (for example -`gpt-5.3-codex`) are not available to ChatGPT-subscription logins. +of Anthropic API key or Vertex flags. After you choose a backend, `ktx setup` +writes that backend's per-role model preset to `ktx.yaml`. To change a model, +edit the matching `llm.models.` value in `ktx.yaml`. ### Embeddings @@ -198,14 +193,13 @@ ktx setup # Run setup for a specific project directory ktx setup --project-dir ./analytics -# Use Claude Code with Opus for ktx LLM calls +# Use Claude Code for ktx LLM calls ktx setup \ --project-dir ./analytics \ - --llm-backend claude-code \ - --llm-model opus + --llm-backend claude-code # Configure **ktx** to use local Codex authentication for LLM work -ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input +ktx setup --llm-backend codex --no-input ``` When you choose `--llm-backend codex`, setup prints a warning if the public diff --git a/docs-site/content/docs/configuration/ktx-yaml.mdx b/docs-site/content/docs/configuration/ktx-yaml.mdx index 831e678a..db74ffa7 100644 --- a/docs-site/content/docs/configuration/ktx-yaml.mdx +++ b/docs-site/content/docs/configuration/ktx-yaml.mdx @@ -377,6 +377,10 @@ llm: models: default: claude-sonnet-4-6 triage: claude-haiku-4-5 + candidateExtraction: claude-sonnet-4-6 + curator: claude-opus-4-7 + reconcile: claude-opus-4-7 + repair: claude-haiku-4-5 promptCaching: enabled: true systemTtl: 1h @@ -404,6 +408,11 @@ llm: backend: codex models: default: gpt-5.5 + triage: gpt-5.5 + candidateExtraction: gpt-5.5 + curator: gpt-5.5 + reconcile: gpt-5.5 + repair: gpt-5.5 ``` ### Model roles @@ -643,6 +652,11 @@ llm: backend: claude-code models: default: sonnet + triage: haiku + candidateExtraction: sonnet + curator: opus + reconcile: opus + repair: haiku ingest: adapters: - live-database diff --git a/docs-site/content/docs/guides/building-context.mdx b/docs-site/content/docs/guides/building-context.mdx index 9bcf2659..24550c85 100644 --- a/docs-site/content/docs/guides/building-context.mdx +++ b/docs-site/content/docs/guides/building-context.mdx @@ -43,7 +43,7 @@ Local-auth backends keep provider credentials out of `ktx.yaml`: ```bash ktx setup --llm-backend claude-code --no-input -ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input +ktx setup --llm-backend codex --no-input ``` With `claude-code`, **ktx** agent loops can invoke only the **ktx** MCP tools diff --git a/docs-site/content/docs/guides/llm-configuration.mdx b/docs-site/content/docs/guides/llm-configuration.mdx index 71ab9d80..776cb275 100644 --- a/docs-site/content/docs/guides/llm-configuration.mdx +++ b/docs-site/content/docs/guides/llm-configuration.mdx @@ -30,19 +30,19 @@ llm: default: sonnet triage: haiku candidateExtraction: sonnet - curator: sonnet - reconcile: sonnet - repair: sonnet + curator: opus + reconcile: opus + repair: haiku ``` -During setup, choose the backend interactively or pass the model in automation: +During setup, choose the backend interactively or pass it in automation: ```bash -ktx setup --llm-backend claude-code --llm-model opus --no-input +ktx setup --llm-backend claude-code --no-input ``` -For Claude Code, `sonnet`, `opus`, and `haiku` map to **ktx** defaults. Full Claude -model IDs are also accepted. +Setup writes `sonnet`, `haiku`, and `opus` aliases into `llm.models`. You can +edit any role to another alias or a full Claude model ID after setup. `claude-code` exposes only **ktx** MCP tools for the current agent loop. SDK init metadata may still list host slash commands, skills, and subagents; **ktx** does not @@ -59,12 +59,17 @@ llm: backend: codex models: default: gpt-5.5 + triage: gpt-5.5 + candidateExtraction: gpt-5.5 + curator: gpt-5.5 + reconcile: gpt-5.5 + repair: gpt-5.5 ``` Configure it non-interactively: ```bash -ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input +ktx setup --llm-backend codex --no-input ``` This is separate from Codex agent-client setup. `ktx setup --agents --target diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index 0302e9ed..418b27f9 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -95,7 +95,6 @@ function shouldShowSetupEntryMenu( llmBackend?: KtxSetupLlmBackend; anthropicApiKeyEnv?: string; anthropicApiKeyFile?: string; - llmModel?: string; vertexProject?: string; vertexLocation?: string; skipLlm?: boolean; @@ -166,7 +165,6 @@ function shouldShowSetupEntryMenu( 'llmBackend', 'anthropicApiKeyEnv', 'anthropicApiKeyFile', - 'llmModel', 'vertexProject', 'vertexLocation', 'skipLlm', @@ -229,7 +227,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo .addOption( new Option('--anthropic-api-key-file ', 'File containing the Anthropic API key').hideHelp(), ) - .addOption(new Option('--llm-model ', 'LLM model ID or backend model alias').hideHelp()) .addOption(new Option('--vertex-project ', 'Google Vertex AI project ID, env:NAME, or file:/path').hideHelp()) .addOption(new Option('--vertex-location ', 'Google Vertex AI location, env:NAME, or file:/path').hideHelp()) .addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false)) @@ -423,7 +420,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo ...(options.llmBackend ? { llmBackend: options.llmBackend } : {}), ...(options.anthropicApiKeyEnv ? { anthropicApiKeyEnv: options.anthropicApiKeyEnv } : {}), ...(options.anthropicApiKeyFile ? { anthropicApiKeyFile: options.anthropicApiKeyFile } : {}), - ...(options.llmModel ? { llmModel: options.llmModel } : {}), ...(options.vertexProject ? { vertexProject: options.vertexProject } : {}), ...(options.vertexLocation ? { vertexLocation: options.vertexLocation } : {}), skipLlm: options.skipLlm === true, diff --git a/packages/cli/src/context/connections/drivers.ts b/packages/cli/src/context/connections/drivers.ts index 1b87984b..3fbeb058 100644 --- a/packages/cli/src/context/connections/drivers.ts +++ b/packages/cli/src/context/connections/drivers.ts @@ -17,7 +17,6 @@ export interface KtxDriverRegistration { readonly driver: KtxConnectionDriver; readonly scopeConfigKey: KtxScopeConfigKey | null; readonly hasHistoricSqlReader: boolean; - readonly hasLocalQueryExecutor: boolean; load(): Promise; } @@ -31,7 +30,6 @@ export const driverRegistrations: Record { const m = await import('../../connectors/bigquery/connector.js'); return { @@ -53,7 +51,6 @@ export const driverRegistrations: Record { const m = await import('../../connectors/clickhouse/connector.js'); return { @@ -75,7 +72,6 @@ export const driverRegistrations: Record { const m = await import('../../connectors/mysql/connector.js'); return { @@ -97,7 +93,6 @@ export const driverRegistrations: Record { const m = await import('../../connectors/postgres/connector.js'); return { @@ -119,7 +114,6 @@ export const driverRegistrations: Record { const m = await import('../../connectors/sqlite/connector.js'); return { @@ -141,7 +135,6 @@ export const driverRegistrations: Record { const m = await import('../../connectors/snowflake/connector.js'); return { @@ -163,7 +156,6 @@ export const driverRegistrations: Record { const m = await import('../../connectors/sqlserver/connector.js'); return { diff --git a/packages/cli/src/context/connections/local-query-executor.ts b/packages/cli/src/context/connections/local-query-executor.ts deleted file mode 100644 index 3a2e34c9..00000000 --- a/packages/cli/src/context/connections/local-query-executor.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { driverRegistrations, getDriverRegistration } from './drivers.js'; -import { createPostgresQueryExecutor } from './postgres-query-executor.js'; -import type { - KtxSqlQueryExecutionInput, - KtxSqlQueryExecutionResult, - KtxSqlQueryExecutorPort, -} from './query-executor.js'; -import { createSqliteQueryExecutor } from './sqlite-query-executor.js'; -import type { KtxConnectionDriver } from '../scan/types.js'; - -export interface DefaultLocalQueryExecutorOptions { - postgres?: KtxSqlQueryExecutorPort; - sqlite?: KtxSqlQueryExecutorPort; -} - -function driverFor(input: KtxSqlQueryExecutionInput): string { - return String(input.connection?.driver ?? '').toLowerCase(); -} - -function localExecutorMap( - options: DefaultLocalQueryExecutorOptions, -): Partial> { - const wiredExecutors: Partial> = { - postgres: options.postgres ?? createPostgresQueryExecutor(), - sqlite: options.sqlite ?? createSqliteQueryExecutor(), - }; - - const executors: Partial> = {}; - for (const registration of Object.values(driverRegistrations)) { - if (!registration.hasLocalQueryExecutor) continue; - const executor = wiredExecutors[registration.driver]; - if (executor) { - executors[registration.driver] = executor; - } - } - return executors; -} - -export function createDefaultLocalQueryExecutor(options: DefaultLocalQueryExecutorOptions = {}): KtxSqlQueryExecutorPort { - const executors = localExecutorMap(options); - - return { - async execute(input: KtxSqlQueryExecutionInput): Promise { - const driver = driverFor(input); - const registration = getDriverRegistration(driver); - if (!registration?.hasLocalQueryExecutor) { - throw new Error(`No local query executor is configured for driver "${input.connection?.driver ?? 'unknown'}".`); - } - - const executor = executors[registration.driver]; - if (!executor) { - throw new Error( - `Local query executor flag is enabled for driver "${registration.driver}", but no executor factory is wired.`, - ); - } - return executor.execute(input); - }, - }; -} diff --git a/packages/cli/src/context/connections/postgres-query-executor.ts b/packages/cli/src/context/connections/postgres-query-executor.ts deleted file mode 100644 index 842609f4..00000000 --- a/packages/cli/src/context/connections/postgres-query-executor.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { Client, type ClientConfig } from 'pg'; -import type { - KtxSqlQueryExecutionInput, - KtxSqlQueryExecutionResult, - KtxSqlQueryExecutorPort, -} from './query-executor.js'; -import { limitSqlForExecution } from './read-only-sql.js'; - -interface PgClientLike { - connect(): Promise; - query(input: string | { text: string; rowMode: 'array' }): Promise<{ - fields: Array<{ name: string }>; - rows: unknown[][]; - command: string; - rowCount: number | null; - }>; - end(): Promise; -} - -interface PostgresQueryExecutorOptions { - statementTimeoutMs?: number; - queryTimeoutMs?: number; - connectionTimeoutMs?: number; - clientFactory?: (config: ClientConfig) => PgClientLike; -} - -function connectionDriver(input: KtxSqlQueryExecutionInput): string { - return String(input.connection?.driver ?? '').toLowerCase(); -} - -function createDefaultClient(config: ClientConfig): PgClientLike { - return new Client(config); -} - -export function createPostgresQueryExecutor(options: PostgresQueryExecutorOptions = {}): KtxSqlQueryExecutorPort { - const clientFactory = options.clientFactory ?? createDefaultClient; - return { - async execute(input: KtxSqlQueryExecutionInput): Promise { - const driver = connectionDriver(input); - const connection = input.connection; - if (driver !== 'postgres') { - throw new Error(`Local Postgres execution cannot run driver "${connection?.driver ?? 'unknown'}".`); - } - if (typeof connection?.url !== 'string' || connection.url.trim().length === 0) { - throw new Error(`Local Postgres execution requires connections.${input.connectionId}.url.`); - } - - const client = clientFactory({ - connectionString: connection.url, - statement_timeout: options.statementTimeoutMs ?? 30_000, - query_timeout: options.queryTimeoutMs ?? 35_000, - connectionTimeoutMillis: options.connectionTimeoutMs ?? 5_000, - application_name: 'ktx-local-query', - }); - await client.connect(); - try { - await client.query('BEGIN READ ONLY'); - const result = await client.query({ - text: limitSqlForExecution(input.sql, input.maxRows), - rowMode: 'array', - }); - await client.query('COMMIT'); - return { - headers: result.fields.map((field) => field.name), - rows: result.rows, - totalRows: result.rows.length, - command: result.command, - rowCount: result.rowCount, - }; - } catch (error) { - await client.query('ROLLBACK').catch(() => undefined); - throw error; - } finally { - await client.end(); - } - }, - }; -} diff --git a/packages/cli/src/context/connections/query-executor.ts b/packages/cli/src/context/connections/query-executor.ts index e169d164..a397dfc3 100644 --- a/packages/cli/src/context/connections/query-executor.ts +++ b/packages/cli/src/context/connections/query-executor.ts @@ -8,7 +8,7 @@ export interface KtxSqlQueryExecutionInput { maxRows?: number; } -export interface KtxSqlQueryExecutionResult { +interface KtxSqlQueryExecutionResult { headers: string[]; rows: unknown[][]; totalRows: number; diff --git a/packages/cli/src/context/connections/sqlite-query-executor.ts b/packages/cli/src/context/connections/sqlite-query-executor.ts deleted file mode 100644 index 40710c96..00000000 --- a/packages/cli/src/context/connections/sqlite-query-executor.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { isAbsolute, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import Database from 'better-sqlite3'; -import { readFileSync } from 'node:fs'; -import { homedir } from 'node:os'; -import type { - KtxSqlQueryExecutionInput, - KtxSqlQueryExecutionResult, - KtxSqlQueryExecutorPort, -} from './query-executor.js'; -import { normalizeQueryRows } from './query-executor.js'; -import { limitSqlForExecution } from './read-only-sql.js'; - -type SqliteConnectionConfig = Record | undefined; - -function connectionDriver(input: KtxSqlQueryExecutionInput): string { - return String(input.connection?.driver ?? '').toLowerCase(); -} - -function stringConfigValue(connection: SqliteConnectionConfig, key: string): string | undefined { - const value = connection?.[key]; - return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(key, value.trim()) : undefined; -} - -function resolveStringReference(key: string, value: string): string { - if (value.startsWith('env:')) { - return process.env[value.slice('env:'.length)] ?? ''; - } - if (key !== 'url' && value.startsWith('file:')) { - const rawPath = value.slice('file:'.length); - const path = rawPath.startsWith('~') ? resolve(homedir(), rawPath.slice(1)) : rawPath; - return readFileSync(path, 'utf-8').trim(); - } - return value; -} - -function sqlitePathFromUrl(url: string): string { - if (url.startsWith('file:')) { - return fileURLToPath(url); - } - - if (url.startsWith('sqlite:')) { - const parsed = new URL(url); - if (parsed.pathname.length > 0) { - return decodeURIComponent(parsed.pathname); - } - } - - return url; -} - -/** @internal */ -export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInput): string { - const driver = connectionDriver(input); - if (driver !== 'sqlite') { - throw new Error(`Local SQLite execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`); - } - - const pathValue = stringConfigValue(input.connection, 'path'); - const urlValue = stringConfigValue(input.connection, 'url'); - if (!pathValue && !urlValue) { - throw new Error( - `Local SQLite execution requires connections.${input.connectionId}.path or connections.${input.connectionId}.url.`, - ); - } - - const candidate = pathValue ?? sqlitePathFromUrl(urlValue as string); - return isAbsolute(candidate) ? candidate : resolve(input.projectDir ?? process.cwd(), candidate); -} - -export function createSqliteQueryExecutor(): KtxSqlQueryExecutorPort { - return { - async execute(input: KtxSqlQueryExecutionInput): Promise { - const sql = limitSqlForExecution(input.sql, input.maxRows); - const dbPath = sqliteDatabasePathFromConnection(input); - const db = new Database(dbPath, { readonly: true, fileMustExist: true }); - try { - const statement = db.prepare(sql); - const rows = statement.all() as unknown[]; - return { - headers: statement.columns().map((column) => column.name), - rows: normalizeQueryRows(rows), - totalRows: rows.length, - command: 'SELECT', - rowCount: rows.length, - }; - } finally { - db.close(); - } - }, - }; -} diff --git a/packages/cli/src/context/ingest/local-bundle-runtime.ts b/packages/cli/src/context/ingest/local-bundle-runtime.ts index e4c45b3f..69b9159a 100644 --- a/packages/cli/src/context/ingest/local-bundle-runtime.ts +++ b/packages/cli/src/context/ingest/local-bundle-runtime.ts @@ -615,8 +615,8 @@ function localIngestLlmProviderGuardMessage(projectDir: string): string { 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, claude-code, or codex, or an injected agentRunner.', 'Configure a local Claude Code/Codex session or API-backed LLM, then rerun ingest:', ` ktx setup --project-dir ${projectDir} --llm-backend claude-code --no-input`, - ` ktx setup --project-dir ${projectDir} --llm-backend codex --llm-model gpt-5.5 --no-input`, - ` ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`, + ` ktx setup --project-dir ${projectDir} --llm-backend codex --no-input`, + ` ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --no-input`, ].join('\n'); } diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index e673cb99..911579a9 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -10,7 +10,7 @@ import { resolveKtxConfigReference } from './context/core/config-reference.js'; import { type KtxProjectConfig, type KtxProjectLlmConfig, serializeKtxProjectConfig } from './context/project/config.js'; import { loadKtxProject } from './context/project/project.js'; import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; -import type { KtxLlmConfig } from './llm/types.js'; +import { type KtxModelRole, KTX_MODEL_ROLES, type KtxLlmConfig } from './llm/types.js'; import { type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from './llm/model-health.js'; import { formatClaudeCodePromptCachingWarning, @@ -37,7 +37,6 @@ export interface KtxSetupModelArgs { llmBackend?: KtxSetupLlmBackend; anthropicApiKeyEnv?: string; anthropicApiKeyFile?: string; - llmModel?: string; vertexProject?: string; vertexLocation?: string; forcePrompt?: boolean; @@ -52,13 +51,6 @@ export type KtxSetupModelResult = | { status: 'missing-input'; projectDir: string } | { status: 'failed'; projectDir: string }; -/** @internal */ -export interface AnthropicModelChoice { - id: string; - label: string; - recommended: boolean; -} - export type KtxSetupLlmBackend = 'anthropic' | 'vertex' | 'claude-code' | 'codex'; /** @internal */ @@ -76,9 +68,7 @@ export interface KtxSetupModelPromptAdapter { export interface KtxSetupModelDeps { env?: NodeJS.ProcessEnv; - fetch?: typeof fetch; prompts?: KtxSetupModelPromptAdapter; - listModels?: (apiKey: string) => Promise; healthCheck?: (config: KtxLlmConfig) => Promise; claudeCodeAuthProbe?: (input: { projectDir: string; @@ -91,91 +81,58 @@ export interface KtxSetupModelDeps { spinner?: () => KtxCliSpinner; } -/** @internal */ -export const BUNDLED_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ - { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }, - { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, - { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, -]; - -const VERTEX_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ - { id: 'claude-opus-4-7', label: 'Claude Opus 4.7', recommended: false }, - { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: false }, - { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, - { id: 'claude-opus-4-5', label: 'Claude Opus 4.5', recommended: false }, - { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, - { id: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5', recommended: false }, - { id: 'claude-opus-4-1', label: 'Claude Opus 4.1', recommended: false }, -]; - -const CLAUDE_CODE_MODELS: AnthropicModelChoice[] = [ - { id: 'sonnet', label: 'Claude Sonnet', recommended: true }, - { id: 'opus', label: 'Claude Opus', recommended: false }, - { id: 'haiku', label: 'Claude Haiku', recommended: false }, -]; - -// Curated Codex models from OpenAI's current lineup that work under both -// ChatGPT-account (subscription) and API-key auth. Intentionally omitted: -// the `*-codex` ids (e.g. gpt-5.3-codex, gpt-5.2-codex) are API-key-only and -// fail on ChatGPT-account auth, and gpt-5.3-codex-spark is a ChatGPT-Pro-only -// research preview. Codex resolves real availability per account at runtime -// (its binary remote-fetches the model list), so this is a convenience -// shortlist only — the manual-entry option accepts any id your account's -// `codex` picker exposes, and the auth probe reports an unsupported choice. -const CODEX_MODELS: AnthropicModelChoice[] = [ - { id: 'gpt-5.5', label: 'GPT-5.5', recommended: true }, - { id: 'gpt-5.4', label: 'GPT-5.4', recommended: false }, - { id: 'gpt-5.4-mini', label: 'GPT-5.4 mini', recommended: false }, -]; - -const HIDDEN_ANTHROPIC_MODEL_PATTERNS = [ - /^claude-sonnet-4$/i, - /^claude-opus-4$/i, - /^Claude Sonnet 4$/i, - /^Claude Opus 4$/i, -]; - const ANTHROPIC_CREDENTIAL_PROMPT_CONTEXT = 'KTX uses the key to verify Anthropic model access now and to run ingest agents that turn schemas, SQL, ' + 'BI metadata, and docs into semantic-layer sources and wiki context. ktx.yaml stores an env: or file: ' + 'reference, not the raw key.'; -const ANTHROPIC_MODEL_PROMPT_CONTEXT = - 'KTX uses this as the default model for ingest agents that turn schemas, SQL, BI metadata, and docs ' + - 'into semantic-layer sources and wiki context.'; - const VERTEX_PROJECT_PROMPT_CONTEXT = 'KTX stores the selected Google Cloud project ID in ktx.yaml and uses Application Default Credentials for ' + 'access. Project visibility depends on the signed-in Google account and organization permissions.'; const DEFAULT_VERTEX_LOCATION = 'us-east5'; +type KtxSetupModelPreset = Record; + +const ANTHROPIC_PRESET = { + default: 'claude-sonnet-4-6', + triage: 'claude-haiku-4-5', + candidateExtraction: 'claude-sonnet-4-6', + curator: 'claude-opus-4-7', + reconcile: 'claude-opus-4-7', + repair: 'claude-haiku-4-5', +} satisfies KtxSetupModelPreset; + +const CLAUDE_CODE_PRESET = { + default: 'sonnet', + triage: 'haiku', + candidateExtraction: 'sonnet', + curator: 'opus', + reconcile: 'opus', + repair: 'haiku', +} satisfies KtxSetupModelPreset; + +const CODEX_PRESET = { + default: DEFAULT_CODEX_MODEL, + triage: DEFAULT_CODEX_MODEL, + candidateExtraction: DEFAULT_CODEX_MODEL, + curator: DEFAULT_CODEX_MODEL, + reconcile: DEFAULT_CODEX_MODEL, + repair: DEFAULT_CODEX_MODEL, +} satisfies KtxSetupModelPreset; + +const MODEL_PRESETS = { + anthropic: ANTHROPIC_PRESET, + vertex: ANTHROPIC_PRESET, + 'claude-code': CLAUDE_CODE_PRESET, + codex: CODEX_PRESET, +} satisfies Record; + +function presetForBackend(backend: KtxSetupLlmBackend): KtxSetupModelPreset { + return MODEL_PRESETS[backend]; +} + const execFileAsync = promisify(execFile); -type AnthropicModelDiscoveryErrorReason = 'authentication' | 'http' | 'empty-response'; - -class AnthropicModelDiscoveryError extends Error { - constructor( - message: string, - public readonly reason: AnthropicModelDiscoveryErrorReason, - public readonly status?: number, - ) { - super(message); - this.name = 'AnthropicModelDiscoveryError'; - } -} - -function isAnthropicModelAuthenticationError(error: unknown): error is AnthropicModelDiscoveryError { - return error instanceof AnthropicModelDiscoveryError && error.reason === 'authentication'; -} - -function isSelectableAnthropicModel(model: AnthropicModelChoice): boolean { - return !HIDDEN_ANTHROPIC_MODEL_PATTERNS.some((pattern) => pattern.test(model.id) || pattern.test(model.label)); -} - -type ChooseModelResult = - | { status: 'ready'; model: string } - | { status: 'back' | 'missing-input' | 'invalid-credential' }; - type ChooseBackendResult = | { status: 'ready'; backend: KtxSetupLlmBackend; prompted: boolean } | { status: 'back' }; @@ -234,47 +191,6 @@ async function defaultListGcloudProjects(): Promise { .filter((project): project is GcloudProjectChoice => Boolean(project)); } -/** @internal */ -export async function fetchAnthropicModels( - apiKey: string, - fetchFn: typeof fetch = fetch, -): Promise { - const response = await fetchFn('https://api.anthropic.com/v1/models?limit=1000', { - headers: { - 'anthropic-version': '2023-06-01', - 'x-api-key': apiKey, - }, - }); - if (!response.ok) { - if (response.status === 401 || response.status === 403) { - throw new AnthropicModelDiscoveryError( - `Anthropic model discovery failed with HTTP ${response.status}`, - 'authentication', - response.status, - ); - } - throw new AnthropicModelDiscoveryError( - `Anthropic model discovery failed with HTTP ${response.status}`, - 'http', - response.status, - ); - } - const body = (await response.json()) as { data?: Array<{ id?: unknown; display_name?: unknown; type?: unknown }> }; - const models = (body.data ?? []) - .map((item) => ({ - id: typeof item.id === 'string' ? item.id : '', - label: typeof item.display_name === 'string' ? item.display_name : typeof item.id === 'string' ? item.id : '', - recommended: false, - })) - .filter((item) => item.id.startsWith('claude-')) - .filter(isSelectableAnthropicModel); - if (models.length === 0) { - throw new AnthropicModelDiscoveryError('Anthropic model discovery returned no Claude models', 'empty-response'); - } - const recommendedIndex = models.findIndex((item) => item.id.includes('sonnet')); - return models.map((item, index) => ({ ...item, recommended: index === Math.max(recommendedIndex, 0) })); -} - export function isKtxSetupLlmConfigReady(config: KtxProjectLlmConfig): boolean { let resolved: KtxLlmConfig | null; try { @@ -309,12 +225,12 @@ function buildProjectLlmConfig( | { backend: 'vertex'; vertex: { project?: string; location: string } } | { backend: 'claude-code' } | { backend: 'codex' }, - model: string, + models: KtxSetupModelPreset, ): KtxProjectLlmConfig { if (provider.backend === 'claude-code') { return { provider: { backend: 'claude-code' }, - models: { ...existing.models, default: model }, + models, promptCaching: existing.promptCaching, }; } @@ -322,7 +238,7 @@ function buildProjectLlmConfig( if (provider.backend === 'codex') { return { provider: { backend: 'codex' }, - models: { ...existing.models, default: model }, + models, promptCaching: existing.promptCaching, }; } @@ -333,7 +249,7 @@ function buildProjectLlmConfig( backend: 'vertex', vertex: provider.vertex, }, - models: { ...existing.models, default: model }, + models, promptCaching: { ...(existing.promptCaching ?? {}), enabled: true, vertexFallbackTo5m: true }, }; } @@ -343,7 +259,7 @@ function buildProjectLlmConfig( backend: 'anthropic', anthropic: { api_key: provider.credentialRef }, }, - models: { ...existing.models, default: model }, + models, promptCaching: { ...(existing.promptCaching ?? {}), enabled: true }, }; } @@ -514,16 +430,12 @@ function requestedBackend(args: KtxSetupModelArgs): KtxSetupLlmBackend | undefin if (args.vertexProject || args.vertexLocation) { return 'vertex'; } - if (args.anthropicApiKeyEnv || args.anthropicApiKeyFile || args.llmModel) { + if (args.anthropicApiKeyEnv || args.anthropicApiKeyFile) { return 'anthropic'; } return undefined; } -function requestedModel(args: KtxSetupModelArgs): string | undefined { - return args.llmModel; -} - async function chooseBackend( args: KtxSetupModelArgs, io: KtxCliIo, @@ -774,187 +686,6 @@ async function chooseVertexConfig( }; } -async function chooseModel( - args: KtxSetupModelArgs, - credentialValue: string, - io: KtxCliIo, - deps: KtxSetupModelDeps, -): Promise { - const providedModel = requestedModel(args); - if (providedModel) { - return { status: 'ready', model: providedModel }; - } - if (args.inputMode === 'disabled') { - io.stderr.write('Missing LLM model: pass --llm-model.\n'); - return { status: 'missing-input' }; - } - - let models: AnthropicModelChoice[]; - try { - models = deps.listModels - ? await deps.listModels(credentialValue) - : await fetchAnthropicModels(credentialValue, deps.fetch); - } catch (error) { - if (isAnthropicModelAuthenticationError(error)) { - const statusSuffix = error.status ? ` (HTTP ${error.status})` : ''; - io.stderr.write(`Anthropic API key is invalid or unauthorized${statusSuffix}. Check the key and try again.\n`); - return { status: 'invalid-credential' }; - } - io.stderr.write( - 'Could not fetch live Anthropic models. Showing bundled defaults. Setup will still test the selected model before saving it.\n', - ); - models = BUNDLED_ANTHROPIC_MODELS; - } - - const selectableModels = models.filter(isSelectableAnthropicModel); - const prompts = deps.prompts ?? createPromptAdapter(); - const modelOptions = [ - ...selectableModels.map((model) => ({ - value: model.id, - label: model.label || model.id, - ...(model.recommended ? { hint: 'recommended' } : {}), - })), - { value: 'manual', label: 'Enter a model ID manually' }, - { value: 'back', label: 'Back' }, - ]; - const choice = await prompts.autocomplete({ - message: `Which Anthropic model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, - placeholder: 'Type to search models', - options: modelOptions, - }); - if (choice === 'back') { - return { status: 'back' }; - } - if (choice === 'manual') { - const manual = await prompts.text({ - message: withTextInputNavigation('Anthropic model ID'), - placeholder: selectableModels.find((model) => model.recommended)?.id ?? selectableModels[0]?.id, - }); - if (manual === undefined) { - return { status: 'back' }; - } - return manual.trim() ? { status: 'ready', model: manual.trim() } : { status: 'missing-input' }; - } - return { status: 'ready', model: choice }; -} - -async function chooseVertexModel(args: KtxSetupModelArgs, io: KtxCliIo, deps: KtxSetupModelDeps): Promise { - const providedModel = requestedModel(args); - if (providedModel) { - return { status: 'ready', model: providedModel }; - } - if (args.inputMode === 'disabled') { - io.stderr.write('Missing LLM model: pass --llm-model.\n'); - return { status: 'missing-input' }; - } - - const selectableModels = VERTEX_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel); - const prompts = deps.prompts ?? createPromptAdapter(); - const choice = await prompts.autocomplete({ - message: `Which Anthropic model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, - placeholder: 'Type to search models', - options: [ - ...selectableModels.map((model) => ({ - value: model.id, - label: model.label || model.id, - ...(model.recommended ? { hint: 'recommended' } : {}), - })), - { value: 'manual', label: 'Enter a model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }); - if (choice === 'back') { - return { status: 'back' }; - } - if (choice === 'manual') { - const manual = await prompts.text({ - message: withTextInputNavigation('Anthropic model ID'), - placeholder: selectableModels.find((model) => model.recommended)?.id ?? selectableModels[0]?.id, - }); - if (manual === undefined) { - return { status: 'back' }; - } - return manual.trim() ? { status: 'ready', model: manual.trim() } : { status: 'missing-input' }; - } - return { status: 'ready', model: choice }; -} - -async function chooseClaudeCodeModel(args: KtxSetupModelArgs, deps: KtxSetupModelDeps): Promise { - const providedModel = requestedModel(args); - if (providedModel) { - return { status: 'ready', model: providedModel }; - } - if (args.inputMode === 'disabled') { - return { status: 'ready', model: 'sonnet' }; - } - - const prompts = deps.prompts ?? createPromptAdapter(); - const choice = await prompts.select({ - message: `Which Claude Code model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, - options: [ - ...CLAUDE_CODE_MODELS.map((model) => ({ - value: model.id, - label: model.label, - ...(model.recommended ? { hint: 'recommended' } : {}), - })), - { value: 'manual', label: 'Enter a Claude Code model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }); - if (choice === 'back') { - return { status: 'back' }; - } - if (choice === 'manual') { - const manual = await prompts.text({ - message: withTextInputNavigation('Claude Code model ID'), - placeholder: CLAUDE_CODE_MODELS.find((model) => model.recommended)?.id ?? CLAUDE_CODE_MODELS[0]?.id, - }); - if (manual === undefined) { - return { status: 'back' }; - } - return manual.trim() ? { status: 'ready', model: manual.trim() } : { status: 'missing-input' }; - } - return { status: 'ready', model: choice }; -} - -async function chooseCodexModel(args: KtxSetupModelArgs, deps: KtxSetupModelDeps): Promise { - const providedModel = requestedModel(args); - if (providedModel) { - return { status: 'ready', model: providedModel }; - } - if (args.inputMode === 'disabled') { - return { status: 'ready', model: DEFAULT_CODEX_MODEL }; - } - - const prompts = deps.prompts ?? createPromptAdapter(); - const choice = await prompts.select({ - message: `Which Codex model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, - options: [ - ...CODEX_MODELS.map((model) => ({ - value: model.id, - label: model.label, - ...(model.recommended ? { hint: 'recommended' } : {}), - })), - { value: 'manual', label: 'Enter a Codex model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }); - if (choice === 'back') { - return { status: 'back' }; - } - if (choice === 'manual') { - const manual = await prompts.text({ - message: withTextInputNavigation('Codex model ID'), - placeholder: CODEX_MODELS.find((model) => model.recommended)?.id ?? CODEX_MODELS[0]?.id, - }); - if (manual === undefined) { - return { status: 'back' }; - } - return manual.trim() ? { status: 'ready', model: manual.trim() } : { status: 'missing-input' }; - } - return { status: 'ready', model: choice }; -} - async function persistLlmConfig( projectDir: string, provider: @@ -962,12 +693,12 @@ async function persistLlmConfig( | { backend: 'vertex'; vertex: { project?: string; location: string } } | { backend: 'claude-code' } | { backend: 'codex' }, - model: string, + models: KtxSetupModelPreset, ): Promise { const project = await loadKtxProject({ projectDir }); const config = { ...project.config, - llm: buildProjectLlmConfig(project.config.llm, provider, model), + llm: buildProjectLlmConfig(project.config.llm, provider, models), scan: { ...project.config.scan, enrichment: { @@ -990,6 +721,61 @@ function buildInteractiveRetryArgs(args: KtxSetupModelArgs, backend?: KtxSetupLl }; } +type PresetModelValidationResult = { ok: true } | { ok: false; message: string }; + +function distinctPresetModels(preset: KtxSetupModelPreset): string[] { + const models: string[] = []; + const seen = new Set(); + for (const role of KTX_MODEL_ROLES) { + const model = preset[role]; + if (!seen.has(model)) { + seen.add(model); + models.push(model); + } + } + return models; +} + +function rolesUsingModel(preset: KtxSetupModelPreset, model: string): KtxModelRole[] { + return KTX_MODEL_ROLES.filter((role) => preset[role] === model); +} + +function formatPresetFallbackWarning(roles: KtxModelRole[], unavailableModel: string, anchorModel: string): string { + return `LLM model ${unavailableModel} is unavailable for ${roles.join(', ')}; using ${anchorModel} for those roles.`; +} + +async function validatePresetModels( + preset: KtxSetupModelPreset, + validateModel: (model: string) => Promise, + io: KtxCliIo, +): Promise<{ status: 'ready'; models: KtxSetupModelPreset } | { status: 'failed'; message: string }> { + const anchorModel = preset.default; + const degraded = { ...preset }; + const models = distinctPresetModels(preset); + + const anchorResult = await validateModel(anchorModel); + if (!anchorResult.ok) { + return { status: 'failed', message: anchorResult.message }; + } + + for (const model of models) { + if (model === anchorModel) { + continue; + } + const result = await validateModel(model); + if (result.ok) { + continue; + } + const affectedRoles = rolesUsingModel(degraded, model); + for (const role of affectedRoles) { + degraded[role] = anchorModel; + } + io.stderr.write(`${formatPresetFallbackWarning(affectedRoles, model, anchorModel)}\n`); + } + + return { status: 'ready', models: degraded }; +} + export async function runKtxSetupAnthropicModelStep( args: KtxSetupModelArgs, io: KtxCliIo, @@ -1007,7 +793,6 @@ export async function runKtxSetupAnthropicModelStep( !args.llmBackend && !args.anthropicApiKeyEnv && !args.anthropicApiKeyFile && - !args.llmModel && !args.vertexProject && !args.vertexLocation ) { @@ -1038,94 +823,74 @@ export async function runKtxSetupAnthropicModelStep( return { status: vertex.status, projectDir: args.projectDir }; } - const model = await chooseVertexModel(backendArgs, io, deps); - if (model.status === 'back' && !backendArgs.vertexLocation) { + const preset = presetForBackend('vertex'); + const validation = await validatePresetModels( + preset, + async (model) => + runLlmHealthCheckWithProgress( + buildVertexHealthConfig(vertex.values, model), + 'Vertex AI', + model, + healthCheck, + deps, + ), + io, + ); + if (validation.status !== 'ready') { + io.stderr.write( + `Vertex AI Anthropic model health check failed: ${formatVertexHealthFailure(validation.message, vertex.values)}\n`, + ); + if (args.inputMode === 'disabled') { + return { status: 'failed', projectDir: args.projectDir }; + } + io.stderr.write('Choose a different Vertex AI project or location, or Back.\n'); attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend); continue; } - if (model.status === 'invalid-credential') { - return { status: 'failed', projectDir: args.projectDir }; - } - if (model.status !== 'ready') { - return { status: model.status, projectDir: args.projectDir }; - } - const health = await runLlmHealthCheckWithProgress( - buildVertexHealthConfig(vertex.values, model.model), - 'Vertex AI', - model.model, - healthCheck, - deps, - ); - if (health.ok) { - await persistLlmConfig(args.projectDir, { backend: 'vertex', vertex: vertex.refs }, model.model); - io.stdout.write(`│ LLM ready: yes (${model.model})\n`); - return { status: 'ready', projectDir: args.projectDir }; - } - - io.stderr.write(`Vertex AI Anthropic model health check failed: ${formatVertexHealthFailure(health.message, vertex.values)}\n`); - if (args.inputMode === 'disabled') { - return { status: 'failed', projectDir: args.projectDir }; - } - io.stderr.write('Choose a different Vertex AI project, location, or model, or Back.\n'); - attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend); - continue; + await persistLlmConfig(args.projectDir, { backend: 'vertex', vertex: vertex.refs }, validation.models); + io.stdout.write(`│ LLM ready: yes (${validation.models.default})\n`); + return { status: 'ready', projectDir: args.projectDir }; } if (backendChoice.backend === 'claude-code') { - const model = await chooseClaudeCodeModel(backendArgs, deps); - if (model.status === 'back' && backendChoice.prompted) { - attemptArgs = buildInteractiveRetryArgs(args); - continue; - } - if (model.status === 'invalid-credential') { - return { status: 'failed', projectDir: args.projectDir }; - } - if (model.status !== 'ready') { - return { status: model.status, projectDir: args.projectDir }; - } + const preset = presetForBackend('claude-code'); const probe = deps.claudeCodeAuthProbe ?? runClaudeCodeAuthProbe; - const health = await probe({ projectDir: args.projectDir, model: model.model, env: deps.env ?? process.env }); - if (!health.ok) { - io.stderr.write(`${health.message}\n`); + const validation = await validatePresetModels( + preset, + async (model) => probe({ projectDir: args.projectDir, model, env: deps.env ?? process.env }), + io, + ); + if (validation.status !== 'ready') { + io.stderr.write(`${validation.message}\n`); return { status: 'failed', projectDir: args.projectDir }; } const warning = formatClaudeCodePromptCachingWarning( ignoredClaudeCodePromptCachingFields( - buildProjectLlmConfig(project.config.llm, { backend: 'claude-code' }, model.model), + buildProjectLlmConfig(project.config.llm, { backend: 'claude-code' }, validation.models), ), ); if (warning) { io.stderr.write(`${warning}\n`); } - await persistLlmConfig(args.projectDir, { backend: 'claude-code' }, model.model); - io.stdout.write(`│ LLM ready: yes (${model.model})\n`); + await persistLlmConfig(args.projectDir, { backend: 'claude-code' }, validation.models); + io.stdout.write(`│ LLM ready: yes (${validation.models.default})\n`); return { status: 'ready', projectDir: args.projectDir }; } if (backendChoice.backend === 'codex') { - const model = await chooseCodexModel(backendArgs, deps); - if (model.status === 'back' && backendChoice.prompted) { - attemptArgs = buildInteractiveRetryArgs(args); - continue; - } - if (model.status === 'invalid-credential') { - return { status: 'failed', projectDir: args.projectDir }; - } - if (model.status !== 'ready') { - return { status: model.status, projectDir: args.projectDir }; - } + const preset = presetForBackend('codex'); const probe = deps.codexAuthProbe ?? runCodexAuthProbe; - const health = await probe({ projectDir: args.projectDir, model: model.model }); - if (!health.ok) { - io.stderr.write(`${health.message}\n`); + const validation = await validatePresetModels(preset, async (model) => probe({ projectDir: args.projectDir, model }), io); + if (validation.status !== 'ready') { + io.stderr.write(`${validation.message}\n`); return { status: 'failed', projectDir: args.projectDir }; } // Prefix the clack gutter so the warning sits inside the setup frame // instead of breaking out of it; kept on stderr for scripted runs. io.stderr.write(`│ ${formatCodexIsolationWarning()}\n`); - await persistLlmConfig(args.projectDir, { backend: 'codex' }, model.model); - io.stdout.write(`│ LLM ready: yes (codex, ${model.model})\n`); + await persistLlmConfig(args.projectDir, { backend: 'codex' }, validation.models); + io.stdout.write(`│ LLM ready: yes (codex, ${validation.models.default})\n`); return { status: 'ready', projectDir: args.projectDir }; } @@ -1138,8 +903,21 @@ export async function runKtxSetupAnthropicModelStep( return { status: credential.status, projectDir: args.projectDir }; } - const model = await chooseModel(backendArgs, credential.value, io, deps); - if (model.status === 'invalid-credential') { + const preset = presetForBackend('anthropic'); + const validation = await validatePresetModels( + preset, + async (model) => + runLlmHealthCheckWithProgress( + buildAnthropicHealthConfig(credential.value, model), + 'Anthropic API', + model, + healthCheck, + deps, + ), + io, + ); + if (validation.status !== 'ready') { + io.stderr.write(`Anthropic model health check failed: ${validation.message}\n`); if (args.inputMode === 'disabled') { return { status: 'failed', projectDir: args.projectDir }; } @@ -1147,32 +925,9 @@ export async function runKtxSetupAnthropicModelStep( attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend); continue; } - if (model.status === 'back' && !backendArgs.anthropicApiKeyEnv && !backendArgs.anthropicApiKeyFile) { - attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend); - continue; - } - if (model.status !== 'ready') { - return { status: model.status, projectDir: args.projectDir }; - } - const health = await runLlmHealthCheckWithProgress( - buildAnthropicHealthConfig(credential.value, model.model), - 'Anthropic API', - model.model, - healthCheck, - deps, - ); - if (health.ok) { - await persistLlmConfig(args.projectDir, { backend: 'anthropic', credentialRef: credential.ref }, model.model); - io.stdout.write(`│ LLM ready: yes (${model.model})\n`); - return { status: 'ready', projectDir: args.projectDir }; - } - - io.stderr.write(`Anthropic model health check failed: ${health.message}\n`); - if (args.inputMode === 'disabled') { - return { status: 'failed', projectDir: args.projectDir }; - } - io.stderr.write('Choose a different credential source or model, or Back.\n'); - attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend); + await persistLlmConfig(args.projectDir, { backend: 'anthropic', credentialRef: credential.ref }, validation.models); + io.stdout.write(`│ LLM ready: yes (${validation.models.default})\n`); + return { status: 'ready', projectDir: args.projectDir }; } } diff --git a/packages/cli/src/setup.ts b/packages/cli/src/setup.ts index fc45abb3..9539a248 100644 --- a/packages/cli/src/setup.ts +++ b/packages/cli/src/setup.ts @@ -86,7 +86,6 @@ export type KtxSetupArgs = llmBackend?: KtxSetupLlmBackend; anthropicApiKeyEnv?: string; anthropicApiKeyFile?: string; - llmModel?: string; vertexProject?: string; vertexLocation?: string; skipLlm: boolean; @@ -700,7 +699,6 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup ...(args.llmBackend ? { llmBackend: args.llmBackend } : {}), ...(args.anthropicApiKeyEnv ? { anthropicApiKeyEnv: args.anthropicApiKeyEnv } : {}), ...(args.anthropicApiKeyFile ? { anthropicApiKeyFile: args.anthropicApiKeyFile } : {}), - ...(args.llmModel ? { llmModel: args.llmModel } : {}), ...(args.vertexProject ? { vertexProject: args.vertexProject } : {}), ...(args.vertexLocation ? { vertexLocation: args.vertexLocation } : {}), forcePrompt: forcePromptSteps.has('models') || runOnly === 'models', diff --git a/packages/cli/src/sl.ts b/packages/cli/src/sl.ts index dcf5e460..6c849265 100644 --- a/packages/cli/src/sl.ts +++ b/packages/cli/src/sl.ts @@ -1,6 +1,5 @@ import { readFile } from 'node:fs/promises'; import type { KtxCliIo } from './cli-runtime.js'; -import { createDefaultLocalQueryExecutor } from './context/connections/local-query-executor.js'; import type { KtxSqlQueryExecutorPort } from './context/connections/query-executor.js'; import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js'; import type { KtxEmbeddingPort } from './context/core/embedding.js'; @@ -20,6 +19,7 @@ import { resolveProjectEmbeddingProvider, type EmbeddingProviderResolution, } from './embedding-resolution.js'; +import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; import type { PrintListColumn } from './io/print-list.js'; import { createManagedPythonSemanticLayerComputePort, @@ -81,7 +81,7 @@ interface KtxSlDeps { io: KtxSlIo; projectDir?: string; }) => Promise; - createQueryExecutor?: () => KtxSqlQueryExecutorPort; + createQueryExecutor?: (project: KtxLocalProject) => KtxSqlQueryExecutorPort; } function resolutionToEmbeddingPort(resolution: EmbeddingProviderResolution): KtxEmbeddingPort | null { @@ -321,7 +321,7 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx io, projectDir: args.projectDir, }); - const queryExecutor = args.execute ? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)() : undefined; + const queryExecutor = args.execute ? (deps.createQueryExecutor ?? createKtxCliIngestQueryExecutor)(project) : undefined; const result = await compileLocalSlQuery(project, { connectionId: args.connectionId, query, diff --git a/packages/cli/test/context/connections/drivers.test.ts b/packages/cli/test/context/connections/drivers.test.ts index 380b2265..5d59db4e 100644 --- a/packages/cli/test/context/connections/drivers.test.ts +++ b/packages/cli/test/context/connections/drivers.test.ts @@ -68,7 +68,6 @@ const connectionFixtures: Record = { const allowedScopeKeys = new Set(['dataset_ids', 'databases', 'schemas', 'schema_names']); const historicSqlReaderDrivers = new Set(['postgres', 'bigquery', 'snowflake']); -const localExecutorDrivers = new Set(['postgres', 'sqlite']); function assertExportedRegistryBoundaryTypes(input: { scopeConfigKey: KtxScopeConfigKey; @@ -140,6 +139,5 @@ describe('driverRegistrations', () => { expect(allowedScopeKeys.has(registration.scopeConfigKey ?? '')).toBe(true); } expect(registration.hasHistoricSqlReader).toBe(historicSqlReaderDrivers.has(registration.driver)); - expect(registration.hasLocalQueryExecutor).toBe(localExecutorDrivers.has(registration.driver)); }); }); diff --git a/packages/cli/test/context/connections/local-query-executor.test.ts b/packages/cli/test/context/connections/local-query-executor.test.ts deleted file mode 100644 index ca700b04..00000000 --- a/packages/cli/test/context/connections/local-query-executor.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { createDefaultLocalQueryExecutor } from '../../../src/context/connections/local-query-executor.js'; - -describe('createDefaultLocalQueryExecutor', () => { - it('dispatches postgres and sqlite drivers to their executors', async () => { - const postgres = { - execute: vi.fn(async () => ({ - headers: ['pg'], - rows: [[1]], - totalRows: 1, - command: 'SELECT', - rowCount: 1, - })), - }; - const sqlite = { - execute: vi.fn(async () => ({ - headers: ['sqlite'], - rows: [[2]], - totalRows: 1, - command: 'SELECT', - rowCount: 1, - })), - }; - const executor = createDefaultLocalQueryExecutor({ postgres, sqlite }); - - await expect( - executor.execute({ - connectionId: 'pg', - connection: { driver: 'postgres' }, - sql: 'select 1', - }), - ).resolves.toMatchObject({ headers: ['pg'] }); - await expect( - executor.execute({ - connectionId: 'local', - connection: { driver: 'sqlite' }, - sql: 'select 1', - }), - ).resolves.toMatchObject({ headers: ['sqlite'] }); - - expect(postgres.execute).toHaveBeenCalledTimes(1); - expect(sqlite.execute).toHaveBeenCalledTimes(1); - }); - - it('rejects unsupported local execution drivers', async () => { - const executor = createDefaultLocalQueryExecutor({ - postgres: { execute: vi.fn() }, - sqlite: { execute: vi.fn() }, - }); - - await expect( - executor.execute({ - connectionId: 'warehouse', - connection: { driver: 'snowflake' }, - sql: 'select 1', - }), - ).rejects.toThrow('No local query executor is configured for driver "snowflake".'); - }); -}); diff --git a/packages/cli/test/context/connections/postgres-query-executor.test.ts b/packages/cli/test/context/connections/postgres-query-executor.test.ts deleted file mode 100644 index fe4ab15c..00000000 --- a/packages/cli/test/context/connections/postgres-query-executor.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { createPostgresQueryExecutor } from '../../../src/context/connections/postgres-query-executor.js'; - -function makeClient() { - const calls: unknown[] = []; - const client = { - connect: vi.fn(async () => undefined), - query: vi.fn(async (input: unknown) => { - calls.push(input); - if (input === 'BEGIN READ ONLY') { - return { rows: [], fields: [], rowCount: null, command: 'BEGIN' }; - } - if (input === 'COMMIT') { - return { rows: [], fields: [], rowCount: null, command: 'COMMIT' }; - } - return { - rows: [ - ['paid', 2], - ['open', 1], - ], - fields: [{ name: 'status' }, { name: 'order_count' }], - rowCount: 2, - command: 'SELECT', - }; - }), - end: vi.fn(async () => undefined), - }; - return { client, calls }; -} - -describe('createPostgresQueryExecutor', () => { - it('runs a read-only transaction in array row mode and closes the client', async () => { - const { client, calls } = makeClient(); - const executor = createPostgresQueryExecutor({ - clientFactory: vi.fn(() => client), - }); - - const result = await executor.execute({ - connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgres://example/db' }, - sql: 'select status, count(*) as order_count from public.orders group by status', - maxRows: 50, - }); - - expect(client.connect).toHaveBeenCalledTimes(1); - expect(calls[0]).toBe('BEGIN READ ONLY'); - expect(calls[1]).toEqual({ - text: 'select * from (select status, count(*) as order_count from public.orders group by status) as ktx_query_result limit 50', - rowMode: 'array', - }); - expect(calls[2]).toBe('COMMIT'); - expect(client.end).toHaveBeenCalledTimes(1); - expect(result).toEqual({ - headers: ['status', 'order_count'], - rows: [ - ['paid', 2], - ['open', 1], - ], - totalRows: 2, - command: 'SELECT', - rowCount: 2, - }); - }); - - it('rolls back and closes the client when query execution fails', async () => { - const client = { - connect: vi.fn(async () => undefined), - query: vi.fn(async (input: unknown) => { - if (input === 'BEGIN READ ONLY' || input === 'ROLLBACK') { - return { rows: [], fields: [], rowCount: null, command: String(input) }; - } - throw new Error('syntax error'); - }), - end: vi.fn(async () => undefined), - }; - const executor = createPostgresQueryExecutor({ - clientFactory: vi.fn(() => client), - }); - - await expect( - executor.execute({ - connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgres://example/db' }, - sql: 'select * from broken', - maxRows: 10, - }), - ).rejects.toThrow('syntax error'); - expect(client.query).toHaveBeenCalledWith('ROLLBACK'); - expect(client.end).toHaveBeenCalledTimes(1); - }); - - it('requires a Postgres url', async () => { - const executor = createPostgresQueryExecutor({ clientFactory: vi.fn() }); - - await expect( - executor.execute({ - connectionId: 'warehouse', - connection: { driver: 'postgres' }, - sql: 'select 1', - }), - ).rejects.toThrow('Local Postgres execution requires connections.warehouse.url'); - }); -}); diff --git a/packages/cli/test/context/connections/sqlite-query-executor.test.ts b/packages/cli/test/context/connections/sqlite-query-executor.test.ts deleted file mode 100644 index a9e61ba5..00000000 --- a/packages/cli/test/context/connections/sqlite-query-executor.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises'; -import { writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import Database from 'better-sqlite3'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { createSqliteQueryExecutor, sqliteDatabasePathFromConnection } from '../../../src/context/connections/sqlite-query-executor.js'; - -describe('createSqliteQueryExecutor', () => { - let tempDir: string; - let dbPath: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-sqlite-query-')); - dbPath = join(tempDir, 'warehouse.db'); - const db = new Database(dbPath); - db.exec(` - CREATE TABLE orders ( - id INTEGER PRIMARY KEY, - status TEXT NOT NULL, - amount INTEGER NOT NULL - ); - INSERT INTO orders (status, amount) VALUES - ('paid', 20), - ('paid', 30), - ('open', 10); - `); - db.close(); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('executes read-only SELECT SQL against a relative SQLite path', async () => { - const executor = createSqliteQueryExecutor(); - - const result = await executor.execute({ - connectionId: 'warehouse', - projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db' }, - sql: 'select status, count(*) as order_count from orders group by status order by status', - maxRows: 10, - }); - - expect(result).toEqual({ - headers: ['status', 'order_count'], - rows: [ - ['open', 1], - ['paid', 2], - ], - totalRows: 2, - command: 'SELECT', - rowCount: 2, - }); - }); - - it('supports file urls for SQLite database paths', async () => { - expect( - sqliteDatabasePathFromConnection({ - connectionId: 'warehouse', - projectDir: tempDir, - connection: { driver: 'sqlite', url: `file://${dbPath}` }, - sql: 'select 1', - }), - ).toBe(dbPath); - }); - - it('resolves file references for SQLite path fields', async () => { - const pointerPath = join(tempDir, 'sqlite-path.txt'); - writeFileSync(pointerPath, dbPath, 'utf-8'); - - expect( - sqliteDatabasePathFromConnection({ - connectionId: 'warehouse', - projectDir: tempDir, - connection: { driver: 'sqlite', path: `file:${pointerPath}` }, - sql: 'select 1', - }), - ).toBe(dbPath); - }); - - it('resolves env references for SQLite database urls', async () => { - const originalDatabaseUrl = process.env.KTX_SQLITE_TEST_URL; - process.env.KTX_SQLITE_TEST_URL = `sqlite:${dbPath}`; - - try { - expect( - sqliteDatabasePathFromConnection({ - connectionId: 'warehouse', - projectDir: tempDir, - connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL' }, - sql: 'select 1', - }), - ).toBe(dbPath); - } finally { - if (originalDatabaseUrl === undefined) { - delete process.env.KTX_SQLITE_TEST_URL; - } else { - process.env.KTX_SQLITE_TEST_URL = originalDatabaseUrl; - } - } - }); - - it('rejects mutating SQL before opening the database', async () => { - const executor = createSqliteQueryExecutor(); - - await expect( - executor.execute({ - connectionId: 'warehouse', - projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db' }, - sql: 'delete from orders', - }), - ).rejects.toThrow('Only read-only SELECT/WITH queries can be executed locally'); - }); - - it('requires a SQLite driver and a database path', async () => { - const executor = createSqliteQueryExecutor(); - - await expect( - executor.execute({ - connectionId: 'warehouse', - projectDir: tempDir, - connection: { driver: 'postgres', path: 'warehouse.db' }, - sql: 'select 1', - }), - ).rejects.toThrow('Local SQLite execution cannot run driver "postgres"'); - - await expect( - executor.execute({ - connectionId: 'warehouse', - projectDir: tempDir, - connection: { driver: 'sqlite' }, - sql: 'select 1', - }), - ).rejects.toThrow('Local SQLite execution requires connections.warehouse.path or connections.warehouse.url'); - }); -}); diff --git a/packages/cli/test/context/ingest/local-bundle-runtime.test.ts b/packages/cli/test/context/ingest/local-bundle-runtime.test.ts index e3031cc5..3ca0c490 100644 --- a/packages/cli/test/context/ingest/local-bundle-runtime.test.ts +++ b/packages/cli/test/context/ingest/local-bundle-runtime.test.ts @@ -80,8 +80,8 @@ describe('createLocalBundleIngestRuntime', () => { 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, claude-code, or codex, or an injected agentRunner.', 'Configure a local Claude Code/Codex session or API-backed LLM, then rerun ingest:', ` ktx setup --project-dir ${project.projectDir} --llm-backend claude-code --no-input`, - ` ktx setup --project-dir ${project.projectDir} --llm-backend codex --llm-model gpt-5.5 --no-input`, - ` ktx setup --project-dir ${project.projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`, + ` ktx setup --project-dir ${project.projectDir} --llm-backend codex --no-input`, + ` ktx setup --project-dir ${project.projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --no-input`, ].join('\n'), ); }); diff --git a/packages/cli/test/index.test.ts b/packages/cli/test/index.test.ts index 57ac4901..eef511fb 100644 --- a/packages/cli/test/index.test.ts +++ b/packages/cli/test/index.test.ts @@ -1136,8 +1136,6 @@ describe('runKtxCli', () => { '--no-input', '--anthropic-api-key-env', 'ANTHROPIC_API_KEY', - '--llm-model', - 'claude-sonnet-4-6', ], setupIo.io, { setup }, @@ -1151,7 +1149,6 @@ describe('runKtxCli', () => { inputMode: 'disabled', cliVersion, anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - llmModel: 'claude-sonnet-4-6', skipLlm: false, }), setupIo.io, @@ -1175,8 +1172,6 @@ describe('runKtxCli', () => { 'local-gcp-project', '--vertex-location', 'us-east5', - '--llm-model', - 'claude-sonnet-4-6', ], setupIo.io, { setup }, @@ -1192,14 +1187,13 @@ describe('runKtxCli', () => { llmBackend: 'vertex', vertexProject: 'local-gcp-project', vertexLocation: 'us-east5', - llmModel: 'claude-sonnet-4-6', skipLlm: false, }), setupIo.io, ); }); - it('dispatches the provider-neutral LLM model setup flag to the setup runner', async () => { + it('rejects the removed --llm-model setup flag', async () => { const setup = vi.fn(async () => 0); const setupIo = makeIo(); @@ -1218,20 +1212,10 @@ describe('runKtxCli', () => { setupIo.io, { setup }, ), - ).resolves.toBe(0); + ).resolves.toBe(1); - expect(setup).toHaveBeenCalledWith( - expect.objectContaining({ - command: 'run', - projectDir: tempDir, - inputMode: 'disabled', - cliVersion, - llmBackend: 'claude-code', - llmModel: 'opus', - skipLlm: false, - }), - setupIo.io, - ); + expect(setup).not.toHaveBeenCalled(); + expect(setupIo.stderr()).toContain("unknown option '--llm-model'"); }); it('rejects conflicting Anthropic credential setup flags', async () => { diff --git a/packages/cli/test/ingest.test.ts b/packages/cli/test/ingest.test.ts index 21037e91..c1abfe8b 100644 --- a/packages/cli/test/ingest.test.ts +++ b/packages/cli/test/ingest.test.ts @@ -341,11 +341,9 @@ describe('runKtxIngest', () => { ); expect(runIo.stderr()).toContain('Configure a local Claude Code/Codex session or API-backed LLM, then rerun ingest:'); expect(runIo.stderr()).toContain(`ktx setup --project-dir ${projectDir} --llm-backend claude-code --no-input`); + expect(runIo.stderr()).toContain(`ktx setup --project-dir ${projectDir} --llm-backend codex --no-input`); expect(runIo.stderr()).toContain( - `ktx setup --project-dir ${projectDir} --llm-backend codex --llm-model gpt-5.5 --no-input`, - ); - expect(runIo.stderr()).toContain( - `ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`, + `ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --no-input`, ); }); diff --git a/packages/cli/test/setup-models.test.ts b/packages/cli/test/setup-models.test.ts index dedf03bd..f09691e0 100644 --- a/packages/cli/test/setup-models.test.ts +++ b/packages/cli/test/setup-models.test.ts @@ -6,8 +6,6 @@ import { parseKtxProjectConfig } from '../src/context/project/config.js'; import { readKtxSetupState, writeKtxSetupState } from '../src/context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { - BUNDLED_ANTHROPIC_MODELS, - fetchAnthropicModels, type KtxSetupModelPromptAdapter, runKtxSetupAnthropicModelStep, } from '../src/setup-models.js'; @@ -97,6 +95,33 @@ function makePromptAdapter(options: { }; } +const anthropicPreset = { + default: 'claude-sonnet-4-6', + triage: 'claude-haiku-4-5', + candidateExtraction: 'claude-sonnet-4-6', + curator: 'claude-opus-4-7', + reconcile: 'claude-opus-4-7', + repair: 'claude-haiku-4-5', +}; + +const claudeCodePreset = { + default: 'sonnet', + triage: 'haiku', + candidateExtraction: 'sonnet', + curator: 'opus', + reconcile: 'opus', + repair: 'haiku', +}; + +const codexPreset = { + default: 'gpt-5.5', + triage: 'gpt-5.5', + candidateExtraction: 'gpt-5.5', + curator: 'gpt-5.5', + reconcile: 'gpt-5.5', + repair: 'gpt-5.5', +}; + describe('setup Anthropic model step', () => { let tempDir: string; @@ -109,66 +134,6 @@ describe('setup Anthropic model step', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('does not expose Claude Sonnet 4 or Claude Opus 4 as selectable Anthropic models', async () => { - const fetchModels = vi.fn( - async () => - new Response( - JSON.stringify({ - data: [ - { id: 'claude-sonnet-4', display_name: 'Claude Sonnet 4' }, - { id: 'claude-opus-4', display_name: 'Claude Opus 4' }, - { id: 'claude-sonnet-4-6', display_name: 'Claude Sonnet 4.6' }, - { id: 'claude-opus-4-6', display_name: 'Claude Opus 4.6' }, - { id: 'claude-haiku-4-5', display_name: 'Claude Haiku 4.5' }, - ], - }), - { status: 200 }, - ), - ); - - await expect(fetchAnthropicModels('sk-ant-test', fetchModels)).resolves.toEqual([ // pragma: allowlist secret - { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }, - { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, - { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, - ]); - expect(BUNDLED_ANTHROPIC_MODELS.map((model) => model.id)).not.toEqual( - expect.arrayContaining(['claude-sonnet-4', 'claude-opus-4']), - ); - }); - - it('filters Claude Sonnet 4 and Claude Opus 4 from Anthropic model prompt choices', async () => { - const prompts = makePromptAdapter({ selectValues: ['env', 'back', 'back'] }); - - await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - makeIo().io, - { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => [ - { id: 'claude-sonnet-4', label: 'Claude Sonnet 4', recommended: true }, - { id: 'claude-opus-4', label: 'Claude Opus 4', recommended: false }, - { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }, - { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, - { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, - ]), - }, - ); - - expect(prompts.autocomplete).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Anthropic model should KTX use?'), - options: [ - { value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', hint: 'recommended' }, - { value: 'claude-opus-4-6', label: 'Claude Opus 4.6' }, - { value: 'claude-haiku-4-5', label: 'Claude Haiku 4.5' }, - { value: 'manual', label: 'Enter a model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }), - ); - }); - it('offers Anthropic provider paths in the preferred order', async () => { const prompts = makePromptAdapter({ providerChoice: 'back' }); @@ -212,9 +177,38 @@ describe('setup Anthropic model step', () => { const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm).toMatchObject({ provider: { backend: 'claude-code' }, - models: { default: 'sonnet' }, + models: claudeCodePreset, }); - expect(authProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'sonnet' })); + expect(authProbe).toHaveBeenCalledTimes(3); + expect(authProbe).toHaveBeenNthCalledWith(1, expect.objectContaining({ projectDir: tempDir, model: 'sonnet' })); + expect(authProbe).toHaveBeenNthCalledWith(2, expect.objectContaining({ projectDir: tempDir, model: 'haiku' })); + expect(authProbe).toHaveBeenNthCalledWith(3, expect.objectContaining({ projectDir: tempDir, model: 'opus' })); + }); + + it('does not prompt for a Claude Code model during interactive setup', async () => { + const io = makeIo(); + const prompts = makePromptAdapter({ selectValues: ['claude-code'] }); + const authProbe = vi.fn(async () => ({ ok: true as const })); + + const result = await runKtxSetupAnthropicModelStep( + { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, + io.io, + { prompts, claudeCodeAuthProbe: authProbe }, + ); + + expect(result.status).toBe('ready'); + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Which LLM provider should KTX use?'), + }), + ); + expect(prompts.select).not.toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Which Claude Code model should KTX use?'), + }), + ); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.llm.models).toMatchObject(claudeCodePreset); }); it('configures Codex backend and validates local auth', async () => { @@ -226,7 +220,6 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', llmBackend: 'codex', - llmModel: 'gpt-5.5', skipLlm: false, }, io.io, @@ -237,8 +230,9 @@ describe('setup Anthropic model step', () => { const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm).toMatchObject({ provider: { backend: 'codex' }, - models: { default: 'gpt-5.5' }, + models: codexPreset, }); + expect(codexAuthProbe).toHaveBeenCalledTimes(1); expect(codexAuthProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'gpt-5.5' })); // The warning carries the clack gutter so it renders inside the setup frame. expect(io.stderr()).toContain('│ Codex backend isolation is limited'); @@ -264,70 +258,12 @@ describe('setup Anthropic model step', () => { const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm).toMatchObject({ provider: { backend: 'codex' }, - models: { default: 'gpt-5.5' }, + models: codexPreset, }); + expect(codexAuthProbe).toHaveBeenCalledTimes(1); expect(codexAuthProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'gpt-5.5' })); }); - it('offers the curated Codex models during interactive setup', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['codex', 'gpt-5.5'] }); - const codexAuthProbe = vi.fn(async () => ({ ok: true as const })); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - io.io, - { prompts, codexAuthProbe }, - ); - - expect(result.status).toBe('ready'); - expect(prompts.select).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Codex model should KTX use?'), - options: [ - { value: 'gpt-5.5', label: 'GPT-5.5', hint: 'recommended' }, - { value: 'gpt-5.4', label: 'GPT-5.4' }, - { value: 'gpt-5.4-mini', label: 'GPT-5.4 mini' }, - { value: 'manual', label: 'Enter a Codex model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }), - ); - expect(codexAuthProbe).toHaveBeenCalledWith(expect.objectContaining({ model: 'gpt-5.5' })); - }); - - it('prompts for the Claude Code model during interactive setup', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['claude-code', 'opus'] }); - const authProbe = vi.fn(async () => ({ ok: true as const })); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - io.io, - { prompts, claudeCodeAuthProbe: authProbe }, - ); - - expect(result.status).toBe('ready'); - expect(prompts.select).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Claude Code model should KTX use?'), - options: [ - { value: 'sonnet', label: 'Claude Sonnet', hint: 'recommended' }, - { value: 'opus', label: 'Claude Opus' }, - { value: 'haiku', label: 'Claude Haiku' }, - { value: 'manual', label: 'Enter a Claude Code model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }), - ); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.llm).toMatchObject({ - provider: { backend: 'claude-code' }, - models: { default: 'opus' }, - }); - expect(authProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'opus' })); - }); - it('warns during Claude Code setup when existing prompt-caching fields will be ignored', async () => { await writeFile( join(tempDir, 'ktx.yaml'), @@ -392,7 +328,6 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -410,7 +345,7 @@ describe('setup Anthropic model step', () => { backend: 'anthropic', anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret }, - models: { default: 'claude-sonnet-4-6' }, + models: anthropicPreset, promptCaching: { enabled: true }, }); expect(config.scan.enrichment.mode).toBe('llm'); @@ -419,11 +354,62 @@ describe('setup Anthropic model step', () => { expect(spinnerEvents).toEqual([ 'start:Checking Anthropic API LLM (claude-sonnet-4-6).', 'stop:LLM test passed (Anthropic API, claude-sonnet-4-6)', + 'start:Checking Anthropic API LLM (claude-haiku-4-5).', + 'stop:LLM test passed (Anthropic API, claude-haiku-4-5)', + 'start:Checking Anthropic API LLM (claude-opus-4-7).', + 'stop:LLM test passed (Anthropic API, claude-opus-4-7)', ]); expect(io.stdout()).toContain('LLM ready: yes'); expect(io.stdout()).not.toContain('sk-ant-test'); }); + it('degrades unavailable Anthropic non-anchor models to the anchor before persisting', async () => { + const io = makeIo(); + const { events: spinnerEvents, spinner } = makeSpinnerEvents(); + const healthCheck = vi + .fn() + .mockResolvedValueOnce({ ok: true as const }) + .mockResolvedValueOnce({ ok: false as const, message: 'model not enabled' }) + .mockResolvedValueOnce({ ok: true as const }); + + const result = await runKtxSetupAnthropicModelStep( + { + projectDir: tempDir, + inputMode: 'disabled', + anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret + skipLlm: false, + }, + io.io, + { + env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret + healthCheck, + spinner, + }, + ); + + expect(result.status).toBe('ready'); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.llm.models).toMatchObject({ + default: 'claude-sonnet-4-6', + triage: 'claude-sonnet-4-6', + candidateExtraction: 'claude-sonnet-4-6', + curator: 'claude-opus-4-7', + reconcile: 'claude-opus-4-7', + repair: 'claude-sonnet-4-6', + }); + expect(io.stderr()).toContain( + 'LLM model claude-haiku-4-5 is unavailable for triage, repair; using claude-sonnet-4-6 for those roles.', + ); + expect(spinnerEvents).toEqual([ + 'start:Checking Anthropic API LLM (claude-sonnet-4-6).', + 'stop:LLM test passed (Anthropic API, claude-sonnet-4-6)', + 'start:Checking Anthropic API LLM (claude-haiku-4-5).', + 'error:LLM test failed', + 'start:Checking Anthropic API LLM (claude-opus-4-7).', + 'stop:LLM test passed (Anthropic API, claude-opus-4-7)', + ]); + }); + it('configures Vertex AI provider, selected model, prompt caching, and llm completion state', async () => { const io = makeIo(); const healthCheck = vi.fn(async () => ({ ok: true as const })); @@ -436,7 +422,6 @@ describe('setup Anthropic model step', () => { llmBackend: 'vertex', vertexProject: 'local-gcp-project', vertexLocation: 'us-east5', - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -444,19 +429,31 @@ describe('setup Anthropic model step', () => { ); expect(result.status).toBe('ready'); - expect(healthCheck).toHaveBeenCalledWith({ + expect(healthCheck).toHaveBeenNthCalledWith(1, { backend: 'vertex', vertex: { project: 'local-gcp-project', location: 'us-east5' }, modelSlots: { default: 'claude-sonnet-4-6' }, promptCaching: { enabled: true, vertexFallbackTo5m: true }, }); + expect(healthCheck).toHaveBeenNthCalledWith(2, { + backend: 'vertex', + vertex: { project: 'local-gcp-project', location: 'us-east5' }, + modelSlots: { default: 'claude-haiku-4-5' }, + promptCaching: { enabled: true, vertexFallbackTo5m: true }, + }); + expect(healthCheck).toHaveBeenNthCalledWith(3, { + backend: 'vertex', + vertex: { project: 'local-gcp-project', location: 'us-east5' }, + modelSlots: { default: 'claude-opus-4-7' }, + promptCaching: { enabled: true, vertexFallbackTo5m: true }, + }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm).toMatchObject({ provider: { backend: 'vertex', vertex: { project: 'local-gcp-project', location: 'us-east5' }, }, - models: { default: 'claude-sonnet-4-6' }, + models: anthropicPreset, promptCaching: { enabled: true, vertexFallbackTo5m: true }, }); expect(config.scan.enrichment.mode).toBe('llm'); @@ -465,13 +462,17 @@ describe('setup Anthropic model step', () => { expect(spinnerEvents).toEqual([ 'start:Checking Vertex AI LLM (claude-sonnet-4-6).', 'stop:LLM test passed (Vertex AI, claude-sonnet-4-6)', + 'start:Checking Vertex AI LLM (claude-haiku-4-5).', + 'stop:LLM test passed (Vertex AI, claude-haiku-4-5)', + 'start:Checking Vertex AI LLM (claude-opus-4-7).', + 'stop:LLM test passed (Vertex AI, claude-opus-4-7)', ]); expect(io.stdout()).toContain('LLM ready: yes (claude-sonnet-4-6)'); }); it('uses existing Vertex AI credentials without an extra auth choice', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'local-gcp-project', 'claude-sonnet-4-6'] }); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'local-gcp-project'] }); const readGcloudProject = vi.fn(async () => 'local-gcp-project'); const listGcloudProjects = vi.fn(async () => [ { projectId: 'local-gcp-project', name: 'Local project' }, @@ -511,22 +512,6 @@ describe('setup Anthropic model step', () => { ], }), ); - expect(prompts.autocomplete).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Anthropic model should KTX use?'), - options: [ - { value: 'claude-opus-4-7', label: 'Claude Opus 4.7' }, - { value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6' }, - { value: 'claude-opus-4-6', label: 'Claude Opus 4.6' }, - { value: 'claude-opus-4-5', label: 'Claude Opus 4.5' }, - { value: 'claude-haiku-4-5', label: 'Claude Haiku 4.5' }, - { value: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5' }, - { value: 'claude-opus-4-1', label: 'Claude Opus 4.1' }, - { value: 'manual', label: 'Enter a model ID manually' }, - { value: 'back', label: 'Back' }, - ], - }), - ); expect(healthCheck).toHaveBeenCalledWith({ backend: 'vertex', vertex: { project: 'local-gcp-project', location: 'us-east5' }, @@ -542,7 +527,7 @@ describe('setup Anthropic model step', () => { it('skips the Vertex AI auth choice when Application Default Credentials are the only option', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'local-gcp-project', 'claude-sonnet-4-6'] }); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'local-gcp-project'] }); const healthCheck = vi.fn(async () => ({ ok: true as const })); const result = await runKtxSetupAnthropicModelStep( @@ -578,7 +563,7 @@ describe('setup Anthropic model step', () => { it('lets users choose a different visible gcloud project for Vertex AI', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'other-gcp-project', 'claude-sonnet-4-6'] }); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'other-gcp-project'] }); const healthCheck = vi.fn(async () => ({ ok: true as const })); const result = await runKtxSetupAnthropicModelStep( @@ -612,10 +597,7 @@ describe('setup Anthropic model step', () => { it('allows manual Vertex AI project entry when gcloud project listing is empty', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ - selectValues: ['vertex', 'manual', 'claude-sonnet-4-6'], - textValues: ['manual-gcp-project'], - }); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'manual'], textValues: ['manual-gcp-project'] }); const healthCheck = vi.fn(async () => ({ ok: true as const })); const result = await runKtxSetupAnthropicModelStep( @@ -654,7 +636,7 @@ describe('setup Anthropic model step', () => { it('lets users retry Vertex AI project listing after gcloud auth fails', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'retry', 'other-gcp-project', 'claude-sonnet-4-6'] }); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'retry', 'other-gcp-project'] }); const listGcloudProjects = vi .fn() .mockRejectedValueOnce(new Error('Reauthentication failed. cannot prompt during non-interactive execution.')) @@ -743,7 +725,6 @@ describe('setup Anthropic model step', () => { llmBackend: 'vertex', vertexProject: 'kaelio-orbit-looker-20260430', vertexLocation: 'us-east5', - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -771,7 +752,6 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyFile: secretPath, - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -779,19 +759,34 @@ describe('setup Anthropic model step', () => { ); expect(result.status).toBe('ready'); - expect(healthCheck).toHaveBeenCalledWith( + expect(healthCheck).toHaveBeenNthCalledWith( + 1, expect.objectContaining({ anthropic: { apiKey: 'sk-ant-file' }, // pragma: allowlist secret modelSlots: { default: 'claude-sonnet-4-6' }, }), ); + expect(healthCheck).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + anthropic: { apiKey: 'sk-ant-file' }, // pragma: allowlist secret + modelSlots: { default: 'claude-haiku-4-5' }, + }), + ); + expect(healthCheck).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ + anthropic: { apiKey: 'sk-ant-file' }, // pragma: allowlist secret + modelSlots: { default: 'claude-opus-4-7' }, + }), + ); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm).toMatchObject({ provider: { backend: 'anthropic', anthropic: { api_key: `file:${secretPath}` }, // pragma: allowlist secret }, - models: { default: 'claude-sonnet-4-6' }, + models: anthropicPreset, }); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); @@ -808,7 +803,6 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyFile: missingSecretPath, - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -835,32 +829,10 @@ describe('setup Anthropic model step', () => { expect(io.stderr()).not.toContain('--skip-llm'); }); - it('does not recommend skipping when non-interactive setup is missing an LLM model', async () => { - const io = makeIo(); - const healthCheck = vi.fn(async () => ({ ok: true as const })); - - const result = await runKtxSetupAnthropicModelStep( - { - projectDir: tempDir, - inputMode: 'disabled', - anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - skipLlm: false, - }, - io.io, - { env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, healthCheck }, // pragma: allowlist secret - ); - - expect(result.status).toBe('missing-input'); - expect(healthCheck).not.toHaveBeenCalled(); - expect(io.stderr()).toContain('Missing LLM model: pass --llm-model.'); - expect(io.stderr()).not.toContain('--skip-llm'); - }); - it('writes pasted keys to .ktx/secrets and never prints the key', async () => { const io = makeIo(); const prompts = makePromptAdapter({ credentialChoice: 'paste', - modelChoice: 'claude-sonnet-4-6', passwordValue: 'sk-ant-pasted', // pragma: allowlist secret }); @@ -870,7 +842,6 @@ describe('setup Anthropic model step', () => { { prompts, env: {}, - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), healthCheck: vi.fn(async () => ({ ok: true as const })), }, ); @@ -888,7 +859,7 @@ describe('setup Anthropic model step', () => { it('opens pasted key entry directly and tells users Escape goes back', async () => { const prompts = makePromptAdapter({ - selectValues: ['paste', 'claude-sonnet-4-6'], + selectValues: ['paste'], passwordValue: 'sk-ant-pasted', // pragma: allowlist secret }); @@ -898,7 +869,6 @@ describe('setup Anthropic model step', () => { { prompts, env: {}, - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), healthCheck: vi.fn(async () => ({ ok: true as const })), }, ); @@ -956,142 +926,6 @@ describe('setup Anthropic model step', () => { expect(io.stdout()).not.toContain('KTX uses the key'); }); - it('does not offer skipping while choosing an Anthropic model', async () => { - const prompts = makePromptAdapter({ selectValues: ['env', 'back', 'back'] }); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - makeIo().io, - { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), - }, - ); - - expect(result.status).toBe('back'); - expect(prompts.autocomplete).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Anthropic model should KTX use?'), - options: expect.not.arrayContaining([expect.objectContaining({ value: 'skip' })]), - }), - ); - }); - - it('explains why KTX asks for an Anthropic model', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ credentialChoice: 'env', modelChoice: 'claude-sonnet-4-6' }); - const expectedPromptMessage = [ - 'Which Anthropic model should KTX use?', - '', - [ - 'KTX uses this as the default model for ingest agents that turn schemas, SQL, BI metadata, and docs', - 'into semantic-layer sources and wiki context.', - ].join(' '), - ].join('\n'); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - io.io, - { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), - healthCheck: vi.fn(async () => ({ ok: true as const })), - }, - ); - - expect(result.status).toBe('ready'); - expect(prompts.autocomplete).toHaveBeenCalledWith( - expect.objectContaining({ - message: expectedPromptMessage, - }), - ); - expect(io.stdout()).not.toContain('KTX uses this as the default model'); - expect(io.stdout()).not.toContain('Setup verifies the selected model now'); - }); - - it('uses the bundled fallback registry when live discovery fails', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ credentialChoice: 'env', modelChoice: 'claude-sonnet-4-6' }); - - await expect( - runKtxSetupAnthropicModelStep({ projectDir: tempDir, inputMode: 'auto', skipLlm: false }, io.io, { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => { - throw new Error('network unavailable'); - }), - healthCheck: vi.fn(async () => ({ ok: true as const })), - }), - ).resolves.toMatchObject({ status: 'ready' }); - - expect(io.stderr()).toContain('Could not fetch live Anthropic models. Showing bundled defaults.'); - }); - - it('shows bundled model choices when live discovery fails', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['env', 'manual'], textValues: [''] }); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - io.io, - { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => { - throw new Error('network unavailable'); - }), - healthCheck: vi.fn(async () => ({ ok: true as const })), - }, - ); - - expect(result.status).toBe('missing-input'); - expect(BUNDLED_ANTHROPIC_MODELS.length).toBeGreaterThan(0); - expect(prompts.autocomplete).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which Anthropic model should KTX use?'), - options: expect.arrayContaining([ - { value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', hint: 'recommended' }, - ]), - }), - ); - expect(prompts.text).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'Anthropic model ID\n│ Press Escape to go back.\n│', - placeholder: 'claude-sonnet-4-6', - }), - ); - }); - - it('reports invalid Anthropic API keys during live discovery instead of showing bundled defaults', async () => { - const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['env', 'back'] }); - const fetchModels = vi.fn( - async () => new Response(JSON.stringify({ error: { message: 'invalid x-api-key' } }), { status: 401 }), - ); - const healthCheck = vi.fn(async () => ({ ok: true as const })); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - io.io, - { - prompts, - env: { ANTHROPIC_API_KEY: 'sk-ant-invalid' }, // pragma: allowlist secret - fetch: fetchModels, - healthCheck, - }, - ); - - expect(result.status).toBe('back'); - expect(fetchModels).toHaveBeenCalledTimes(1); - expect(healthCheck).not.toHaveBeenCalled(); - expect(io.stderr()).toContain('Anthropic API key is invalid or unauthorized'); - expect(io.stderr()).toContain('Choose a different credential source or Back.'); - expect(io.stderr()).not.toContain('Could not fetch live Anthropic models. Showing bundled defaults.'); - expect(io.stderr()).not.toContain('sk-ant-invalid'); - }); - it('does not persist llm completion when the health check fails', async () => { const io = makeIo(); const result = await runKtxSetupAnthropicModelStep( @@ -1099,7 +933,6 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -1117,12 +950,12 @@ describe('setup Anthropic model step', () => { it('re-prompts after an interactive health-check failure and saves after retry success', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ - selectValues: ['env', 'claude-haiku-3-5', 'env', 'claude-sonnet-4-6'], - }); + const prompts = makePromptAdapter({ selectValues: ['env', 'env'] }); const healthCheck = vi .fn() .mockResolvedValueOnce({ ok: false as const, message: 'model not found' }) + .mockResolvedValueOnce({ ok: true as const }) + .mockResolvedValueOnce({ ok: true as const }) .mockResolvedValueOnce({ ok: true as const }); const result = await runKtxSetupAnthropicModelStep( @@ -1131,22 +964,22 @@ describe('setup Anthropic model step', () => { { prompts, env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret - listModels: vi.fn(async () => [ - { id: 'claude-haiku-3-5', label: 'Claude Haiku 3.5', recommended: false }, - { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }, - ]), healthCheck, }, ); expect(result.status).toBe('ready'); - expect(healthCheck).toHaveBeenCalledTimes(2); + expect(healthCheck).toHaveBeenCalledTimes(4); expect(prompts.select).toHaveBeenCalledTimes(3); - expect(prompts.autocomplete).toHaveBeenCalledTimes(2); + expect(prompts.autocomplete).not.toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Which Anthropic model should KTX use?'), + }), + ); expect(io.stderr()).toContain('Anthropic model health check failed: model not found'); - expect(io.stderr()).toContain('Choose a different credential source or model, or Back.'); + expect(io.stderr()).toContain('Choose a different credential source or Back.'); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.llm.models.default).toBe('claude-sonnet-4-6'); + expect(config.llm.models).toMatchObject(anthropicPreset); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); expect(io.stderr()).not.toContain('sk-ant-test'); @@ -1175,39 +1008,8 @@ describe('setup Anthropic model step', () => { expect(config.llm.provider.backend).toBe('none'); }); - it('returns from model selection Back to credential selection instead of exiting setup', async () => { - const prompts = makePromptAdapter({ - selectValues: ['paste', 'back', 'back'], - passwordValue: 'sk-ant-pasted', // pragma: allowlist secret - }); - - const result = await runKtxSetupAnthropicModelStep( - { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, - makeIo().io, - { - prompts, - env: {}, - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), - healthCheck: vi.fn(async () => ({ ok: true as const })), - }, - ); - - expect(result.status).toBe('back'); - expect(prompts.select).toHaveBeenNthCalledWith( - 3, - expect.objectContaining({ - message: expect.stringContaining('How should KTX find your Anthropic API key?'), - }), - ); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.llm.provider.backend).toBe('none'); - }); - it('returns from pasted key entry Escape to credential selection and can use env credentials', async () => { - const prompts = makePromptAdapter({ - selectValues: ['paste', 'env', 'claude-sonnet-4-6'], - passwordValues: [undefined], - }); + const prompts = makePromptAdapter({ selectValues: ['paste', 'env'], passwordValues: [undefined] }); const result = await runKtxSetupAnthropicModelStep( { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, @@ -1215,7 +1017,6 @@ describe('setup Anthropic model step', () => { { prompts, env: { ANTHROPIC_API_KEY: 'sk-ant-env' }, // pragma: allowlist secret - listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]), healthCheck: vi.fn(async () => ({ ok: true as const })), }, ); diff --git a/packages/cli/test/setup.test.ts b/packages/cli/test/setup.test.ts index 9b8bf689..ecb20520 100644 --- a/packages/cli/test/setup.test.ts +++ b/packages/cli/test/setup.test.ts @@ -1305,7 +1305,6 @@ describe('setup status', () => { yes: true, cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - llmModel: 'claude-sonnet-4-6', skipLlm: false, skipEmbeddings: true, databaseSchemas: [], @@ -1322,7 +1321,6 @@ describe('setup status', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - llmModel: 'claude-sonnet-4-6', skipLlm: false, }), testIo.io, @@ -1347,7 +1345,6 @@ describe('setup status', () => { llmBackend: 'vertex', vertexProject: 'local-gcp-project', vertexLocation: 'us-east5', - llmModel: 'claude-sonnet-4-6', skipLlm: false, skipEmbeddings: true, databaseSchemas: [], @@ -1366,7 +1363,6 @@ describe('setup status', () => { llmBackend: 'vertex', vertexProject: 'local-gcp-project', vertexLocation: 'us-east5', - llmModel: 'claude-sonnet-4-6', skipLlm: false, }), testIo.io, @@ -1390,7 +1386,6 @@ describe('setup status', () => { yes: true, cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - llmModel: 'claude-sonnet-4-6', skipLlm: false, embeddingBackend: 'openai', embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret @@ -1658,7 +1653,6 @@ describe('setup status', () => { yes: true, cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - llmModel: 'claude-sonnet-4-6', skipLlm: false, embeddingBackend: 'openai', embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret @@ -2657,7 +2651,6 @@ describe('setup status', () => { yes: true, cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - llmModel: 'claude-sonnet-4-6', skipLlm: false, skipEmbeddings: false, databaseSchemas: [], diff --git a/scripts/codex-backend-live-smoke.mjs b/scripts/codex-backend-live-smoke.mjs index 7793fefc..0d25a10a 100644 --- a/scripts/codex-backend-live-smoke.mjs +++ b/scripts/codex-backend-live-smoke.mjs @@ -68,8 +68,6 @@ async function runSetupSmoke(projectDir) { projectDir, '--llm-backend', 'codex', - '--llm-model', - 'gpt-5.3-codex', '--no-input', '--yes', '--skip-databases', @@ -79,7 +77,7 @@ async function runSetupSmoke(projectDir) { { timeoutMs: 600_000 }, ); requireSuccess('ktx setup codex backend', result); - if (!result.stdout.includes('LLM ready: yes (codex, gpt-5.3-codex)')) { + if (!result.stdout.includes('LLM ready: yes (codex, gpt-5.5)')) { throw new Error(`setup did not report Codex LLM readiness\nstdout:\n${result.stdout}`); } } @@ -91,7 +89,14 @@ async function runRuntimeSmoke(projectDir) { const { z } = await import(zodUrl); const runtime = new CodexKtxLlmRuntime({ projectDir, - modelSlots: { default: 'gpt-5.3-codex' }, + modelSlots: { + default: 'gpt-5.5', + triage: 'gpt-5.5', + candidateExtraction: 'gpt-5.5', + curator: 'gpt-5.5', + reconcile: 'gpt-5.5', + repair: 'gpt-5.5', + }, }); const text = await runtime.generateText({