From e50fef851f4c717a228836ebf0a61e85ddfb400a Mon Sep 17 00:00:00 2001 From: Luca Martial Date: Wed, 13 May 2026 09:16:35 -0700 Subject: [PATCH 1/8] fix(cli): hide setup project banner --- packages/cli/src/cli-program.ts | 4 ++++ packages/cli/src/index.test.ts | 16 ++++++++++++++++ packages/cli/src/project-dir.test.ts | 2 +- packages/cli/src/standalone-smoke.test.ts | 10 +++++++--- scripts/package-artifacts.mjs | 4 ++-- 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index 69437aec..efe2e5bb 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -162,6 +162,10 @@ function shouldSuppressProjectDirLine(path: string[], options: Record { expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`); }); + it('does not print the command-level project directory line for setup', async () => { + const setup = vi.fn(async () => 0); + const testIo = makeIo(); + + await expect(runKtxCli(['--project-dir', tempDir, 'setup', '--no-input'], testIo.io, { setup })).resolves.toBe(0); + + expect(setup).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'run', + projectDir: tempDir, + }), + testIo.io, + ); + expect(testIo.stderr()).toBe(''); + }); + it('skips the project directory line for JSON and TUI output modes', async () => { const ingest = vi.fn(async () => 0); const jsonIo = makeIo(); diff --git a/packages/cli/src/project-dir.test.ts b/packages/cli/src/project-dir.test.ts index c0022d4d..02502b35 100644 --- a/packages/cli/src/project-dir.test.ts +++ b/packages/cli/src/project-dir.test.ts @@ -65,7 +65,7 @@ describe('project directory defaults', () => { argv: ['setup', '--no-input'], spy: setup, expected: { command: 'run', projectDir: '/tmp/ktx-env-project' }, - expectedStderr: 'Project: /tmp/ktx-env-project\n', + expectedStderr: '', }, { argv: ['scan', 'warehouse'], diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/src/standalone-smoke.test.ts index c6fefd96..6f878617 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/src/standalone-smoke.test.ts @@ -130,6 +130,10 @@ function expectProjectStderr(result: CliResult, projectDir: string): void { expect(result).toMatchObject({ code: 0, stderr: `Project: ${projectDir}\n` }); } +function expectSetupStderr(result: CliResult): void { + expect(result).toMatchObject({ code: 0, stderr: '' }); +} + async function runSetupNewProject(projectDir: string): Promise { return await runBuiltCli([ 'setup', @@ -162,7 +166,7 @@ describe('standalone built ktx CLI smoke', () => { const sourceDir = join(tempDir, 'source'); const init = await runSetupNewProject(projectDir); - expectProjectStderr(init, projectDir); + expectSetupStderr(init); expect(init.stdout).toContain(`Project: ${projectDir}`); await writeWarehouseConfig(projectDir); @@ -207,7 +211,7 @@ describe('standalone built ktx CLI smoke', () => { it('runs structural and enriched scans through the built binary with manifest artifacts', async () => { const projectDir = join(tempDir, 'scan-project'); const init = await runSetupNewProject(projectDir); - expectProjectStderr(init, projectDir); + expectSetupStderr(init); const dbPath = join(projectDir, 'warehouse.db'); createSqliteWarehouse(dbPath); @@ -310,7 +314,7 @@ describe('standalone built ktx CLI smoke', () => { it('rejects the removed connection add command through the built binary', async () => { const projectDir = join(tempDir, 'notion-project'); const init = await runSetupNewProject(projectDir); - expectProjectStderr(init, projectDir); + expectSetupStderr(init); const add = await runBuiltCli([ 'connection', diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index b74b4277..e7998c58 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -618,7 +618,7 @@ try { '--skip-sources', '--skip-agents', ]); - requireProjectStderr('ktx setup', init, projectDir); + requireSuccess('ktx setup', init); requireOutput('ktx setup', init, /Project: /); const emptyProjectDir = join(root, 'empty-project'); @@ -637,7 +637,7 @@ try { '--skip-sources', '--skip-agents', ]); - requireProjectStderr('ktx setup empty project', emptyInit, emptyProjectDir); + requireSuccess('ktx setup empty project', emptyInit); await writeFile( join(projectDir, 'ktx.yaml'), [ From d1b59364412b70ebec1f9b0d7180a5b518ac63bc Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Wed, 13 May 2026 19:32:49 +0200 Subject: [PATCH 2/8] feat(cli): add text ingest command (#72) --- packages/cli/src/cli-program.ts | 4 + packages/cli/src/cli-runtime.ts | 2 + packages/cli/src/commands/ingest-commands.ts | 31 +- packages/cli/src/context-build-view.test.ts | 24 ++ packages/cli/src/context-build-view.ts | 48 ++- packages/cli/src/index.test.ts | 59 ++++ packages/cli/src/text-ingest.test.ts | 339 ++++++++++++++++++ packages/cli/src/text-ingest.ts | 354 +++++++++++++++++++ 8 files changed, 850 insertions(+), 11 deletions(-) create mode 100644 packages/cli/src/text-ingest.test.ts create mode 100644 packages/cli/src/text-ingest.ts diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index efe2e5bb..c1935495 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -316,6 +316,10 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command { registerIngestCommands(program, context, { runIngestWithProgress: async (ingestArgs, ingestIo, ingestDeps, defaultRunIngest) => await (ingestDeps.ingest ?? defaultRunIngest)(ingestArgs, ingestIo), + runTextIngest: async (textIngestArgs, ingestIo, ingestDeps) => { + const { runKtxTextIngest } = await import('./text-ingest.js'); + return await (ingestDeps.textIngest ?? runKtxTextIngest)(textIngestArgs, ingestIo); + }, }); registerScanCommands(program, context); registerWikiCommands(program, context); diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index 5e2430cf..2712558f 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -9,6 +9,7 @@ import type { KtxScanArgs } from './scan.js'; import type { KtxSetupArgs } from './setup.js'; import type { KtxSlArgs } from './sl.js'; import { profileMark, profileSpan } from './startup-profile.js'; +import type { KtxTextIngestArgs } from './text-ingest.js'; profileMark('module:cli-runtime'); @@ -30,6 +31,7 @@ export interface KtxCliDeps { connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise; doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise; ingest?: (args: KtxIngestArgs, io: KtxCliIo) => Promise; + textIngest?: (args: KtxTextIngestArgs, io: KtxCliIo) => Promise; runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise; scan?: (args: KtxScanArgs, io: KtxCliIo) => Promise; knowledge?: (args: KtxKnowledgeArgs, io: KtxCliIo) => Promise; diff --git a/packages/cli/src/commands/ingest-commands.ts b/packages/cli/src/commands/ingest-commands.ts index 5ad357e1..952b6aa0 100644 --- a/packages/cli/src/commands/ingest-commands.ts +++ b/packages/cli/src/commands/ingest-commands.ts @@ -1,10 +1,11 @@ import { resolve } from 'node:path'; import { type Command, Option } from '@commander-js/extra-typings'; -import { type KtxCliCommandContext, type OutputModeOptions, resolveCommandProjectDir } from '../cli-program.js'; +import { collectOption, type KtxCliCommandContext, type OutputModeOptions, resolveCommandProjectDir } from '../cli-program.js'; import type { KtxCliDeps, KtxCliIo } from '../index.js'; import type { KtxIngestArgs, KtxIngestOutputMode } from '../ingest.js'; import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; import { profileMark } from '../startup-profile.js'; +import type { KtxTextIngestArgs } from '../text-ingest.js'; profileMark('module:commands/ingest-commands'); @@ -15,6 +16,7 @@ interface IngestCommandOptions { deps: KtxCliDeps, defaultRunIngest: (args: KtxIngestArgs, io: KtxCliIo) => Promise, ) => Promise; + runTextIngest: (args: KtxTextIngestArgs, io: KtxCliIo, deps: KtxCliDeps) => Promise; } function outputMode(options: OutputModeOptions): KtxIngestOutputMode { @@ -101,6 +103,33 @@ export function registerIngestCommands( ); }); + ingest + .command('text') + .description('Ingest free-form text artifacts into KTX memory') + .argument('[files...]', 'Files to ingest; use - to read one item from stdin') + .option('--text ', 'Text content to ingest; repeat for a batch', collectOption, []) + .option('--connection-id ', 'Optional KTX connection id for semantic-layer capture') + .option('--user-id ', 'Memory user id for capture attribution', 'local-cli') + .option('--json', 'Print JSON output') + .option('--fail-fast', 'Stop after the first failed text item', false) + .action(async (files: string[], options, command) => { + context.setExitCode( + await commandOptions.runTextIngest( + { + projectDir: resolveCommandProjectDir(command), + texts: options.text, + files, + ...(options.connectionId ? { connectionId: options.connectionId } : {}), + userId: options.userId, + json: options.json === true, + failFast: options.failFast === true, + }, + context.io, + context.deps, + ), + ); + }); + ingest .command('status') .description('Print status for the latest or selected stored local ingest run or report file') diff --git a/packages/cli/src/context-build-view.test.ts b/packages/cli/src/context-build-view.test.ts index c8dc5130..db172484 100644 --- a/packages/cli/src/context-build-view.test.ts +++ b/packages/cli/src/context-build-view.test.ts @@ -158,6 +158,30 @@ describe('renderContextBuildView', () => { expect(output).toContain('dbt-main'); }); + it('supports text ingest labels while preserving the shared compact progress view', () => { + const state = initViewState([ + { connectionId: 'text-1', driver: 'text', operation: 'source-ingest', debugCommand: '', steps: ['memory-update'] }, + { connectionId: 'schema.md', driver: 'text', operation: 'source-ingest', debugCommand: '', steps: ['memory-update'] }, + ]); + state.contextSources[0].status = 'running'; + state.contextSources[0].detailLine = 'capturing...'; + + const output = renderContextBuildView(state, { + styled: false, + title: 'Ingesting text memory', + contextGroupLabel: 'Texts', + sourceIngestRunningText: 'capturing...', + completedItemName: { singular: 'text', plural: 'texts' }, + }); + + expect(output).toContain('Ingesting text memory'); + expect(output).toContain('Texts:'); + expect(output).toContain('text-1'); + expect(output).toContain('schema.md'); + expect(output).toContain('capturing...'); + expect(output).not.toContain('Context sources:'); + }); + it('renders header with total elapsed time when set', () => { const state = initViewState([ { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, diff --git a/packages/cli/src/context-build-view.ts b/packages/cli/src/context-build-view.ts index 38f3d674..e1f43ead 100644 --- a/packages/cli/src/context-build-view.ts +++ b/packages/cli/src/context-build-view.ts @@ -65,6 +65,24 @@ export interface ContextBuildSourceProgressUpdate { summaryText?: string; } +interface CompletedItemName { + singular: string; + plural: string; +} + +interface ContextBuildRenderOptions { + styled?: boolean; + showHint?: boolean; + hintText?: string; + projectDir?: string; + title?: string; + primaryGroupLabel?: string; + contextGroupLabel?: string; + scanRunningText?: string; + sourceIngestRunningText?: string; + completedItemName?: CompletedItemName; +} + export interface ContextBuildDeps { executeTarget?: typeof executePublicIngestTarget; now?: () => number; @@ -148,7 +166,7 @@ function staleProgressText(target: ContextBuildTargetState, styled: boolean): st return styled ? dim(text) : text; } -function targetDetail(target: ContextBuildTargetState, styled: boolean): string { +function targetDetail(target: ContextBuildTargetState, styled: boolean, options: ContextBuildRenderOptions): string { if (target.status === 'done') { const parts: string[] = []; if (target.summaryText) parts.push(target.summaryText); @@ -162,7 +180,9 @@ function targetDetail(target: ContextBuildTargetState, styled: boolean): string if (target.status === 'running') { const percent = extractPercent(target.detailLine); const progressText = target.detailLine?.replace(/^\[\d+%\]\s*/, '') - ?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...'); + ?? (target.target.operation === 'scan' + ? (options.scanRunningText ?? 'scanning...') + : (options.sourceIngestRunningText ?? 'ingesting...')); const elapsed = target.elapsedMs > 0 ? `(${formatDuration(target.elapsedMs)})` : null; const parts: string[] = []; if (percent !== null) { @@ -182,8 +202,14 @@ function columnWidth(state: ContextBuildViewState): number { return Math.max(12, ...all.map((t) => t.target.connectionId.length)) + 2; } -function renderTargetLine(target: ContextBuildTargetState, frame: number, styled: boolean, width: number): string { - return ` ${statusIcon(target.status, frame, styled)} ${target.target.connectionId.padEnd(width)} ${targetDetail(target, styled)}`; +function renderTargetLine( + target: ContextBuildTargetState, + frame: number, + styled: boolean, + width: number, + options: ContextBuildRenderOptions, +): string { + return ` ${statusIcon(target.status, frame, styled)} ${target.target.connectionId.padEnd(width)} ${targetDetail(target, styled, options)}`; } function renderTargetGroup( @@ -192,9 +218,10 @@ function renderTargetGroup( frame: number, styled: boolean, width: number, + options: ContextBuildRenderOptions, ): string[] { if (targets.length === 0) return []; - return ['', ` ${label}:`, ...targets.map((t) => renderTargetLine(t, frame, styled, width))]; + return ['', ` ${label}:`, ...targets.map((t) => renderTargetLine(t, frame, styled, width, options))]; } function resumeCommand(projectDir?: string): string { @@ -203,7 +230,7 @@ function resumeCommand(projectDir?: string): string { export function renderContextBuildView( state: ContextBuildViewState, - options: { styled?: boolean; showHint?: boolean; hintText?: string; projectDir?: string } = {}, + options: ContextBuildRenderOptions = {}, ): string { const styled = options.styled ?? true; const width = columnWidth(state); @@ -213,7 +240,7 @@ export function renderContextBuildView( const hasActive = allTargets.some((t) => t.status === 'running' || t.status === 'queued'); const allDone = totalCount > 0 && !hasActive; - const headerParts = ['Building KTX context']; + const headerParts = [options.title ?? 'Building KTX context']; if (totalCount > 0) { const progressParts: string[] = [`${doneCount}/${totalCount}`]; if (state.totalElapsedMs > 0) progressParts.push(formatDuration(state.totalElapsedMs)); @@ -229,13 +256,14 @@ export function renderContextBuildView( header, separator, ...(options.projectDir ? [` Project: ${options.projectDir}`] : []), - ...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width), - ...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width), + ...renderTargetGroup(options.primaryGroupLabel ?? 'Primary sources', state.primarySources, state.frame, styled, width, options), + ...renderTargetGroup(options.contextGroupLabel ?? 'Context sources', state.contextSources, state.frame, styled, width, options), '', ]; if (allDone && state.totalElapsedMs > 0) { - const sourcesLabel = totalCount === 1 ? '1 source' : `${totalCount} sources`; + const itemName = options.completedItemName ?? { singular: 'source', plural: 'sources' }; + const sourcesLabel = totalCount === 1 ? `1 ${itemName.singular}` : `${totalCount} ${itemName.plural}`; const summary = ` Done in ${formatDuration(state.totalElapsedMs)} · ${sourcesLabel} processed`; lines.push(styled ? green(summary) : summary); lines.push(''); diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 83f11639..cd635d78 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -734,14 +734,73 @@ describe('runKtxCli', () => { expect(testIo.stdout()).toContain('Usage: ktx ingest [options] [command]'); expect(testIo.stdout()).toContain('Run or inspect local ingest memory-flow output'); expect(testIo.stdout()).toContain('run'); + expect(testIo.stdout()).toContain('text'); expect(testIo.stdout()).toContain('status'); expect(testIo.stdout()).toContain('watch'); expect(testIo.stdout()).toContain('replay'); + expect(testIo.stdout()).not.toContain('--manifest'); expect(testIo.stdout()).not.toContain('--all'); expect(testIo.stderr()).toBe(''); expect(ingest).not.toHaveBeenCalled(); }); + it('routes text memory ingest through Commander without exposing chat ids', async () => { + const textIngest = vi.fn(async () => 0); + const testIo = makeIo(); + + await expect( + runKtxCli( + [ + '--project-dir', + tempDir, + 'ingest', + 'text', + '--text', + 'Revenue means gross receipts.', + '--text', + 'Orders are completed purchases.', + '--connection-id', + 'warehouse', + '--user-id', + 'agent', + '--json', + '--fail-fast', + ], + testIo.io, + { textIngest }, + ), + ).resolves.toBe(0); + + expect(textIngest).toHaveBeenCalledWith( + { + projectDir: tempDir, + texts: ['Revenue means gross receipts.', 'Orders are completed purchases.'], + files: [], + connectionId: 'warehouse', + userId: 'agent', + json: true, + failFast: true, + }, + testIo.io, + ); + expect(testIo.stderr()).toBe(''); + }); + + it('documents text ingest inputs without a manifest option', async () => { + const textIngest = vi.fn(async () => 0); + const testIo = makeIo(); + + await expect(runKtxCli(['ingest', 'text', '--help'], testIo.io, { textIngest })).resolves.toBe(0); + + expect(testIo.stdout()).toContain('Usage: ktx ingest text [options] [files...]'); + expect(testIo.stdout()).toContain('--text '); + expect(testIo.stdout()).toContain('--connection-id '); + expect(testIo.stdout()).toContain('--user-id '); + expect(testIo.stdout()).toContain('--fail-fast'); + expect(testIo.stdout()).not.toContain('--manifest'); + expect(textIngest).not.toHaveBeenCalled(); + }); + it('routes ingest run at the top level and rejects removed dev ingest', async () => { const runIo = makeIo(); const devRunIo = makeIo(); diff --git a/packages/cli/src/text-ingest.test.ts b/packages/cli/src/text-ingest.test.ts new file mode 100644 index 00000000..55dbe9e3 --- /dev/null +++ b/packages/cli/src/text-ingest.test.ts @@ -0,0 +1,339 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { MemoryCaptureStatus } from '@ktx/context/memory'; +import type { KtxLocalProject } from '@ktx/context/project'; +import { runKtxTextIngest, type TextMemoryCapturePort } from './text-ingest.js'; + +function makeIo(options: { isTTY?: boolean } = {}) { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + isTTY: options.isTTY, + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +function fakeCapture( + options: { + failRunIds?: Set; + missingStatusRunIds?: Set; + events?: string[]; + } = {}, +): TextMemoryCapturePort { + let next = 1; + return { + capture: vi.fn(async () => { + const runId = `run-${next++}`; + options.events?.push(`capture:${runId}`); + return { runId }; + }), + waitForRun: vi.fn(async (runId: string) => { + options.events?.push(`wait:${runId}`); + }), + status: vi.fn(async (runId: string) => { + options.events?.push(`status:${runId}`); + if (options.missingStatusRunIds?.has(runId)) { + return null; + } + if (options.failRunIds?.has(runId)) { + return { + runId, + status: 'error', + stage: 'capturing', + done: true, + captured: { wiki: [], sl: [], xrefs: [] }, + error: `${runId} failed`, + commitHash: null, + skillsLoaded: [], + signalDetected: false, + } satisfies MemoryCaptureStatus; + } + return { + runId, + status: 'done', + stage: 'capturing', + done: true, + captured: { wiki: [`wiki-${runId}`], sl: [`sl-${runId}`], xrefs: [] }, + error: null, + commitHash: `commit-${runId}`, + skillsLoaded: ['wiki_capture', 'sl'], + signalDetected: true, + } satisfies MemoryCaptureStatus; + }), + }; +} + +function fakeProject(projectDir = '/tmp/project'): KtxLocalProject { + return { projectDir } as KtxLocalProject; +} + +describe('runKtxTextIngest', () => { + it('captures repeated inline text sequentially with generated internal chat ids', async () => { + const io = makeIo(); + const events: string[] = []; + const capture = fakeCapture({ events }); + const createMemoryCapture = vi.fn(() => capture); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['Revenue means gross receipts.', 'Orders are completed purchases.'], + files: [], + userId: 'local-cli', + json: true, + failFast: false, + }, + io.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture, + now: () => 1_700_000_000_000, + }, + ), + ).resolves.toBe(0); + + expect(createMemoryCapture).toHaveBeenCalledWith({ projectDir: '/tmp/project' }); + expect(capture.capture).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + userId: 'local-cli', + chatId: 'cli-text-ingest-1700000000000-1', + userMessage: 'Ingest external text artifact "Revenue means gross receipts." into KTX memory.', + assistantMessage: 'Revenue means gross receipts.', + sourceType: 'external_ingest', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + chatId: 'cli-text-ingest-1700000000000-2', + userMessage: 'Ingest external text artifact "Orders are completed purchases." into KTX memory.', + assistantMessage: 'Orders are completed purchases.', + }), + ); + expect(capture.capture).not.toHaveBeenCalledWith(expect.objectContaining({ connectionId: expect.anything() })); + expect(events).toEqual(['capture:run-1', 'wait:run-1', 'status:run-1', 'capture:run-2', 'wait:run-2', 'status:run-2']); + expect(JSON.parse(io.stdout())).toMatchObject({ + status: 'done', + results: [ + { + label: '"Revenue means gross receipts."', + runId: 'run-1', + status: 'done', + captured: { wiki: ['wiki-run-1'], sl: ['sl-run-1'] }, + }, + { + label: '"Orders are completed purchases."', + runId: 'run-2', + status: 'done', + captured: { wiki: ['wiki-run-2'], sl: ['sl-run-2'] }, + }, + ], + }); + }); + + it('loads files and stdin as batch items and passes a global connection id', async () => { + const io = makeIo(); + const capture = fakeCapture(); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: [], + files: ['/tmp/docs/revenue.md', '-'], + connectionId: 'warehouse', + userId: 'agent', + json: false, + failFast: false, + }, + io.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => capture), + readFile: vi.fn(async (path) => `file:${path}`), + readStdin: vi.fn(async () => 'stdin content'), + now: () => 10, + }, + ), + ).resolves.toBe(0); + + expect(capture.capture).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + connectionId: 'warehouse', + userId: 'agent', + userMessage: 'Ingest external text artifact "revenue.md" into KTX memory.', + assistantMessage: 'file:/tmp/docs/revenue.md', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + connectionId: 'warehouse', + userMessage: 'Ingest external text artifact "stdin" into KTX memory.', + assistantMessage: 'stdin content', + }), + ); + expect(io.stdout()).toContain('Ingesting text memory'); + expect(io.stdout()).toContain('Texts:'); + expect(io.stdout()).toContain('revenue.md'); + expect(io.stdout()).toContain('stdin'); + }); + + it('uses bounded inline text previews as labels in plain output and capture metadata', async () => { + const io = makeIo(); + const capture = fakeCapture(); + const longText = `This inline note is intentionally long ${'x'.repeat(120)}`; + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['remember to call me Andrey', ' first line\n\tsecond line ', longText], + files: [], + userId: 'local-cli', + json: false, + failFast: false, + }, + io.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => capture), + now: () => 10, + }, + ), + ).resolves.toBe(0); + + const output = io.stdout(); + expect(output).toContain('"remember to call me Andrey"'); + expect(output).toContain('"first line second line"'); + expect(output).toContain('"This inline note is intentionally long xxxxxxxx..."'); + expect(output).not.toContain('text-1'); + expect(output).not.toContain(longText); + + expect(capture.capture).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + userMessage: 'Ingest external text artifact "remember to call me Andrey" into KTX memory.', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + userMessage: 'Ingest external text artifact "first line second line" into KTX memory.', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ + userMessage: 'Ingest external text artifact "This inline note is intentionally long xxxxxxxx..." into KTX memory.', + }), + ); + }); + + it('continues after an item failure by default and stops when failFast is set', async () => { + const continueIo = makeIo(); + const continueCapture = fakeCapture({ failRunIds: new Set(['run-1']) }); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['bad', 'good'], + files: [], + userId: 'local-cli', + json: true, + failFast: false, + }, + continueIo.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => continueCapture), + }, + ), + ).resolves.toBe(1); + + expect(continueCapture.capture).toHaveBeenCalledTimes(2); + expect(JSON.parse(continueIo.stdout())).toMatchObject({ + status: 'failed', + results: [ + { label: '"bad"', status: 'error', error: 'run-1 failed' }, + { label: '"good"', status: 'done' }, + ], + }); + + const failFastIo = makeIo(); + const failFastCapture = fakeCapture({ failRunIds: new Set(['run-1']) }); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['bad', 'skipped'], + files: [], + userId: 'local-cli', + json: true, + failFast: true, + }, + failFastIo.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => failFastCapture), + }, + ), + ).resolves.toBe(1); + + expect(failFastCapture.capture).toHaveBeenCalledTimes(1); + expect(JSON.parse(failFastIo.stdout()).results).toHaveLength(1); + }); + + it('rejects empty batches and empty text items', async () => { + const noInputIo = makeIo(); + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: [], + files: [], + userId: 'local-cli', + json: false, + failFast: false, + }, + noInputIo.io, + { loadProject: vi.fn(), createMemoryCapture: vi.fn() }, + ), + ).resolves.toBe(1); + expect(noInputIo.stderr()).toContain('Provide at least one text item'); + + const emptyIo = makeIo(); + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: [' '], + files: [], + userId: 'local-cli', + json: false, + failFast: false, + }, + emptyIo.io, + { loadProject: vi.fn(), createMemoryCapture: vi.fn() }, + ), + ).resolves.toBe(1); + expect(emptyIo.stderr()).toContain('Text item "text-1" is empty'); + }); +}); diff --git a/packages/cli/src/text-ingest.ts b/packages/cli/src/text-ingest.ts new file mode 100644 index 00000000..d48ee24b --- /dev/null +++ b/packages/cli/src/text-ingest.ts @@ -0,0 +1,354 @@ +import { readFile as fsReadFile } from 'node:fs/promises'; +import { basename, resolve } from 'node:path'; +import { createLocalProjectMemoryCapture, type MemoryAgentInput, type MemoryCaptureStatus } from '@ktx/context/memory'; +import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; +import type { KtxCliIo } from './cli-runtime.js'; +import { createRepainter, initViewState, renderContextBuildView, type ContextBuildTargetState } from './context-build-view.js'; +import { formatDuration } from './demo-metrics.js'; +import type { KtxPublicIngestPlanTarget } from './public-ingest.js'; + +export interface KtxTextIngestArgs { + projectDir: string; + texts: string[]; + files: string[]; + connectionId?: string; + userId: string; + json: boolean; + failFast: boolean; +} + +export interface TextMemoryCapturePort { + capture(input: MemoryAgentInput): Promise<{ runId: string }>; + waitForRun(runId: string): Promise; + status(runId: string): Promise; +} + +interface TextIngestItem { + label: string; + content: string; +} + +interface TextIngestResult { + label: string; + runId: string | null; + status: 'done' | 'error'; + captured: MemoryCaptureStatus['captured']; + commitHash: string | null; + error: string | null; +} + +export interface KtxTextIngestDeps { + loadProject?: (options: { projectDir: string }) => Promise; + createMemoryCapture?: (project: KtxLocalProject) => TextMemoryCapturePort; + readFile?: (path: string) => Promise; + readStdin?: () => Promise; + now?: () => number; +} + +const INLINE_TEXT_LABEL_MAX_LENGTH = 50; +const ANSI_ESCAPE_PATTERN = /\x1B\[[0-?]*[ -/]*[@-~]/g; + +function defaultCreateMemoryCapture(project: KtxLocalProject): TextMemoryCapturePort { + return createLocalProjectMemoryCapture(project); +} + +async function defaultReadStdin(): Promise { + const chunks: string[] = []; + process.stdin.setEncoding('utf-8'); + for await (const chunk of process.stdin) { + chunks.push(String(chunk)); + } + return chunks.join(''); +} + +async function defaultReadFile(path: string): Promise { + return await fsReadFile(path, 'utf-8'); +} + +function emptyCaptured(): MemoryCaptureStatus['captured'] { + return { wiki: [], sl: [], xrefs: [] }; +} + +function normalizedTextPreview(content: string): string { + return content + .replace(ANSI_ESCAPE_PATTERN, '') + .replace(/[\u0000-\u001f\u007f-\u009f]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function truncateLabel(label: string, maxLength = INLINE_TEXT_LABEL_MAX_LENGTH): string { + const chars = Array.from(label); + if (chars.length <= maxLength) { + return label; + } + return `${chars.slice(0, maxLength - 3).join('').trimEnd()}...`; +} + +function quoteInlineTextLabel(label: string): string { + return JSON.stringify(label); +} + +function makeUniqueLabel(label: string, usedLabels: Set): string { + if (!usedLabels.has(label)) { + return label; + } + + for (let index = 2; ; index++) { + const suffix = ` (${index})`; + const candidate = `${truncateLabel(label, INLINE_TEXT_LABEL_MAX_LENGTH - suffix.length)}${suffix}`; + if (!usedLabels.has(candidate)) { + return candidate; + } + } +} + +function textLabel(content: string, index: number, usedLabels: Set): string { + const preview = normalizedTextPreview(content); + const baseLabel = preview.length > 0 ? quoteInlineTextLabel(truncateLabel(preview)) : `text-${index + 1}`; + return makeUniqueLabel(baseLabel, usedLabels); +} + +function artifactReference(label: string): string { + return label.startsWith('"') ? label : `"${label}"`; +} + +function stdinLabel(items: TextIngestItem[]): string { + if (!items.some((item) => item.label === 'stdin')) { + return 'stdin'; + } + return `stdin-${items.filter((item) => item.label.startsWith('stdin')).length + 1}`; +} + +async function loadItems(args: KtxTextIngestArgs, deps: KtxTextIngestDeps): Promise { + const items: TextIngestItem[] = []; + const usedTextLabels = new Set(); + args.texts.forEach((content, index) => { + const label = textLabel(content, index, usedTextLabels); + usedTextLabels.add(label); + items.push({ label, content }); + }); + + const readFile = deps.readFile ?? defaultReadFile; + const readStdin = deps.readStdin ?? defaultReadStdin; + for (const file of args.files) { + if (file === '-') { + items.push({ label: stdinLabel(items), content: await readStdin() }); + } else { + const path = resolve(file); + items.push({ label: basename(path), content: await readFile(path) }); + } + } + + return items; +} + +function validateItems(items: TextIngestItem[], io: KtxCliIo): boolean { + if (items.length === 0) { + io.stderr.write('Provide at least one text item with --text, a file path, or - for stdin.\n'); + return false; + } + + for (const item of items) { + if (item.content.trim().length === 0) { + io.stderr.write(`Text item "${item.label}" is empty.\n`); + return false; + } + } + return true; +} + +function makeTarget(label: string): KtxPublicIngestPlanTarget { + return { + connectionId: label, + driver: 'text', + operation: 'source-ingest', + debugCommand: '', + steps: ['memory-update'], + }; +} + +function allTargets(state: ReturnType): ContextBuildTargetState[] { + return [...state.primarySources, ...state.contextSources]; +} + +function renderTextIngestView(state: ReturnType, styled: boolean): string { + return renderContextBuildView(state, { + styled, + title: 'Ingesting text memory', + contextGroupLabel: 'Texts', + sourceIngestRunningText: 'capturing...', + completedItemName: { singular: 'text', plural: 'texts' }, + }); +} + +function summarizeCaptured(captured: MemoryCaptureStatus['captured']): string { + const parts = [ + `wiki=${captured.wiki.length}`, + `sl=${captured.sl.length}`, + `xrefs=${captured.xrefs.length}`, + ]; + return parts.join(', '); +} + +function resultFromStatus(label: string, status: MemoryCaptureStatus): TextIngestResult { + return { + label, + runId: status.runId, + status: status.status === 'done' ? 'done' : 'error', + captured: status.captured, + commitHash: status.commitHash, + error: status.error, + }; +} + +function errorResult(label: string, runId: string | null, error: unknown): TextIngestResult { + return { + label, + runId, + status: 'error', + captured: emptyCaptured(), + commitHash: null, + error: error instanceof Error ? error.message : String(error), + }; +} + +function writeJsonResult(args: KtxTextIngestArgs, results: TextIngestResult[], io: KtxCliIo): void { + io.stdout.write( + `${JSON.stringify( + { + status: results.some((result) => result.status === 'error') ? 'failed' : 'done', + projectDir: args.projectDir, + connectionId: args.connectionId ?? null, + results, + }, + null, + 2, + )}\n`, + ); +} + +function writePlainFailures(results: TextIngestResult[], io: KtxCliIo): void { + const failures = results.filter((result) => result.status === 'error'); + if (failures.length === 0) { + return; + } + + io.stdout.write('\nFailed text items:\n'); + for (const result of failures) { + io.stdout.write(` ${result.label}: ${result.error ?? 'failed'}\n`); + } +} + +export async function runKtxTextIngest( + args: KtxTextIngestArgs, + io: KtxCliIo, + deps: KtxTextIngestDeps = {}, +): Promise { + const items = await loadItems(args, deps); + if (!validateItems(items, io)) { + return 1; + } + + const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir }); + const memoryCapture = (deps.createMemoryCapture ?? defaultCreateMemoryCapture)(project); + const now = deps.now ?? (() => Date.now()); + const batchId = now(); + const state = initViewState(items.map((item) => makeTarget(item.label))); + const targets = allTargets(state); + const isTTY = io.stdout.isTTY === true && args.json !== true; + const repainter = isTTY ? createRepainter(io) : null; + const results: TextIngestResult[] = []; + + state.startedAt = now(); + const paint = () => repainter?.paint(renderTextIngestView(state, true)); + paint(); + + let spinnerInterval: ReturnType | null = null; + if (repainter) { + spinnerInterval = setInterval(() => { + const current = now(); + state.frame++; + state.totalElapsedMs = state.startedAt === null ? 0 : current - state.startedAt; + for (const target of targets) { + if (target.status === 'running' && target.startedAt !== null) { + target.elapsedMs = current - target.startedAt; + } + } + paint(); + }, 140); + } + + try { + for (let index = 0; index < items.length; index++) { + const item = items[index]!; + const target = targets[index]!; + target.status = 'running'; + target.startedAt = now(); + target.detailLine = 'capturing...'; + target.progressUpdatedAtMs = target.startedAt; + paint(); + + let runId: string | null = null; + let result: TextIngestResult; + try { + const captureInput: MemoryAgentInput = { + userId: args.userId, + chatId: `cli-text-ingest-${batchId}-${index + 1}`, + userMessage: `Ingest external text artifact ${artifactReference(item.label)} into KTX memory.`, + assistantMessage: item.content.trim(), + ...(args.connectionId ? { connectionId: args.connectionId } : {}), + sourceType: 'external_ingest', + }; + const capture = await memoryCapture.capture(captureInput); + runId = capture.runId; + await memoryCapture.waitForRun(runId); + const status = await memoryCapture.status(runId); + if (!status) { + throw new Error(`Memory capture run "${runId}" was not found.`); + } + result = resultFromStatus(item.label, status); + } catch (error) { + result = errorResult(item.label, runId, error); + } + + results.push(result); + target.elapsedMs = now() - (target.startedAt ?? now()); + target.detailLine = null; + target.status = result.status === 'done' ? 'done' : 'failed'; + target.summaryText = result.status === 'done' ? summarizeCaptured(result.captured) : null; + target.failureText = result.status === 'error' ? result.error : null; + paint(); + + if (result.status === 'error' && args.failFast) { + break; + } + } + } finally { + if (spinnerInterval) { + clearInterval(spinnerInterval); + } + } + + if (state.startedAt !== null) { + state.totalElapsedMs = now() - state.startedAt; + } + + if (args.json) { + writeJsonResult(args, results, io); + } else if (repainter) { + repainter.paint(renderTextIngestView(state, true)); + writePlainFailures(results, io); + } else { + io.stdout.write(renderTextIngestView(state, false)); + writePlainFailures(results, io); + } + + if (!args.json && results.length > 0) { + const duration = state.totalElapsedMs > 0 ? ` in ${formatDuration(state.totalElapsedMs)}` : ''; + const outcome = results.some((result) => result.status === 'error') ? 'finished with failures' : 'finished'; + io.stdout.write(`Text memory ingest ${outcome}${duration}.\n`); + } + + return results.some((result) => result.status === 'error') ? 1 : 0; +} From 3fde4438b1ad524daf72bc19483411939e3e3c5c Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Wed, 13 May 2026 19:37:25 +0200 Subject: [PATCH 3/8] fix: stop requiring readonly connection config (#71) --- .../docs/cli-reference/ktx-connection.mdx | 3 +-- .../docs/integrations/context-sources.mdx | 9 ------- .../docs/integrations/primary-sources.mdx | 12 --------- examples/local-warehouse/ktx.yaml | 1 - .../orbit-relationship-verification/ktx.yaml | 1 - packages/cli/src/connection.test.ts | 6 ++--- packages/cli/src/demo-assets.ts | 1 - packages/cli/src/doctor.test.ts | 1 - packages/cli/src/historic-sql-doctor.test.ts | 8 +----- packages/cli/src/historic-sql-doctor.ts | 3 ++- packages/cli/src/local-adapters.test.ts | 3 --- packages/cli/src/local-adapters.ts | 20 ++++++-------- .../cli/src/local-scan-connectors.test.ts | 3 --- packages/cli/src/scan.test.ts | 7 ----- packages/cli/src/setup-databases.test.ts | 13 ---------- packages/cli/src/setup-databases.ts | 10 +------ packages/cli/src/setup-sources.test.ts | 3 +-- packages/cli/src/setup.test.ts | 3 --- packages/cli/src/sl.test.ts | 11 ++++---- packages/cli/src/standalone-smoke.test.ts | 1 - .../connector-bigquery/src/connector.test.ts | 7 ----- packages/connector-bigquery/src/connector.ts | 11 ++++---- .../src/connector.test.ts | 10 ------- .../connector-clickhouse/src/connector.ts | 11 ++++---- .../connector-mysql/src/connector.test.ts | 12 +-------- packages/connector-mysql/src/connector.ts | 11 ++++---- .../connector-postgres/src/connector.test.ts | 17 +++++------- packages/connector-postgres/src/connector.ts | 11 ++++---- .../src/historic-sql-query-client.test.ts | 1 - .../connector-snowflake/src/connector.test.ts | 11 -------- packages/connector-snowflake/src/connector.ts | 13 +++++----- .../connector-sqlite/src/connector.test.ts | 26 +++++++++---------- packages/connector-sqlite/src/connector.ts | 11 ++++---- .../connector-sqlserver/src/connector.test.ts | 11 -------- packages/connector-sqlserver/src/connector.ts | 11 ++++---- .../connections/local-query-executor.test.ts | 6 ++--- .../postgres-query-executor.test.ts | 16 +++--------- .../connections/postgres-query-executor.ts | 10 +++---- .../connections/sqlite-query-executor.test.ts | 25 ++++++------------ .../src/connections/sqlite-query-executor.ts | 3 --- .../daemon-introspection.test.ts | 9 ++----- .../live-database/daemon-introspection.ts | 3 --- .../src/ingest/local-stage-ingest.test.ts | 1 - .../src/mcp/local-project-ports.test.ts | 7 ----- .../context/src/memory/local-memory.test.ts | 2 +- packages/context/src/project/config.ts | 1 - packages/context/src/scan/local-scan.test.ts | 6 ----- .../src/scan/relationship-artifacts.test.ts | 1 - .../relationship-review-decisions.test.ts | 1 - packages/context/src/sl/local-query.test.ts | 6 ++--- scripts/build-adventureworks-oltp-fixture.mjs | 1 - scripts/examples-docs.test.mjs | 1 - scripts/installed-live-database-smoke.mjs | 1 - .../installed-live-database-smoke.test.mjs | 1 - scripts/package-artifacts.mjs | 1 - 55 files changed, 103 insertions(+), 292 deletions(-) diff --git a/docs-site/content/docs/cli-reference/ktx-connection.mdx b/docs-site/content/docs/cli-reference/ktx-connection.mdx index 0cec3eae..68b7f496 100644 --- a/docs-site/content/docs/cli-reference/ktx-connection.mdx +++ b/docs-site/content/docs/cli-reference/ktx-connection.mdx @@ -63,8 +63,7 @@ agents. "connections": [ { "id": "my-warehouse", - "driver": "postgres", - "readonly": false + "driver": "postgres" } ] } diff --git a/docs-site/content/docs/integrations/context-sources.mdx b/docs-site/content/docs/integrations/context-sources.mdx index 5b85bff2..be72c941 100644 --- a/docs-site/content/docs/integrations/context-sources.mdx +++ b/docs-site/content/docs/integrations/context-sources.mdx @@ -23,7 +23,6 @@ Agents should configure and ingest context sources in this order: | Field | Required | Description | |-------|----------|-------------| | `driver` | Yes | Source adapter: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, or `notion` | -| `readonly` | Strongly recommended | Marks the source as read-only for KTX | | `source_dir` | For local file sources | Absolute or project-relative source directory | | `repo_url` | For Git-hosted sources | Git repository URL | | `branch` | No | Git branch to read | @@ -49,7 +48,6 @@ connections: my-dbt: driver: dbt source_dir: /path/to/dbt/project - readonly: true ``` For a Git-hosted project: @@ -62,7 +60,6 @@ connections: branch: main path: analytics/dbt # For monorepos auth_token_ref: env:GITHUB_TOKEN - readonly: true ``` ### Authentication @@ -110,7 +107,6 @@ connections: branch: main path: dbt_metrics # Subdirectory for monorepos auth_token_ref: env:GITHUB_TOKEN - readonly: true ``` For a local path: @@ -157,7 +153,6 @@ connections: branch: main path: analytics # Subdirectory for monorepos auth_token_ref: env:GITHUB_TOKEN - readonly: true ``` For a local path: @@ -219,7 +214,6 @@ connections: syncEnabled: "3": true syncMode: ONLY # Only ingest mapped databases - readonly: true ``` ### Authentication @@ -276,7 +270,6 @@ connections: mappings: connectionMappings: postgres_connection: postgres-main # Looker conn → KTX conn - readonly: true ``` ### Authentication @@ -329,7 +322,6 @@ connections: crawl_mode: selected_roots root_page_ids: - "abc123def456..." - readonly: true ``` For crawling all accessible pages: @@ -340,7 +332,6 @@ connections: driver: notion auth_token_ref: env:NOTION_TOKEN crawl_mode: all_accessible - readonly: true ``` ### Authentication diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 94dc4e44..653c4e38 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -21,7 +21,6 @@ Agents should prefer environment or file references over literal secrets. | `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` | | `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, ClickHouse, SQL Server | Field-by-field connection values | | `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan | -| `readonly` | Strongly recommended | all primary sources | Marks the connection as read-only in KTX config | | `historicSql` | No | supported warehouses | Enables query-history ingestion when the warehouse supports it | | `path` | Yes for path-style SQLite | SQLite | Local SQLite database path or `env:NAME` reference | @@ -37,7 +36,6 @@ connections: driver: postgres url: postgresql://user:password@host:5432/database schema: public - readonly: true ``` Or with individual fields: @@ -55,7 +53,6 @@ connections: - public - analytics ssl: true - readonly: true ``` ### Authentication @@ -123,7 +120,6 @@ connections: username: KTX_SERVICE password: env:SNOWFLAKE_PASSWORD role: ANALYST - readonly: true ``` For multiple schemas: @@ -196,7 +192,6 @@ connections: credentials_json: file:~/.config/gcloud/bq-service-account.json dataset_id: analytics location: US - readonly: true ``` For multiple datasets: @@ -269,7 +264,6 @@ connections: my-clickhouse: driver: clickhouse url: http://localhost:8123/analytics - readonly: true ``` Or with individual fields: @@ -284,7 +278,6 @@ connections: username: default password: env:CH_PASSWORD ssl: false - readonly: true ``` ### Authentication @@ -328,7 +321,6 @@ connections: my-mysql: driver: mysql url: mysql://user:password@host:3306/database - readonly: true ``` Or with individual fields: @@ -343,7 +335,6 @@ connections: username: ktx_reader password: env:MYSQL_PASSWORD ssl: true - readonly: true ``` ### Authentication @@ -387,7 +378,6 @@ connections: my-sqlserver: driver: sqlserver url: mssql://user:password@host:1433/database?trustServerCertificate=true - readonly: true ``` Or with individual fields: @@ -403,7 +393,6 @@ connections: password: env:MSSQL_PASSWORD schema: dbo trustServerCertificate: true - readonly: true ``` For multiple schemas: @@ -455,7 +444,6 @@ connections: my-sqlite: driver: sqlite path: ./data/warehouse.sqlite - readonly: true ``` Path supports multiple formats: diff --git a/examples/local-warehouse/ktx.yaml b/examples/local-warehouse/ktx.yaml index 00ccffbd..7e814188 100644 --- a/examples/local-warehouse/ktx.yaml +++ b/examples/local-warehouse/ktx.yaml @@ -2,7 +2,6 @@ project: local-warehouse connections: warehouse: driver: postgres - readonly: true storage: state: sqlite search: sqlite-fts5 diff --git a/examples/orbit-relationship-verification/ktx.yaml b/examples/orbit-relationship-verification/ktx.yaml index 5f826daf..082e0835 100644 --- a/examples/orbit-relationship-verification/ktx.yaml +++ b/examples/orbit-relationship-verification/ktx.yaml @@ -3,7 +3,6 @@ connections: orbit: driver: sqlite path: ../../packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite - readonly: true storage: state: sqlite search: sqlite-fts5 diff --git a/packages/cli/src/connection.test.ts b/packages/cli/src/connection.test.ts index 57ed8742..6eb3a08c 100644 --- a/packages/cli/src/connection.test.ts +++ b/packages/cli/src/connection.test.ts @@ -94,7 +94,7 @@ describe('runKtxConnection', () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); await writeConnections(projectDir, { - warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true }, + warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' }, docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' }, }); const io = makeIo(); @@ -123,7 +123,7 @@ describe('runKtxConnection', () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); await writeConnections(projectDir, { - warehouse: { driver: 'sqlite', readonly: true }, + warehouse: { driver: 'sqlite' }, }); const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']); const createScanConnector = vi.fn(async () => connector); @@ -202,7 +202,7 @@ describe('runKtxConnection', () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); await writeConnections(projectDir, { - warehouse: { driver: 'sqlite', readonly: true }, + warehouse: { driver: 'sqlite' }, }); const cleanup = vi.fn(async () => undefined); const connector: KtxScanConnector = { diff --git a/packages/cli/src/demo-assets.ts b/packages/cli/src/demo-assets.ts index 1e972ef7..aae9f1a2 100644 --- a/packages/cli/src/demo-assets.ts +++ b/packages/cli/src/demo-assets.ts @@ -57,7 +57,6 @@ function demoConfig(databasePath: string): string { ` ${DEMO_CONNECTION_ID}:`, ' driver: sqlite', ` path: ${JSON.stringify(databasePath)}`, - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5', diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/src/doctor.test.ts index d0ebdb95..cd96e6d2 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/src/doctor.test.ts @@ -275,7 +275,6 @@ describe('runKtxDoctor', () => { ' warehouse:', ' driver: postgres', ' url: env:WAREHOUSE_DATABASE_URL', - ' readonly: true', ' historicSql:', ' enabled: true', ' dialect: postgres', diff --git a/packages/cli/src/historic-sql-doctor.test.ts b/packages/cli/src/historic-sql-doctor.test.ts index b6f5f0fa..f3bc347e 100644 --- a/packages/cli/src/historic-sql-doctor.test.ts +++ b/packages/cli/src/historic-sql-doctor.test.ts @@ -25,7 +25,7 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { it('passes when no Postgres historic-SQL connections are enabled', async () => { const checks = await runPostgresHistoricSqlDoctorChecks( projectWithConnections({ - warehouse: { driver: 'sqlite', path: './warehouse.db', readonly: true }, + warehouse: { driver: 'sqlite', path: './warehouse.db' }, }), { postgresHistoricSqlProbe: vi.fn(), @@ -53,7 +53,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, }), @@ -66,7 +65,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { connection: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, env: process.env, @@ -87,7 +85,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, }), @@ -119,7 +116,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, }), @@ -154,7 +150,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'mysql', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, }), @@ -180,7 +175,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, }), diff --git a/packages/cli/src/historic-sql-doctor.ts b/packages/cli/src/historic-sql-doctor.ts index c83e7fb4..bb9a681c 100644 --- a/packages/cli/src/historic-sql-doctor.ts +++ b/packages/cli/src/historic-sql-doctor.ts @@ -86,8 +86,9 @@ async function defaultPostgresHistoricSqlProbe( const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]); + const inputDriver = input.connection.driver ?? 'unknown'; if (!isKtxPostgresConnectionConfig(input.connection)) { - throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection.driver ?? 'unknown'}"`); + throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`); } const client = new KtxPostgresHistoricSqlQueryClient({ diff --git a/packages/cli/src/local-adapters.test.ts b/packages/cli/src/local-adapters.test.ts index 517c0588..b7491920 100644 --- a/packages/cli/src/local-adapters.test.ts +++ b/packages/cli/src/local-adapters.test.ts @@ -45,7 +45,6 @@ describe('CLI local ingest adapters', () => { ' warehouse:', ' driver: postgres', ' url: env:WAREHOUSE_DATABASE_URL', - ' readonly: true', ' historicSql:', ' enabled: true', ' dialect: postgres', @@ -76,7 +75,6 @@ describe('CLI local ingest adapters', () => { 'connections:', ' bq:', ' driver: bigquery', - ' readonly: true', ' dataset_id: analytics', ' location: us', ' credentials_json: \'{"project_id":"demo-project"}\'', @@ -110,7 +108,6 @@ describe('CLI local ingest adapters', () => { 'connections:', ' sf:', ' driver: snowflake', - ' readonly: true', ' account: acct', ' warehouse: wh', ' database: ANALYTICS', diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index 9a6915c2..010a7188 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -190,10 +190,9 @@ function enabledHistoricSqlDialect(connection: unknown): 'postgres' | 'bigquery' function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, connectionId: string) { const connection = project.config.connections[connectionId] as KtxPostgresConnectionConfig | undefined; + const inputDriver = connection?.driver ?? 'unknown'; if (!isKtxPostgresConnectionConfig(connection)) { - throw new Error( - `Historic SQL local ingest requires a Postgres connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Historic SQL local ingest requires a Postgres connection, got ${String(inputDriver)}`); } return { async executeQuery(sql: string, params?: unknown[]) { @@ -212,10 +211,9 @@ function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, conn function createEphemeralBigQueryHistoricSqlClient(project: KtxLocalProject, connectionId: string) { const connection = project.config.connections[connectionId] as KtxBigQueryConnectionConfig | undefined; + const inputDriver = connection?.driver ?? 'unknown'; if (!isKtxBigQueryConnectionConfig(connection)) { - throw new Error( - `Historic SQL local ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Historic SQL local ingest requires a BigQuery connection, got ${String(inputDriver)}`); } return { async executeQuery(query: string) { @@ -243,10 +241,9 @@ async function createEphemeralSnowflakeHistoricSqlClient( connectorModule: SnowflakeConnectorModule, ) { const connection = project.config.connections[connectionId]; + const inputDriver = connection?.driver ?? 'unknown'; if (!connectorModule.isKtxSnowflakeConnectionConfig(connection)) { - throw new Error( - `Historic SQL local ingest requires a Snowflake connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Historic SQL local ingest requires a Snowflake connection, got ${String(inputDriver)}`); } return { async executeQuery(query: string) { @@ -308,10 +305,9 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli } if (dialect === 'bigquery') { + const inputDriver = connection?.driver ?? 'unknown'; if (!isKtxBigQueryConnectionConfig(connection)) { - throw new Error( - `Historic SQL local ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Historic SQL local ingest requires a BigQuery connection, got ${String(inputDriver)}`); } return { ...base, diff --git a/packages/cli/src/local-scan-connectors.test.ts b/packages/cli/src/local-scan-connectors.test.ts index 087e978d..e8a5c1e9 100644 --- a/packages/cli/src/local-scan-connectors.test.ts +++ b/packages/cli/src/local-scan-connectors.test.ts @@ -49,7 +49,6 @@ describe('createKtxCliScanConnector', () => { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -72,7 +71,6 @@ describe('createKtxCliScanConnector', () => { ' warehouse:', ' driver: bigquery', ' dataset_id: analytics', - ' readonly: true', ' max_bytes_billed: "987654321"', '', ].join('\n'), @@ -123,7 +121,6 @@ describe('createKtxCliScanConnector', () => { ' warehouse:', ' type: postgres', ' url: postgresql://example/db', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/scan.test.ts b/packages/cli/src/scan.test.ts index 28c60ea0..c4cbaf70 100644 --- a/packages/cli/src/scan.test.ts +++ b/packages/cli/src/scan.test.ts @@ -861,7 +861,6 @@ describe('runKtxScan', () => { ' warehouse:', ' driver: mysql', ' url: env:MYSQL_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -910,7 +909,6 @@ describe('runKtxScan', () => { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -968,7 +966,6 @@ describe('runKtxScan', () => { ' database: analytics', ' username: reader', ' password: env:POSTGRES_PASSWORD', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1035,7 +1032,6 @@ describe('runKtxScan', () => { ' database: analytics', ' username: reader', ' password: env:CLICKHOUSE_PASSWORD', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1087,7 +1083,6 @@ describe('runKtxScan', () => { ' database: analytics', ' username: reader', ' schema: dbo', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1153,7 +1148,6 @@ describe('runKtxScan', () => { ' dataset_id: analytics', ' credentials_json: env:BIGQUERY_CREDENTIALS_JSON', ' location: US', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1222,7 +1216,6 @@ describe('runKtxScan', () => { ' database: ANALYTICS', ' schema_name: PUBLIC', ' username: reader', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index 95d1e3fb..d010a908 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -218,7 +218,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -281,7 +280,6 @@ describe('setup databases step', () => { expect(config.connections['postgres-warehouse']).toEqual({ driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }); }); @@ -542,7 +540,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -583,7 +580,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -698,7 +694,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -843,7 +838,6 @@ describe('setup databases step', () => { port: 5432, database: 'analytics', username: 'readonly', - readonly: true, }); expect(connection.password).toMatch(/^file:/); const secretPath = join(tempDir, '.ktx/secrets/postgres-warehouse-password'); @@ -998,7 +992,6 @@ describe('setup databases step', () => { expect(config.connections['postgres-warehouse']).toMatchObject({ driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }); }); @@ -1115,7 +1108,6 @@ describe('setup databases step', () => { driver: 'postgres', url: 'env:DATABASE_URL', schemas: ['public'], - readonly: true, }); expect(config.setup).toEqual({ database_connection_ids: ['warehouse'], @@ -1153,7 +1145,6 @@ describe('setup databases step', () => { expect(config.connections.warehouse).toEqual({ driver: 'sqlite', path: './warehouse.sqlite', - readonly: true, }); expect(config.setup).toEqual({ database_connection_ids: ['warehouse'], @@ -1170,7 +1161,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', ' analytics:', ' driver: snowflake', ' authMethod: password', @@ -1180,7 +1170,6 @@ describe('setup databases step', () => { ' schema_name: PUBLIC', ' username: reader', ' password: env:SNOWFLAKE_PASSWORD', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1443,7 +1432,6 @@ describe('setup databases step', () => { ' driver: bigquery', ' dataset_id: analytics', ' credentials_json: env:BIGQUERY_CREDENTIALS_JSON', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1492,7 +1480,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 85be2620..5b5b5f8a 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -593,7 +593,6 @@ async function buildFieldsConnectionConfig(input: { username, ...(passwordRef ? { password: passwordRef } : {}), ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -615,7 +614,6 @@ async function buildPastedUrlConnectionConfig(input: { driver: input.driver, url, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -629,7 +627,6 @@ async function buildPastedUrlConnectionConfig(input: { driver: input.driver, url: ref, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -637,7 +634,6 @@ async function buildPastedUrlConnectionConfig(input: { driver: input.driver, url, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -661,14 +657,12 @@ async function buildUrlConnectionConfig(input: { driver: input.driver, url: ref, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } return { driver: input.driver, url, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -706,7 +700,7 @@ async function buildConnectionConfig(input: { 'SQLite database file\nEnter a relative or absolute path, for example ./warehouse.sqlite.', )); if (path === undefined) return 'back'; - return path ? { driver: 'sqlite', path, readonly: true } : null; + return path ? { driver: 'sqlite', path } : null; } if (driver === 'postgres' || driver === 'mysql' || driver === 'clickhouse' || driver === 'sqlserver') { return await buildUrlConnectionConfig({ driver, connectionId: input.connectionId, args, prompts }); @@ -728,7 +722,6 @@ async function buildConnectionConfig(input: { dataset_id: datasetId, credentials_json: normalizeFileReference(credentialsPath), ...(location ? { location } : {}), - readonly: true, }; } if (driver === 'snowflake') { @@ -767,7 +760,6 @@ async function buildConnectionConfig(input: { username, password: passwordRef, ...(role ? { role } : {}), - readonly: true, }; } throw new Error(`Unsupported database driver: ${driver}`); diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index c0fb0227..7fe61f76 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -98,7 +98,7 @@ describe('setup sources step', () => { ...config, connections: { ...config.connections, - warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true }, + warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' }, }, setup: { ...config.setup, @@ -455,7 +455,6 @@ describe('setup sources step', () => { driver: 'snowflake', account: 'acme', database: 'analytics', - readonly: true, }); const cases: Array<{ diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index dd134fce..246484d3 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -170,7 +170,6 @@ describe('setup status', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -192,7 +191,6 @@ describe('setup status', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1373,7 +1371,6 @@ describe('setup status', () => { ' warehouse:', ' driver: postgres', ' url: env:DEMO_DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/sl.test.ts b/packages/cli/src/sl.test.ts index ff4132b4..14f18337 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/src/sl.test.ts @@ -190,7 +190,7 @@ joins: [] it('runs sl query and prints SQL output', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -247,7 +247,7 @@ joins: [] it('runs sl query from a JSON query file', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -314,7 +314,7 @@ joins: [] it('creates default sl query compute through the managed runtime helper', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -375,7 +375,7 @@ joins: [] it('executes sl query through the injected query executor', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -471,7 +471,7 @@ joins: [] `); db.close(); - project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db', readonly: true }; + project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db' }; await writeFile( join(projectDir, 'ktx.yaml'), [ @@ -480,7 +480,6 @@ joins: [] ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/src/standalone-smoke.test.ts index 6f878617..0f53cca0 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/src/standalone-smoke.test.ts @@ -106,7 +106,6 @@ async function writeSqliteScanConfig(projectDir: string, dbPath: string, enrich ' warehouse:', ' driver: sqlite', ` path: ${JSON.stringify(dbPath)}`, - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/connector-bigquery/src/connector.test.ts b/packages/connector-bigquery/src/connector.test.ts index a1c23864..46dc3b53 100644 --- a/packages/connector-bigquery/src/connector.test.ts +++ b/packages/connector-bigquery/src/connector.test.ts @@ -100,7 +100,6 @@ const connection = { dataset_id: 'analytics', credentials_json: JSON.stringify({ project_id: 'project-1', client_email: 'reader@example.test' }), location: 'US', - readonly: true, }; describe('KtxBigQueryScanConnector', () => { @@ -112,12 +111,6 @@ describe('KtxBigQueryScanConnector', () => { datasetIds: ['analytics'], location: 'US', }); - expect(() => - bigQueryConnectionConfigFromConfig({ - connectionId: 'warehouse', - connection: { ...connection, readonly: false }, - }), - ).toThrow('Native BigQuery connector requires connections.warehouse.readonly: true'); }); it('introspects datasets, table metadata, primary keys, and normalized types', async () => { diff --git a/packages/connector-bigquery/src/connector.ts b/packages/connector-bigquery/src/connector.ts index a994912e..72cb8129 100644 --- a/packages/connector-bigquery/src/connector.ts +++ b/packages/connector-bigquery/src/connector.ts @@ -30,7 +30,6 @@ export interface KtxBigQueryConnectionConfig { dataset_ids?: string[]; credentials_json?: string; location?: string; - readonly?: boolean; [key: string]: unknown; } @@ -194,7 +193,9 @@ function normalizeValue(value: unknown): unknown { return value; } -export function isKtxBigQueryConnectionConfig(connection: KtxBigQueryConnectionConfig | undefined): boolean { +export function isKtxBigQueryConnectionConfig( + connection: KtxBigQueryConnectionConfig | undefined, +): connection is KtxBigQueryConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'bigquery'; } @@ -203,11 +204,9 @@ export function bigQueryConnectionConfigFromConfig(input: { connection: KtxBigQueryConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxBigQueryResolvedConnectionConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxBigQueryConnectionConfig(input.connection)) { - throw new Error(`Native BigQuery connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native BigQuery connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native BigQuery connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-clickhouse/src/connector.test.ts b/packages/connector-clickhouse/src/connector.test.ts index 7ed60efa..4f8b7f52 100644 --- a/packages/connector-clickhouse/src/connector.test.ts +++ b/packages/connector-clickhouse/src/connector.test.ts @@ -112,7 +112,6 @@ describe('KtxClickHouseScanConnector', () => { username: 'reader', password: 'test-pass', // pragma: allowlist secret ssl: true, - readonly: true, }, }), ).toMatchObject({ @@ -123,12 +122,6 @@ describe('KtxClickHouseScanConnector', () => { password: 'test-pass', // pragma: allowlist secret ssl: true, }); - expect(() => - clickHouseClientConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'clickhouse', host: 'ch.example.test', database: 'analytics', readonly: false }, - }), - ).toThrow('Native ClickHouse connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, and views', async () => { @@ -140,7 +133,6 @@ describe('KtxClickHouseScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-pass', // pragma: allowlist secret - readonly: true, }, clientFactory: fakeClientFactory(), now: () => new Date('2026-04-29T14:00:00.000Z'), @@ -189,7 +181,6 @@ describe('KtxClickHouseScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-pass', // pragma: allowlist secret - readonly: true, }, clientFactory, }); @@ -253,7 +244,6 @@ describe('KtxClickHouseScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-pass', // pragma: allowlist secret - readonly: true, }, }, clientFactory: fakeClientFactory(), diff --git a/packages/connector-clickhouse/src/connector.ts b/packages/connector-clickhouse/src/connector.ts index 0273a62b..4b39c943 100644 --- a/packages/connector-clickhouse/src/connector.ts +++ b/packages/connector-clickhouse/src/connector.ts @@ -35,7 +35,6 @@ export interface KtxClickHouseConnectionConfig { password?: string; url?: string; ssl?: boolean; - readonly?: boolean; [key: string]: unknown; } @@ -193,7 +192,9 @@ function isNullableClickHouseType(type: string): boolean { return type.startsWith('Nullable(') || type.startsWith('LowCardinality(Nullable('); } -export function isKtxClickHouseConnectionConfig(connection: KtxClickHouseConnectionConfig | undefined): boolean { +export function isKtxClickHouseConnectionConfig( + connection: KtxClickHouseConnectionConfig | undefined, +): connection is KtxClickHouseConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'clickhouse'; } @@ -202,11 +203,9 @@ export function clickHouseClientConfigFromConfig(input: { connection: KtxClickHouseConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxClickHouseResolvedClientConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxClickHouseConnectionConfig(input.connection)) { - throw new Error(`Native ClickHouse connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native ClickHouse connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native ClickHouse connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-mysql/src/connector.test.ts b/packages/connector-mysql/src/connector.test.ts index 3e7ac1e1..c5c5a3fa 100644 --- a/packages/connector-mysql/src/connector.test.ts +++ b/packages/connector-mysql/src/connector.test.ts @@ -92,7 +92,7 @@ function fakePoolFactory(): KtxMysqlPoolFactory { describe('KtxMysqlScanConnector', () => { it('resolves MySQL connection configuration safely', () => { - expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics', readonly: true })).toBe(true); + expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(true); expect(isKtxMysqlConnectionConfig({ driver: 'postgres', host: 'localhost', database: 'analytics' })).toBe(false); expect( mysqlConnectionPoolConfigFromConfig({ @@ -105,7 +105,6 @@ describe('KtxMysqlScanConnector', () => { username: 'reader', password: 'secret', // pragma: allowlist secret ssl: true, - readonly: true, }, }), ).toMatchObject({ @@ -116,12 +115,6 @@ describe('KtxMysqlScanConnector', () => { password: 'secret', // pragma: allowlist secret ssl: { rejectUnauthorized: false }, }); - expect(() => - mysqlConnectionPoolConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'mysql', host: 'db.example.test', database: 'analytics', readonly: false }, - }), - ).toThrow('Native MySQL connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => { @@ -133,7 +126,6 @@ describe('KtxMysqlScanConnector', () => { database: 'analytics', username: 'reader', password: 'secret', // pragma: allowlist secret - readonly: true, }, poolFactory: fakePoolFactory(), now: () => new Date('2026-04-29T12:00:00.000Z'), @@ -192,7 +184,6 @@ describe('KtxMysqlScanConnector', () => { database: 'analytics', username: 'reader', password: 'secret', // pragma: allowlist secret - readonly: true, }, poolFactory, }); @@ -249,7 +240,6 @@ describe('KtxMysqlScanConnector', () => { database: 'analytics', username: 'reader', password: 'secret', // pragma: allowlist secret - readonly: true, }, }, poolFactory: fakePoolFactory(), diff --git a/packages/connector-mysql/src/connector.ts b/packages/connector-mysql/src/connector.ts index 69a09272..62bb1880 100644 --- a/packages/connector-mysql/src/connector.ts +++ b/packages/connector-mysql/src/connector.ts @@ -35,7 +35,6 @@ export interface KtxMysqlConnectionConfig { password?: string; url?: string; ssl?: boolean | { rejectUnauthorized?: boolean }; - readonly?: boolean; [key: string]: unknown; } @@ -232,7 +231,9 @@ function queryParams(params: Record | unknown[] | undefined): u return Array.isArray(params) ? params : Object.values(params); } -export function isKtxMysqlConnectionConfig(connection: KtxMysqlConnectionConfig | undefined): boolean { +export function isKtxMysqlConnectionConfig( + connection: KtxMysqlConnectionConfig | undefined, +): connection is KtxMysqlConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'mysql'; } @@ -241,11 +242,9 @@ export function mysqlConnectionPoolConfigFromConfig(input: { connection: KtxMysqlConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxMysqlPoolConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxMysqlConnectionConfig(input.connection)) { - throw new Error(`Native MySQL connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native MySQL connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native MySQL connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-postgres/src/connector.test.ts b/packages/connector-postgres/src/connector.test.ts index 96443c90..8093acda 100644 --- a/packages/connector-postgres/src/connector.test.ts +++ b/packages/connector-postgres/src/connector.test.ts @@ -102,7 +102,7 @@ function metadataResults(): Map { describe('KtxPostgresScanConnector', () => { it('resolves configuration safely', () => { - expect(isKtxPostgresConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL', readonly: true })).toBe(true); + expect(isKtxPostgresConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL' })).toBe(true); expect(isKtxPostgresConnectionConfig({ driver: 'postgresql', host: 'db', database: 'analytics' })).toBe(true); expect(isKtxPostgresConnectionConfig({ driver: 'mysql', host: 'db' })).toBe(false); expect( @@ -115,7 +115,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schemas: ['analytics', 'public'], - readonly: true, ssl: true, rejectUnauthorized: false, }, @@ -134,7 +133,6 @@ describe('KtxPostgresScanConnector', () => { connection: { driver: 'postgres', url: 'env:DEMO_DATABASE_URL', - readonly: true, }, env: { DEMO_DATABASE_URL: 'postgresql://reader@demo.example.test:5432/demo?sslmode=prefer', @@ -148,12 +146,16 @@ describe('KtxPostgresScanConnector', () => { }); expect(libpqPreferConfig).not.toHaveProperty('connectionString'); expect(libpqPreferConfig).not.toHaveProperty('ssl'); - expect(() => + expect( postgresPoolConfigFromConfig({ connectionId: 'warehouse', connection: { driver: 'postgres', host: 'db.example.test', database: 'analytics', username: 'reader' }, }), - ).toThrow('Native PostgreSQL connector requires connections.warehouse.readonly: true'); + ).toMatchObject({ + host: 'db.example.test', + database: 'analytics', + user: 'reader', + }); }); it('introspects schemas, tables, views, primary keys, comments, row counts, and foreign keys', async () => { @@ -166,7 +168,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, poolFactory: fakePoolFactory(metadataResults()), now: () => new Date('2026-04-29T10:00:00.000Z'), @@ -225,7 +226,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, poolFactory: fakePoolFactory(metadataResults()), }); @@ -274,7 +274,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, }, poolFactory: fakePoolFactory(metadataResults()), @@ -347,7 +346,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, }, poolFactory: endAwarePoolFactory, @@ -383,7 +381,6 @@ describe('KtxPostgresScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-password', // pragma: allowlist secret - readonly: true, }, poolFactory, }); diff --git a/packages/connector-postgres/src/connector.ts b/packages/connector-postgres/src/connector.ts index 65490040..7f5ed65c 100644 --- a/packages/connector-postgres/src/connector.ts +++ b/packages/connector-postgres/src/connector.ts @@ -61,7 +61,6 @@ export interface KtxPostgresConnectionConfig { sslmode?: string; sslMode?: string; rejectUnauthorized?: boolean; - readonly?: boolean; [key: string]: unknown; } @@ -291,7 +290,9 @@ function searchPathSchemasFromConnection(connection: KtxPostgresConnectionConfig return schemas.includes('public') ? schemas : [...schemas, 'public']; } -export function isKtxPostgresConnectionConfig(connection: KtxPostgresConnectionConfig | undefined): boolean { +export function isKtxPostgresConnectionConfig( + connection: KtxPostgresConnectionConfig | undefined, +): connection is KtxPostgresConnectionConfig { const driver = String(connection?.driver ?? '').toLowerCase(); return driver === 'postgres' || driver === 'postgresql'; } @@ -301,11 +302,9 @@ export function postgresPoolConfigFromConfig(input: { connection: KtxPostgresConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxPostgresPoolConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxPostgresConnectionConfig(input.connection)) { - throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native PostgreSQL connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-postgres/src/historic-sql-query-client.test.ts b/packages/connector-postgres/src/historic-sql-query-client.test.ts index f02cb9c3..b9c9fd40 100644 --- a/packages/connector-postgres/src/historic-sql-query-client.test.ts +++ b/packages/connector-postgres/src/historic-sql-query-client.test.ts @@ -30,7 +30,6 @@ describe('KtxPostgresHistoricSqlQueryClient', () => { connectionId: 'warehouse', connection: { driver: 'postgres', - readonly: true, url: 'postgresql://readonly:secret@pg.example.test/warehouse', // pragma: allowlist secret }, poolFactory, diff --git a/packages/connector-snowflake/src/connector.test.ts b/packages/connector-snowflake/src/connector.test.ts index 91bb33d4..a49be885 100644 --- a/packages/connector-snowflake/src/connector.test.ts +++ b/packages/connector-snowflake/src/connector.test.ts @@ -78,7 +78,6 @@ describe('KtxSnowflakeScanConnector', () => { warehouse: 'WH', database: 'ANALYTICS', username: 'reader', - readonly: true, }), ).toBe(true); expect(isKtxSnowflakeConnectionConfig({ driver: 'bigquery' })).toBe(false); @@ -94,7 +93,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, }), ).toMatchObject({ @@ -105,12 +103,6 @@ describe('KtxSnowflakeScanConnector', () => { username: 'reader', authMethod: 'password', }); - expect(() => - snowflakeConnectionConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'snowflake', account: 'acct', readonly: false }, - }), - ).toThrow('Native Snowflake connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, and dimensions', async () => { @@ -125,7 +117,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, driverFactory: fakeDriverFactory(), now: () => new Date('2026-04-29T18:00:00.000Z'), @@ -185,7 +176,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, driverFactory, }); @@ -243,7 +233,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, }, driverFactory: fakeDriverFactory(), diff --git a/packages/connector-snowflake/src/connector.ts b/packages/connector-snowflake/src/connector.ts index 063976f7..76fc34fd 100644 --- a/packages/connector-snowflake/src/connector.ts +++ b/packages/connector-snowflake/src/connector.ts @@ -38,7 +38,6 @@ export interface KtxSnowflakeConnectionConfig { privateKey?: string; passphrase?: string; role?: string; - readonly?: boolean; [key: string]: unknown; } @@ -191,7 +190,9 @@ function toSnowflakeBinds(params: unknown[] | undefined): snowflake.Binds | unde return params?.map((value) => toSnowflakeBind(value)); } -export function isKtxSnowflakeConnectionConfig(connection: KtxSnowflakeConnectionConfig | undefined): boolean { +export function isKtxSnowflakeConnectionConfig( + connection: KtxSnowflakeConnectionConfig | undefined, +): connection is KtxSnowflakeConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'snowflake'; } @@ -200,11 +201,9 @@ export function snowflakeConnectionConfigFromConfig(input: { connection: KtxSnowflakeConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxSnowflakeResolvedConnectionConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxSnowflakeConnectionConfig(input.connection)) { - throw new Error(`Native Snowflake connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native Snowflake connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; const authMethod = input.connection?.authMethod ?? 'password'; @@ -395,7 +394,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver { private async createConnection(): Promise { const patch = await this.sdkOptionsProvider?.resolve({ account: this.resolved.account, - connection: { ...this.resolved, driver: 'snowflake', readonly: true }, + connection: { ...this.resolved, driver: 'snowflake' }, }); if (patch?.close) { this.closeSdkOptions.push(patch.close); diff --git a/packages/connector-sqlite/src/connector.test.ts b/packages/connector-sqlite/src/connector.test.ts index 4bc26ec9..9bee5567 100644 --- a/packages/connector-sqlite/src/connector.test.ts +++ b/packages/connector-sqlite/src/connector.test.ts @@ -53,45 +53,43 @@ describe('KtxSqliteScanConnector', () => { writeFileSync(pointerPath, dbPath, 'utf-8'); try { - expect(isKtxSqliteConnectionConfig({ driver: 'sqlite', path: 'warehouse.db', readonly: true })).toBe(true); - expect(isKtxSqliteConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL', readonly: true })).toBe( - false, - ); + expect(isKtxSqliteConnectionConfig({ driver: 'sqlite', path: 'warehouse.db' })).toBe(true); + expect(isKtxSqliteConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL' })).toBe(false); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, }), ).toBe(dbPath); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true }, + connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL' }, }), ).toBe(dbPath); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: `file://${dbPath}`, readonly: true }, + connection: { driver: 'sqlite', url: `file://${dbPath}` }, }), ).toBe(dbPath); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: `file:${pointerPath}`, readonly: true }, + connection: { driver: 'sqlite', path: `file:${pointerPath}` }, }), ).toBe(dbPath); - expect(() => + expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: false }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, }), - ).toThrow('Native SQLite connector requires connections.warehouse.readonly: true'); + ).toBe(dbPath); } finally { if (originalDatabaseUrl === undefined) { delete process.env.KTX_SQLITE_TEST_URL; @@ -104,7 +102,7 @@ describe('KtxSqliteScanConnector', () => { it('introspects schema, primary keys, row counts, views, and foreign keys', async () => { const connector = new KtxSqliteScanConnector({ connectionId: 'warehouse', - connection: { driver: 'sqlite', path: dbPath, readonly: true }, + connection: { driver: 'sqlite', path: dbPath }, now: () => new Date('2026-04-29T10:00:00.000Z'), }); @@ -151,7 +149,7 @@ describe('KtxSqliteScanConnector', () => { it('runs samples, distinct values, statistics, and read-only SQL', async () => { const connector = new KtxSqliteScanConnector({ connectionId: 'warehouse', - connection: { driver: 'sqlite', path: dbPath, readonly: true }, + connection: { driver: 'sqlite', path: dbPath }, }); await expect( @@ -199,7 +197,7 @@ describe('KtxSqliteScanConnector', () => { const introspection = createSqliteLiveDatabaseIntrospection({ projectDir: tempDir, connections: { - warehouse: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + warehouse: { driver: 'sqlite', path: 'warehouse.db' }, }, now: () => new Date('2026-04-29T10:00:00.000Z'), }); diff --git a/packages/connector-sqlite/src/connector.ts b/packages/connector-sqlite/src/connector.ts index c42db002..2a0d997e 100644 --- a/packages/connector-sqlite/src/connector.ts +++ b/packages/connector-sqlite/src/connector.ts @@ -29,7 +29,6 @@ export interface KtxSqliteConnectionConfig { path?: string; url?: string; file_path?: string; - readonly?: boolean; [key: string]: unknown; } @@ -135,17 +134,17 @@ function stripLeadingSqlComments(sql: string): string { return sql.slice(index); } -export function isKtxSqliteConnectionConfig(connection: KtxSqliteConnectionConfig | undefined): boolean { +export function isKtxSqliteConnectionConfig( + connection: KtxSqliteConnectionConfig | undefined, +): connection is KtxSqliteConnectionConfig { const driver = String(connection?.driver ?? '').toLowerCase(); return driver === 'sqlite' || driver === 'sqlite3'; } export function sqliteDatabasePathFromConfig(input: SqliteDatabasePathInput): string { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxSqliteConnectionConfig(input.connection)) { - throw new Error(`Native SQLite connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native SQLite connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native SQLite connector cannot run driver "${inputDriver}"`); } const configuredPath = stringConfigValue(input.connection, 'path') ?? diff --git a/packages/connector-sqlserver/src/connector.test.ts b/packages/connector-sqlserver/src/connector.test.ts index eebab0ba..b7915fa8 100644 --- a/packages/connector-sqlserver/src/connector.test.ts +++ b/packages/connector-sqlserver/src/connector.test.ts @@ -145,7 +145,6 @@ describe('KtxSqlServerScanConnector', () => { driver: 'sqlserver', host: 'localhost', database: 'analytics', - readonly: true, }), ).toBe(true); expect(isKtxSqlServerConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(false); @@ -159,7 +158,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', trustServerCertificate: false, - readonly: true, }, }), ).toMatchObject({ @@ -169,12 +167,6 @@ describe('KtxSqlServerScanConnector', () => { user: 'reader', options: { encrypt: true, trustServerCertificate: false }, }); - expect(() => - sqlServerConnectionPoolConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'sqlserver', host: 'db.example.test', database: 'analytics', readonly: false }, - }), - ).toThrow('Native SQL Server connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => { @@ -186,7 +178,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', schema: 'dbo', - readonly: true, }, poolFactory: fakePoolFactory(), now: () => new Date('2026-04-29T16:00:00.000Z'), @@ -246,7 +237,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', schema: 'dbo', - readonly: true, }, poolFactory, }); @@ -315,7 +305,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', schema: 'dbo', - readonly: true, }, }, poolFactory: fakePoolFactory(), diff --git a/packages/connector-sqlserver/src/connector.ts b/packages/connector-sqlserver/src/connector.ts index 189ff98b..73a46aab 100644 --- a/packages/connector-sqlserver/src/connector.ts +++ b/packages/connector-sqlserver/src/connector.ts @@ -37,7 +37,6 @@ export interface KtxSqlServerConnectionConfig { schema?: string; schemas?: string[]; trustServerCertificate?: boolean; - readonly?: boolean; [key: string]: unknown; } @@ -234,7 +233,9 @@ function limitSqlForSqlServerExecution(sqlText: string, maxRows: number | undefi return `SELECT TOP ${maxRows} * FROM (${trimmed}) AS ktx_query_result`; } -export function isKtxSqlServerConnectionConfig(connection: KtxSqlServerConnectionConfig | undefined): boolean { +export function isKtxSqlServerConnectionConfig( + connection: KtxSqlServerConnectionConfig | undefined, +): connection is KtxSqlServerConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'sqlserver'; } @@ -243,11 +244,9 @@ export function sqlServerConnectionPoolConfigFromConfig(input: { connection: KtxSqlServerConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxSqlServerPoolConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxSqlServerConnectionConfig(input.connection)) { - throw new Error(`Native SQL Server connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native SQL Server connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native SQL Server connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/context/src/connections/local-query-executor.test.ts b/packages/context/src/connections/local-query-executor.test.ts index fd94c6dc..d2f77975 100644 --- a/packages/context/src/connections/local-query-executor.test.ts +++ b/packages/context/src/connections/local-query-executor.test.ts @@ -26,14 +26,14 @@ describe('createDefaultLocalQueryExecutor', () => { await expect( executor.execute({ connectionId: 'pg', - connection: { driver: 'postgres', readonly: true }, + connection: { driver: 'postgres' }, sql: 'select 1', }), ).resolves.toMatchObject({ headers: ['pg'] }); await expect( executor.execute({ connectionId: 'local', - connection: { driver: 'sqlite', readonly: true }, + connection: { driver: 'sqlite' }, sql: 'select 1', }), ).resolves.toMatchObject({ headers: ['sqlite'] }); @@ -51,7 +51,7 @@ describe('createDefaultLocalQueryExecutor', () => { await expect( executor.execute({ connectionId: 'warehouse', - connection: { driver: 'snowflake', readonly: true }, + connection: { driver: 'snowflake' }, sql: 'select 1', }), ).rejects.toThrow('No local query executor is configured for driver "snowflake".'); diff --git a/packages/context/src/connections/postgres-query-executor.test.ts b/packages/context/src/connections/postgres-query-executor.test.ts index 2c52d97e..6bb522cf 100644 --- a/packages/context/src/connections/postgres-query-executor.test.ts +++ b/packages/context/src/connections/postgres-query-executor.test.ts @@ -37,7 +37,7 @@ describe('createPostgresQueryExecutor', () => { const result = await executor.execute({ connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgres://example/db', readonly: true }, + connection: { driver: 'postgres', url: 'postgres://example/db' }, sql: 'select status, count(*) as order_count from public.orders group by status', maxRows: 50, }); @@ -80,7 +80,7 @@ describe('createPostgresQueryExecutor', () => { await expect( executor.execute({ connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgres://example/db', readonly: true }, + connection: { driver: 'postgres', url: 'postgres://example/db' }, sql: 'select * from broken', maxRows: 10, }), @@ -89,23 +89,15 @@ describe('createPostgresQueryExecutor', () => { expect(client.end).toHaveBeenCalledTimes(1); }); - it('requires a Postgres url and read-only connection config', async () => { + it('requires a Postgres url', async () => { const executor = createPostgresQueryExecutor({ clientFactory: vi.fn() }); await expect( executor.execute({ connectionId: 'warehouse', - connection: { driver: 'postgres', readonly: true }, + connection: { driver: 'postgres' }, sql: 'select 1', }), ).rejects.toThrow('Local Postgres execution requires connections.warehouse.url'); - - await expect( - executor.execute({ - connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgres://example/db', readonly: false }, - sql: 'select 1', - }), - ).rejects.toThrow('Local query execution requires connections.warehouse.readonly: true'); }); }); diff --git a/packages/context/src/connections/postgres-query-executor.ts b/packages/context/src/connections/postgres-query-executor.ts index 2ab142a5..b5f2d02e 100644 --- a/packages/context/src/connections/postgres-query-executor.ts +++ b/packages/context/src/connections/postgres-query-executor.ts @@ -37,18 +37,16 @@ export function createPostgresQueryExecutor(options: PostgresQueryExecutorOption return { async execute(input: KtxSqlQueryExecutionInput): Promise { const driver = connectionDriver(input); + const connection = input.connection; if (driver !== 'postgres' && driver !== 'postgresql') { - throw new Error(`Local Postgres execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`); + throw new Error(`Local Postgres execution cannot run driver "${connection?.driver ?? 'unknown'}".`); } - if (input.connection?.readonly !== true) { - throw new Error(`Local query execution requires connections.${input.connectionId}.readonly: true.`); - } - if (typeof input.connection.url !== 'string' || input.connection.url.trim().length === 0) { + if (typeof connection?.url !== 'string' || connection.url.trim().length === 0) { throw new Error(`Local Postgres execution requires connections.${input.connectionId}.url.`); } const client = clientFactory({ - connectionString: input.connection.url, + connectionString: connection.url, statement_timeout: options.statementTimeoutMs ?? 30_000, query_timeout: options.queryTimeoutMs ?? 35_000, connectionTimeoutMillis: options.connectionTimeoutMs ?? 5_000, diff --git a/packages/context/src/connections/sqlite-query-executor.test.ts b/packages/context/src/connections/sqlite-query-executor.test.ts index 3046f9bb..facb5139 100644 --- a/packages/context/src/connections/sqlite-query-executor.test.ts +++ b/packages/context/src/connections/sqlite-query-executor.test.ts @@ -38,7 +38,7 @@ describe('createSqliteQueryExecutor', () => { const result = await executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, sql: 'select status, count(*) as order_count from orders group by status order by status', maxRows: 10, }); @@ -60,7 +60,7 @@ describe('createSqliteQueryExecutor', () => { sqliteDatabasePathFromConnection({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: `file://${dbPath}`, readonly: true }, + connection: { driver: 'sqlite', url: `file://${dbPath}` }, sql: 'select 1', }), ).toBe(dbPath); @@ -74,7 +74,7 @@ describe('createSqliteQueryExecutor', () => { sqliteDatabasePathFromConnection({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: `file:${pointerPath}`, readonly: true }, + connection: { driver: 'sqlite', path: `file:${pointerPath}` }, sql: 'select 1', }), ).toBe(dbPath); @@ -89,7 +89,7 @@ describe('createSqliteQueryExecutor', () => { sqliteDatabasePathFromConnection({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true }, + connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL' }, sql: 'select 1', }), ).toBe(dbPath); @@ -109,20 +109,20 @@ describe('createSqliteQueryExecutor', () => { executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, sql: 'delete from orders', }), ).rejects.toThrow('Only read-only SELECT/WITH queries can be executed locally'); }); - it('requires a SQLite driver, read-only config, and a database path', async () => { + it('requires a SQLite driver and a database path', async () => { const executor = createSqliteQueryExecutor(); await expect( executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'postgres', path: 'warehouse.db', readonly: true }, + connection: { driver: 'postgres', path: 'warehouse.db' }, sql: 'select 1', }), ).rejects.toThrow('Local SQLite execution cannot run driver "postgres"'); @@ -131,16 +131,7 @@ describe('createSqliteQueryExecutor', () => { executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: false }, - sql: 'select 1', - }), - ).rejects.toThrow('Local query execution requires connections.warehouse.readonly: true'); - - await expect( - executor.execute({ - connectionId: 'warehouse', - projectDir: tempDir, - connection: { driver: 'sqlite', readonly: true }, + connection: { driver: 'sqlite' }, sql: 'select 1', }), ).rejects.toThrow('Local SQLite execution requires connections.warehouse.path or connections.warehouse.url'); diff --git a/packages/context/src/connections/sqlite-query-executor.ts b/packages/context/src/connections/sqlite-query-executor.ts index d32a37ba..2a87ef7d 100644 --- a/packages/context/src/connections/sqlite-query-executor.ts +++ b/packages/context/src/connections/sqlite-query-executor.ts @@ -54,9 +54,6 @@ export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInpu if (driver !== 'sqlite' && driver !== 'sqlite3') { throw new Error(`Local SQLite execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`); } - if (input.connection?.readonly !== true) { - throw new Error(`Local query execution requires connections.${input.connectionId}.readonly: true.`); - } const pathValue = stringConfigValue(input.connection, 'path'); const urlValue = stringConfigValue(input.connection, 'url'); diff --git a/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts b/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts index fe65920e..93a9739d 100644 --- a/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts +++ b/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts @@ -45,7 +45,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => { warehouse: { driver: 'postgres', url: 'postgres://localhost:5432/warehouse', - readonly: true, }, }, schemas: ['public'], @@ -157,7 +156,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => { warehouse: { driver: 'postgresql', url: 'postgres://localhost:5432/warehouse', - readonly: true, }, }, baseUrl: `http://127.0.0.1:${address.port}`, @@ -186,20 +184,18 @@ describe('createDaemonLiveDatabaseIntrospection', () => { } }); - it('requires a configured read-only postgres connection with a url', async () => { + it('requires a configured postgres connection with a url', async () => { const introspection = createDaemonLiveDatabaseIntrospection({ connections: { warehouse: { driver: 'postgres', - url: 'postgres://localhost:5432/warehouse', - readonly: false, }, }, runJson: vi.fn(async () => daemonResponse), }); await expect(introspection.extractSchema('warehouse')).rejects.toThrow( - 'Local live-database ingest requires connections.warehouse.readonly: true.', + 'Local live-database ingest requires connections.warehouse.url.', ); }); @@ -210,7 +206,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => { warehouse: { driver: 'snowflake', url: 'snowflake://example', - readonly: true, }, }, runJson, diff --git a/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts b/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts index 531c1a66..6c333385 100644 --- a/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts +++ b/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts @@ -162,9 +162,6 @@ function requirePostgresConnection( if (driver !== 'postgres') { throw new Error(`Local live-database ingest cannot run driver "${connection?.driver ?? 'unknown'}".`); } - if (connection?.readonly !== true) { - throw new Error(`Local live-database ingest requires connections.${connectionId}.readonly: true.`); - } if (typeof connection.url !== 'string' || connection.url.trim().length === 0) { throw new Error(`Local live-database ingest requires connections.${connectionId}.url.`); } diff --git a/packages/context/src/ingest/local-stage-ingest.test.ts b/packages/context/src/ingest/local-stage-ingest.test.ts index 5f0ee501..3a0beaa5 100644 --- a/packages/context/src/ingest/local-stage-ingest.test.ts +++ b/packages/context/src/ingest/local-stage-ingest.test.ts @@ -39,7 +39,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise { ' warehouse:', ' driver: postgres', ' url: postgres://localhost:5432/warehouse', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/mcp/local-project-ports.test.ts b/packages/context/src/mcp/local-project-ports.test.ts index b95e4ad1..fab2f076 100644 --- a/packages/context/src/mcp/local-project-ports.test.ts +++ b/packages/context/src/mcp/local-project-ports.test.ts @@ -75,7 +75,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const ports = createLocalProjectMcpContextPorts(project); @@ -89,7 +88,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const connector = testConnector(); const createConnector = vi.fn(async () => connector); @@ -125,7 +123,6 @@ describe('createLocalProjectMcpContextPorts', () => { const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); project.config.connections.warehouse = { driver: 'postgres', - readonly: true, }; project.config.ingest.adapters = ['fake']; project.config.ingest.embeddings = { @@ -633,7 +630,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const shapeOnlyPorts = createLocalProjectMcpContextPorts(project); await shapeOnlyPorts.semanticLayer?.writeSource({ @@ -720,7 +716,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const shapeOnlyPorts = createLocalProjectMcpContextPorts(project); await shapeOnlyPorts.semanticLayer?.writeSource({ @@ -958,7 +953,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://localhost:5432/warehouse', - readonly: true, }; project.config.ingest.adapters = ['live-database']; project.config.llm = { @@ -1034,7 +1028,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; project.config.ingest.adapters = ['live-database']; const ports = createLocalProjectMcpContextPorts(project, { diff --git a/packages/context/src/memory/local-memory.test.ts b/packages/context/src/memory/local-memory.test.ts index e44a5bf1..83b22146 100644 --- a/packages/context/src/memory/local-memory.test.ts +++ b/packages/context/src/memory/local-memory.test.ts @@ -145,7 +145,7 @@ describe('createLocalProjectMemoryCapture', () => { it('captures a semantic-layer source for a named local connection id', async () => { const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; const agentRunner = { runLoop: async ({ toolSet, diff --git a/packages/context/src/project/config.ts b/packages/context/src/project/config.ts index ad5ecb8a..0c345473 100644 --- a/packages/context/src/project/config.ts +++ b/packages/context/src/project/config.ts @@ -69,7 +69,6 @@ export interface KtxProjectScanConfig { export interface KtxProjectConnectionConfig { driver: string; url?: string; - readonly?: boolean; [key: string]: unknown; } diff --git a/packages/context/src/scan/local-scan.test.ts b/packages/context/src/scan/local-scan.test.ts index 6c3e877f..581f02d7 100644 --- a/packages/context/src/scan/local-scan.test.ts +++ b/packages/context/src/scan/local-scan.test.ts @@ -110,7 +110,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1006,7 +1005,6 @@ describe('local scan', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1363,7 +1361,6 @@ describe('local scan', () => { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1396,7 +1393,6 @@ describe('local scan', () => { ' warehouse:', ' driver: mysql', ' url: env:MYSQL_URL', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1432,7 +1428,6 @@ describe('local scan', () => { ' database: analytics', ' username: reader', ' password: env:CLICKHOUSE_PASSWORD', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1468,7 +1463,6 @@ describe('local scan', () => { ' database: analytics', ' username: reader', ' schema: dbo', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/scan/relationship-artifacts.test.ts b/packages/context/src/scan/relationship-artifacts.test.ts index e5fd0190..c366f1b2 100644 --- a/packages/context/src/scan/relationship-artifacts.test.ts +++ b/packages/context/src/scan/relationship-artifacts.test.ts @@ -24,7 +24,6 @@ async function writeWarehouseConfig(projectDir: string): Promise { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/scan/relationship-review-decisions.test.ts b/packages/context/src/scan/relationship-review-decisions.test.ts index 3b92e345..30277c57 100644 --- a/packages/context/src/scan/relationship-review-decisions.test.ts +++ b/packages/context/src/scan/relationship-review-decisions.test.ts @@ -28,7 +28,6 @@ async function createProject(projectDir: string): Promise { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/sl/local-query.test.ts b/packages/context/src/sl/local-query.test.ts index 80209329..4105c9f6 100644 --- a/packages/context/src/sl/local-query.test.ts +++ b/packages/context/src/sl/local-query.test.ts @@ -14,7 +14,7 @@ describe('compileLocalSlQuery', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-query-')); project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -222,7 +222,7 @@ grain: [] expect(queryExecutor.execute).toHaveBeenCalledWith({ connectionId: 'warehouse', projectDir: project.projectDir, - connection: { driver: 'postgres', readonly: true }, + connection: { driver: 'postgres' }, sql: 'select status, count(*) as order_count from public.orders group by status', maxRows: 10, }); @@ -248,7 +248,7 @@ grain: [] }); it('requires connectionId when multiple connections are configured', async () => { - project.config.connections.analytics = { driver: 'bigquery', readonly: true }; + project.config.connections.analytics = { driver: 'bigquery' }; await expect( compileLocalSlQuery(project, { diff --git a/scripts/build-adventureworks-oltp-fixture.mjs b/scripts/build-adventureworks-oltp-fixture.mjs index 4ac2e5ba..2a57fe85 100644 --- a/scripts/build-adventureworks-oltp-fixture.mjs +++ b/scripts/build-adventureworks-oltp-fixture.mjs @@ -231,7 +231,6 @@ async function main() { driver: 'sqlserver', url, schemas: ['dbo', 'HumanResources', 'Person', 'Production', 'Purchasing', 'Sales'], - readonly: true, trustServerCertificate: true, }, now: () => new Date('2026-05-07T00:00:00.000Z'), diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index 79e26f74..47fe3f10 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -50,7 +50,6 @@ describe('standalone example docs', () => { config, /path: \.\.\/\.\.\/packages\/context\/test\/fixtures\/relationship-benchmarks\/orbit_style_product_no_declared_constraints\/data\.sqlite/, ); - assert.match(config, /readonly: true/); assert.match(config, /llm_proposals: false/); assert.match(config, /validation_required_for_manifest: true/); }); diff --git a/scripts/installed-live-database-smoke.mjs b/scripts/installed-live-database-smoke.mjs index 653f53b9..88213366 100644 --- a/scripts/installed-live-database-smoke.mjs +++ b/scripts/installed-live-database-smoke.mjs @@ -92,7 +92,6 @@ export function buildKtxYaml(postgresUrl) { ' warehouse:', ' driver: postgres', ` url: "${postgresUrl}"`, - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5', diff --git a/scripts/installed-live-database-smoke.test.mjs b/scripts/installed-live-database-smoke.test.mjs index c62e98b0..0a45cdf5 100644 --- a/scripts/installed-live-database-smoke.test.mjs +++ b/scripts/installed-live-database-smoke.test.mjs @@ -59,7 +59,6 @@ describe('installed live-database artifact smoke helpers', () => { ' warehouse:', ' driver: postgres', ' url: "postgresql://ktx:postgres@127.0.0.1:15432/warehouse"', // pragma: allowlist secret - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5', diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index e7998c58..ee785c27 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -646,7 +646,6 @@ try { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5', From fa9237956eda616a457834e18c9efd89b060c27b Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Wed, 13 May 2026 19:49:25 +0200 Subject: [PATCH 4/8] ci: run pre-commit checks in CI (#74) * ci: run pre-commit in CI * test: update CI workflow guardrail --- .github/workflows/ci.yml | 50 +++++++++++++++++-- .github/workflows/release.yml | 2 +- LICENSE | 1 - .../docs/integrations/primary-sources.mdx | 6 +-- ...11-historic-sql-cross-dialect-readiness.md | 32 ++++++------ ...ric-sql-end-to-end-retrieval-acceptance.md | 2 +- .../2026-05-11-historic-sql-foundations.md | 42 ++++++++-------- ...1-historic-sql-pattern-shard-smoke-docs.md | 8 +-- ...-historic-sql-pattern-workunit-sharding.md | 18 +++---- ...-05-11-historic-sql-redaction-hardening.md | 12 ++--- ...26-05-11-historic-sql-search-enrichment.md | 22 ++++---- ...-historic-sql-skills-projection-cutover.md | 50 +++++++++---------- packages/cli/src/connection.test.ts | 2 +- packages/cli/src/setup-embeddings.test.ts | 14 +++--- .../adapters/historic-sql/redaction.test.ts | 2 +- .../historic-sql/stage-unified.test.ts | 2 +- .../ingest/adapters/metabase/client.test.ts | 2 +- .../metabase/local-metabase.adapter.test.ts | 2 +- packages/llm/src/embedding-health.test.ts | 10 ++-- packages/llm/src/model-health.test.ts | 6 +-- pyproject.toml | 1 + .../ktx-daemon/src/ktx_daemon/sql_analysis.py | 4 +- scripts/public-benchmark-manifest.json | 6 +-- scripts/standalone-ci-workflow.test.mjs | 7 ++- uv.lock | 2 + 25 files changed, 177 insertions(+), 128 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3da14c7b..ff0b7843 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,46 @@ concurrency: cancel-in-progress: true jobs: + pre-commit-checks: + name: Pre-commit checks + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24" + cache: "pnpm" + cache-dependency-path: "pnpm-lock.yaml" + + - name: Install TypeScript dependencies + run: pnpm install --frozen-lockfile + + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Setup uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.11.11" + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install Python dependencies + run: uv sync --all-packages --all-groups + + - name: Run pre-commit hooks + run: uv run pre-commit run --all-files + typescript-checks: name: TypeScript checks runs-on: ubuntu-latest @@ -23,7 +63,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -51,7 +91,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -79,7 +119,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -107,7 +147,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -156,7 +196,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2a8f696e..36eaf49c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false diff --git a/LICENSE b/LICENSE index 57bc88a1..261eeb9e 100644 --- a/LICENSE +++ b/LICENSE @@ -199,4 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 653c4e38..f3abf9f0 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -34,7 +34,7 @@ The most full-featured connector. Supports schema introspection, foreign key det connections: my-postgres: driver: postgres - url: postgresql://user:password@host:5432/database + url: env:DATABASE_URL schema: public ``` @@ -320,7 +320,7 @@ Standard MySQL/MariaDB connector with full foreign key support and schema intros connections: my-mysql: driver: mysql - url: mysql://user:password@host:3306/database + url: env:MYSQL_DATABASE_URL ``` Or with individual fields: @@ -377,7 +377,7 @@ Connects to Microsoft SQL Server and Azure SQL. Supports multi-schema scanning w connections: my-sqlserver: driver: sqlserver - url: mssql://user:password@host:1433/database?trustServerCertificate=true + url: env:SQLSERVER_DATABASE_URL ``` Or with individual fields: diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md b/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md index a7a5cc6c..3fc3e496 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md @@ -40,37 +40,37 @@ This plan does not update `examples/postgres-historic/README.md` or `examples/po Modify: -- `packages/context/src/ingest/adapters/historic-sql/types.ts` +- `packages/context/src/ingest/adapters/historic-sql/types.ts` Adds optional probe `info` notes and lets injected historic-SQL dependencies use any reader/query client pair while preserving the existing Postgres-specific option. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` Moves low `pg_stat_statements.max` from `warnings` to `info`. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` Locks `track = none` as warning and low `max` as info. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` Returns `{ warnings: [], info: [] }` from `probe()`. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` Locks the BigQuery probe return object. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` Returns `{ warnings: [], info: [] }` from `probe()`. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` Locks the Snowflake probe return object. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Updates test readers to return the normalized probe shape. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` Updates test readers to return the normalized probe shape. -- `packages/context/src/ingest/local-adapters.ts` +- `packages/context/src/ingest/local-adapters.ts` Accepts generic historic-SQL reader/query-client dependencies while keeping `postgresQueryClient` as the compatibility input used by current callers. -- `packages/context/src/ingest/local-adapters.test.ts` +- `packages/context/src/ingest/local-adapters.test.ts` Verifies generic reader/query-client injection and the existing Postgres compatibility path. -- `packages/cli/src/local-adapters.ts` +- `packages/cli/src/local-adapters.ts` Chooses Postgres, BigQuery, or Snowflake historic-SQL readers/query clients from the configured connection. -- `packages/cli/src/local-adapters.test.ts` +- `packages/cli/src/local-adapters.test.ts` Adds direct tests for CLI local adapter registration for Postgres, BigQuery, and Snowflake. -- `packages/cli/src/historic-sql-doctor.ts` +- `packages/cli/src/historic-sql-doctor.ts` Treats info-only Postgres probe notes as a passing doctor check, and warnings as warnings. -- `packages/cli/src/historic-sql-doctor.test.ts` +- `packages/cli/src/historic-sql-doctor.test.ts` Verifies low `pg_stat_statements.max` is pass/detail, while `track = none` remains warn. -- `packages/cli/src/doctor.test.ts` +- `packages/cli/src/doctor.test.ts` Updates the project doctor integration expectation for the new info-only behavior. ## Task 1: Normalize Historic-SQL Probe Results diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md b/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md index c9c40fd9..106131ed 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md @@ -44,7 +44,7 @@ Remaining acceptance gap this plan covers: Create: -- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` Owns the end-to-end local regression for the redesigned historic-SQL pipeline. It uses the real adapter and local ingest runner, with fake deterministic reader/analysis/agent components so the test does not need a live database or LLM provider. ## Task 1: Add Real-Adapter Local Ingest Acceptance Coverage diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md b/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md index fdd97d3f..6705d56f 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md @@ -41,50 +41,50 @@ The next plan after this one should cover search enrichment from spec §6.2.3-§ Create: -- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` +- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` Owns the shared zod schemas for historic-SQL LLM outputs. -- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts` Locks schema acceptance, JSON schema generation, and future-key tolerance. -- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` +- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` Implements batch sqlglot parsing for table and clause-level column extraction. -- `python/ktx-daemon/tests/test_sql_analysis.py` +- `python/ktx-daemon/tests/test_sql_analysis.py` Tests batch parser behavior without FastAPI. Modify: -- `packages/context/src/ingest/index.ts` +- `packages/context/src/ingest/index.ts` Exports the new historic-SQL skill schemas. -- `packages/context/src/sl/types.ts` +- `packages/context/src/sl/types.ts` Adds `usage?: TableUsageOutput` to `SemanticLayerSource`. -- `packages/context/src/sl/schemas.ts` +- `packages/context/src/sl/schemas.ts` Accepts `usage` in standalone and overlay semantic-layer source validation. -- `packages/context/src/sl/semantic-layer.service.ts` +- `packages/context/src/sl/semantic-layer.service.ts` Projects manifest `usage` onto `SemanticLayerSource` and composes overlay usage intentionally. -- `packages/context/src/sl/semantic-layer.service.test.ts` +- `packages/context/src/sl/semantic-layer.service.test.ts` Tests source schema acceptance, manifest projection, and overlay composition. -- `packages/context/src/ingest/adapters/live-database/manifest.ts` +- `packages/context/src/ingest/adapters/live-database/manifest.ts` Adds `LiveDatabaseManifestTableEntry.usage`, existing-usage inputs, and `mergeUsagePreservingExternal()`. -- `packages/context/src/ingest/adapters/live-database/manifest.test.ts` +- `packages/context/src/ingest/adapters/live-database/manifest.test.ts` Tests scan-managed usage replacement while preserving external keys. -- `packages/context/src/scan/local-enrichment-artifacts.ts` +- `packages/context/src/scan/local-enrichment-artifacts.ts` Loads existing manifest usage and passes it through scan manifest rebuilds. -- `packages/context/src/scan/local-enrichment-artifacts.test.ts` +- `packages/context/src/scan/local-enrichment-artifacts.test.ts` Tests that structural scan rewrites preserve existing usage. -- `python/ktx-daemon/src/ktx_daemon/app.py` +- `python/ktx-daemon/src/ktx_daemon/app.py` Registers `/sql/analyze-batch`. -- `python/ktx-daemon/tests/test_app.py` +- `python/ktx-daemon/tests/test_app.py` Tests the FastAPI endpoint. -- `packages/context/src/sql-analysis/ports.ts` +- `packages/context/src/sql-analysis/ports.ts` Adds batch analysis types and `SqlAnalysisPort.analyzeBatch()`. -- `packages/context/src/sql-analysis/index.ts` +- `packages/context/src/sql-analysis/index.ts` Exports the new batch analysis types. -- `packages/context/src/sql-analysis/http-sql-analysis-port.ts` +- `packages/context/src/sql-analysis/http-sql-analysis-port.ts` Maps `/sql/analyze-batch` request and response payloads. -- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts` +- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts` Tests HTTP mapping and malformed response rejection. -- `packages/cli/src/managed-python-http.test.ts` +- `packages/cli/src/managed-python-http.test.ts` Verifies the managed daemon wrapper routes `analyzeBatch()`. -- Existing test files with `SqlAnalysisPort` object literals +- Existing test files with `SqlAnalysisPort` object literals Add a no-op `analyzeBatch: async () => new Map()` while legacy paths still use `analyzeForFingerprint()`. ## Task 1: Add Historic SQL Skill Schemas diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md index 9e386a16..b5382ff4 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md @@ -39,13 +39,13 @@ Remaining gap this plan fixes: ## File Structure -- Modify `scripts/examples-docs.test.mjs` +- Modify `scripts/examples-docs.test.mjs` Pins docs and smoke script to the sharded pattern WorkUnit contract. -- Modify `examples/postgres-historic/scripts/smoke.sh` +- Modify `examples/postgres-historic/scripts/smoke.sh` Validates `patterns-input/part-*.json` shard files and `historic-sql-patterns-part-*` stage-only WorkUnits. -- Modify `examples/postgres-historic/README.md` +- Modify `examples/postgres-historic/README.md` Documents `patterns-input.json` as the full audit artifact and `patterns-input/part-*.json` as bounded pattern WorkUnit input. -- Modify `examples/README.md` +- Modify `examples/README.md` Updates the short example catalog entry with the same audit-vs-shard wording. ### Task 1: Pin Example Tests To Pattern Shards diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md index ee7604a7..c67f6d78 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md @@ -30,23 +30,23 @@ No existing spec-derived plan is currently unimplemented in this worktree. This ## File Structure -- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts` +- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts` Owns deterministic pattern audit ordering, cross-table candidate filtering, byte-bounded shard creation, shard path constants, and shard path detection. -- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` +- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` Covers deterministic shard ordering, single-table exclusion from WorkUnit shards, byte limits, and oversize-template manifest warnings. -- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Writes full `patterns-input.json` plus bounded `patterns-input/part-0001.json` shard files, and appends shard warnings to `manifest.json`. -- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Adds a regression for audit file preservation and sharded WorkUnit input creation. -- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` Emits one patterns WorkUnit per changed shard path, treats root `patterns-input.json` as audit-only, and includes shard paths in the scope descriptor and eviction calculation. -- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` Updates root-file expectations and adds multi-shard diff behavior. -- Modify `packages/context/skills/historic_sql_patterns/SKILL.md` +- Modify `packages/context/skills/historic_sql_patterns/SKILL.md` Tells the skill to read the exact pattern shard in `rawFiles` and emit evidence with that shard as `rawPath`. -- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` Updates the fake agent to emit pattern evidence for `historic-sql-patterns-part-0001`. -- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts` +- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts` Keeps packaged skill assertions aligned with sharded pattern file guidance. ## Task 1: Add Pattern Input Sharding Helper diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md b/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md index 1adcdfd3..e59e164b 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md @@ -55,16 +55,16 @@ Remaining spec gap this plan covers: Create: -- `packages/context/src/ingest/adapters/historic-sql/redaction.ts` +- `packages/context/src/ingest/adapters/historic-sql/redaction.ts` Owns compilation and application of historic-SQL SQL-text redaction patterns. Supports JavaScript regex strings and the documented `(?i)` case-insensitive prefix used by setup tests/docs. -- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts` Tests raw regex replacement, `(?i)` compatibility, empty config behavior, and invalid-pattern diagnostics. Modify: -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Compiles `config.redactionPatterns` once per fetch. Keeps original SQL for filtering and `SqlAnalysisPort.analyzeBatch()`, then stores redacted SQL in `ParsedTemplate.template.canonicalSql` before `toStagedTable()` and `toPatternsInput()` serialize files. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Adds a regression proving raw secrets are absent from staged artifacts while `analyzeBatch()` still receives the original SQL. ## Task 1: Add Historic SQL Redaction Helper @@ -89,7 +89,7 @@ describe('historic-SQL redaction', () => { ]); const sql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret expect(redactHistoricSqlText(sql, redactors)).toBe( "select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'", @@ -202,7 +202,7 @@ Append this test inside the existing `describe('stageHistoricSqlAggregatedSnapsh it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => { const stagedDir = await tempDir(); const originalSql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret const reader: HistoricSqlReader = { async probe() { return { warnings: [], info: [] }; diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md b/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md index ee960bb8..cafc234b 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md @@ -37,27 +37,27 @@ This plan does not rewrite the historic-SQL adapter, readers, skills, projection Modify: -- `packages/context/src/sl/sl-search.service.ts` +- `packages/context/src/sl/sl-search.service.ts` Adds usage narrative, frequency, filters, group-bys, joins, and stale marker to the canonical SL search text. Preserves snippets returned by repository search for direct `SlSearchService.search()` callers. -- `packages/context/src/sl/sl-search.service.test.ts` +- `packages/context/src/sl/sl-search.service.test.ts` Tests usage search-text content and direct service snippet pass-through. -- `packages/context/src/sl/ports.ts` +- `packages/context/src/sl/ports.ts` Extends `SlSourcesIndexPort.search()` rows with optional `snippet`. -- `packages/context/src/sl/sqlite-sl-sources-index.ts` +- `packages/context/src/sl/sqlite-sl-sources-index.ts` Adds FTS5 `snippet()` selection to lexical candidate search and direct index search. -- `packages/context/src/sl/sqlite-sl-sources-index.test.ts` +- `packages/context/src/sl/sqlite-sl-sources-index.test.ts` Locks snippet behavior for both direct search and lexical lane candidates. -- `packages/context/src/sl/local-sl.ts` +- `packages/context/src/sl/local-sl.ts` Adds `frequencyTier` and `snippet` to query-mode `LocalSlSourceSearchResult`; collects snippets from the lexical lane and hydrates frequency from `SemanticLayerSource.usage`. -- `packages/context/src/sl/local-sl.test.ts` +- `packages/context/src/sl/local-sl.test.ts` Tests that usage-only terms can find a source and that results include `frequencyTier` and FTS snippet. -- `packages/context/src/sl/pglite-sl-search-prototype.ts` +- `packages/context/src/sl/pglite-sl-search-prototype.ts` Propagates `frequencyTier` for the prototype backend so the shared result type stays truthful. -- `packages/context/src/mcp/types.ts` +- `packages/context/src/mcp/types.ts` Adds `frequencyTier` and `snippet` to `KtxSemanticLayerSourceSummary`. -- `packages/context/src/mcp/local-project-ports.ts` +- `packages/context/src/mcp/local-project-ports.ts` Includes `frequencyTier` and `snippet` in `semanticLayer.listSources()` output. -- `packages/context/src/mcp/local-project-ports.test.ts` +- `packages/context/src/mcp/local-project-ports.test.ts` Tests the agent/MCP-facing list response. ## Task 1: Index Historic SQL Usage In SL Search Text diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md b/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md index a892542e..a7494e2d 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md @@ -52,58 +52,58 @@ Still not implemented: Create: -- `packages/context/src/ingest/adapters/historic-sql/evidence.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence.ts` Owns typed evidence envelopes, ignored evidence path helpers, and load/write helpers for table usage and pattern evidence. -- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts` Tests evidence schema validation, path normalization, and loader rejection of malformed evidence. -- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts` Adds `emit_historic_sql_evidence`, the only write tool the two new historic-SQL skills use. -- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts` Tests the tool writes ignored run-local JSON with `skipLock: true` and rejects non-historic ingest sessions. -- `packages/context/src/ingest/adapters/historic-sql/projection.ts` +- `packages/context/src/ingest/adapters/historic-sql/projection.ts` Projects table usage evidence into manifest shards, writes pattern wiki pages, marks stale usage/pages, and deletes legacy query pages. -- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` Tests `_schema` merge, stale usage, pattern slug reuse, stale page tagging, archive movement, and legacy page cleanup. -- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts` +- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts` Implements `IngestBundlePostProcessorPort` for the deterministic projection phase. -- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts` Tests post-processor path resolution from `workdir`, `connectionId`, `sourceKey`, and `syncId`. -- `packages/context/skills/historic_sql_table_digest/SKILL.md` +- `packages/context/skills/historic_sql_table_digest/SKILL.md` Skill for one changed `tables/*.json` WorkUnit; emits one table usage evidence object. -- `packages/context/skills/historic_sql_patterns/SKILL.md` +- `packages/context/skills/historic_sql_patterns/SKILL.md` Skill for `patterns-input.json`; emits one pattern evidence object per recurring cross-table intent. Modify: -- `packages/context/src/ingest/adapters/historic-sql/types.ts` +- `packages/context/src/ingest/adapters/historic-sql/types.ts` Keep only unified config/staged schemas and reader contracts; extend config preprocessing for existing `serviceAccountUserPatterns` and `minCalls` aliases. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Add `staleArchiveAfterDays` to `manifest.json` so projection can archive stale pattern pages deterministically. -- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` Keep the same WorkUnits, but mention `emit_historic_sql_evidence` in `notes`. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` Switch production fetch/chunk/scope to the unified hot path, replace skills, remove legacy triage support, and run legacy PGSS baseline cache cleanup. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` Rewrite around unified staging and new skills. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` Inline the PGSS probe logic so `postgres-pgss-query-history-reader.ts` can be deleted. -- `packages/context/src/ingest/local-adapters.ts` +- `packages/context/src/ingest/local-adapters.ts` Use `PostgresPgssReader` for local Postgres historic SQL and return unified pull config. -- `packages/context/src/ingest/local-bundle-runtime.ts` +- `packages/context/src/ingest/local-bundle-runtime.ts` Add the source-specific evidence tool to historic-SQL WorkUnits and register the historic-SQL post-processor. -- `packages/context/src/ingest/ingest-runtime-assets.test.ts` +- `packages/context/src/ingest/ingest-runtime-assets.test.ts` Replace old skill asset assertions with the two new skills. -- `packages/context/src/memory/memory-runtime-assets.test.ts` +- `packages/context/src/memory/memory-runtime-assets.test.ts` Replace old historic-SQL skill heading with the two new skill headings. -- `packages/context/src/package-exports.test.ts` +- `packages/context/src/package-exports.test.ts` Remove legacy export assertions and add evidence/projection export assertions. -- `packages/context/src/ingest/index.ts` +- `packages/context/src/ingest/index.ts` Export new evidence/projection/post-processor helpers and remove legacy historic-SQL exports. -- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts` +- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts` Import `PostgresPgssReader` instead of `PostgresPgssQueryHistoryReader`. -- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts` +- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts` Rename generated config to `minExecutions` while accepting the old `--historic-sql-min-calls` flag for one release. -- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts` +- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts` Remove historic-SQL template triage examples because the new adapter no longer uses page triage. Delete: diff --git a/packages/cli/src/connection.test.ts b/packages/cli/src/connection.test.ts index 6eb3a08c..0d592b00 100644 --- a/packages/cli/src/connection.test.ts +++ b/packages/cli/src/connection.test.ts @@ -159,7 +159,7 @@ describe('runKtxConnection', () => { prod_metabase: { driver: 'metabase', api_url: 'http://metabase.example.test', - api_key: 'mb_test', + api_key: 'mb_test', // pragma: allowlist secret }, }); const testConnection = vi.fn(async () => ({ success: true as const })); diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index c2d5cad2..4c91ddd5 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -319,14 +319,14 @@ describe('setup embeddings step', () => { projectDir: tempDir, inputMode: 'disabled', embeddingBackend: 'openai', - embeddingApiKeyEnv: 'OPENAI_API_KEY', + embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret cliVersion: '0.2.0', runtimeInstallPolicy: 'auto', skipEmbeddings: false, }, io.io, { - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, }, ); @@ -336,14 +336,14 @@ describe('setup embeddings step', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.ingest.embeddings).toMatchObject({ backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { api_key: 'env:OPENAI_API_KEY' }, + openai: { api_key: 'env:OPENAI_API_KEY' }, // pragma: allowlist secret }); expect(io.stdout()).not.toContain('sk-openai-test'); }); @@ -367,7 +367,7 @@ describe('setup embeddings step', () => { io.io, { prompts, - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()), }, @@ -384,7 +384,7 @@ describe('setup embeddings step', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }); expect(prompts.select).toHaveBeenCalledWith( expect.objectContaining({ @@ -478,7 +478,7 @@ describe('setup embeddings step', () => { }, makeIo().io, { - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, }, ), diff --git a/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts b/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts index c8f1d78b..d27015a6 100644 --- a/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts @@ -9,7 +9,7 @@ describe('historic-SQL redaction', () => { ]); const sql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret expect(redactHistoricSqlText(sql, redactors)).toBe( "select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'", diff --git a/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts b/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts index 421970bf..d1610054 100644 --- a/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts @@ -169,7 +169,7 @@ describe('stageHistoricSqlAggregatedSnapshot', () => { it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => { const stagedDir = await tempDir(); const originalSql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret const reader: HistoricSqlReader = { async probe() { return { warnings: [], info: [] }; diff --git a/packages/context/src/ingest/adapters/metabase/client.test.ts b/packages/context/src/ingest/adapters/metabase/client.test.ts index 1c0fdfa9..3d45a276 100644 --- a/packages/context/src/ingest/adapters/metabase/client.test.ts +++ b/packages/context/src/ingest/adapters/metabase/client.test.ts @@ -92,7 +92,7 @@ describe('MetabaseClient retry exhaustion', () => { .mockResolvedValueOnce(new Response(JSON.stringify([]), { status: 200 })); const client = new MetabaseClient( - { apiUrl: 'https://metabase.example.test', apiKey: 'key' }, + { apiUrl: 'https://metabase.example.test', apiKey: 'key' }, // pragma: allowlist secret { ...DEFAULT_METABASE_CLIENT_CONFIG, baseDelayMs: 0, diff --git a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts index 7cbe913b..b25ea18b 100644 --- a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts +++ b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts @@ -39,7 +39,7 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => { const connection: KtxProjectConnectionConfig = { driver: 'metabase', api_url: 'https://metabase.example.com', - api_key_ref: `file:${keyPath}`, + api_key_ref: `file:${keyPath}`, // pragma: allowlist secret }; expect(metabaseRuntimeConfigFromLocalConnection('prod-metabase', connection)).toEqual({ diff --git a/packages/llm/src/embedding-health.test.ts b/packages/llm/src/embedding-health.test.ts index ca998aa9..65956311 100644 --- a/packages/llm/src/embedding-health.test.ts +++ b/packages/llm/src/embedding-health.test.ts @@ -17,13 +17,13 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), ).resolves.toEqual({ ok: true }); - expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined }); + expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined }); // pragma: allowlist secret }); it('returns failed when the provider returns the wrong dimensions', async () => { @@ -41,7 +41,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), @@ -66,7 +66,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-secret' }, + openai: { apiKey: 'sk-openai-secret' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), @@ -94,7 +94,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { timeoutMs: 1, deps: { createOpenAIClient } }, ), diff --git a/packages/llm/src/model-health.test.ts b/packages/llm/src/model-health.test.ts index d1b3df47..8752b09e 100644 --- a/packages/llm/src/model-health.test.ts +++ b/packages/llm/src/model-health.test.ts @@ -14,7 +14,7 @@ describe('KTX LLM health check', () => { runKtxLlmHealthCheck( { backend: 'anthropic', - anthropic: { apiKey: 'sk-ant-test' }, + anthropic: { apiKey: 'sk-ant-test' }, // pragma: allowlist secret modelSlots: { default: 'claude-sonnet-4-6' }, }, { deps: { createAnthropic, generateText, devtoolsEnabled: true, wrapLanguageModel } }, @@ -23,7 +23,7 @@ describe('KTX LLM health check', () => { expect(createAnthropic).toHaveBeenCalledWith( expect.objectContaining({ - apiKey: 'sk-ant-test', + apiKey: 'sk-ant-test', // pragma: allowlist secret }), ); expect(generateText).toHaveBeenCalledWith( @@ -46,7 +46,7 @@ describe('KTX LLM health check', () => { runKtxLlmHealthCheck( { backend: 'anthropic', - anthropic: { apiKey: 'sk-ant-secret' }, + anthropic: { apiKey: 'sk-ant-secret' }, // pragma: allowlist secret modelSlots: { default: 'claude-sonnet-4-6' }, }, { diff --git a/pyproject.toml b/pyproject.toml index 1c4816b7..e6422fb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ Issues = "https://github.com/kaelio/ktx/issues" [dependency-groups] dev = [ + "pre-commit>=4.6.0", "pytest>=9.0.2", "ruff>=0.8.4", ] diff --git a/python/ktx-daemon/src/ktx_daemon/sql_analysis.py b/python/ktx-daemon/src/ktx_daemon/sql_analysis.py index 9a222098..d5deb240 100644 --- a/python/ktx-daemon/src/ktx_daemon/sql_analysis.py +++ b/python/ktx-daemon/src/ktx_daemon/sql_analysis.py @@ -130,7 +130,9 @@ def _analyze_one( ) -def _analyze_payload(payload: tuple[str, str, str]) -> tuple[str, AnalyzeSqlBatchResult]: +def _analyze_payload( + payload: tuple[str, str, str], +) -> tuple[str, AnalyzeSqlBatchResult]: item_id, sql, dialect = payload return _analyze_one(item_id, sql, dialect) diff --git a/scripts/public-benchmark-manifest.json b/scripts/public-benchmark-manifest.json index e106e24e..fdd97e59 100644 --- a/scripts/public-benchmark-manifest.json +++ b/scripts/public-benchmark-manifest.json @@ -4,7 +4,7 @@ "id": "chinook_with_declared_metadata", "displayName": "Chinook (SQLite, declared metadata)", "url": "https://raw.githubusercontent.com/lerocha/chinook-database/master/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite", - "sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15", + "sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15", "_allowlist": "// pragma: allowlist secret", "license": "MIT", "source": "https://github.com/lerocha/chinook-database" }, @@ -12,7 +12,7 @@ "id": "northwind_with_declared_metadata", "displayName": "Northwind (SQLite, declared metadata)", "url": "https://github.com/jpwhite3/northwind-SQLite3/raw/main/dist/northwind.db", - "sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877", + "sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877", "_allowlist": "// pragma: allowlist secret", "license": "MIT", "source": "https://github.com/jpwhite3/northwind-SQLite3" }, @@ -20,7 +20,7 @@ "id": "sakila_with_declared_metadata", "displayName": "Sakila (SQLite, declared metadata)", "url": "https://raw.githubusercontent.com/bradleygrant/sakila-sqlite3/master/sakila_master.db", - "sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268", + "sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268", "_allowlist": "// pragma: allowlist secret", "license": "BSD-2-Clause", "source": "https://github.com/bradleygrant/sakila-sqlite3" }, diff --git a/scripts/standalone-ci-workflow.test.mjs b/scripts/standalone-ci-workflow.test.mjs index 195fce53..5aa4cc02 100644 --- a/scripts/standalone-ci-workflow.test.mjs +++ b/scripts/standalone-ci-workflow.test.mjs @@ -20,6 +20,8 @@ describe('standalone KTX CI workflow', () => { assertIncludesAll(workflow, [ 'permissions:', 'contents: read', + 'pre-commit-checks:', + 'name: Pre-commit checks', 'typescript-checks:', 'name: TypeScript checks', 'slow-context-tests:', @@ -33,7 +35,7 @@ describe('standalone KTX CI workflow', () => { 'artifact-checks:', 'name: Artifact checks', 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd', - 'pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0', + 'pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093', 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e', 'node-version: "24"', 'cache-dependency-path: "pnpm-lock.yaml"', @@ -46,7 +48,10 @@ describe('standalone KTX CI workflow', () => { 'actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405', 'python-version: "3.13"', 'astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b', + 'version: "0.11.11"', 'cache-dependency-glob: "uv.lock"', + 'uv sync --all-packages --all-groups', + 'uv run pre-commit run --all-files', 'uv sync --all-packages', 'uv run pytest', 'pnpm run artifacts:check', diff --git a/uv.lock b/uv.lock index 5458900e..5531c8e3 100644 --- a/uv.lock +++ b/uv.lock @@ -546,6 +546,7 @@ source = { virtual = "." } [package.dev-dependencies] dev = [ + { name = "pre-commit" }, { name = "pytest" }, { name = "ruff" }, ] @@ -554,6 +555,7 @@ dev = [ [package.metadata.requires-dev] dev = [ + { name = "pre-commit", specifier = ">=4.6.0" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "ruff", specifier = ">=0.8.4" }, ] From ea33e51e8f56ac426fae9383bebec2f97962fd10 Mon Sep 17 00:00:00 2001 From: Luca Martial Date: Wed, 13 May 2026 13:29:50 -0700 Subject: [PATCH 5/8] refactor(cli): remove interactive gcloud auth from Vertex AI setup Instead of spawning an interactive gcloud login flow, tell users to run gcloud auth application-default login themselves before continuing. Also adds a Vertex-specific model list and spinner progress for LLM health checks. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/cli/src/setup-models.test.ts | 87 +++++++------ packages/cli/src/setup-models.ts | 176 ++++++++++++-------------- 2 files changed, 129 insertions(+), 134 deletions(-) diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/src/setup-models.test.ts index e310ea90..e4425d69 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/src/setup-models.test.ts @@ -7,10 +7,8 @@ import { BUNDLED_ANTHROPIC_MODELS, fetchAnthropicModels, type KtxSetupModelPromptAdapter, - runKtxSetupGcloudApplicationDefaultAuth, runKtxSetupAnthropicModelStep, } from './setup-models.js'; -import type { KtxCliIo } from './cli-runtime.js'; function makeIo() { let stdout = ''; @@ -34,6 +32,17 @@ function makeIo() { }; } +function makeSpinnerEvents() { + const events: string[] = []; + const spinner = vi.fn(() => ({ + start: (msg: string) => events.push(`start:${msg}`), + message: (msg: string) => events.push(`message:${msg}`), + stop: (msg: string) => events.push(`stop:${msg}`), + error: (msg: string) => events.push(`error:${msg}`), + })); + return { events, spinner }; +} + function makePromptAdapter(options: { providerChoice?: string; selectValues?: string[]; @@ -191,6 +200,7 @@ describe('setup Anthropic model step', () => { it('configures env credentials, selected model, prompt caching, and llm completion state', async () => { const io = makeIo(); + const { events: spinnerEvents, spinner } = makeSpinnerEvents(); const result = await runKtxSetupAnthropicModelStep( { projectDir: tempDir, @@ -203,6 +213,7 @@ describe('setup Anthropic model step', () => { { env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret healthCheck: vi.fn(async () => ({ ok: true as const })), + spinner, }, ); @@ -219,6 +230,10 @@ describe('setup Anthropic model step', () => { expect(config.scan.enrichment.mode).toBe('llm'); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); + expect(spinnerEvents).toEqual([ + 'start:Checking Anthropic API LLM (claude-sonnet-4-6).', + 'stop:LLM test passed (Anthropic API, claude-sonnet-4-6)', + ]); expect(io.stdout()).toContain('LLM ready: yes'); expect(io.stdout()).not.toContain('sk-ant-test'); }); @@ -226,6 +241,7 @@ describe('setup Anthropic model step', () => { it('configures Vertex AI provider, selected model, prompt caching, and llm completion state', async () => { const io = makeIo(); const healthCheck = vi.fn(async () => ({ ok: true as const })); + const { events: spinnerEvents, spinner } = makeSpinnerEvents(); const result = await runKtxSetupAnthropicModelStep( { @@ -238,7 +254,7 @@ describe('setup Anthropic model step', () => { skipLlm: false, }, io.io, - { env: {}, healthCheck }, + { env: {}, healthCheck, spinner }, ); expect(result.status).toBe('ready'); @@ -260,13 +276,16 @@ describe('setup Anthropic model step', () => { expect(config.scan.enrichment.mode).toBe('llm'); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); + expect(spinnerEvents).toEqual([ + 'start:Checking Vertex AI LLM (claude-sonnet-4-6).', + 'stop:LLM test passed (Vertex AI, claude-sonnet-4-6)', + ]); expect(io.stdout()).toContain('LLM ready: yes (claude-sonnet-4-6)'); }); - it('can run gcloud auth for Vertex AI and infer project and default location', async () => { + it('uses existing Vertex AI credentials without offering to run gcloud auth', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'gcloud', 'local-gcp-project', 'claude-sonnet-4-6'] }); - const runGcloudAuth = vi.fn(async () => ({ ok: true as const })); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'existing', 'local-gcp-project', 'claude-sonnet-4-6'] }); const readGcloudProject = vi.fn(async () => 'local-gcp-project'); const listGcloudProjects = vi.fn(async () => [ { projectId: 'local-gcp-project', name: 'Local project' }, @@ -280,7 +299,6 @@ describe('setup Anthropic model step', () => { { prompts, env: {}, - runGcloudAuth, readGcloudProject, listGcloudProjects, healthCheck, @@ -288,7 +306,15 @@ describe('setup Anthropic model step', () => { ); expect(result.status).toBe('ready'); - expect(runGcloudAuth).toHaveBeenCalledWith(io.io); + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('How should KTX authenticate with Google Vertex AI?'), + options: [ + { value: 'existing', label: 'Use existing gcloud/Application Default Credentials' }, + { value: 'back', label: 'Back' }, + ], + }), + ); expect(readGcloudProject).toHaveBeenCalled(); expect(listGcloudProjects).toHaveBeenCalled(); expect(prompts.text).not.toHaveBeenCalled(); @@ -303,6 +329,22 @@ describe('setup Anthropic model step', () => { ], }), ); + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Which Anthropic model should KTX use?'), + options: [ + { value: 'claude-opus-4-7', label: 'Claude Opus 4.7' }, + { value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6' }, + { value: 'claude-opus-4-6', label: 'Claude Opus 4.6' }, + { value: 'claude-opus-4-5', label: 'Claude Opus 4.5' }, + { value: 'claude-haiku-4-5', label: 'Claude Haiku 4.5' }, + { value: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5' }, + { value: 'claude-opus-4-1', label: 'Claude Opus 4.1' }, + { value: 'manual', label: 'Enter a model ID manually' }, + { value: 'back', label: 'Back' }, + ], + }), + ); expect(healthCheck).toHaveBeenCalledWith({ backend: 'vertex', vertex: { project: 'local-gcp-project', location: 'us-east5' }, @@ -415,35 +457,6 @@ describe('setup Anthropic model step', () => { ); }); - it('runs only gcloud application-default login for Vertex AI auth', async () => { - const io = makeIo(); - const runGcloud = vi.fn(async () => ({ ok: true as const })); - - await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true }); - - expect(runGcloud).toHaveBeenCalledTimes(1); - expect(runGcloud).toHaveBeenCalledWith(['auth', 'application-default', 'login'], expect.anything()); - expect(runGcloud).not.toHaveBeenCalledWith(['auth', 'login'], expect.anything()); - expect(io.stdout()).toContain('gcloud auth application-default login'); - expect(io.stdout()).not.toContain('gcloud auth login'); - }); - - it('indents gcloud auth output inside the setup gutter', async () => { - const io = makeIo(); - const runGcloud = vi.fn(async (_args: string[], commandIo: KtxCliIo) => { - commandIo.stdout.write('Your browser has been opened to visit:\n\n https://accounts.example/auth\n'); - commandIo.stderr.write('Credentials saved to file: [/tmp/application_default_credentials.json]\n'); - return { ok: true as const }; - }); - - await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true }); - - expect(io.stdout()).toContain('│ Your browser has been opened to visit:'); - expect(io.stdout()).toContain('│ https://accounts.example/auth'); - expect(io.stderr()).toContain('│ Credentials saved to file: [/tmp/application_default_credentials.json]'); - expect(io.stdout()).not.toContain('\nYour browser has been opened'); - }); - it('explains common Vertex AI Forbidden health-check causes', async () => { const io = makeIo(); diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index bd05bd44..e4c7fcd2 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -1,4 +1,4 @@ -import { execFile, spawn } from 'node:child_process'; +import { execFile } from 'node:child_process'; import { writeFile } from 'node:fs/promises'; import { promisify } from 'node:util'; import { resolveLocalKtxLlmConfig } from '@ktx/context'; @@ -11,6 +11,7 @@ import { serializeKtxProjectConfig, } from '@ktx/context/project'; import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from '@ktx/llm'; +import { createClackSpinner, type KtxCliSpinner } from './clack.js'; import type { KtxCliIo } from './cli-runtime.js'; import { withTextInputNavigation } from './prompt-navigation.js'; import { envCredentialReference, writeProjectLocalSecretReference } from './setup-secrets.js'; @@ -61,9 +62,9 @@ export interface KtxSetupModelDeps { prompts?: KtxSetupModelPromptAdapter; listModels?: (apiKey: string) => Promise; healthCheck?: (config: KtxLlmConfig) => Promise; - runGcloudAuth?: (io: KtxCliIo) => Promise; readGcloudProject?: () => Promise; listGcloudProjects?: () => Promise; + spinner?: () => KtxCliSpinner; } export const BUNDLED_ANTHROPIC_MODEL_REGISTRY_VERSION = '2026-05-07'; @@ -74,6 +75,16 @@ export const BUNDLED_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, ]; +const VERTEX_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ + { id: 'claude-opus-4-7', label: 'Claude Opus 4.7', recommended: false }, + { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: false }, + { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, + { id: 'claude-opus-4-5', label: 'Claude Opus 4.5', recommended: false }, + { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, + { id: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5', recommended: false }, + { id: 'claude-opus-4-1', label: 'Claude Opus 4.1', recommended: false }, +]; + const HIDDEN_ANTHROPIC_MODEL_PATTERNS = [ /^claude-sonnet-4$/i, /^claude-opus-4$/i, @@ -91,8 +102,8 @@ const ANTHROPIC_MODEL_PROMPT_CONTEXT = 'into semantic-layer sources and wiki context.'; const VERTEX_AUTH_PROMPT_CONTEXT = - 'KTX can use Google Cloud Application Default Credentials for local Vertex AI access. This opens the normal ' + - 'gcloud browser login flow and does not store Google credentials in ktx.yaml.'; + 'KTX uses Google Cloud Application Default Credentials for local Vertex AI access and does not store Google ' + + 'credentials in ktx.yaml. If needed, run gcloud auth application-default login before continuing.'; const VERTEX_PROJECT_PROMPT_CONTEXT = 'KTX stores the selected Google Cloud project ID in ktx.yaml and uses Application Default Credentials for ' + 'access. Project visibility depends on the signed-in Google account and organization permissions.'; @@ -137,94 +148,17 @@ type VertexConfigChoice = } | { status: 'back' | 'missing-input' }; -type VertexAuthChoice = { status: 'ready' } | { status: 'back' | 'missing-input' }; +type VertexAuthChoice = { status: 'ready' } | { status: 'back' }; -export type GcloudAuthResult = { ok: true } | { ok: false; message: string }; interface GcloudProjectChoice { projectId: string; name?: string; } -type GcloudCommandRunner = (args: string[], io: KtxCliIo) => Promise; function createPromptAdapter(): KtxSetupModelPromptAdapter { return createKtxSetupPromptAdapter({ selectCancelValue: 'back' }); } -function createIndentedCommandIo(io: KtxCliIo): KtxCliIo { - const indentedWriter = (write: (chunk: string) => void) => { - let atLineStart = true; - return (chunk: string) => { - for (const char of chunk) { - if (atLineStart) { - write('│ '); - atLineStart = false; - } - write(char); - if (char === '\n') { - atLineStart = true; - } - } - }; - }; - - return { - stdout: { - isTTY: io.stdout.isTTY, - columns: io.stdout.columns, - write: indentedWriter((chunk) => io.stdout.write(chunk)), - }, - stderr: { - write: indentedWriter((chunk) => io.stderr.write(chunk)), - }, - }; -} - -function runInteractiveGcloud(args: string[], io: KtxCliIo): Promise { - return new Promise((resolve) => { - let settled = false; - const child = spawn('gcloud', args, { stdio: ['inherit', 'pipe', 'pipe'] }); - child.stdout?.on('data', (chunk: Buffer) => { - io.stdout.write(chunk.toString('utf8')); - }); - child.stderr?.on('data', (chunk: Buffer) => { - io.stderr.write(chunk.toString('utf8')); - }); - child.on('error', (error: NodeJS.ErrnoException) => { - if (settled) { - return; - } - settled = true; - if (error.code === 'ENOENT') { - resolve({ ok: false, message: 'gcloud CLI was not found on PATH.' }); - return; - } - resolve({ ok: false, message: error.message }); - }); - child.on('close', (code, signal) => { - if (settled) { - return; - } - settled = true; - if (code === 0) { - resolve({ ok: true }); - return; - } - resolve({ - ok: false, - message: signal ? `gcloud exited after signal ${signal}.` : `gcloud exited with code ${code ?? 'unknown'}.`, - }); - }); - }); -} - -export async function runKtxSetupGcloudApplicationDefaultAuth( - io: KtxCliIo, - runGcloud: GcloudCommandRunner = runInteractiveGcloud, -): Promise { - io.stdout.write('│ Running gcloud auth application-default login...\n'); - return await runGcloud(['auth', 'application-default', 'login'], createIndentedCommandIo(io)); -} - async function defaultReadGcloudProject(): Promise { try { const { stdout } = await execFileAsync('gcloud', ['config', 'get-value', 'project'], { encoding: 'utf8' }); @@ -374,6 +308,53 @@ function buildVertexHealthConfig(vertex: { project?: string; location: string }, }; } +type LlmHealthProvider = 'Anthropic API' | 'Vertex AI'; + +function llmHealthCheckStartText(provider: LlmHealthProvider, model: string): string { + return `Checking ${provider} LLM (${model}).`; +} + +function startLlmHealthCheckProgress( + spinner: KtxCliSpinner, + message: string, +): { succeed(msg: string): void; fail(msg: string): void } { + spinner.start(message); + return { + succeed(msg: string) { + spinner.stop(msg); + }, + fail(msg: string) { + spinner.error(msg); + }, + }; +} + +async function runLlmHealthCheckWithProgress( + config: KtxLlmConfig, + provider: LlmHealthProvider, + model: string, + healthCheck: (config: KtxLlmConfig) => Promise, + deps: KtxSetupModelDeps, +): Promise { + const progress = startLlmHealthCheckProgress( + (deps.spinner ?? createClackSpinner)(), + llmHealthCheckStartText(provider, model), + ); + let health: KtxLlmHealthCheckResult; + try { + health = await healthCheck(config); + } catch (error) { + progress.fail('LLM test failed'); + throw error; + } + if (health.ok) { + progress.succeed(`LLM test passed (${provider}, ${model})`); + } else { + progress.fail('LLM test failed'); + } + return health; +} + function formatVertexHealthFailure(message: string, vertex: { project?: string; location: string }): string { const trimmed = message.trim() || 'unknown error'; if (!/(forbidden|permission|permission_denied|403)/i.test(trimmed)) { @@ -516,7 +497,6 @@ async function chooseBackend( async function chooseVertexAuth( args: KtxSetupModelArgs, - io: KtxCliIo, deps: KtxSetupModelDeps, ): Promise { if (args.inputMode === 'disabled' || args.vertexProject || args.vertexLocation) { @@ -527,7 +507,6 @@ async function chooseVertexAuth( const choice = await prompts.select({ message: `How should KTX authenticate with Google Vertex AI?\n\n${VERTEX_AUTH_PROMPT_CONTEXT}`, options: [ - { value: 'gcloud', label: 'Run gcloud Application Default Credentials login' }, { value: 'existing', label: 'Use existing gcloud/Application Default Credentials' }, { value: 'back', label: 'Back' }, ], @@ -535,15 +514,6 @@ async function chooseVertexAuth( if (choice === 'back') { return { status: 'back' }; } - if (choice !== 'gcloud') { - return { status: 'ready' }; - } - - const result = await (deps.runGcloudAuth ?? runKtxSetupGcloudApplicationDefaultAuth)(io); - if (!result.ok) { - io.stderr.write(`gcloud authentication failed: ${result.message}\n`); - return { status: 'missing-input' }; - } return { status: 'ready' }; } @@ -799,7 +769,7 @@ async function chooseVertexModel(args: KtxSetupModelArgs, io: KtxCliIo, deps: Kt return { status: 'missing-input' }; } - const selectableModels = BUNDLED_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel); + const selectableModels = VERTEX_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel); const prompts = deps.prompts ?? createPromptAdapter(); const choice = await prompts.select({ message: `Which Anthropic model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, @@ -901,7 +871,7 @@ export async function runKtxSetupAnthropicModelStep( : attemptArgs; if (backendChoice.backend === 'vertex') { - const auth = await chooseVertexAuth(backendArgs, io, deps); + const auth = await chooseVertexAuth(backendArgs, deps); if (auth.status === 'back' && backendChoice.prompted) { attemptArgs = buildInteractiveRetryArgs(args); continue; @@ -931,7 +901,13 @@ export async function runKtxSetupAnthropicModelStep( return { status: model.status, projectDir: args.projectDir }; } - const health = await healthCheck(buildVertexHealthConfig(vertex.values, model.model)); + const health = await runLlmHealthCheckWithProgress( + buildVertexHealthConfig(vertex.values, model.model), + 'Vertex AI', + model.model, + healthCheck, + deps, + ); if (health.ok) { await persistLlmConfig(args.projectDir, { backend: 'vertex', vertex: vertex.refs }, model.model); io.stdout.write(`│ LLM ready: yes (${model.model})\n`); @@ -973,7 +949,13 @@ export async function runKtxSetupAnthropicModelStep( return { status: model.status, projectDir: args.projectDir }; } - const health = await healthCheck(buildAnthropicHealthConfig(credential.value, model.model)); + const health = await runLlmHealthCheckWithProgress( + buildAnthropicHealthConfig(credential.value, model.model), + 'Anthropic API', + model.model, + healthCheck, + deps, + ); if (health.ok) { await persistLlmConfig(args.projectDir, { backend: 'anthropic', credentialRef: credential.ref }, model.model); io.stdout.write(`│ LLM ready: yes (${model.model})\n`); From 9ecb8cb119411ab6808cd9c3f1806fc817fa17b3 Mon Sep 17 00:00:00 2001 From: Luca Martial <48870843+luca-martial@users.noreply.github.com> Date: Wed, 13 May 2026 17:22:59 -0400 Subject: [PATCH 6/8] feat(cli): add edit flow for setup connections (#77) * feat(cli): add edit flow for primary database connections in setup Allow users to edit existing primary database connections during setup instead of only adding new ones. Preselects existing values (URL, schemas, tables) so users can adjust without re-entering everything. Co-Authored-By: Claude Opus 4.6 (1M context) * feat(cli): add edit flow for context source connections in setup Allow users to edit existing context source connections during setup. Preselects existing values (URLs, credentials, repo details) and offers a "Keep existing credential" option for sensitive fields. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(cli): rename "Add more" to "Add additional" in primary sources menu Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- packages/cli/src/setup-databases.test.ts | 404 ++++++++++++++++++- packages/cli/src/setup-databases.ts | 490 +++++++++++++++++++---- packages/cli/src/setup-sources.test.ts | 313 +++++++++++++++ packages/cli/src/setup-sources.ts | 488 +++++++++++++++++++--- 4 files changed, 1553 insertions(+), 142 deletions(-) diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index d010a908..fa4ca3f2 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -240,8 +240,9 @@ describe('setup databases step', () => { expect(prompts.select).toHaveBeenCalledWith({ message: 'Configure PostgreSQL', options: [ - { value: 'existing:warehouse', label: 'Use existing PostgreSQL connection: warehouse' }, - { value: 'new', label: 'Add new PostgreSQL connection' }, + { value: 'existing:warehouse', label: 'Keep existing PostgreSQL connection: warehouse' }, + { value: 'edit:warehouse', label: 'Edit PostgreSQL connection: warehouse' }, + { value: 'new', label: 'Add another PostgreSQL connection' }, { value: 'back', label: 'Back' }, ], }); @@ -564,7 +565,8 @@ describe('setup databases step', () => { message: 'Primary sources already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); expect(testConnection).not.toHaveBeenCalled(); @@ -608,11 +610,16 @@ describe('setup databases step', () => { connectionIds: ['warehouse', 'mysql-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(1); + expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(prompts.select).toHaveBeenCalledWith({ message: 'Primary sources already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); expect(testConnection).toHaveBeenCalledTimes(1); @@ -642,11 +649,16 @@ describe('setup databases step', () => { connectionIds: ['postgres-warehouse', 'mysql-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(2); + expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(prompts.select).toHaveBeenCalledWith({ message: 'Primary sources already configured: postgres-warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); @@ -675,12 +687,17 @@ describe('setup databases step', () => { connectionIds: ['postgres-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(2); + expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(io.stdout()).not.toContain('KTX cannot work without at least one primary source'); expect(prompts.select).toHaveBeenNthCalledWith(2, { message: 'Primary sources already configured: postgres-warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); }); @@ -715,16 +732,389 @@ describe('setup databases step', () => { ); expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(io.stdout()).not.toContain('KTX cannot work without at least one primary source'); expect(prompts.select).toHaveBeenNthCalledWith(2, { message: 'Primary sources already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); }); + it('returns from primary source edit selection back to the configured source menu', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + selectValues: ['edit', 'back', 'continue'], + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.select).toHaveBeenNthCalledWith(2, { + message: 'Primary source to edit', + options: [ + { value: 'warehouse', label: 'warehouse (PostgreSQL)' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(prompts.select).toHaveBeenNthCalledWith(3, { + message: 'Primary sources already configured: warehouse\nWhat would you like to do?', + options: [ + { value: 'continue', label: 'Continue to knowledge sources' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, + ], + }); + expect(testConnection).not.toHaveBeenCalled(); + expect(scanConnection).not.toHaveBeenCalled(); + }); + + it('reruns table selection after editing schema scope so stale enabled tables are removed', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['analytics']], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['analytics', 'public']); + const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection, listSchemas, listTables }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('PostgreSQL connection URL'), + placeholder: 'env:DATABASE_URL', + initialValue: 'env:DATABASE_URL', + }); + expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse'); + expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + expect(scanConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + schemas: ['analytics'], + enabled_tables: ['analytics.customers'], + }); + }); + + it('preselects existing schema and table choices when editing a primary source', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.customers', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['public'], ['public.customers', 'public.orders']], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + if (options.message.startsWith('Tables found in selected schemas')) return 'customize'; + return 'back'; + }); + const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + { schema: 'public', name: 'products', kind: 'table' as const }, + ]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + listSchemas, + listTables, + }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.multiselect).toHaveBeenNthCalledWith(1, { + message: expect.stringContaining('PostgreSQL schemas to scan'), + options: [ + { value: 'orbit_analytics', label: 'orbit_analytics' }, + { value: 'orbit_raw', label: 'orbit_raw' }, + { value: 'public', label: 'public' }, + ], + initialValues: ['public'], + required: true, + }); + expect(prompts.multiselect).toHaveBeenNthCalledWith(2, { + message: expect.stringContaining('Tables to enable for warehouse'), + options: [ + { value: 'public.customers', label: 'public.customers' }, + { value: 'public.orders', label: 'public.orders' }, + { value: 'public.products', label: 'public.products' }, + ], + initialValues: ['public.customers', 'public.orders'], + required: true, + }); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + schemas: ['public'], + enabled_tables: ['public.customers', 'public.orders'], + }); + }); + + it('returns to the configured primary menu when backing out of schema review during edit', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['back']], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['analytics', 'public']); + const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection, listSchemas, listTables }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(primaryMenuCount).toBe(2); + expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + expect(scanConnection).not.toHaveBeenCalled(); + expect(listTables).not.toHaveBeenCalled(); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + url: 'env:DATABASE_URL', + schemas: ['public'], + enabled_tables: ['public.orders'], + }); + }); + + it('returns to the configured primary menu when backing out of table review during edit', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ textValues: ['env:DATABASE_URL'] }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + if (options.message.startsWith('Tables found in selected schemas')) return 'back'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['public']); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + ]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection, listSchemas, listTables }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(primaryMenuCount).toBe(2); + expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse'); + expect(scanConnection).not.toHaveBeenCalled(); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + url: 'env:DATABASE_URL', + schemas: ['public'], + enabled_tables: ['public.orders'], + }); + }); + + it('restores an existing primary source edit when the follow-up scan fails', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['public']], + }); + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') return 'edit'; + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + if (options.message.startsWith('Tables found in selected schemas')) return 'all'; + return 'back'; + }); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + ]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 1), + listTables, + }, + ); + + expect(result).toEqual({ status: 'failed', projectDir: tempDir }); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + enabled_tables: ['public.orders'], + }); + }); + it('lets Escape from connection fields return to connection method selection', async () => { const prompts = makePromptAdapter({ selectValues: ['fields', 'url'], diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 5b5b5f8a..9db80689 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -176,6 +176,7 @@ const SCOPE_DISCOVERY_SPECS: Partial; +type ConnectionSetupStatus = 'ready' | 'back' | 'failed'; const DRIVER_CONNECTION_DEFAULTS: Record = { postgres: { port: '5432' }, @@ -227,6 +228,16 @@ function unique(values: string[]): string[] { return [...new Set(values.filter((value) => value.trim().length > 0))]; } +function stringConfigField(connection: KtxProjectConnectionConfig | undefined, field: string): string | undefined { + const value = connection?.[field]; + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; +} + +function numberConfigField(connection: KtxProjectConnectionConfig | undefined, field: string): number | undefined { + const value = connection?.[field]; + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + function historicSqlConfigRecord(connection: KtxProjectConnectionConfig | undefined): Record | null { const historicSql = connection?.historicSql; return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql) @@ -454,6 +465,18 @@ function configuredPrimaryConnectionIds( .sort((left, right) => left.localeCompare(right)); } +function configuredPrimaryDrivers( + connections: Record, + connectionIds: string[], +): KtxSetupDatabaseDriver[] { + const configured = new Set( + connectionIds + .map((connectionId) => normalizeDriver(connections[connectionId]?.driver)) + .filter((driver): driver is KtxSetupDatabaseDriver => driver !== null), + ); + return DRIVER_OPTIONS.map((option) => option.value).filter((driver) => configured.has(driver)); +} + function configuredPrimarySourcesPrompt(connectionIds: string[]): { message: string; options: Array<{ value: string; label: string }>; @@ -462,7 +485,8 @@ function configuredPrimarySourcesPrompt(connectionIds: string[]): { message: `Primary sources already configured: ${connectionIds.join(', ')}\nWhat would you like to do?`, options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }; } @@ -552,23 +576,40 @@ async function buildFieldsConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const label = driverLabel(input.driver); const defaults = DRIVER_CONNECTION_DEFAULTS[input.driver]; - const host = await promptText(input.prompts, `${label} host`, 'localhost'); + const host = await promptText( + input.prompts, + `${label} host`, + stringConfigField(input.existingConnection, 'host') ?? 'localhost', + ); if (host === undefined) return 'back'; if (!host) return null; - const portStr = await promptText(input.prompts, `${label} port`, defaults.port); + const portStr = await promptText( + input.prompts, + `${label} port`, + String(numberConfigField(input.existingConnection, 'port') ?? defaults.port), + ); if (portStr === undefined) return 'back'; const port = Number(portStr || defaults.port); - const database = await promptText(input.prompts, `${label} database name`); + const database = await promptText( + input.prompts, + `${label} database name`, + stringConfigField(input.existingConnection, 'database'), + ); if (database === undefined) return 'back'; if (!database) return null; - const username = await promptText(input.prompts, `${label} username`); + const username = await promptText( + input.prompts, + `${label} username`, + stringConfigField(input.existingConnection, 'username'), + ); if (username === undefined) return 'back'; if (!username) return null; @@ -583,6 +624,7 @@ async function buildFieldsConnectionConfig(input: { }); if (credentialResult === 'back') return 'back'; if (credentialResult) passwordRef = credentialResult; + if (!credentialResult) passwordRef = stringConfigField(input.existingConnection, 'password'); } return { @@ -601,9 +643,14 @@ async function buildPastedUrlConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const label = driverLabel(input.driver); - const rawUrl = await promptText(input.prompts, `${label} connection URL`); + const rawUrl = await promptText( + input.prompts, + `${label} connection URL`, + stringConfigField(input.existingConnection, 'url'), + ); if (rawUrl === undefined) return 'back'; if (!rawUrl) return null; @@ -642,6 +689,7 @@ async function buildUrlConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { if (input.args.inputMode === 'disabled' && !input.args.databaseUrl) return null; @@ -689,6 +737,7 @@ async function buildConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const { driver, args, prompts } = input; if (driver === 'sqlite') { @@ -698,22 +747,37 @@ async function buildConnectionConfig(input: { (await promptText( prompts, 'SQLite database file\nEnter a relative or absolute path, for example ./warehouse.sqlite.', + stringConfigField(input.existingConnection, 'path'), )); if (path === undefined) return 'back'; return path ? { driver: 'sqlite', path } : null; } if (driver === 'postgres' || driver === 'mysql' || driver === 'clickhouse' || driver === 'sqlserver') { - return await buildUrlConnectionConfig({ driver, connectionId: input.connectionId, args, prompts }); + return await buildUrlConnectionConfig({ + driver, + connectionId: input.connectionId, + args, + prompts, + existingConnection: input.existingConnection, + }); } if (driver === 'bigquery') { - const datasetId = await promptText(prompts, 'BigQuery dataset\nFor example analytics.'); + const datasetId = await promptText( + prompts, + 'BigQuery dataset\nFor example analytics.', + stringConfigField(input.existingConnection, 'dataset_id'), + ); if (datasetId === undefined) return 'back'; - const credentialsPath = await promptText(prompts, 'Path to service account JSON file'); + const credentialsPath = await promptText( + prompts, + 'Path to service account JSON file', + stringConfigField(input.existingConnection, 'credentials_json'), + ); if (credentialsPath === undefined) return 'back'; const location = await promptText( prompts, 'BigQuery location\nPress Enter for US, or enter a location like EU.', - 'US', + stringConfigField(input.existingConnection, 'location') ?? 'US', ); if (location === undefined) return 'back'; if (!datasetId || !credentialsPath) return null; @@ -725,19 +789,35 @@ async function buildConnectionConfig(input: { }; } if (driver === 'snowflake') { - const account = await promptText(prompts, 'Snowflake account identifier'); + const account = await promptText( + prompts, + 'Snowflake account identifier', + stringConfigField(input.existingConnection, 'account'), + ); if (account === undefined) return 'back'; - const warehouse = await promptText(prompts, 'Snowflake warehouse\nFor example ANALYTICS_WH.'); + const warehouse = await promptText( + prompts, + 'Snowflake warehouse\nFor example ANALYTICS_WH.', + stringConfigField(input.existingConnection, 'warehouse'), + ); if (warehouse === undefined) return 'back'; - const database = await promptText(prompts, 'Snowflake database name'); + const database = await promptText( + prompts, + 'Snowflake database name', + stringConfigField(input.existingConnection, 'database'), + ); if (database === undefined) return 'back'; const schemaName = await promptText( prompts, 'Snowflake schema\nPress Enter for PUBLIC, or enter a schema name.', - 'PUBLIC', + stringConfigField(input.existingConnection, 'schema_name') ?? 'PUBLIC', ); if (schemaName === undefined) return 'back'; - const username = await promptText(prompts, 'Snowflake username'); + const username = await promptText( + prompts, + 'Snowflake username', + stringConfigField(input.existingConnection, 'username'), + ); if (username === undefined) return 'back'; const passwordRef = await promptCredential({ prompts, @@ -747,9 +827,14 @@ async function buildConnectionConfig(input: { secretName: 'password', // pragma: allowlist secret }); if (passwordRef === 'back') return 'back'; // pragma: allowlist secret - const role = await promptText(prompts, 'Snowflake role (optional)\nPress Enter to skip.'); + const role = await promptText( + prompts, + 'Snowflake role (optional)\nPress Enter to skip.', + stringConfigField(input.existingConnection, 'role'), + ); if (role === undefined) return 'back'; - if (!account || !warehouse || !database || !schemaName || !username || !passwordRef) return null; + const resolvedPasswordRef = passwordRef ?? stringConfigField(input.existingConnection, 'password'); + if (!account || !warehouse || !database || !schemaName || !username || !resolvedPasswordRef) return null; return { driver: 'snowflake', authMethod: 'password', @@ -758,7 +843,7 @@ async function buildConnectionConfig(input: { database, schema_name: schemaName, username, - password: passwordRef, + password: resolvedPasswordRef, ...(role ? { role } : {}), }; } @@ -1096,6 +1181,59 @@ async function writeConnectionConfig(input: { } } +async function createConnectionConfigRollback(projectDir: string, connectionId: string): Promise<() => Promise> { + const project = await loadKtxProject({ projectDir }); + const previousConnection = project.config.connections[connectionId]; + const hadPreviousConnection = previousConnection !== undefined; + return async () => { + const latest = await loadKtxProject({ projectDir }); + const connections = { ...latest.config.connections }; + if (hadPreviousConnection) { + connections[connectionId] = previousConnection; + } else { + delete connections[connectionId]; + } + await writeFile( + latest.configPath, + serializeKtxProjectConfig({ + ...latest.config, + connections, + }), + 'utf-8', + ); + }; +} + +function withExistingPrimaryEditPromptDefaults(input: { + previous: KtxProjectConnectionConfig; + next: KtxProjectConnectionConfig; + driver: KtxSetupDatabaseDriver; +}): KtxProjectConnectionConfig { + const merged: KtxProjectConnectionConfig = { ...input.next }; + const spec = SCOPE_DISCOVERY_SPECS[input.driver]; + if (spec) { + const nextArray = input.next[spec.configArrayField]; + const previousArray = input.previous[spec.configArrayField]; + if ( + !(Array.isArray(nextArray) && nextArray.length > 0) && + Array.isArray(previousArray) && + previousArray.length > 0 + ) { + delete merged[spec.configSingleField]; + merged[spec.configArrayField] = previousArray; + } else if (!Object.hasOwn(input.next, spec.configArrayField) && !Object.hasOwn(input.next, spec.configSingleField)) { + const previousSingle = input.previous[spec.configSingleField]; + if (typeof previousSingle === 'string' && previousSingle.trim().length > 0) { + merged[spec.configSingleField] = previousSingle; + } + } + } + if (!Object.hasOwn(input.next, 'enabled_tables') && Array.isArray(input.previous.enabled_tables)) { + merged.enabled_tables = input.previous.enabled_tables; + } + return merged; +} + function configuredScopeValues( connection: KtxProjectConnectionConfig | undefined, spec: ScopeDiscoverySpec, @@ -1156,18 +1294,19 @@ async function maybeConfigureSchemaScope(input: { prompts: KtxSetupDatabasesPromptAdapter; deps: KtxSetupDatabasesDeps; io: KtxCliIo; -}): Promise { + forcePrompt?: boolean; +}): Promise { const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; const driver = normalizeDriver(connection?.driver); - if (!driver) return true; + if (!driver) return 'ready'; const spec = SCOPE_DISCOVERY_SPECS[driver]; - if (!spec) return true; + if (!spec) return 'ready'; const arrayVal = connection?.[spec.configArrayField]; - if (Array.isArray(arrayVal) && arrayVal.length > 0) { - return true; + if (Array.isArray(arrayVal) && arrayVal.length > 0 && input.forcePrompt !== true) { + return 'ready'; } if (input.args.databaseSchemas.length > 0) { @@ -1177,7 +1316,7 @@ async function maybeConfigureSchemaScope(input: { values: input.args.databaseSchemas, spec, }); - return true; + return 'ready'; } writeSetupSection(input.io, `Discovering ${spec.promptLabel.toLowerCase()}`, [ @@ -1190,14 +1329,18 @@ async function maybeConfigureSchemaScope(input: { await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId), ); } catch (error) { + const detail = error instanceof Error ? error.message : String(error); input.io.stderr.write( - `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; continuing with existing ${spec.noun} scope. ` + - `Pass --database-schema to set it explicitly. ${error instanceof Error ? error.message : String(error)}\n`, + input.forcePrompt === true + ? `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; edit was not saved. ` + + `Pass --database-schema to set it explicitly. ${detail}\n` + : `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; continuing with existing ${spec.noun} scope. ` + + `Pass --database-schema to set it explicitly. ${detail}\n`, ); - return true; + return input.forcePrompt === true ? 'failed' : 'ready'; } if (discovered.length === 0) { - return true; + return 'ready'; } let selected: string[]; @@ -1217,7 +1360,7 @@ async function maybeConfigureSchemaScope(input: { required: true, }); if (choices.includes('back')) { - return false; + return 'back'; } selected = choices.length > 0 ? choices : initialValues; } @@ -1232,7 +1375,7 @@ async function maybeConfigureSchemaScope(input: { writeSetupSection(input.io, `${capitalNounPlural} saved for ${input.connectionId}`, [ `✓ ${selected.join(', ')}`, ]); - return true; + return 'ready'; } async function maybeConfigureTableScope(input: { @@ -1242,19 +1385,20 @@ async function maybeConfigureTableScope(input: { prompts: KtxSetupDatabasesPromptAdapter; io: KtxCliIo; deps: KtxSetupDatabasesDeps; -}): Promise { + forcePrompt?: boolean; +}): Promise { const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; const driver = normalizeDriver(connection?.driver); - if (!driver || driver === 'sqlite') return true; + if (!driver || driver === 'sqlite') return 'ready'; const existingTables = connection?.enabled_tables; - if (Array.isArray(existingTables) && existingTables.length > 0) { - return true; + if (Array.isArray(existingTables) && existingTables.length > 0 && input.forcePrompt !== true) { + return 'ready'; } if (input.args.inputMode === 'disabled') { - return true; + return 'ready'; } writeSetupSection(input.io, 'Discovering tables', [ @@ -1268,15 +1412,20 @@ async function maybeConfigureTableScope(input: { input.connectionId, ); } catch (error) { + const detail = error instanceof Error ? error.message : String(error); input.io.stderr.write( - `Could not discover tables for ${input.connectionId}; continuing without table filter. ` + - `${error instanceof Error ? error.message : String(error)}\n`, + input.forcePrompt === true + ? `Could not discover tables for ${input.connectionId}; edit was not saved. ${detail}\n` + : `Could not discover tables for ${input.connectionId}; continuing without table filter. ${detail}\n`, ); - return true; + return input.forcePrompt === true ? 'failed' : 'ready'; } if (discovered.length === 0) { - return true; + if (input.forcePrompt === true) { + input.io.stderr.write(`No tables discovered for ${input.connectionId}; edit was not saved.\n`); + } + return input.forcePrompt === true ? 'failed' : 'ready'; } const allQualified = discovered.map((t) => `${t.schema}.${t.name}`); @@ -1290,7 +1439,7 @@ async function maybeConfigureTableScope(input: { writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ `✓ ${allQualified[0]}`, ]); - return true; + return 'ready'; } const bySchema = new Map(); @@ -1316,7 +1465,7 @@ async function maybeConfigureTableScope(input: { }); if (action === 'back') { - return false; + return 'back'; } if (action === 'all') { @@ -1332,7 +1481,10 @@ async function maybeConfigureTableScope(input: { const suffix = t.kind === 'view' ? ' (view)' : ''; return { value: qualified, label: `${qualified}${suffix}` }; }), - initialValues: allQualified, + initialValues: + Array.isArray(existingTables) && input.forcePrompt === true + ? existingTables.filter((table): table is string => typeof table === 'string' && allQualified.includes(table)) + : allQualified, required: true, }); @@ -1356,7 +1508,7 @@ async function maybeConfigureTableScope(input: { writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ `✓ ${selected.length}/${discovered.length} tables enabled`, ]); - return true; + return 'ready'; } async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise { @@ -1466,7 +1618,8 @@ async function validateAndScanConnection(input: { deps: KtxSetupDatabasesDeps; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; -}): Promise { + forceScopeAndTables?: boolean; +}): Promise { const testConnection = input.deps.testConnection ?? defaultTestConnection; const scanConnection = input.deps.scanConnection ?? defaultScanConnection; const project = await loadKtxProject({ projectDir: input.projectDir }); @@ -1477,7 +1630,7 @@ async function validateAndScanConnection(input: { if (testCode !== 0) { flushBufferedCommandOutput(input.io, testIo); input.io.stderr.write(`Connection test failed for ${input.connectionId}.\n`); - return false; + return 'failed'; } const testOutput = testIo.stdoutText(); const outputDriver = normalizeDriver(readOutputValue(testOutput, 'Driver')); @@ -1486,14 +1639,24 @@ async function validateAndScanConnection(input: { writeSetupSection(input.io, `Testing ${input.connectionId}`, testLines); while (true) { - if (!(await maybeConfigureSchemaScope(input))) { - return false; + const schemaStatus = await maybeConfigureSchemaScope({ ...input, forcePrompt: input.forceScopeAndTables }); + if (schemaStatus !== 'ready') { + return schemaStatus; } - if (await maybeConfigureTableScope(input)) { + const tableStatus = await maybeConfigureTableScope({ ...input, forcePrompt: input.forceScopeAndTables }); + if (tableStatus === 'ready') { break; } + if (input.forceScopeAndTables) { + return tableStatus; + } + + if (tableStatus === 'failed') { + return 'failed'; + } + await clearScopeConfig(input.projectDir, input.connectionId); } @@ -1554,7 +1717,7 @@ async function validateAndScanConnection(input: { ); } if (scanCode !== 0) { - return false; + return 'failed'; } } const scanOutput = scanIo.stdoutText(); @@ -1570,14 +1733,14 @@ async function validateAndScanConnection(input: { writeSetupSection(input.io, 'Primary source ready', [ `${input.connectionId} · ${driverDisplay} · structural scan complete`, ]); - return true; + return 'ready'; } async function chooseDrivers( args: KtxSetupDatabasesArgs, io: KtxCliIo, prompts: KtxSetupDatabasesPromptAdapter, - options?: { hasPrimarySources?: boolean }, + options?: { hasPrimarySources?: boolean; initialDrivers?: KtxSetupDatabaseDriver[] }, ): Promise { if (args.databaseDrivers && args.databaseDrivers.length > 0) { return [...new Set(args.databaseDrivers)]; @@ -1592,10 +1755,12 @@ async function chooseDrivers( return 'missing-input'; } while (true) { + const initialValues = unique(options?.initialDrivers ?? []); const choices = await prompts.multiselect({ message: withMultiselectNavigation('Which primary sources should KTX connect to?'), options: [...DRIVER_OPTIONS], - required: false, + ...(initialValues.length > 0 ? { initialValues } : {}), + required: options?.hasPrimarySources === true, }); if (choices.includes('back')) { return 'back'; @@ -1617,7 +1782,7 @@ async function chooseConnectionIdForDriver(input: { connections: Record; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; -}): Promise<{ kind: 'existing' | 'new'; connectionId: string } | 'back' | 'missing-input'> { +}): Promise<{ kind: 'existing' | 'new' | 'edit'; connectionId: string } | 'back' | 'missing-input'> { if (input.args.databaseConnectionId) { return { kind: 'new', connectionId: input.args.databaseConnectionId }; } @@ -1647,14 +1812,19 @@ async function chooseConnectionIdForDriver(input: { options: [ ...existingIds.map((connectionId) => ({ value: `existing:${connectionId}`, - label: `Use existing ${label} connection: ${connectionId}`, + label: `Keep existing ${label} connection: ${connectionId}`, })), - { value: 'new', label: `Add new ${label} connection` }, + ...existingIds.map((connectionId) => ({ + value: `edit:${connectionId}`, + label: `Edit ${label} connection: ${connectionId}`, + })), + { value: 'new', label: `Add another ${label} connection` }, { value: 'back', label: 'Back' }, ], }); if (choice === 'back') return 'back'; if (choice.startsWith('existing:')) return { kind: 'existing', connectionId: choice.slice('existing:'.length) }; + if (choice.startsWith('edit:')) return { kind: 'edit', connectionId: choice.slice('edit:'.length) }; const entered = await input.prompts.text({ message: withTextInputNavigation(connectionNamePrompt(label)), placeholder: defaultId, @@ -1666,6 +1836,102 @@ async function chooseConnectionIdForDriver(input: { } } +async function choosePrimarySourceToEdit(input: { + projectDir: string; + connectionIds: string[]; + prompts: KtxSetupDatabasesPromptAdapter; +}): Promise { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const options = input.connectionIds + .map((connectionId) => { + const driver = normalizeDriver(project.config.connections[connectionId]?.driver); + if (!driver) return null; + return { value: connectionId, label: `${connectionId} (${driverLabel(driver)})` }; + }) + .filter((option): option is { value: string; label: string } => option !== null); + if (options.length === 0) return 'back'; + const choice = await input.prompts.select({ + message: 'Primary source to edit', + options: [...options, { value: 'back', label: 'Back' }], + }); + return choice === 'back' ? 'back' : choice; +} + +async function runPrimarySourceFullEdit(input: { + projectDir: string; + connectionId: string; + args: KtxSetupDatabasesArgs; + prompts: KtxSetupDatabasesPromptAdapter; + io: KtxCliIo; + deps: KtxSetupDatabasesDeps; +}): Promise<'ready' | 'back' | 'failed'> { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const existing = project.config.connections[input.connectionId]; + const driver = normalizeDriver(existing?.driver); + if (!existing || !driver) { + input.io.stderr.write(`Connection "${input.connectionId}" is not a configured primary source.\n`); + return 'failed'; + } + + const rollback = await createConnectionConfigRollback(input.projectDir, input.connectionId); + const replacement = await buildConnectionConfig({ + driver, + connectionId: input.connectionId, + args: input.args, + prompts: input.prompts, + existingConnection: existing, + }); + if (replacement === 'back') { + await rollback(); + return 'back'; + } + if (!replacement) { + await rollback(); + return 'failed'; + } + + const withHistoricSql = await maybeApplyHistoricSqlConfig({ + connection: replacement, + driver, + args: input.args, + prompts: input.prompts, + }); + if (withHistoricSql === 'back') { + await rollback(); + return 'back'; + } + + await writeConnectionConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + connection: withExistingPrimaryEditPromptDefaults({ + previous: existing, + next: { + ...withHistoricSql, + ...(!Object.hasOwn(withHistoricSql, 'historicSql') && existing.historicSql !== undefined + ? { historicSql: existing.historicSql } + : {}), + }, + driver, + }), + }); + + const validated = await validateAndScanConnection({ + projectDir: input.projectDir, + connectionId: input.connectionId, + io: input.io, + deps: input.deps, + args: input.args, + prompts: input.prompts, + forceScopeAndTables: true, + }); + if (validated !== 'ready') { + await rollback(); + return validated; + } + return 'ready'; +} + export async function runKtxSetupDatabasesStep( args: KtxSetupDatabasesArgs, io: KtxCliIo, @@ -1688,7 +1954,18 @@ export async function runKtxSetupDatabasesStep( prompts, }); if (historicSqlResult === 'back') return { status: 'back', projectDir: args.projectDir }; - if (!(await validateAndScanConnection({ projectDir: args.projectDir, connectionId, io, deps, args, prompts }))) { + const setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId, + io, + deps, + args, + prompts, + }); + if (setupStatus === 'back') { + return { status: 'back', projectDir: args.projectDir }; + } + if (setupStatus === 'failed') { return { status: 'failed', projectDir: args.projectDir }; } selectedConnectionIds.push(connectionId); @@ -1712,10 +1989,43 @@ export async function runKtxSetupDatabasesStep( await markDatabasesComplete(args.projectDir, selectedConnectionIds); return { status: 'ready', projectDir: args.projectDir, connectionIds: selectedConnectionIds }; } + if (action === 'edit') { + const connectionId = await choosePrimarySourceToEdit({ + projectDir: args.projectDir, + connectionIds: selectedConnectionIds, + prompts, + }); + if (connectionId === 'back') { + showConfiguredPrimaryMenu = true; + continue; + } + const editResult = await runPrimarySourceFullEdit({ + projectDir: args.projectDir, + connectionId, + args, + prompts, + io, + deps, + }); + if (editResult === 'back') { + showConfiguredPrimaryMenu = true; + continue; + } + if (editResult === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + pushUniqueConnectionId(selectedConnectionIds, connectionId); + showConfiguredPrimaryMenu = true; + continue; + } } showConfiguredPrimaryMenu = false; - const drivers = await chooseDrivers(args, io, prompts, { hasPrimarySources: selectedConnectionIds.length > 0 }); + const driverProject = await loadKtxProject({ projectDir: args.projectDir }); + const drivers = await chooseDrivers(args, io, prompts, { + hasPrimarySources: selectedConnectionIds.length > 0, + initialDrivers: configuredPrimaryDrivers(driverProject.config.connections, selectedConnectionIds), + }); if (drivers === 'back') { if (selectedConnectionIds.length > 0 && canReturnToDriverSelection && args.inputMode !== 'disabled') { showConfiguredPrimaryMenu = true; @@ -1750,7 +2060,26 @@ export async function runKtxSetupDatabasesStep( return { status: 'missing-input', projectDir: args.projectDir }; } - if (connectionChoice.kind === 'new') { + let connectionAlreadyValidated = false; + if (connectionChoice.kind === 'edit') { + const editResult = await runPrimarySourceFullEdit({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + args, + prompts, + io, + deps, + }); + if (editResult === 'back') { + if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; + returnToDriverSelection = true; + break; + } + if (editResult === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + connectionAlreadyValidated = true; + } else if (connectionChoice.kind === 'new') { let connection = await buildConnectionConfig({ driver, connectionId: connectionChoice.connectionId, @@ -1819,16 +2148,22 @@ export async function runKtxSetupDatabasesStep( } let connectionSkipped = false; - while ( - !(await validateAndScanConnection({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - io, - deps, - args, - prompts, - })) - ) { + let setupStatus: ConnectionSetupStatus = connectionAlreadyValidated + ? 'ready' + : await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); + while (!connectionAlreadyValidated && setupStatus !== 'ready') { + if (setupStatus === 'back') { + if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; + returnToDriverSelection = true; + break; + } if (args.inputMode === 'disabled') return { status: 'failed', projectDir: args.projectDir }; const action = await prompts.select({ message: `Primary source setup failed for ${connectionChoice.connectionId}`, @@ -1848,7 +2183,16 @@ export async function runKtxSetupDatabasesStep( connectionSkipped = true; break; } - if (action === 're-enter') { + if (action === 'retry') { + setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); + } else if (action === 're-enter') { const connection = await buildConnectionConfig({ driver, connectionId: connectionChoice.connectionId, @@ -1872,6 +2216,14 @@ export async function runKtxSetupDatabasesStep( connectionId: connectionChoice.connectionId, connection: withHistoricSql, }); + setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); } } if (returnToDriverSelection) break; diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index 7fe61f76..0a0eab2c 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -861,6 +861,7 @@ describe('setup sources step', () => { message: 'Configure dbt', options: [ { value: 'existing:dbt-main', label: 'Use existing dbt connection: dbt-main' }, + { value: 'edit:dbt-main', label: 'Edit existing dbt connection: dbt-main' }, { value: 'new', label: 'Add new dbt connection' }, { value: 'back', label: 'Back' }, ], @@ -988,6 +989,10 @@ describe('setup sources step', () => { value: `existing:${testCase.connectionId}`, label: `Use existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`, }, + { + value: `edit:${testCase.connectionId}`, + label: `Edit existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`, + }, { value: 'new', label: `Add new ${testCase.expectedLabel} connection` }, { value: 'back', label: 'Back' }, ], @@ -996,6 +1001,314 @@ describe('setup sources step', () => { } }); + it('edits an existing Notion source and reopens the page picker with stored pages selected', async () => { + await addPrimarySource(); + await addConnection('notion-main', { + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['old-page'], + root_database_ids: [], + root_data_source_ids: [], + }); + const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=1' })); + const pickNotionRootPages = vi.fn(async () => ({ kind: 'selected' as const, rootPageIds: ['new-page'] })); + const testPrompts = prompts({ + multiselect: [['notion']], + select: ['edit:notion-main', 'keep', 'selected_roots', 'done'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateNotion, + pickNotionRootPages, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'How should KTX find your Notion integration token?', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use NOTION_TOKEN from the environment' }, + { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(pickNotionRootPages).toHaveBeenCalledWith( + { + connectionId: 'notion-main', + connection: expect.objectContaining({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['old-page'], + }), + }, + expect.anything(), + ); + expect((await readConfig()).connections['notion-main']).toMatchObject({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['new-page'], + }); + }); + + it('edits an existing Metabase source with the current URL and credential as defaults', async () => { + await addPrimarySource(); + await addConnection('metabase-main', { + driver: 'metabase', + api_url: 'https://metabase-old.example.com', + api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret + mappings: { + databaseMappings: { '1': 'warehouse' }, + syncEnabled: { '1': true }, + syncMode: 'ALL', + }, + }); + const testPrompts = prompts({ + multiselect: [['metabase']], + select: ['edit:metabase-main', 'keep', 'done'], + text: ['https://metabase-new.example.com'], + }); + const discoverMetabaseDatabases = vi.fn(async () => [ + { id: 2, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' }, + ]); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + discoverMetabaseDatabases, + validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })), + runMapping: vi.fn(async () => 0), + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metabase-main'] }); + + expect(testPrompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('Metabase URL'), + initialValue: 'https://metabase-old.example.com', + }); + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'How should KTX find your Metabase API key?', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use METABASE_API_KEY from the environment' }, + { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(discoverMetabaseDatabases).toHaveBeenCalledWith({ + sourceUrl: 'https://metabase-new.example.com', + sourceApiKeyRef: 'env:METABASE_API_KEY', + sourceConnectionId: 'metabase-main', + }); + expect((await readConfig()).connections['metabase-main']).toMatchObject({ + driver: 'metabase', + api_url: 'https://metabase-new.example.com', + api_key_ref: 'env:METABASE_API_KEY', + mappings: { + databaseMappings: { '2': 'warehouse' }, + syncEnabled: { '2': true }, + syncMode: 'ALL', + }, + }); + }); + + it('rolls back an edited context source when validation fails', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: false as const, message: 'dbt project not found' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['edit:dbt-main', 'path'], + text: ['/repo/new-dbt', ''], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'failed', projectDir }); + + expect(validateDbt).toHaveBeenCalledWith(expect.objectContaining({ + driver: 'dbt', + source_dir: '/repo/new-dbt', + })); + const config = await readConfig(); + expect(config.connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + expect(config.ingest.adapters).not.toContain('dbt'); + }); + + it('lets git-backed context source edits keep the existing repo credential', async () => { + await addPrimarySource(); + await addConnection('metricflow-main', { + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/private-metricflow', + branch: 'main', + path: 'metrics', + auth_token_ref: 'env:METRICFLOW_REPO_TOKEN', // pragma: allowlist secret + }, + }); + const testGitRepo = vi.fn(async () => ({ ok: false as const, error: 'authentication required' })); + const testPrompts = prompts({ + multiselect: [['metricflow']], + select: ['edit:metricflow-main', 'git', 'keep', 'done'], + text: ['https://github.com/acme/private-metricflow', 'main', 'metrics'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + testGitRepo, + validateMetricflow: vi.fn(async () => ({ ok: true as const, detail: 'metrics=1' })), + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metricflow-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'This MetricFlow repo requires authentication.', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, + { value: 'paste', label: 'Paste a token and save it as a local secret file' }, + { value: 'skip', label: 'Skip — try without authentication' }, + { value: 'back', label: 'Back' }, + ], + }); + expect((await readConfig()).connections['metricflow-main']).toMatchObject({ + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/private-metricflow', + branch: 'main', + path: 'metrics', + auth_token_ref: 'env:METRICFLOW_REPO_TOKEN', + }, + }); + }); + + it('edits an existing context source from the configured-source follow-up menu', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['existing:dbt-main', 'edit', 'dbt-main', 'path', 'done'], + text: ['/repo/edited-dbt', ''], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: '1 context source configured (dbt-main). Add another?', + options: [ + { value: 'done', label: 'Done — continue to context build' }, + { value: 'edit', label: 'Edit an existing context source' }, + { value: 'add', label: 'Add another context source' }, + ], + }); + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'Context source to edit', + options: [ + { value: 'dbt-main', label: 'dbt-main (dbt)' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(testPrompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('dbt local path'), + initialValue: '/repo/existing-dbt', + }); + expect(validateDbt).toHaveBeenLastCalledWith(expect.objectContaining({ + driver: 'dbt', + source_dir: '/repo/edited-dbt', + })); + expect((await readConfig()).connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/edited-dbt', + project_name: 'analytics', + }); + }); + + it('backs out of editing an existing context source to the source connection menu', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['edit:dbt-main', 'back', 'existing:dbt-main'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); + + expect( + vi + .mocked(testPrompts.select) + .mock.calls.map(([options]) => options.message) + .filter((message) => message === 'Configure dbt'), + ).toHaveLength(2); + expect(validateDbt).toHaveBeenCalledWith({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + expect((await readConfig()).connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + }); + it('lets Escape from dbt git URL return to source location selection', async () => { await addPrimarySource(); const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index d18004d9..e0819e4a 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -224,17 +224,20 @@ async function chooseSourceCredentialRef(input: { label: string; envName: string; secretFileName: string; + existingRef?: string; }): Promise { while (true) { const choice = await input.prompts.select({ message: `How should KTX find your ${input.label}?`, options: [ + ...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []), { value: 'env', label: `Use ${input.envName} from the environment` }, { value: 'paste', label: 'Paste a key and save it as a local secret file' }, { value: 'back', label: 'Back' }, ], }); if (choice === 'back') return 'back'; + if (choice === 'keep' && input.existingRef) return input.existingRef; if (choice === 'paste') { const value = await input.prompts.password({ message: input.label }); if (value === undefined) continue; @@ -256,12 +259,14 @@ async function chooseGitAuthCredentialRef(input: { projectDir: string; source: KtxSetupSourceType; connectionId: string; + existingRef?: string; }): Promise { const label = input.source === 'dbt' ? 'This' : `This ${sourceLabel(input.source)}`; while (true) { const choice = await input.prompts.select({ message: `${label} repo requires authentication.`, options: [ + ...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []), { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, { value: 'paste', label: 'Paste a token and save it as a local secret file' }, { value: 'skip', label: 'Skip — try without authentication' }, @@ -269,6 +274,7 @@ async function chooseGitAuthCredentialRef(input: { ], }); if (choice === 'back') return 'back'; + if (choice === 'keep' && input.existingRef) return input.existingRef; if (choice === 'skip') return undefined; if (choice === 'paste') { const value = await input.prompts.password({ message: 'Git access token' }); @@ -793,8 +799,14 @@ interface WarehouseConnectionChoice { type InteractiveSourceConnectionChoice = | { kind: 'existing'; connectionId: string; connection: KtxProjectConnectionConfig } | { kind: 'new'; args: KtxSetupSourcesArgs } + | { kind: 'edited'; connectionId: string; args: KtxSetupSourcesArgs } | 'back'; +type SourceSetupChoiceResult = + | { status: 'ready'; connectionId: string } + | { status: 'back' } + | { status: 'failed' }; + async function runSourcePromptSteps( initialState: SourcePromptState, stepsForState: (state: SourcePromptState) => SourcePromptStep[], @@ -828,6 +840,12 @@ function resetRepoLocationFields(state: SourcePromptState): void { delete state.sourceProjectName; } +function sourceLocationFromArgs(args: KtxSetupSourcesArgs): SourceLocationChoice | undefined { + if (args.sourcePath) return 'path'; + if (args.sourceGitUrl) return 'git'; + return undefined; +} + function warehouseConnectionChoices(config: KtxProjectConfig): WarehouseConnectionChoice[] { return Object.entries(config.connections) .filter(([, connection]) => PRIMARY_SOURCE_DRIVERS.has(String(connection.driver ?? '').toLowerCase())) @@ -964,7 +982,7 @@ async function promptForInteractiveSource( testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection, discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'], ): Promise { - const initialState: SourcePromptState = { ...args, source }; + const initialState: SourcePromptState = { ...args, source, sourceLocation: sourceLocationFromArgs(args) }; if (args.sourceConnectionId) { initialState.sourceConnectionId = args.sourceConnectionId; } @@ -994,7 +1012,10 @@ async function promptForInteractiveSource( ...(state.sourceLocation === 'path' ? [ async (currentState: SourcePromptState) => { - const sourcePath = await promptText(prompts, { message: `${source} local path` }); + const sourcePath = await promptText(prompts, { + message: `${source} local path`, + ...(currentState.sourcePath ? { initialValue: currentState.sourcePath } : {}), + }); if (sourcePath === undefined) return 'back'; currentState.sourcePath = sourcePath; return 'next'; @@ -1004,13 +1025,19 @@ async function promptForInteractiveSource( ...(state.sourceLocation === 'git' ? [ async (currentState: SourcePromptState) => { - const sourceGitUrl = await promptText(prompts, { message: `${source} git URL` }); + const sourceGitUrl = await promptText(prompts, { + message: `${source} git URL`, + ...(currentState.sourceGitUrl ? { initialValue: currentState.sourceGitUrl } : {}), + }); if (sourceGitUrl === undefined) return 'back'; currentState.sourceGitUrl = sourceGitUrl; return 'next'; }, async (currentState: SourcePromptState) => { - const branch = await promptText(prompts, { message: `${source} git branch`, initialValue: 'main' }); + const branch = await promptText(prompts, { + message: `${source} git branch`, + initialValue: currentState.sourceBranch ?? 'main', + }); if (branch === undefined) return 'back'; currentState.sourceBranch = branch || 'main'; return 'next'; @@ -1031,6 +1058,7 @@ async function promptForInteractiveSource( projectDir: args.projectDir, source, connectionId: currentState.sourceConnectionId ?? `${source}-main`, + existingRef: currentState.sourceAuthTokenRef, }); if (authRef === 'back') return 'back'; if (authRef) { @@ -1104,6 +1132,7 @@ async function promptForInteractiveSource( const subpath = await promptText(prompts, { message: sourceSubpathPrompt(source), placeholder: 'optional', + ...(currentState.sourceSubpath ? { initialValue: currentState.sourceSubpath } : {}), }); if (subpath === undefined) return 'back'; if (subpath) { @@ -1122,7 +1151,10 @@ async function promptForInteractiveSource( return await runSourcePromptSteps(initialState, () => [ ...connectionSteps, async (state) => { - const sourceUrl = await promptText(prompts, { message: 'Metabase URL' }); + const sourceUrl = await promptText(prompts, { + message: 'Metabase URL', + ...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}), + }); if (sourceUrl === undefined) return 'back'; state.sourceUrl = sourceUrl; return 'next'; @@ -1134,6 +1166,7 @@ async function promptForInteractiveSource( label: 'Metabase API key', envName: 'METABASE_API_KEY', secretFileName: `${state.sourceConnectionId ?? 'metabase-main'}-api-key`, + existingRef: state.sourceApiKeyRef, }); if (ref === 'back') return 'back'; state.sourceApiKeyRef = ref; @@ -1165,13 +1198,19 @@ async function promptForInteractiveSource( return await runSourcePromptSteps(initialState, () => [ ...connectionSteps, async (state) => { - const sourceUrl = await promptText(prompts, { message: 'Looker base URL' }); + const sourceUrl = await promptText(prompts, { + message: 'Looker base URL', + ...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}), + }); if (sourceUrl === undefined) return 'back'; state.sourceUrl = sourceUrl; return 'next'; }, async (state) => { - const sourceClientId = await promptText(prompts, { message: 'Looker client id' }); + const sourceClientId = await promptText(prompts, { + message: 'Looker client id', + ...(state.sourceClientId ? { initialValue: state.sourceClientId } : {}), + }); if (sourceClientId === undefined) return 'back'; state.sourceClientId = sourceClientId; return 'next'; @@ -1183,6 +1222,7 @@ async function promptForInteractiveSource( label: 'Looker client secret', envName: 'LOOKER_CLIENT_SECRET', secretFileName: `${state.sourceConnectionId ?? 'looker-main'}-client-secret`, + existingRef: state.sourceClientSecretRef, }); if (ref === 'back') return 'back'; state.sourceClientSecretRef = ref; @@ -1201,6 +1241,7 @@ async function promptForInteractiveSource( const lookerConnectionName = await promptText(prompts, { message: 'Looker connection name', placeholder: 'optional', + ...(state.sourceTarget ? { initialValue: state.sourceTarget } : {}), }); if (lookerConnectionName === undefined) return 'back'; if (lookerConnectionName) { @@ -1222,6 +1263,7 @@ async function promptForInteractiveSource( label: 'Notion integration token', envName: 'NOTION_TOKEN', secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`, + existingRef: currentState.sourceApiKeyRef, }); if (ref === 'back') return 'back'; currentState.sourceApiKeyRef = ref; @@ -1286,6 +1328,24 @@ function existingConnectionIdsBySource( .sort((left, right) => left.localeCompare(right)); } +function sourceTypeForConnection(connection: KtxProjectConnectionConfig): KtxSetupSourceType | null { + const driver = String(connection.driver ?? '').toLowerCase(); + return SOURCE_OPTIONS.some((option) => option.value === driver) ? (driver as KtxSetupSourceType) : null; +} + +function contextSourceEditTargets(connections: Record): Array<{ + connectionId: string; + source: KtxSetupSourceType; +}> { + return Object.entries(connections) + .map(([connectionId, connection]) => { + const source = sourceTypeForConnection(connection); + return source ? { connectionId, source } : null; + }) + .filter((target): target is { connectionId: string; source: KtxSetupSourceType } => target !== null) + .sort((left, right) => left.connectionId.localeCompare(right.connectionId)); +} + function sourceChecklistForConnections(connections: Record): { options: Array<{ value: KtxSetupSourceType; label: string; hint?: string }>; initialValues: KtxSetupSourceType[]; @@ -1317,6 +1377,180 @@ function defaultConnectionIdForSource( return `${base}-${index}`; } +function firstStringRecordEntry(value: unknown): [string, string] | undefined { + if (!isRecord(value)) return undefined; + for (const [key, raw] of Object.entries(value)) { + if (typeof raw === 'string' && raw.trim().length > 0) { + return [key, raw.trim()]; + } + } + return undefined; +} + +function applyRepoSourceArgs( + args: KtxSetupSourcesArgs, + input: { repoUrl?: string; sourceDir?: string; branch?: string; subpath?: string; authTokenRef?: string }, +): void { + if (input.sourceDir) { + args.sourcePath = input.sourceDir; + } else if (input.repoUrl?.startsWith('file:')) { + args.sourcePath = fileURLToPath(input.repoUrl); + } else if (input.repoUrl) { + args.sourceGitUrl = input.repoUrl; + } + if (input.branch) args.sourceBranch = input.branch; + if (input.subpath) args.sourceSubpath = input.subpath; + if (input.authTokenRef) args.sourceAuthTokenRef = input.authTokenRef; +} + +function sourceArgsFromExistingConnection(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + connectionId: string; + connection: KtxProjectConnectionConfig; +}): KtxSetupSourcesArgs { + const sourceArgs: KtxSetupSourcesArgs = { + projectDir: input.args.projectDir, + inputMode: input.args.inputMode, + source: input.source, + sourceConnectionId: input.connectionId, + runInitialSourceIngest: input.args.runInitialSourceIngest, + skipSources: input.args.skipSources, + }; + + if (input.source === 'dbt') { + applyRepoSourceArgs(sourceArgs, { + sourceDir: stringField(input.connection.source_dir), + repoUrl: stringField(input.connection.repo_url), + branch: stringField(input.connection.branch), + subpath: stringField(input.connection.path), + authTokenRef: stringField(input.connection.auth_token_ref), + }); + const profilesPath = stringField(input.connection.profiles_path); + const target = stringField(input.connection.target); + const projectName = stringField(input.connection.project_name); + if (profilesPath) sourceArgs.sourceProfilesPath = profilesPath; + if (target) sourceArgs.sourceTarget = target; + if (projectName) sourceArgs.sourceProjectName = projectName; + return sourceArgs; + } + + if (input.source === 'metricflow') { + const metricflow = isRecord(input.connection.metricflow) ? input.connection.metricflow : {}; + applyRepoSourceArgs(sourceArgs, { + repoUrl: stringField(metricflow.repoUrl), + branch: stringField(metricflow.branch), + subpath: stringField(metricflow.path), + authTokenRef: stringField(metricflow.auth_token_ref), + }); + return sourceArgs; + } + + if (input.source === 'lookml') { + applyRepoSourceArgs(sourceArgs, { + repoUrl: stringField(input.connection.repoUrl), + branch: stringField(input.connection.branch), + subpath: stringField(input.connection.path), + authTokenRef: stringField(input.connection.auth_token_ref), + }); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const expectedLookerConnectionName = stringField(mappings.expectedLookerConnectionName); + if (expectedLookerConnectionName) sourceArgs.sourceTarget = expectedLookerConnectionName; + return sourceArgs; + } + + if (input.source === 'metabase') { + sourceArgs.sourceUrl = stringField(input.connection.api_url); + sourceArgs.sourceApiKeyRef = stringField(input.connection.api_key_ref); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const databaseMapping = firstStringRecordEntry(mappings.databaseMappings); + if (databaseMapping) { + sourceArgs.metabaseDatabaseId = Number.parseInt(databaseMapping[0], 10); + sourceArgs.sourceWarehouseConnectionId = databaseMapping[1]; + } + return sourceArgs; + } + + if (input.source === 'looker') { + sourceArgs.sourceUrl = stringField(input.connection.base_url); + sourceArgs.sourceClientId = stringField(input.connection.client_id); + sourceArgs.sourceClientSecretRef = stringField(input.connection.client_secret_ref); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const connectionMapping = firstStringRecordEntry(mappings.connectionMappings); + if (connectionMapping) { + sourceArgs.sourceTarget = connectionMapping[0]; + sourceArgs.sourceWarehouseConnectionId = connectionMapping[1]; + } + return sourceArgs; + } + + sourceArgs.sourceApiKeyRef = stringField(input.connection.auth_token_ref); + sourceArgs.notionCrawlMode = + input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots'; + if (Array.isArray(input.connection.root_page_ids)) { + sourceArgs.notionRootPageIds = input.connection.root_page_ids.filter( + (pageId): pageId is string => typeof pageId === 'string', + ); + } + return sourceArgs; +} + +async function promptEditedSourceConnection(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + connectionId: string; + connection: KtxProjectConnectionConfig; + prompts: KtxSetupSourcesPromptAdapter; + io: KtxCliIo; + testGitRepo?: KtxSetupSourcesDeps['testGitRepo']; + pickNotionRootPages?: KtxSetupSourcesDeps['pickNotionRootPages']; + discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases']; +}): Promise | 'back'> { + const sourceArgs = await promptForInteractiveSource( + sourceArgsFromExistingConnection({ + args: input.args, + source: input.source, + connectionId: input.connectionId, + connection: input.connection, + }), + input.source, + input.prompts, + input.io, + { + pickNotionRootPages: input.pickNotionRootPages, + discoverMetabaseDatabases: input.discoverMetabaseDatabases, + }, + input.connectionId, + input.testGitRepo, + input.discoverMetabaseDatabases, + ); + return sourceArgs === 'back' + ? 'back' + : { kind: 'edited', connectionId: input.connectionId, args: sourceArgs }; +} + +async function chooseContextSourceToEdit(input: { + projectDir: string; + prompts: KtxSetupSourcesPromptAdapter; +}): Promise<{ connectionId: string; source: KtxSetupSourceType } | 'back'> { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const targets = contextSourceEditTargets(project.config.connections); + if (targets.length === 0) return 'back'; + const choice = await input.prompts.select({ + message: 'Context source to edit', + options: [ + ...targets.map((target) => ({ + value: target.connectionId, + label: `${target.connectionId} (${sourceLabel(target.source)})`, + })), + { value: 'back', label: 'Back' }, + ], + }); + if (choice === 'back') return 'back'; + const target = targets.find((candidate) => candidate.connectionId === choice); + return target ?? 'back'; +} + async function chooseInteractiveSourceConnection(input: { args: KtxSetupSourcesArgs; source: KtxSetupSourceType; @@ -1356,6 +1590,10 @@ async function chooseInteractiveSourceConnection(input: { value: `existing:${connectionId}`, label: `Use existing ${label} connection: ${connectionId}`, })), + ...existingIds.map((connectionId) => ({ + value: `edit:${connectionId}`, + label: `Edit existing ${label} connection: ${connectionId}`, + })), { value: 'new', label: `Add new ${label} connection` }, { value: 'back', label: 'Back' }, ], @@ -1369,6 +1607,28 @@ async function chooseInteractiveSourceConnection(input: { } continue; } + if (choice.startsWith('edit:')) { + const connectionId = choice.slice('edit:'.length); + const connection = input.connections[connectionId]; + if (!connection) { + continue; + } + const edited = await promptEditedSourceConnection({ + args: input.args, + source: input.source, + connectionId, + connection, + prompts: input.prompts, + io: input.io, + testGitRepo: input.testGitRepo, + pickNotionRootPages: input.pickNotionRootPages, + discoverMetabaseDatabases: input.discoverMetabaseDatabases, + }); + if (edited === 'back') { + continue; + } + return edited; + } const sourceArgs = await promptForInteractiveSource( input.args, input.source, @@ -1433,6 +1693,85 @@ async function validateSource( return await (deps.validateNotion ?? defaultValidateNotion)(args.connection); } +async function saveValidateAndMaybeBuildSource(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + sourceChoice: Exclude; + prompts: KtxSetupSourcesPromptAdapter; + io: KtxCliIo; + deps: KtxSetupSourcesDeps; +}): Promise { + const connectionId = + input.sourceChoice.kind === 'existing' + ? input.sourceChoice.connectionId + : input.sourceChoice.kind === 'edited' + ? input.sourceChoice.connectionId + : (input.sourceChoice.args.sourceConnectionId ?? `${input.source}-main`); + const connection = + input.sourceChoice.kind === 'existing' + ? input.sourceChoice.connection + : buildConnection(input.source, input.sourceChoice.args); + const rollback = + input.sourceChoice.kind === 'existing' + ? undefined + : await writeSourceConnection( + input.args.projectDir, + connectionId, + connection, + sourceAdapter(input.source), + ); + + if (input.sourceChoice.kind === 'existing') { + await ensureSourceAdapterEnabled(input.args.projectDir, input.source); + } + + const validation = await validateSource( + input.source, + { projectDir: input.args.projectDir, connectionId, connection }, + input.deps, + ); + if (!validation.ok) { + await rollback?.(); + input.io.stderr.write(`${validation.message}\n`); + return { status: 'failed' }; + } + + if (input.source === 'metabase' || input.source === 'looker') { + input.prompts.log?.(`Validating ${sourceLabel(input.source)} mapping…`); + const mappingCode = await (input.deps.runMapping ?? defaultRunMapping)( + input.args.projectDir, + connectionId, + createSetupPrefixedIo(input.io), + ); + if (mappingCode !== 0) { + await rollback?.(); + return { status: 'failed' }; + } + } + + if (input.args.runInitialSourceIngest) { + const ingestResult = await runInitialSourceIngestWithRecovery({ + args: input.args, + connectionId, + io: input.io, + prompts: input.prompts, + deps: input.deps, + }); + if (ingestResult === 'failed') { + await rollback?.(); + return { status: 'failed' }; + } + if (ingestResult === 'back') { + await rollback?.(); + return { status: 'back' }; + } + } else { + input.io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`); + } + + return { status: 'ready', connectionId }; +} + export async function runKtxSetupSourcesStep( args: KtxSetupSourcesArgs, io: KtxCliIo, @@ -1510,62 +1849,27 @@ export async function runKtxSetupSourcesStep( returnToSourceSelection = true; break; } - const connectionId = - sourceChoice.kind === 'existing' - ? sourceChoice.connectionId - : (sourceChoice.args.sourceConnectionId ?? `${source}-main`); - const connection = - sourceChoice.kind === 'existing' ? sourceChoice.connection : buildConnection(source, sourceChoice.args); - const rollback = - sourceChoice.kind === 'existing' - ? undefined - : await writeSourceConnection(args.projectDir, connectionId, connection, sourceAdapter(source)); - if (sourceChoice.kind === 'existing') { - await ensureSourceAdapterEnabled(args.projectDir, source); - } - const validation = await validateSource(source, { projectDir: args.projectDir, connectionId, connection }, deps); - - if (!validation.ok) { - await rollback?.(); - io.stderr.write(`${validation.message}\n`); + const choiceResult = await saveValidateAndMaybeBuildSource({ + args, + source, + sourceChoice, + prompts, + io, + deps, + }); + if (choiceResult.status === 'failed') { return { status: 'failed', projectDir: args.projectDir }; } - if (source === 'metabase' || source === 'looker') { - prompts.log?.(`Validating ${sourceLabel(source)} mapping…`); - const mappingCode = await (deps.runMapping ?? defaultRunMapping)( - args.projectDir, - connectionId, - createSetupPrefixedIo(io), - ); - if (mappingCode !== 0) { - await rollback?.(); - return { status: 'failed', projectDir: args.projectDir }; + if (choiceResult.status === 'back') { + if (args.source) { + return { status: 'back', projectDir: args.projectDir }; } + returnToSourceSelection = true; + break; } - if (args.runInitialSourceIngest) { - const ingestResult = await runInitialSourceIngestWithRecovery({ - args, - connectionId, - io, - prompts, - deps, - }); - if (ingestResult === 'failed') { - await rollback?.(); - return { status: 'failed', projectDir: args.projectDir }; - } - if (ingestResult === 'back') { - await rollback?.(); - if (args.source) { - return { status: 'back', projectDir: args.projectDir }; - } - returnToSourceSelection = true; - break; - } - } else { - io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`); + if (!readyConnectionIds.includes(choiceResult.connectionId)) { + readyConnectionIds.push(choiceResult.connectionId); } - readyConnectionIds.push(connectionId); } if (returnToSourceSelection) { @@ -1573,14 +1877,66 @@ export async function runKtxSetupSourcesStep( } if (readyConnectionIds.length > 0 && !args.source && args.inputMode !== 'disabled') { - const addMore = await prompts.select({ - message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`, - options: [ - { value: 'done', label: 'Done — continue to context build' }, - { value: 'add', label: 'Add another context source' }, - ], - }); - if (addMore === 'add') { + let restartSourceSelection = false; + while (true) { + const addMore = await prompts.select({ + message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`, + options: [ + { value: 'done', label: 'Done — continue to context build' }, + { value: 'edit', label: 'Edit an existing context source' }, + { value: 'add', label: 'Add another context source' }, + ], + }); + if (addMore === 'add') { + restartSourceSelection = true; + break; + } + if (addMore === 'edit') { + const editTarget = await chooseContextSourceToEdit({ projectDir: args.projectDir, prompts }); + if (editTarget === 'back') { + continue; + } + const projectForEdit = await loadKtxProject({ projectDir: args.projectDir }); + const connection = projectForEdit.config.connections[editTarget.connectionId]; + if (!connection) { + continue; + } + const sourceChoice = await promptEditedSourceConnection({ + args, + source: editTarget.source, + connectionId: editTarget.connectionId, + connection, + prompts, + io, + testGitRepo: deps.testGitRepo, + pickNotionRootPages: deps.pickNotionRootPages, + discoverMetabaseDatabases: deps.discoverMetabaseDatabases, + }); + if (sourceChoice === 'back') { + continue; + } + const choiceResult = await saveValidateAndMaybeBuildSource({ + args, + source: editTarget.source, + sourceChoice, + prompts, + io, + deps, + }); + if (choiceResult.status === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + if (choiceResult.status === 'back') { + continue; + } + if (!readyConnectionIds.includes(choiceResult.connectionId)) { + readyConnectionIds.push(choiceResult.connectionId); + } + continue; + } + break; + } + if (restartSourceSelection) { continue; } } From f219ba22a61e68fa1bcda3dfeb921955aba36aa4 Mon Sep 17 00:00:00 2001 From: Luca Martial <48870843+luca-martial@users.noreply.github.com> Date: Wed, 13 May 2026 17:28:48 -0400 Subject: [PATCH 7/8] feat(cli): redesign Notion page picker UI and add skip-empty flow (#78) Rework the inline picker to use a cleaner visual style (filled/empty square glyphs, bordered layout, clack-style header) and streamlined keybindings (Enter to confirm, Escape to quit, Right Arrow to expand). Replace the transient "select at least one" hint with a skip-empty confirmation prompt that exits cleanly via quit-without-save. Co-authored-by: Claude Opus 4.6 (1M context) --- .../cli/src/notion-page-picker-tree.test.ts | 25 ++--- packages/cli/src/notion-page-picker-tree.ts | 15 +-- .../cli/src/notion-page-picker-tui.test.tsx | 70 ++++++------ packages/cli/src/notion-page-picker-tui.tsx | 102 ++++++++++++------ 4 files changed, 126 insertions(+), 86 deletions(-) diff --git a/packages/cli/src/notion-page-picker-tree.test.ts b/packages/cli/src/notion-page-picker-tree.test.ts index 94b46b57..58e8c7ca 100644 --- a/packages/cli/src/notion-page-picker-tree.test.ts +++ b/packages/cli/src/notion-page-picker-tree.test.ts @@ -11,7 +11,6 @@ import { selectAllVisible, selectNone, toggleChecked, - TRANSIENT_HINT_DURATION_MS, visibleNodeIds, type NotionPickerPageInput, } from './notion-page-picker-tree.js'; @@ -223,22 +222,24 @@ describe('bulk actions and reducer effects', () => { }); }); - it('blocks empty saves, updates search state, and quits without saving', () => { + it('prompts skip-empty confirmation on empty save, updates search state, and quits without saving', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), existingRootPageIds: [], currentCrawlMode: 'selected_roots', }); - const blockedSave = reducer(state, 'save-request', 9000); - expect(blockedSave).toEqual({ - next: { - ...state, - transientHint: { - text: 'Select at least one page or press q to quit', - expiresAt: 9000 + TRANSIENT_HINT_DURATION_MS, - }, - }, + const emptySave = reducer(state, 'save-request'); + expect(emptySave).toEqual({ + next: { ...state, pendingConfirm: 'skip-empty' }, + effect: null, + }); + expect(reducer(emptySave.next, 'save-confirm')).toEqual({ + next: { ...state, pendingConfirm: null }, + effect: 'quit-without-save', + }); + expect(reducer(emptySave.next, 'save-cancel')).toEqual({ + next: { ...state, pendingConfirm: null }, effect: null, }); expect( @@ -262,7 +263,7 @@ describe('bulk actions and reducer effects', () => { const withHint = { ...state, transientHint: { - text: 'Select at least one page or press q to quit', + text: 'Select at least one page or press esc to cancel', expiresAt: 11500, }, }; diff --git a/packages/cli/src/notion-page-picker-tree.ts b/packages/cli/src/notion-page-picker-tree.ts index 379ac938..738ab723 100644 --- a/packages/cli/src/notion-page-picker-tree.ts +++ b/packages/cli/src/notion-page-picker-tree.ts @@ -22,7 +22,7 @@ export interface PickerState { checked: Set; cursorId: string; search: { editing: boolean; query: string }; - pendingConfirm: 'mode-switch' | null; + pendingConfirm: 'mode-switch' | 'skip-empty' | null; preLoadWarnings: string[]; transientHint: { text: string; expiresAt: number } | null; currentCrawlMode: 'all_accessible' | 'selected_roots'; @@ -61,7 +61,7 @@ interface MutableNode { childIds: string[]; } -export const TRANSIENT_HINT_DURATION_MS = 2500; +const TRANSIENT_HINT_DURATION_MS = 2500; const collator = new Intl.Collator('en', { sensitivity: 'base', numeric: true }); @@ -444,7 +444,8 @@ export function buildInitialState(args: { export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()): { next: PickerState; effect: PickerEffect } { if (state.pendingConfirm) { if (cmd === 'save-confirm') { - return { next: cloneState(state, { pendingConfirm: null }), effect: 'save' }; + const effect: PickerEffect = state.pendingConfirm === 'skip-empty' ? 'quit-without-save' : 'save'; + return { next: cloneState(state, { pendingConfirm: null }), effect }; } if (cmd === 'save-cancel') { return { next: cloneState(state, { pendingConfirm: null }), effect: null }; @@ -498,19 +499,13 @@ export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now() }; case 'save-request': if (state.checked.size === 0) { - return { - next: cloneState(state, { - transientHint: transientHint('Select at least one page or press q to quit', now), - }), - effect: null, - }; + return { next: cloneState(state, { pendingConfirm: 'skip-empty' }), effect: null }; } if (state.currentCrawlMode === 'all_accessible') { return { next: cloneState(state, { pendingConfirm: 'mode-switch' }), effect: null }; } return { next: state, effect: 'save' }; case 'save-confirm': - return { next: state, effect: 'save' }; case 'save-cancel': return { next: state, effect: null }; case 'quit': diff --git a/packages/cli/src/notion-page-picker-tui.test.tsx b/packages/cli/src/notion-page-picker-tui.test.tsx index 2d4dffc3..16ad93db 100644 --- a/packages/cli/src/notion-page-picker-tui.test.tsx +++ b/packages/cli/src/notion-page-picker-tui.test.tsx @@ -1,7 +1,7 @@ /* @jsxImportSource react */ import { render as renderInkTest } from 'ink-testing-library'; -import { act, type ReactNode } from 'react'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { type ReactNode } from 'react'; +import { describe, expect, it, vi } from 'vitest'; import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js'; import { NotionPickerApp, @@ -70,13 +70,9 @@ function fakeInkInstance(): NotionPickerInkInstance { } function normalizeFrameWrap(frame: string | undefined): string { - return frame?.replace(/\n/g, ' ') ?? ''; + return frame?.replace(/\n/g, ' ').replace(/│ /g, '').replace(/ +/g, ' ') ?? ''; } -afterEach(() => { - vi.useRealTimers(); -}); - describe('notionPickerCommandForInkInput', () => { it('maps browse, search, and confirm input to reducer commands', () => { expect(notionPickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down'); @@ -87,9 +83,11 @@ describe('notionPickerCommandForInkInput', () => { expect(notionPickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start'); expect(notionPickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible'); expect(notionPickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none'); - expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBe('save-request'); - expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBe('quit'); + expect(notionPickerCommandForInkInput('', { return: true }, state().search, null)).toBe('save-request'); + expect(notionPickerCommandForInkInput('', { escape: true }, state().search, null)).toBe('quit'); expect(notionPickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit'); + expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBeNull(); + expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBeNull(); expect(notionPickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({ type: 'search-input', @@ -145,13 +143,16 @@ describe('NotionPickerApp', () => { ); const frame = lastFrame() ?? ''; - expect(frame).toContain('Notion pages visible to integration "Design Workspace"'); + expect(frame).toContain('Select Notion pages to ingest'); + expect(frame).toContain('Workspace: Design Workspace'); expect(frame).toContain('5000-page cap reached - some pages not shown'); expect(frame).toContain('1 stored root_page_ids no longer visible - they will be removed if you save'); - expect(frame).toContain('▸ [ ] Engineering Docs ▸ (1)'); - expect(frame).toContain(' [ ] Marketing'); + expect(frame).toContain('◻ Engineering Docs ▸ (1)'); + expect(frame).toContain('◻ Marketing'); expect(frame).not.toContain('Search ready: -'); - expect(frame).toContain('space toggle · enter expand · / search · a all · n none · s save & exit · q quit'); + expect(normalizeFrameWrap(frame)).toContain( + 'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.', + ); }); it('renders partial discovery warnings without stale-root save suffix', () => { @@ -199,8 +200,8 @@ describe('NotionPickerApp', () => { ); const frame = lastFrame() ?? ''; - expect(frame).toContain('▸ [×] Engineering Docs ▾'); - expect(frame).toContain(' [~] Architecture'); + expect(frame).toContain('◼ Engineering Docs ▾'); + expect(frame).toContain(' ◼ Architecture'); }); it('supports keyboard selection, all_accessible confirmation, and save callback', async () => { @@ -220,12 +221,12 @@ describe('NotionPickerApp', () => { stdin.write(' '); await waitForInkInput(); - expect(lastFrame()).toContain('[×] Engineering Docs'); + expect(lastFrame()).toContain('◼ Engineering Docs'); - stdin.write('s'); + stdin.write('\r'); await waitForInkInput(); expect(normalizeFrameWrap(lastFrame())).toContain( - 'Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to 1 selected page. [y] confirm [esc] back', + 'Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to 1 selected page. Press Enter to confirm or Escape to go back.', ); stdin.write('y'); @@ -233,8 +234,7 @@ describe('NotionPickerApp', () => { expect(onExit).toHaveBeenCalledWith({ kind: 'save', rootPageIds: [IDS.engineering] }); }); - it('removes transient hints after their expiry time', async () => { - vi.useFakeTimers(); + it('prompts skip-empty confirmation on empty submit and dismisses on cancel', async () => { const onExit = vi.fn(); const { stdin, lastFrame } = renderInkTest( { />, ); - await act(async () => { - stdin.write('s'); - await vi.advanceTimersByTimeAsync(10); - }); - expect(lastFrame()).toContain('Select at least one page or press q to quit'); - - await act(async () => { - await vi.advanceTimersByTimeAsync(2500); - }); - expect(lastFrame()).not.toContain('Select at least one page or press q to quit'); + stdin.write('\r'); + await waitForInkInput(); + expect(normalizeFrameWrap(lastFrame())).toContain( + 'Nothing selected. Skip this step? Press Enter to skip or Escape to go back.', + ); expect(onExit).not.toHaveBeenCalled(); + + stdin.write('n'); + await waitForInkInput(); + expect(lastFrame()).not.toContain('Nothing selected. Skip this step?'); + expect(onExit).not.toHaveBeenCalled(); + + stdin.write('\r'); + await waitForInkInput(); + expect(lastFrame()).toContain('Nothing selected. Skip this step?'); + + stdin.write('\r'); + await waitForInkInput(); + expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); }); it('renders row-window overflow indicators when the visible list is clipped', async () => { @@ -312,7 +320,7 @@ describe('NotionPickerApp', () => { />, ); - stdin.write('q'); + stdin.write('\u0003'); await waitForInkInput(); expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); }); diff --git a/packages/cli/src/notion-page-picker-tui.tsx b/packages/cli/src/notion-page-picker-tui.tsx index 30af7522..d627d200 100644 --- a/packages/cli/src/notion-page-picker-tui.tsx +++ b/packages/cli/src/notion-page-picker-tui.tsx @@ -16,6 +16,7 @@ const COLOR_THEME = { text: 'white', muted: 'gray', active: 'cyan', + selected: 'green', warning: 'yellow', } as const; @@ -23,6 +24,7 @@ const NO_COLOR_THEME = { text: 'white', muted: 'white', active: 'white', + selected: 'white', warning: 'white', } as const; @@ -158,13 +160,12 @@ export function notionPickerCommandForInkInput( if (key.downArrow) return 'cursor-down'; if (key.leftArrow) return 'cursor-left'; if (key.rightArrow) return 'cursor-right'; - if (key.return) return 'expand'; + if (key.return) return 'save-request'; if (input === ' ') return 'toggle-check'; if (input === '/') return 'search-start'; if (input === 'a') return 'select-all-visible'; if (input === 'n') return 'select-none'; - if (input === 's') return 'save-request'; - if (input === 'q' || key.escape) return 'quit'; + if (key.escape) return 'quit'; return null; } @@ -174,18 +175,27 @@ function PickerRow(props: { state: PickerState; nodeId: string; width: number; t const focused = props.state.cursorId === node.id; const locked = isAncestorChecked(node.id, props.state.checked, props.state.byId); const checked = props.state.checked.has(node.id); - const glyph = locked ? '[~]' : checked ? '[×]' : '[ ]'; - const children = + const isSelected = checked || locked; + const glyph = isSelected ? '◼' : '◻'; + const glyphColor = locked ? props.theme.muted : checked ? props.theme.selected : props.theme.muted; + const childAffordance = node.childIds.length > 0 ? (props.state.expanded.has(node.id) ? ' ▾' : ` ▸ (${node.childIds.length})`) : ''; - const prefix = `${focused ? '▸' : ' '} ${glyph} ${' '.repeat(node.depth * 2)}`; - const color = focused ? props.theme.active : locked || node.archived ? props.theme.muted : props.theme.text; - const title = truncateText(`${node.title}${children}`, Math.max(10, props.width - prefix.length)); + const indent = ' '.repeat(node.depth * 2); + const titleColor = focused ? props.theme.text : props.theme.muted; const inverse = rowMatchesSearch(props.state, node.id); + const prefixWidth = indent.length + 2; + const title = truncateText(`${node.title}${childAffordance}`, Math.max(10, props.width - prefixWidth)); return ( - - {prefix} - {title} + + + {indent} + {glyph} + + + {' '} + {title} + ); } @@ -198,7 +208,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { const visibleIds = visibleNodeIds(state); const selectedIndex = Math.max(0, visibleIds.indexOf(state.cursorId)); const reservedRows = state.pendingConfirm === 'mode-switch' ? 9 : 8; - const visibleRows = Math.max(5, Math.min(20, (props.terminalRows ?? 24) - reservedRows)); + const visibleRows = Math.max(5, Math.min(12, (props.terminalRows ?? 24) - reservedRows)); const rows = windowItems(visibleIds, selectedIndex, visibleRows); const hiddenAbove = rows.offset; const hiddenBelow = Math.max(0, visibleIds.length - rows.offset - rows.items.length); @@ -254,34 +264,60 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { return ( - Notion pages visible to integration "{props.workspaceLabel}" - {props.cappedAtCount ? {props.cappedAtCount}-page cap reached - some pages not shown : null} - {state.preLoadWarnings.map((warning) => ( - - {staleWarningText(warning)} - - ))} - {showSearch ? ( + + ◆ + Select Notion pages to ingest + + - / {state.search.query} - {state.search.editing ? '█' : ''} ({searchMatchCount} matches) + Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape + to go back, or Ctrl+C to exit. - ) : null} - + + Workspace: {props.workspaceLabel} + {props.cappedAtCount ? ( + {props.cappedAtCount}-page cap reached - some pages not shown + ) : null} + {state.preLoadWarnings.map((warning) => ( + + {staleWarningText(warning)} + + ))} + {showSearch ? ( + + / + + {state.search.query} + {state.search.editing ? '█' : ''} + + ({searchMatchCount} matches) + + ) : null} {hiddenAbove > 0 ? ↑ {hiddenAbove} more : null} {rows.items.map((nodeId) => ( ))} {hiddenBelow > 0 ? ↓ {hiddenBelow} more : null} + {state.pendingConfirm === 'mode-switch' ? ( + + Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to{' '} + {selectedPageCountText(selectedCount)}. Press Enter to confirm or Escape to go back. + + ) : null} + {state.pendingConfirm === 'skip-empty' ? ( + Nothing selected. Skip this step? Press Enter to skip or Escape to go back. + ) : null} + {state.transientHint ? {state.transientHint.text} : null} - {state.pendingConfirm === 'mode-switch' ? ( - - Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to{' '} - {selectedPageCountText(selectedCount)}. [y] confirm [esc] back - - ) : null} - {state.transientHint ? {state.transientHint.text} : null} - space toggle · enter expand · / search · a all · n none · s save & exit · q quit + └ ); } @@ -323,7 +359,7 @@ export async function renderNotionPickerTui( exitOnCtrlC: false, patchConsole: false, maxFps: 30, - alternateScreen: true, + alternateScreen: false, }, ); await instance.waitUntilExit(); From c2750dd7970be18a18f4cf3fb595258c88be323f Mon Sep 17 00:00:00 2001 From: Luca Martial <48870843+luca-martial@users.noreply.github.com> Date: Wed, 13 May 2026 17:55:25 -0400 Subject: [PATCH 8/8] refactor(cli): hide internal setup options and remove dead flags (#79) Hide advanced/internal `ktx setup` options from --help output using .hideHelp() so the command surface is approachable for new users. Remove the --project, --agent-scope, and --skip-initial-source-ingest flags that are no longer needed. Update docs and tests to match. Co-authored-by: Claude Opus 4.6 (1M context) --- .../content/docs/cli-reference/ktx-setup.mdx | 94 +------- .../docs/getting-started/quickstart.mdx | 2 +- packages/cli/src/commands/setup-commands.ts | 213 ++++++++++-------- packages/cli/src/index.test.ts | 72 +++++- packages/cli/src/setup-agents.ts | 2 +- 5 files changed, 185 insertions(+), 198 deletions(-) diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index f490988a..59fbe666 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -18,8 +18,6 @@ ktx setup [options] | Flag | Description | Default | |------|-------------|---------| | `--project-dir ` | KTX project directory | `KTX_PROJECT_DIR`, nearest `ktx.yaml`, or cwd | -| `--new` | Create a new KTX project before setup | `false` | -| `--existing` | Use an existing KTX project | `false` | | `--yes` | Accept safe defaults in non-interactive setup | `false` | | `--no-input` | Disable interactive terminal input | — | @@ -29,76 +27,11 @@ ktx setup [options] |------|-------------|---------| | `--agents` | Install agent integration only | `false` | | `--target ` | Agent target (`claude-code`, `codex`, `cursor`, `opencode`, `universal`) | — | -| `--agent-scope ` | Agent install scope (`project` or `global`) | `project` | -| `--project` | Install agent integration into the project scope | `false` | | `--global` | Install agent integration into the global target scope (Claude Code and Codex only) | `false` | -| `--skip-agents` | Leave agent integration incomplete for now | `false` | -### LLM Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--anthropic-api-key-env ` | Environment variable containing the Anthropic API key | — | -| `--anthropic-api-key-file ` | File containing the Anthropic API key | — | -| `--anthropic-model ` | Anthropic model ID to validate and save | — | -| `--skip-llm` | Leave LLM setup incomplete for now | `false` | - -### Embedding Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--embedding-backend ` | Embedding backend (`openai` or `sentence-transformers`) | — | -| `--embedding-api-key-env ` | Environment variable containing the embedding provider API key | — | -| `--embedding-api-key-file ` | File containing the embedding provider API key | — | -| `--skip-embeddings` | Leave embedding setup incomplete for now | `false` | - -### Database Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--database ` | Database driver to configure; repeatable (`sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake`) | — | -| `--database-connection-id ` | Existing or new connection id; repeatable | — | -| `--new-database-connection-id ` | Connection id for one new database connection | — | -| `--database-url ` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection | — | -| `--database-schema ` | Database schema to include; repeatable | — | -| `--skip-databases` | Leave database setup incomplete | `false` | - -### Historic SQL - -| Flag | Description | Default | -|------|-------------|---------| -| `--enable-historic-sql` | Enable Historic SQL when the selected database supports it | `false` | -| `--disable-historic-sql` | Disable Historic SQL for the selected database | `false` | -| `--historic-sql-window-days ` | Historic SQL query-history window in days | — | -| `--historic-sql-min-executions ` | Minimum executions for a Historic SQL template | — | -| `--historic-sql-min-calls ` | Alias for `--historic-sql-min-executions` for one release | — | -| `--historic-sql-service-account-pattern ` | Historic SQL service-account regex; repeatable | — | -| `--historic-sql-redaction-pattern ` | Historic SQL SQL-literal redaction regex; repeatable | — | - -### Context Source Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--source ` | Source connector type (`dbt`, `metricflow`, `metabase`, `looker`, `lookml`, `notion`) | — | -| `--source-connection-id ` | Connection id for source setup | — | -| `--source-path ` | Local source path for dbt, MetricFlow, or LookML | — | -| `--source-git-url ` | Git URL for dbt, MetricFlow, or LookML | — | -| `--source-branch ` | Git branch for source setup | — | -| `--source-subpath ` | Repo subpath for source setup | — | -| `--source-auth-token-ref ` | `env:` or `file:` credential ref for source repo auth | — | -| `--source-url ` | Source service URL for Metabase or Looker | — | -| `--source-api-key-ref ` | `env:` or `file:` API key ref for Metabase or Notion | — | -| `--source-client-id ` | Looker client id | — | -| `--source-client-secret-ref ` | `env:` or `file:` Looker client secret ref | — | -| `--source-warehouse-connection-id ` | Mapped warehouse connection id | — | -| `--source-project-name ` | dbt project name override | — | -| `--source-profiles-path ` | dbt profiles path | — | -| `--source-target ` | dbt target or source-specific mapping target | — | -| `--metabase-database-id ` | Metabase database id to map | — | -| `--notion-crawl-mode ` | Notion crawl mode (`all_accessible` or `selected_roots`) | — | -| `--notion-root-page-id ` | Notion root page id; repeatable | — | -| `--skip-initial-source-ingest` | Validate source setup without building source context during setup | `false` | -| `--skip-sources` | Mark optional source setup complete with no sources | `false` | +The setup wizard is the public configuration interface. It prompts for LLM +credentials, embeddings, database connections, context sources, Historic SQL, +and agent integration when those values are needed. ## Examples @@ -106,17 +39,8 @@ ktx setup [options] # Run the interactive setup wizard ktx setup -# Create a new project and run setup -ktx setup --new - -# Resume setup in an existing project -ktx setup --existing - -# Non-interactive setup with Anthropic key from environment -ktx setup --yes --anthropic-api-key-env ANTHROPIC_API_KEY - -# Set up a Postgres connection -ktx setup --database postgres --database-url "env:DATABASE_URL" +# Run setup for a specific project directory +ktx setup --project-dir ./analytics # Install agent integration for Claude Code only ktx setup --agents --target claude-code @@ -124,12 +48,6 @@ ktx setup --agents --target claude-code # Install agent integration globally for Codex ktx setup --agents --target codex --global -# Add a dbt source from a local path -ktx setup --source dbt --source-path ./my-dbt-project - -# Skip optional steps for a minimal setup -ktx setup --skip-sources --skip-agents - # Check setup readiness ktx status ``` @@ -156,5 +74,5 @@ Agent integration ready: yes (codex:project) |-------|-------|----------| | Setup resumes an unexpected project | `KTX_PROJECT_DIR` or nearest `ktx.yaml` points to another directory | Pass `--project-dir ` explicitly | | Health check for model fails | Provider key or model id is invalid | Set the correct environment variable or secret file and rerun setup | -| Setup cannot run in CI | Interactive prompts need a TTY | Use `--yes --no-input` with explicit flags for required values | +| Setup cannot run in CI | Interactive prompts need a TTY | Run setup interactively before CI, or provide a fixture `ktx.yaml` for automated tests | | Agent integration missing | Setup skipped the agents step | Run `ktx setup --agents --target ` | diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 7aba00fd..635c666b 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -242,7 +242,7 @@ Agent integration ready: yes (claude-code:project) | Local embeddings hang or fail | The managed Python runtime cannot start or the local model runtime is unavailable | Install `uv`, run `ktx dev runtime status`, then run `ktx dev runtime install --feature local-embeddings --yes` and rerun setup | | Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection | | `KTX context built: no` in `ktx status` | Setup saved configuration but did not build context | Run `ktx setup` and choose to build context now | -| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex --project` using the target you need | +| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex` using the target you need | ## Next steps diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index 1688724d..3da8d094 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -64,13 +64,6 @@ function sourceType(value: string): KtxSetupSourceType { throw new InvalidArgumentError(`invalid choice '${value}'`); } -function agentScope(value: string): 'project' | 'global' { - if (value === 'project' || value === 'global') { - return value; - } - throw new InvalidArgumentError(`invalid choice '${value}'`); -} - function positiveNumber(value: string): number { const parsed = Number.parseInt(value, 10); if (!Number.isInteger(parsed) || parsed <= 0) { @@ -97,7 +90,6 @@ function shouldShowSetupEntryMenu( agents?: boolean; target?: string; global?: boolean; - project?: boolean; skipAgents?: boolean; yes?: boolean; input?: boolean; @@ -142,7 +134,6 @@ function shouldShowSetupEntryMenu( metabaseDatabaseId?: number; notionCrawlMode?: string; notionRootPageId?: string[]; - skipInitialSourceIngest?: boolean; skipSources?: boolean; }, command: Command, @@ -172,7 +163,6 @@ function shouldShowSetupEntryMenu( 'agents', 'target', 'global', - 'project', 'skipAgents', 'yes', 'input', @@ -211,7 +201,6 @@ function shouldShowSetupEntryMenu( 'sourceTarget', 'metabaseDatabaseId', 'notionCrawlMode', - 'skipInitialSourceIngest', 'skipSources', ].some((optionName) => optionWasSpecified(command, optionName)); } @@ -220,9 +209,9 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo const setup = program .command('setup') .description('Set up or resume a local KTX project') - .option('--project-dir ', 'KTX project directory') - .option('--new', 'Create a new KTX project before setup', false) - .option('--existing', 'Use an existing KTX project', false) + .addOption(new Option('--project-dir ', 'KTX project directory').hideHelp()) + .addOption(new Option('--new', 'Create a new KTX project before setup').hideHelp().default(false)) + .addOption(new Option('--existing', 'Use an existing KTX project').hideHelp().default(false)) .option('--agents', 'Install agent integration only', false) .addOption( new Option('--target ', 'Agent target').choices([ @@ -233,94 +222,124 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo 'universal', ]), ) - .addOption(new Option('--agent-scope ', 'Agent install scope').argParser(agentScope).default('project')) - .option('--project', 'Install agent integration into the project scope', false) .option('--global', 'Install agent integration into the global target scope', false) - .option('--skip-agents', 'Leave agent integration incomplete for now', false) + .addOption(new Option('--skip-agents', 'Leave agent integration incomplete for now').hideHelp().default(false)) .option('--yes', 'Accept safe defaults in non-interactive setup', false) .option('--no-input', 'Disable interactive terminal input') - .addOption(new Option('--llm-backend ', 'LLM backend').argParser(llmBackend)) - .option('--anthropic-api-key-env ', 'Environment variable containing the Anthropic API key') - .option('--anthropic-api-key-file ', 'File containing the Anthropic API key') - .option('--anthropic-model ', 'Anthropic model ID to validate and save') - .option('--vertex-project ', 'Google Vertex AI project ID, env:NAME, or file:/path') - .option('--vertex-location ', 'Google Vertex AI location, env:NAME, or file:/path') - .addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false)) - .addOption(new Option('--embedding-backend ', 'Embedding backend').argParser(embeddingBackend)) - .option('--embedding-api-key-env ', 'Environment variable containing the embedding provider API key') - .option('--embedding-api-key-file ', 'File containing the embedding provider API key') - .addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false)) - .option( - '--database ', - 'Database driver to configure; repeatable', - (value, previous: KtxSetupDatabaseDriver[]) => { - return [...previous, databaseDriver(value)]; - }, - [] as KtxSetupDatabaseDriver[], - ) - .option( - '--database-connection-id ', - 'Existing selected connection id or new connection id', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--new-database-connection-id ', 'Connection id for one new database connection', (value) => { - if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) { - throw new InvalidArgumentError(`Unsafe connection id: ${value}`); - } - return value; - }) - .option('--database-url ', 'URL, env:NAME, or file:/path for one new URL-style database connection') - .option( - '--database-schema ', - 'Database schema to include; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--enable-historic-sql', 'Enable Historic SQL when the selected database supports it', false) - .option('--disable-historic-sql', 'Disable Historic SQL for the selected database', false) - .option('--historic-sql-window-days ', 'Historic SQL query-history window', positiveInteger) - .option('--historic-sql-min-executions ', 'Minimum Historic SQL executions for a template', positiveInteger) - .option( - '--historic-sql-service-account-pattern ', - 'Historic SQL service-account regex; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option( - '--historic-sql-redaction-pattern ', - 'Historic SQL SQL-literal redaction regex; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a primary source is added', false) - .addOption(new Option('--source ', 'Source connector type').argParser(sourceType)) - .option('--source-connection-id ', 'Connection id for source setup') - .option('--source-path ', 'Local source path for dbt, MetricFlow, or LookML') - .option('--source-git-url ', 'Git URL for dbt, MetricFlow, or LookML') - .option('--source-branch ', 'Git branch for source setup') - .option('--source-subpath ', 'Repo subpath for source setup') - .option('--source-auth-token-ref ', 'env: or file: credential ref for source repo auth') - .option('--source-url ', 'Source service URL for Metabase or Looker') - .option('--source-api-key-ref ', 'env: or file: API key ref for Metabase or Notion') - .option('--source-client-id ', 'Looker client id') - .option('--source-client-secret-ref ', 'env: or file: Looker client secret ref') - .option('--source-warehouse-connection-id ', 'Mapped warehouse connection id') - .option('--source-project-name ', 'dbt project name override') - .option('--source-profiles-path ', 'dbt profiles path') - .option('--source-target ', 'dbt target or source-specific mapping target') - .option('--metabase-database-id ', 'Metabase database id to map', positiveNumber) + .addOption(new Option('--llm-backend ', 'LLM backend').argParser(llmBackend).hideHelp()) .addOption( - new Option('--notion-crawl-mode ', 'Notion crawl mode').choices(['all_accessible', 'selected_roots']), + new Option('--anthropic-api-key-env ', 'Environment variable containing the Anthropic API key').hideHelp(), ) - .option( - '--notion-root-page-id ', - 'Notion root page id; repeatable', - (value, previous: string[]) => [...previous, value], - [], + .addOption( + new Option('--anthropic-api-key-file ', 'File containing the Anthropic API key').hideHelp(), ) - .option('--skip-initial-source-ingest', 'Validate source setup without building source context during setup', false) - .option('--skip-sources', 'Mark optional source setup complete with no sources', false) + .addOption(new Option('--anthropic-model ', 'Anthropic model ID to validate and save').hideHelp()) + .addOption(new Option('--vertex-project ', 'Google Vertex AI project ID, env:NAME, or file:/path').hideHelp()) + .addOption(new Option('--vertex-location ', 'Google Vertex AI location, env:NAME, or file:/path').hideHelp()) + .addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false)) + .addOption(new Option('--embedding-backend ', 'Embedding backend').argParser(embeddingBackend).hideHelp()) + .addOption( + new Option( + '--embedding-api-key-env ', + 'Environment variable containing the embedding provider API key', + ).hideHelp(), + ) + .addOption( + new Option('--embedding-api-key-file ', 'File containing the embedding provider API key').hideHelp(), + ) + .addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false)) + .addOption( + new Option('--database ', 'Database driver to configure; repeatable') + .argParser((value, previous: KtxSetupDatabaseDriver[]) => { + return [...previous, databaseDriver(value)]; + }) + .default([] as KtxSetupDatabaseDriver[]) + .hideHelp(), + ) + .addOption( + new Option('--database-connection-id ', 'Existing selected connection id or new connection id') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--new-database-connection-id ', 'Connection id for one new database connection') + .argParser((value) => { + if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) { + throw new InvalidArgumentError(`Unsafe connection id: ${value}`); + } + return value; + }) + .hideHelp(), + ) + .addOption( + new Option('--database-url ', 'URL, env:NAME, or file:/path for one new URL-style database connection').hideHelp(), + ) + .addOption( + new Option('--database-schema ', 'Database schema to include; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--enable-historic-sql', 'Enable Historic SQL when the selected database supports it') + .hideHelp() + .default(false), + ) + .addOption( + new Option('--disable-historic-sql', 'Disable Historic SQL for the selected database').hideHelp().default(false), + ) + .addOption(new Option('--historic-sql-window-days ', 'Historic SQL query-history window').argParser(positiveInteger).hideHelp()) + .addOption( + new Option('--historic-sql-min-executions ', 'Minimum Historic SQL executions for a template') + .argParser(positiveInteger) + .hideHelp(), + ) + .addOption( + new Option('--historic-sql-service-account-pattern ', 'Historic SQL service-account regex; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--historic-sql-redaction-pattern ', 'Historic SQL SQL-literal redaction regex; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a primary source is added') + .hideHelp() + .default(false), + ) + .addOption(new Option('--source ', 'Source connector type').argParser(sourceType).hideHelp()) + .addOption(new Option('--source-connection-id ', 'Connection id for source setup').hideHelp()) + .addOption(new Option('--source-path ', 'Local source path for dbt, MetricFlow, or LookML').hideHelp()) + .addOption(new Option('--source-git-url ', 'Git URL for dbt, MetricFlow, or LookML').hideHelp()) + .addOption(new Option('--source-branch ', 'Git branch for source setup').hideHelp()) + .addOption(new Option('--source-subpath ', 'Repo subpath for source setup').hideHelp()) + .addOption(new Option('--source-auth-token-ref ', 'env: or file: credential ref for source repo auth').hideHelp()) + .addOption(new Option('--source-url ', 'Source service URL for Metabase or Looker').hideHelp()) + .addOption(new Option('--source-api-key-ref ', 'env: or file: API key ref for Metabase or Notion').hideHelp()) + .addOption(new Option('--source-client-id ', 'Looker client id').hideHelp()) + .addOption(new Option('--source-client-secret-ref ', 'env: or file: Looker client secret ref').hideHelp()) + .addOption(new Option('--source-warehouse-connection-id ', 'Mapped warehouse connection id').hideHelp()) + .addOption(new Option('--source-project-name ', 'dbt project name override').hideHelp()) + .addOption(new Option('--source-profiles-path ', 'dbt profiles path').hideHelp()) + .addOption(new Option('--source-target ', 'dbt target or source-specific mapping target').hideHelp()) + .addOption(new Option('--metabase-database-id ', 'Metabase database id to map').argParser(positiveNumber).hideHelp()) + .addOption( + new Option('--notion-crawl-mode ', 'Notion crawl mode') + .choices(['all_accessible', 'selected_roots']) + .hideHelp(), + ) + .addOption( + new Option('--notion-root-page-id ', 'Notion root page id; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption(new Option('--skip-sources', 'Mark optional source setup complete with no sources').hideHelp().default(false)) .showHelpAfterError(); setup.hook('preAction', (_thisCommand, actionCommand) => { @@ -371,7 +390,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo } const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto'; - const resolvedAgentScope = options.global ? 'global' : options.agentScope; + const resolvedAgentScope = options.global ? 'global' : 'project'; await runSetupArgs(context, { command: 'run', projectDir: resolveCommandProjectDir(command), diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index cd635d78..305cf30e 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -444,20 +444,54 @@ describe('runKtxCli', () => { expect(io.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); }); - it('documents setup as a bare command without subcommands', async () => { + it('documents setup with only the common interactive options visible', async () => { const testIo = makeIo(); await expect(runKtxCli(['setup', '--help'], testIo.io)).resolves.toBe(0); - expect(testIo.stdout()).toContain('Usage: ktx setup [options]'); - expect(testIo.stdout()).not.toContain('Commands:'); - expect(testIo.stdout()).not.toContain('setup demo'); - expect(testIo.stdout()).not.toContain('setup context'); - expect(testIo.stdout()).not.toContain('--skip-llm'); - expect(testIo.stdout()).not.toContain('--skip-embeddings'); - expect(testIo.stdout()).not.toContain('--embedding-model'); - expect(testIo.stdout()).not.toContain('--embedding-dimensions'); - expect(testIo.stdout()).not.toContain('--embedding-base-url'); + const stdout = testIo.stdout(); + expect(stdout).toContain('Usage: ktx setup [options]'); + expect(stdout).toContain('--agents'); + expect(stdout).toContain('--target '); + expect(stdout).toContain('--global'); + expect(stdout).toContain('--yes'); + expect(stdout).toContain('--no-input'); + expect(stdout).toContain('Global Options:'); + expect(stdout.match(/--project-dir /g)).toHaveLength(1); + expect(stdout).not.toContain('Commands:'); + expect(stdout).not.toContain('setup demo'); + expect(stdout).not.toContain('setup context'); + + for (const hiddenFlag of [ + '--new', + '--existing', + '--agent-scope', + '--skip-agents', + '--llm-backend', + '--anthropic-api-key-env', + '--vertex-project', + '--embedding-backend', + '--database ', + '--database-connection-id', + '--new-database-connection-id', + '--enable-historic-sql', + '--historic-sql-min-executions', + '--skip-databases', + '--source ', + '--source-connection-id', + '--metabase-database-id', + '--notion-root-page-id', + '--skip-initial-source-ingest', + '--skip-sources', + '--skip-llm', + '--skip-embeddings', + '--embedding-model', + '--embedding-dimensions', + '--embedding-base-url', + ]) { + expect(stdout).not.toContain(hiddenFlag); + } + expect(stdout).not.toMatch(/^ --project\s/m); expect(testIo.stderr()).toBe(''); }); @@ -725,6 +759,23 @@ describe('runKtxCli', () => { expect(setup).not.toHaveBeenCalled(); }); + it('rejects removed setup options', async () => { + const setup = vi.fn(async () => 0); + const cases = [ + ['setup', '--project'], + ['setup', '--agent-scope', 'global'], + ['setup', '--skip-initial-source-ingest'], + ]; + + for (const args of cases) { + const testIo = makeIo(); + await expect(runKtxCli(['--project-dir', tempDir, ...args], testIo.io, { setup })).resolves.toBe(1); + expect(testIo.stderr()).toMatch(/unknown option|error:/i); + } + + expect(setup).not.toHaveBeenCalled(); + }); + it('prints ingest help without invoking ingest execution', async () => { const testIo = makeIo(); const ingest = vi.fn(); @@ -1250,7 +1301,6 @@ describe('runKtxCli', () => { '--agents', '--target', 'codex', - '--project', '--no-input', '--yes', ], diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index 9505307d..7a18a969 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -82,7 +82,7 @@ export function plannedKtxAgentFiles(input: { { kind: 'file', path: join(codexHome, 'instructions/ktx.md'), role: 'rule' as const }, ]; } - throw new Error(`Global ${input.target} installation is not supported; use --project.`); + throw new Error(`Global ${input.target} installation is not supported; omit --global.`); } const root = resolve(input.projectDir);