diff --git a/packages/cli/src/prompt-navigation.test.ts b/packages/cli/src/prompt-navigation.test.ts index 4dd428df..9338b56e 100644 --- a/packages/cli/src/prompt-navigation.test.ts +++ b/packages/cli/src/prompt-navigation.test.ts @@ -28,12 +28,12 @@ describe('prompt navigation helpers', () => { 'Name this PostgreSQL connection\nKTX will use this short name in commands and config. You can rename it now.', ), ).toBe( - 'Name this PostgreSQL connection\n\nKTX will use this short name in commands and config. You can rename it now.\nPress Escape to go back.\n', + 'Name this PostgreSQL connection\n│\n│ KTX will use this short name in commands and config. You can rename it now.\n│ Press Escape to go back.\n│', ); }); it('adds a blank separator before compact text input values', () => { - expect(withTextInputNavigation('Project folder path')).toBe('Project folder path\nPress Escape to go back.\n'); + expect(withTextInputNavigation('Project folder path')).toBe('Project folder path\n│ Press Escape to go back.\n│'); }); it('normalizes already hinted text input prompts without duplicating the hint', () => { @@ -42,7 +42,19 @@ describe('prompt navigation helpers', () => { 'Name this PostgreSQL connection\nKTX will use this short name in commands and config. You can rename it now.\nPress Escape to go back.', ), ).toBe( - 'Name this PostgreSQL connection\n\nKTX will use this short name in commands and config. You can rename it now.\nPress Escape to go back.\n', + 'Name this PostgreSQL connection\n│\n│ KTX will use this short name in commands and config. You can rename it now.\n│ Press Escape to go back.\n│', ); }); + + it('is idempotent when text input navigation is applied twice', () => { + const once = withTextInputNavigation('Project folder path'); + expect(withTextInputNavigation(once)).toBe(once); + }); + + it('is idempotent when text input navigation with body is applied twice', () => { + const once = withTextInputNavigation( + 'Name this PostgreSQL connection\nKTX will use this short name in commands and config.', + ); + expect(withTextInputNavigation(once)).toBe(once); + }); }); diff --git a/packages/cli/src/prompt-navigation.ts b/packages/cli/src/prompt-navigation.ts index d80f2f97..c3644338 100644 --- a/packages/cli/src/prompt-navigation.ts +++ b/packages/cli/src/prompt-navigation.ts @@ -6,6 +6,26 @@ function removeTrailingBlankLines(message: string): string { return message.replace(/\n+$/, ''); } +function prefixContinuationLines(message: string): string { + const lines = message.split('\n'); + if (lines.length <= 1) return message; + const [title, ...body] = lines; + let trailingEmptyCount = 0; + while (trailingEmptyCount < body.length && body[body.length - 1 - trailingEmptyCount] === '') { + trailingEmptyCount++; + } + const contentBody = trailingEmptyCount > 0 ? body.slice(0, -trailingEmptyCount) : body; + const trailingBody = trailingEmptyCount > 0 ? body.slice(-trailingEmptyCount) : []; + return [ + title, + ...contentBody.map((line) => { + const stripped = line.replace(/^│\s*/, ''); + return stripped === '' ? '│' : `│ ${stripped}`; + }), + ...trailingBody, + ].join('\n'); +} + function withTextInputBodySpacing(message: string): string { const normalized = removeTrailingBlankLines(message); if (!normalized.includes('\n')) { @@ -39,7 +59,9 @@ export function withMultiselectNavigation(message: string): string { export function withTextInputNavigation(message: string): string { const messageWithoutHint = removeTrailingBlankLines(message) .split('\n') - .filter((line) => line !== TEXT_INPUT_NAVIGATION_HINT) + .filter((line) => !line.includes(TEXT_INPUT_NAVIGATION_HINT)) + .map((line) => line.replace(/^│\s*/, '')) .join('\n'); - return `${withTextInputBodySpacing(messageWithoutHint)}\n${TEXT_INPUT_NAVIGATION_HINT}\n`; + const full = `${withTextInputBodySpacing(messageWithoutHint)}\n${TEXT_INPUT_NAVIGATION_HINT}`; + return `${prefixContinuationLines(full)}\n│`; } diff --git a/packages/cli/src/setup-agents.test.ts b/packages/cli/src/setup-agents.test.ts index 9a984352..322db2aa 100644 --- a/packages/cli/src/setup-agents.test.ts +++ b/packages/cli/src/setup-agents.test.ts @@ -1,6 +1,7 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { readKtxSetupState } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { formatInstallSummary, @@ -89,7 +90,7 @@ describe('setup agents', () => { projectDir: tempDir, installs: [{ target: 'universal', scope: 'project', mode: 'cli' }], }); - expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).toContain('agents'); + expect(await readKtxSetupState(tempDir)).toEqual({ completed_steps: ['agents'] }); expect(io.stderr()).toBe(''); }); @@ -143,7 +144,7 @@ describe('setup agents', () => { await expect(readKtxAgentInstallManifest(tempDir)).resolves.toEqual(null); }); - it('uses prompts in interactive mode and supports Back', async () => { + it('treats cancel as skip in interactive mode', async () => { const io = makeIo(); const prompts = { select: vi.fn(async () => 'back'), @@ -165,7 +166,7 @@ describe('setup agents', () => { io.io, { prompts }, ), - ).resolves.toEqual({ status: 'back', projectDir: tempDir }); + ).resolves.toEqual({ status: 'skipped', projectDir: tempDir }); }); it('explains how to select multiple agent targets in interactive mode', async () => { diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index 1ce9b2e6..151967aa 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -1,7 +1,7 @@ import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { dirname, join, relative, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { cancel, isCancel, multiselect, select } from '@clack/prompts'; +import { cancel, confirm, isCancel, multiselect, select } from '@clack/prompts'; import { loadKtxProject, markKtxSetupStateStepComplete, @@ -277,12 +277,23 @@ function createPromptAdapter(): KtxSetupAgentsPromptAdapter { return String(value); }, async multiselect(options) { - const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options))); - if (isCancel(value)) { - cancel('Setup cancelled.'); - return ['back']; + while (true) { + const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options))); + if (isCancel(value)) { + cancel('Setup cancelled.'); + return ['back']; + } + const selected = [...value] as string[]; + if (selected.length === 0 && !options.required) { + const skipConfirmed = await confirm({ message: 'Nothing selected. Skip this step?', initialValue: false }); + if (isCancel(skipConfirmed)) { + cancel('Setup cancelled.'); + return ['back']; + } + if (!skipConfirmed) continue; + } + return selected; } - return [...value] as string[]; }, cancel(message) { cancel(message); @@ -375,7 +386,7 @@ export async function runKtxSetupAgentsStep( deps: KtxSetupAgentsDeps = {}, ): Promise { if (args.skipAgents) { - io.stdout.write('Agent integration skipped.\n'); + io.stdout.write('│ Agent integration skipped.\n'); return { status: 'skipped', projectDir: args.projectDir }; } if (!args.agents && args.inputMode === 'disabled') { @@ -391,10 +402,9 @@ export async function runKtxSetupAgentsStep( options: [ { value: 'cli', label: 'CLI tools and skills' }, { value: 'skip', label: 'Skip' }, - { value: 'back', label: 'Back' }, ], })) as KtxAgentInstallMode | 'skip' | 'back'); - if (mode === 'back') return { status: 'back', projectDir: args.projectDir }; + if (mode === 'back') return { status: 'skipped', projectDir: args.projectDir }; if (mode === 'skip') return { status: 'skipped', projectDir: args.projectDir }; const targets = diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index 76f1ca62..46506ae7 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -58,10 +58,10 @@ function connectionNamePrompt(label: string): string { function textInputPrompt(message: string): string { const normalized = message.replace(/\n+$/, ''); if (!normalized.includes('\n')) { - return `${normalized}\nPress Escape to go back.\n`; + return `${normalized}\n│ Press Escape to go back.\n│`; } const [title, ...bodyLines] = normalized.split('\n'); - return `${title}\n\n${bodyLines.join('\n')}\nPress Escape to go back.\n`; + return `${title}\n│\n│ ${bodyLines.join('\n│ ')}\n│ Press Escape to go back.\n│`; } const legacyHistoricSqlServiceAccountPatternsKey = ['serviceAccount', 'UserPatterns'].join(''); @@ -142,8 +142,8 @@ describe('setup databases step', () => { expect(prompts.select).toHaveBeenCalledWith({ message: 'How do you want to connect to PostgreSQL?', options: [ - { value: 'fields', label: 'Enter connection details (host, port, database, user)' }, { value: 'url', label: 'Paste a connection URL' }, + { value: 'fields', label: 'Enter connection details (host, port, database, user)' }, { value: 'back', label: 'Back' }, ], }); @@ -154,6 +154,43 @@ describe('setup databases step', () => { ); }); + it('offers connection URL paste first for URL-capable primary sources', async () => { + const cases: Array<{ driver: KtxSetupDatabaseDriver; label: string }> = [ + { driver: 'postgres', label: 'PostgreSQL' }, + { driver: 'mysql', label: 'MySQL' }, + { driver: 'clickhouse', label: 'ClickHouse' }, + { driver: 'sqlserver', label: 'SQL Server' }, + ]; + + for (const testCase of cases) { + const prompts = makePromptAdapter({ + selectValues: ['back'], + }); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'auto', + databaseDrivers: [testCase.driver], + skipDatabases: false, + databaseSchemas: [], + }, + makeIo().io, + { prompts }, + ); + + expect(result.status).toBe('back'); + expect(prompts.select).toHaveBeenCalledWith({ + message: `How do you want to connect to ${testCase.label}?`, + options: [ + { value: 'url', label: 'Paste a connection URL' }, + { value: 'fields', label: 'Enter connection details (host, port, database, user)' }, + { value: 'back', label: 'Back' }, + ], + }); + } + }); + it('lets Back leave database setup when the driver came from flags', async () => { const prompts = makePromptAdapter({ selectValues: ['back'] }); @@ -488,8 +525,8 @@ describe('setup databases step', () => { expect(prompts.select).toHaveBeenNthCalledWith(1, { message: 'How do you want to connect to PostgreSQL?', options: [ - { value: 'fields', label: 'Enter connection details (host, port, database, user)' }, { value: 'url', label: 'Paste a connection URL' }, + { value: 'fields', label: 'Enter connection details (host, port, database, user)' }, { value: 'back', label: 'Back' }, ], }); @@ -534,7 +571,6 @@ describe('setup databases step', () => { options: [ { value: 'continue', label: 'Continue to knowledge sources' }, { value: 'add', label: 'Add another primary source' }, - { value: 'back', label: 'Back' }, ], }); expect(testConnection).not.toHaveBeenCalled(); @@ -585,7 +621,6 @@ describe('setup databases step', () => { options: [ { value: 'continue', label: 'Continue to knowledge sources' }, { value: 'add', label: 'Add another primary source' }, - { value: 'back', label: 'Back' }, ], }); expect(testConnection).toHaveBeenCalledTimes(1); @@ -620,7 +655,6 @@ describe('setup databases step', () => { options: [ { value: 'continue', label: 'Continue to knowledge sources' }, { value: 'add', label: 'Add another primary source' }, - { value: 'back', label: 'Back' }, ], }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); @@ -655,7 +689,6 @@ describe('setup databases step', () => { options: [ { value: 'continue', label: 'Continue to knowledge sources' }, { value: 'add', label: 'Add another primary source' }, - { value: 'back', label: 'Back' }, ], }); }); @@ -698,7 +731,6 @@ describe('setup databases step', () => { options: [ { value: 'continue', label: 'Continue to knowledge sources' }, { value: 'add', label: 'Add another primary source' }, - { value: 'back', label: 'Back' }, ], }); }); @@ -918,10 +950,11 @@ describe('setup databases step', () => { [ '◇ Testing postgres-warehouse', '│ ✓ Connection test passed', - '│ Driver: PostgreSQL · Tables: 2', + '│ Driver: PostgreSQL', '│', ].join('\n'), ); + expect(io.stdout()).not.toContain('Tables: 2'); expect(io.stdout()).toContain( [ '◇ Scanning postgres-warehouse', diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index f0012e34..f770c5c4 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -1,5 +1,5 @@ import { writeFile } from 'node:fs/promises'; -import { cancel, isCancel, multiselect, password, select, text } from '@clack/prompts'; +import { cancel, confirm, isCancel, multiselect, password, select, text } from '@clack/prompts'; import type { HistoricSqlDialect } from '@ktx/context/ingest'; import { type KtxProjectConnectionConfig, @@ -9,6 +9,7 @@ import { setKtxSetupDatabaseConnectionIds, stripKtxSetupCompletedSteps, } from '@ktx/context/project'; +import type { KtxTableListEntry } from '@ktx/context/scan'; import type { KtxCliIo } from './cli-runtime.js'; import { runKtxConnection } from './connection.js'; import { withMenuOptionsSpacing, withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js'; @@ -83,6 +84,7 @@ export interface KtxSetupDatabasesDeps { testConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise; scanConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise; listSchemas?: (projectDir: string, connectionId: string) => Promise; + listTables?: (projectDir: string, connectionId: string) => Promise; historicSqlProbe?: KtxSetupHistoricSqlProbe; } @@ -203,12 +205,23 @@ function missingConnectionDetailsPrompt( function createPromptAdapter(): KtxSetupDatabasesPromptAdapter { return { async multiselect(options) { - const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options))); - if (isCancel(value)) { - cancel('Setup cancelled.'); - return ['back']; + while (true) { + const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options))); + if (isCancel(value)) { + cancel('Setup cancelled.'); + return ['back']; + } + const selected = [...value] as string[]; + if (selected.length === 0 && !options.required) { + const skipConfirmed = await confirm({ message: 'Nothing selected. Skip this step?', initialValue: false }); + if (isCancel(skipConfirmed)) { + cancel('Setup cancelled.'); + return ['back']; + } + if (!skipConfirmed) continue; + } + return selected; } - return [...value] as string[]; }, async select(options) { const value = await withSetupInterruptConfirmation(() => select(withMenuOptionsSpacing(options))); @@ -364,6 +377,89 @@ async function defaultListSchemas(projectDir: string, connectionId: string): Pro return []; } +function configuredSchemas(connection: KtxProjectConnectionConfig | undefined, driver: KtxSetupDatabaseDriver): string[] | undefined { + if (!connection) return undefined; + const spec = SCOPE_DISCOVERY_SPECS[driver]; + if (!spec) return undefined; + const values = configuredScopeValues(connection, spec); + return values.length > 0 ? values : undefined; +} + +async function defaultListTables(projectDir: string, connectionId: string): Promise { + const project = await loadKtxProject({ projectDir }); + const connection = project.config.connections[connectionId]; + const driver = normalizeDriver(connection?.driver); + const schemas = driver ? configuredSchemas(connection, driver) : undefined; + + if (driver === 'postgres') { + const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres'); + if (!isKtxPostgresConnectionConfig(connection)) return []; + const connector = new KtxPostgresScanConnector({ connectionId, connection }); + try { + return await connector.listTables(schemas); + } finally { + await connector.cleanup(); + } + } + + if (driver === 'mysql') { + const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('@ktx/connector-mysql'); + if (!isKtxMysqlConnectionConfig(connection)) return []; + const connector = new KtxMysqlScanConnector({ connectionId, connection }); + try { + return await connector.listTables(schemas); + } finally { + await connector.cleanup(); + } + } + + if (driver === 'sqlserver') { + const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('@ktx/connector-sqlserver'); + if (!isKtxSqlServerConnectionConfig(connection)) return []; + const connector = new KtxSqlServerScanConnector({ connectionId, connection }); + try { + return await connector.listTables(schemas); + } finally { + await connector.cleanup(); + } + } + + if (driver === 'bigquery') { + const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('@ktx/connector-bigquery'); + if (!isKtxBigQueryConnectionConfig(connection)) return []; + const connector = new KtxBigQueryScanConnector({ connectionId, connection }); + try { + return await connector.listTables(schemas); + } finally { + await connector.cleanup(); + } + } + + if (driver === 'snowflake') { + const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('@ktx/connector-snowflake'); + if (!isKtxSnowflakeConnectionConfig(connection)) return []; + const connector = new KtxSnowflakeScanConnector({ connectionId, connection }); + try { + return await connector.listTables(schemas); + } finally { + await connector.cleanup(); + } + } + + if (driver === 'clickhouse') { + const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('@ktx/connector-clickhouse'); + if (!isKtxClickHouseConnectionConfig(connection)) return []; + const connector = new KtxClickHouseScanConnector({ connectionId, connection }); + try { + return await connector.listTables(schemas); + } finally { + await connector.cleanup(); + } + } + + return []; +} + function existingConnectionIdsByDriver( connections: Record, driver: KtxSetupDatabaseDriver, @@ -400,7 +496,6 @@ function configuredPrimarySourcesPrompt(connectionIds: string[]): { options: [ { value: 'continue', label: 'Continue to knowledge sources' }, { value: 'add', label: 'Add another primary source' }, - { value: 'back', label: 'Back' }, ], }; } @@ -615,8 +710,8 @@ async function buildUrlConnectionConfig(input: { const choice = await input.prompts.select({ message: `How do you want to connect to ${label}?`, options: [ - { value: 'fields', label: 'Enter connection details (host, port, database, user)' }, { value: 'url', label: 'Paste a connection URL' }, + { value: 'fields', label: 'Enter connection details (host, port, database, user)' }, { value: 'back', label: 'Back' }, ], }); @@ -975,6 +1070,22 @@ async function writeScopeConfig(input: { }); } +async function clearScopeConfig(projectDir: string, connectionId: string): Promise { + const project = await loadKtxProject({ projectDir }); + const connection = project.config.connections[connectionId]; + if (!connection) return; + const driver = normalizeDriver(connection.driver); + if (!driver) return; + const spec = SCOPE_DISCOVERY_SPECS[driver]; + if (!spec) return; + const cleaned = Object.fromEntries( + Object.entries(connection).filter( + ([key]) => key !== spec.configArrayField && key !== spec.configSingleField && key !== 'enabled_tables', + ), + ) as KtxProjectConnectionConfig; + await writeConnectionConfig({ projectDir, connectionId, connection: cleaned }); +} + async function maybeConfigureSchemaScope(input: { projectDir: string; connectionId: string; @@ -1061,6 +1172,130 @@ async function maybeConfigureSchemaScope(input: { return true; } +async function maybeConfigureTableScope(input: { + projectDir: string; + connectionId: string; + args: KtxSetupDatabasesArgs; + prompts: KtxSetupDatabasesPromptAdapter; + io: KtxCliIo; + deps: KtxSetupDatabasesDeps; +}): Promise { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const connection = project.config.connections[input.connectionId]; + const driver = normalizeDriver(connection?.driver); + if (!driver || driver === 'sqlite') return true; + + const existingTables = connection?.enabled_tables; + if (Array.isArray(existingTables) && existingTables.length > 0) { + return true; + } + + if (input.args.inputMode === 'disabled') { + return true; + } + + writeSetupSection(input.io, 'Discovering tables', [ + `Connecting to ${input.connectionId}…`, + ]); + + let discovered: KtxTableListEntry[]; + try { + discovered = await (input.deps.listTables ?? defaultListTables)( + input.projectDir, + input.connectionId, + ); + } catch (error) { + input.io.stderr.write( + `Could not discover tables for ${input.connectionId}; continuing without table filter. ` + + `${error instanceof Error ? error.message : String(error)}\n`, + ); + return true; + } + + if (discovered.length === 0) { + return true; + } + + const allQualified = discovered.map((t) => `${t.schema}.${t.name}`); + + if (discovered.length === 1) { + await writeConnectionConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + connection: { ...connection!, enabled_tables: allQualified }, + }); + writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ + `✓ ${allQualified[0]}`, + ]); + return true; + } + + const bySchema = new Map(); + for (const entry of discovered) { + const existing = bySchema.get(entry.schema) ?? []; + existing.push(entry); + bySchema.set(entry.schema, existing); + } + const schemaList = [...bySchema.keys()].sort(); + const schemaSummary = schemaList.map((s) => `${s} (${bySchema.get(s)!.length})`).join(', '); + + let selected: string[] | null = null; + + while (selected === null) { + const action = await input.prompts.select({ + message: `Tables found in selected schemas\n` + + `${discovered.length} tables across ${schemaList.length} ${schemaList.length === 1 ? 'schema' : 'schemas'}: ${schemaSummary}`, + options: [ + { value: 'all', label: 'Enable all tables' }, + { value: 'customize', label: 'Customize which tables to enable' }, + { value: 'back', label: 'Back' }, + ], + }); + + if (action === 'back') { + return false; + } + + if (action === 'all') { + selected = allQualified; + } else { + const choices = await input.prompts.multiselect({ + message: withMultiselectNavigation( + `Tables to enable for ${input.connectionId}\n` + + `Deselect any tables agents should not use.`, + ), + options: discovered.map((t) => { + const qualified = `${t.schema}.${t.name}`; + const suffix = t.kind === 'view' ? ' (view)' : ''; + return { value: qualified, label: `${qualified}${suffix}` }; + }), + initialValues: allQualified, + required: true, + }); + + if (choices.includes('back')) { + continue; + } + if (choices.length === 0) { + input.io.stdout.write('│ KTX needs at least one table enabled. Select a table or press Escape to go back.\n'); + continue; + } + selected = choices; + } + } + + await writeConnectionConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + connection: { ...connection!, enabled_tables: selected }, + }); + + writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ + `✓ ${selected.length}/${discovered.length} tables enabled`, + ]); + return true; +} + async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); const adapters = project.config.ingest.adapters.includes('historic-sql') @@ -1115,7 +1350,7 @@ async function maybeRunHistoricSqlSetupProbe(input: { return; } - input.io.stdout.write('Historic SQL probe...\n'); + input.io.stdout.write('│ Historic SQL probe...\n'); const probe = input.deps.historicSqlProbe ?? defaultHistoricSqlProbe; const result = await probe({ projectDir: input.projectDir, @@ -1123,10 +1358,10 @@ async function maybeRunHistoricSqlSetupProbe(input: { dialect: 'postgres', }); for (const line of result.lines) { - input.io.stdout.write(`${line}\n`); + input.io.stdout.write(`│${line}\n`); } if (!result.ok) { - input.io.stdout.write('Setup written; first ingest run will fail until fixed.\n'); + input.io.stdout.write('│ Setup written; first ingest run will fail until fixed.\n'); } } @@ -1184,13 +1419,19 @@ async function validateAndScanConnection(input: { const testOutput = testIo.stdoutText(); const outputDriver = normalizeDriver(readOutputValue(testOutput, 'Driver')); const driverDisplay = outputDriver ? driverLabel(outputDriver) : (configuredDriverLabel ?? 'Unknown driver'); - const tableCount = Number(readOutputValue(testOutput, 'Tables') ?? NaN); - const testLines = ['✓ Connection test passed']; - testLines.push(`Driver: ${driverDisplay}${Number.isFinite(tableCount) ? ` · Tables: ${tableCount}` : ''}`); + const testLines = ['✓ Connection test passed', `Driver: ${driverDisplay}`]; writeSetupSection(input.io, `Testing ${input.connectionId}`, testLines); - if (!(await maybeConfigureSchemaScope(input))) { - return false; + while (true) { + if (!(await maybeConfigureSchemaScope(input))) { + return false; + } + + if (await maybeConfigureTableScope(input)) { + break; + } + + await clearScopeConfig(input.projectDir, input.connectionId); } await maybeRunHistoricSqlSetupProbe({ @@ -1261,7 +1502,7 @@ async function chooseDrivers( return 'back'; } - io.stdout.write('KTX cannot work without at least one primary source. Select a source or press Escape to go back.\n'); + io.stdout.write('│ KTX cannot work without at least one primary source. Select a source or press Escape to go back.\n'); } } @@ -1325,7 +1566,7 @@ export async function runKtxSetupDatabasesStep( deps: KtxSetupDatabasesDeps = {}, ): Promise { if (args.skipDatabases) { - io.stdout.write('Primary source setup skipped. KTX cannot work until you add a primary source.\n'); + io.stdout.write('│ Primary source setup skipped. KTX cannot work until you add a primary source.\n'); return { status: 'skipped', projectDir: args.projectDir }; } @@ -1361,13 +1602,10 @@ export async function runKtxSetupDatabasesStep( while (true) { if (showConfiguredPrimaryMenu) { const action = await prompts.select(configuredPrimarySourcesPrompt(selectedConnectionIds)); - if (action === 'continue') { + if (action === 'continue' || action === 'back') { await markDatabasesComplete(args.projectDir, selectedConnectionIds); return { status: 'ready', projectDir: args.projectDir, connectionIds: selectedConnectionIds }; } - if (action === 'back') { - return { status: 'back', projectDir: args.projectDir }; - } } showConfiguredPrimaryMenu = false; @@ -1382,7 +1620,7 @@ export async function runKtxSetupDatabasesStep( if (drivers === 'missing-input') return { status: 'missing-input', projectDir: args.projectDir }; if (drivers.length === 0) { await markDatabasesComplete(args.projectDir, []); - io.stdout.write('KTX cannot work without a primary source.\n'); + io.stdout.write('│ KTX cannot work without a primary source.\n'); return { status: 'skipped', projectDir: args.projectDir }; } diff --git a/packages/cli/src/setup-demo-tour.test.ts b/packages/cli/src/setup-demo-tour.test.ts index d3f7e3f4..a8b63974 100644 --- a/packages/cli/src/setup-demo-tour.test.ts +++ b/packages/cli/src/setup-demo-tour.test.ts @@ -209,7 +209,7 @@ describe('runDemoTour', () => { }, ); expect(result).toBe(0); - // Navigation called once for databases step, then exits + // Navigation called once for intro, then exits on back expect(navigation).toHaveBeenCalledTimes(1); }); @@ -218,10 +218,11 @@ describe('runDemoTour', () => { let callCount = 0; const navigation = vi.fn().mockImplementation(() => { callCount++; - // First call (databases): forward - // Second call (sources): back - // Third call (databases again): back (exit) - if (callCount === 1) return Promise.resolve('forward'); + // First call (intro): forward + // Second call (databases): forward + // Third call (sources): back + // Fourth call (databases again): back (exit) + if (callCount <= 2) return Promise.resolve('forward'); return Promise.resolve('back'); }); @@ -235,7 +236,7 @@ describe('runDemoTour', () => { }, ); expect(result).toBe(0); - expect(navigation).toHaveBeenCalledTimes(3); + expect(navigation).toHaveBeenCalledTimes(4); }); it('handles agent step returning back', async () => { @@ -243,10 +244,10 @@ describe('runDemoTour', () => { let navCount = 0; const navigation = vi.fn().mockImplementation(() => { navCount++; - // Forward through databases, sources, context + // Forward through intro, databases, sources, context // Then back from context (after agents returns back) // Then back from sources, then back from databases (exit) - if (navCount <= 3) return Promise.resolve('forward'); + if (navCount <= 4) return Promise.resolve('forward'); return Promise.resolve('back'); }); diff --git a/packages/cli/src/setup-demo-tour.ts b/packages/cli/src/setup-demo-tour.ts index f3d71f70..fe211e67 100644 --- a/packages/cli/src/setup-demo-tour.ts +++ b/packages/cli/src/setup-demo-tour.ts @@ -62,12 +62,15 @@ function createTargetState(target: KtxPublicIngestPlanTarget): ContextBuildTarge // Pure rendering functions // --------------------------------------------------------------------------- -export function renderDemoBanner(): string { +export function renderDemoBanner(projectDir?: string): string { const lines = [ '', `┌ ${cyan('Demo mode')} — data has been pre-processed and KTX context is already built.`, '│ This walkthrough illustrates the setup steps. Selections are pre-filled and read-only.', ]; + if (projectDir) { + lines.push(`│ Project directory: ${dim(projectDir)}`); + } return lines.join('\n'); } @@ -144,16 +147,15 @@ export async function waitForDemoNavigation( }; const onData = (data: Buffer) => { - const char = data.toString(); - if (char === '\r' || char === '\n') { - cleanup(); - resolve('forward'); - } else if (char === '\x1b') { - cleanup(); - resolve('back'); - } else if (char === '\x03') { + if (data[0] === 0x03) { cleanup(); reject(new KtxSetupExitError()); + } else if (data[0] === 0x0d || data[0] === 0x0a) { + cleanup(); + resolve('forward'); + } else if (data[0] === 0x1b) { + cleanup(); + resolve('back'); } }; @@ -171,8 +173,9 @@ export async function renderDemoCard( io: KtxCliIo, stdin?: NodeJS.ReadStream, waitNav: (stdin?: NodeJS.ReadStream) => Promise<'forward' | 'back'> = waitForDemoNavigation, + projectDir?: string, ): Promise<'forward' | 'back'> { - io.stdout.write(renderDemoBanner() + '\n\n'); + io.stdout.write(renderDemoBanner(projectDir) + '\n\n'); io.stdout.write(renderDemoCardContent(title, selections) + '\n'); return waitNav(stdin); } @@ -337,6 +340,11 @@ export async function runDemoTour( const projectDir = defaultDemoProjectDir(); await ensureProject({ projectDir, force: false }); + io.stdout.write(renderDemoBanner(projectDir) + '\n'); + io.stdout.write(`\n│ ${dim('Press Enter to continue, Escape to go back')}\n└\n`); + const introDirection = await waitNav(); + if (introDirection === 'back') return 0; + let stepIndex = 0; while (stepIndex < DEMO_STEPS.length) { @@ -344,11 +352,11 @@ export async function runDemoTour( let direction: 'forward' | 'back'; if (step === 'databases') { - direction = await renderDemoCard('Database connection', ['PostgreSQL — Orbit Analytics (56 tables, 2 schemas)'], io, undefined, waitNav); + direction = await renderDemoCard('Database connection', ['PostgreSQL — Orbit Analytics (56 tables, 2 schemas)'], io, undefined, waitNav, projectDir); } else if (step === 'sources') { - direction = await renderDemoCard('Context sources', ['dbt — 34 transformation models', 'Metabase — 80 dashboard cards', 'Notion — 9 knowledge pages'], io, undefined, waitNav); + direction = await renderDemoCard('Context sources', ['dbt — 34 transformation models', 'Metabase — 80 dashboard cards', 'Notion — 9 knowledge pages'], io, undefined, waitNav, projectDir); } else if (step === 'context') { - io.stdout.write(renderDemoBanner() + '\n\n'); + io.stdout.write(renderDemoBanner(projectDir) + '\n\n'); if (deps.skipReplayAnimation) { direction = await waitNav(); } else { diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index 24c567ad..67ef83b3 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -133,6 +133,12 @@ describe('setup embeddings step', () => { const healthCheck = vi.fn(async () => ({ ok: true as const })); const prompts = makePromptAdapter({ selectValues: ['sentence-transformers'] }); const ensureLocalEmbeddings = vi.fn(async () => managedDaemon()); + const spinnerEvents: string[] = []; + const spinner = vi.fn(() => ({ + start: (msg: string) => spinnerEvents.push(`start:${msg}`), + stop: (msg: string) => spinnerEvents.push(`stop:${msg}`), + error: (msg: string) => spinnerEvents.push(`error:${msg}`), + })); const result = await runKtxSetupEmbeddingsStep( { @@ -143,7 +149,7 @@ describe('setup embeddings step', () => { skipEmbeddings: false, }, io.io, - { prompts, env: {}, healthCheck, ensureLocalEmbeddings }, + { prompts, env: {}, healthCheck, ensureLocalEmbeddings, spinner }, ); expect(result.status).toBe('ready'); @@ -168,8 +174,8 @@ describe('setup embeddings step', () => { expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings); expect(config.setup?.completed_steps).toEqual(undefined); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings'); - expect(io.stdout()).toContain( - 'Testing local sentence-transformers embeddings (all-MiniLM-L6-v2, 384 dimensions). First run may take up to 60 seconds.', + expect(spinnerEvents).toContainEqual( + 'start:Testing local sentence-transformers embeddings (all-MiniLM-L6-v2, 384 dimensions). First run may take up to 60 seconds.', ); expect(io.stdout()).toContain('Embeddings ready: yes'); }); @@ -184,6 +190,12 @@ describe('setup embeddings step', () => { resolveHealthCheck = resolve; }), ); + const spinnerEvents: string[] = []; + const spinner = vi.fn(() => ({ + start: (msg: string) => spinnerEvents.push(`start:${msg}`), + stop: (msg: string) => spinnerEvents.push(`stop:${msg}`), + error: (msg: string) => spinnerEvents.push(`error:${msg}`), + })); const result = runKtxSetupEmbeddingsStep( { @@ -194,12 +206,12 @@ describe('setup embeddings step', () => { skipEmbeddings: false, }, io.io, - { prompts, env: {}, healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()) }, + { prompts, env: {}, healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()), spinner }, ); await vi.waitFor(() => { - expect(io.stdout()).toContain( - '\r- Testing local sentence-transformers embeddings (all-MiniLM-L6-v2, 384 dimensions). First run may take up to 60 seconds.', + expect(spinnerEvents).toContainEqual( + 'start:Testing local sentence-transformers embeddings (all-MiniLM-L6-v2, 384 dimensions). First run may take up to 60 seconds.', ); }); diff --git a/packages/cli/src/setup-embeddings.ts b/packages/cli/src/setup-embeddings.ts index f7995796..1b6a2381 100644 --- a/packages/cli/src/setup-embeddings.ts +++ b/packages/cli/src/setup-embeddings.ts @@ -13,6 +13,7 @@ import { } from '@ktx/context/project'; import { type KtxEmbeddingConfig, type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from '@ktx/llm'; import type { KtxCliIo } from './cli-runtime.js'; +import { createClackSpinner, type KtxCliSpinner } from './clack.js'; import { ensureManagedLocalEmbeddingsDaemon, managedLocalEmbeddingHealthConfig, @@ -61,6 +62,7 @@ export interface KtxSetupEmbeddingsDeps { installPolicy: KtxManagedPythonInstallPolicy; io: KtxCliIo; }) => Promise; + spinner?: () => KtxCliSpinner; } type BackendChoice = KtxSetupEmbeddingBackend | 'back'; @@ -83,14 +85,6 @@ const EMBEDDING_OPTION_PROMPT_CONTEXT = 'KTX uses embeddings for semantic search over semantic-layer sources, wiki context, schema metadata, ' + 'and relationship evidence.'; const LOCAL_EMBEDDING_HEALTH_TIMEOUT_MS = 120_000; -const HEALTH_CHECK_SPINNER_FRAMES = ['-', '\\', '|', '/'] as const; -const HEALTH_CHECK_SPINNER_INTERVAL_MS = 120; -const CLEAR_CURRENT_LINE = '\x1b[2K\r'; - -interface HealthCheckProgress { - succeed(message: string): void; - fail(message: string): void; -} function createPromptAdapter(): KtxSetupEmbeddingsPromptAdapter { return { @@ -260,7 +254,7 @@ async function chooseCredentialRef( } if (choice === 'paste') { io.stdout.write( - `${[ + `│ ${[ `KTX will save the key in .ktx/secrets/${backend}-api-key with local file permissions,`, 'then write a file: reference in ktx.yaml.', ].join(' ')}\n`, @@ -350,42 +344,17 @@ function healthCheckStartText(backend: KtxSetupEmbeddingBackend, model: string, return `Checking ${backend} embeddings (${model}, ${dimensions} dimensions).`; } -function startHealthCheckProgress(io: KtxCliIo, message: string): HealthCheckProgress { - if (io.stdout.isTTY !== true) { - io.stdout.write(`${message}\n`); - const noop = () => undefined; - return { - succeed: noop, - fail: noop, - }; - } - - let frameIndex = 0; - let stopped = false; - const writeFrame = () => { - io.stdout.write(`${CLEAR_CURRENT_LINE}${HEALTH_CHECK_SPINNER_FRAMES[frameIndex]} ${message}`); - }; - writeFrame(); - const interval = setInterval(() => { - frameIndex = (frameIndex + 1) % HEALTH_CHECK_SPINNER_FRAMES.length; - writeFrame(); - }, HEALTH_CHECK_SPINNER_INTERVAL_MS); - - const stop = (finalMessage: string) => { - if (stopped) { - return; - } - stopped = true; - clearInterval(interval); - io.stdout.write(`${CLEAR_CURRENT_LINE}${finalMessage}\n`); - }; - +function startHealthCheckProgress( + spinner: KtxCliSpinner, + message: string, +): { succeed(msg: string): void; fail(msg: string): void } { + spinner.start(message); return { - succeed(message) { - stop(message); + succeed(msg: string) { + spinner.stop(msg); }, - fail(message) { - stop(message); + fail(msg: string) { + spinner.error(msg); }, }; } @@ -396,7 +365,7 @@ export async function runKtxSetupEmbeddingsStep( deps: KtxSetupEmbeddingsDeps = {}, ): Promise { if (args.skipEmbeddings) { - io.stdout.write('Embeddings setup skipped.\n'); + io.stdout.write('│ Embeddings setup skipped.\n'); return { status: 'skipped', projectDir: args.projectDir }; } @@ -408,7 +377,7 @@ export async function runKtxSetupEmbeddingsStep( !args.embeddingApiKeyEnv && !args.embeddingApiKeyFile ) { - io.stdout.write(`Embeddings ready: yes (${project.config.ingest.embeddings.model})\n`); + io.stdout.write(`│ Embeddings ready: yes (${project.config.ingest.embeddings.model})\n`); return { status: 'ready', projectDir: args.projectDir }; } @@ -474,7 +443,8 @@ export async function runKtxSetupEmbeddingsStep( dimensions, credentialValue, }); - const progress = startHealthCheckProgress(io, healthCheckStartText(selectedBackend, model, dimensions)); + const healthSpinner = (deps.spinner ?? createClackSpinner)(); + const progress = startHealthCheckProgress(healthSpinner, healthCheckStartText(selectedBackend, model, dimensions)); let health: KtxEmbeddingHealthCheckResult; try { health = await healthCheck(healthConfig); @@ -495,7 +465,7 @@ export async function runKtxSetupEmbeddingsStep( credentialRef, }), ); - io.stdout.write(`Embeddings ready: yes (${model}, ${dimensions} dimensions)\n`); + io.stdout.write(`│ Embeddings ready: yes (${model}, ${dimensions} dimensions)\n`); return { status: 'ready', projectDir: args.projectDir }; } diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/src/setup-models.test.ts index 0c7686f7..82f82875 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/src/setup-models.test.ts @@ -312,7 +312,7 @@ describe('setup Anthropic model step', () => { expect(result.status).toBe('ready'); expect(prompts.select).not.toHaveBeenCalledWith(expect.objectContaining({ message: 'Paste Anthropic API key now?' })); expect(prompts.password).toHaveBeenCalledWith({ - message: 'Anthropic API key\nPress Escape to go back.\n', + message: 'Anthropic API key\n│ Press Escape to go back.\n│', }); }); @@ -464,7 +464,7 @@ describe('setup Anthropic model step', () => { ); expect(prompts.text).toHaveBeenCalledWith( expect.objectContaining({ - message: 'Anthropic model ID\nPress Escape to go back.\n', + message: 'Anthropic model ID\n│ Press Escape to go back.\n│', placeholder: 'claude-sonnet-4-6', }), ); @@ -629,7 +629,7 @@ describe('setup Anthropic model step', () => { expect(result.status).toBe('ready'); expect(prompts.password).toHaveBeenCalledWith({ - message: 'Anthropic API key\nPress Escape to go back.\n', + message: 'Anthropic API key\n│ Press Escape to go back.\n│', }); await expect(readFile(join(tempDir, '.ktx/secrets/anthropic-api-key'), 'utf-8')).rejects.toMatchObject({ code: 'ENOENT', diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index 843691cd..6d3c6757 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -255,7 +255,7 @@ async function chooseCredentialRef( const prompts = deps.prompts ?? createPromptAdapter(); if (args.showPromptInstructions !== false) { io.stdout.write( - 'Use Up/Down to move, Enter to confirm the current selection, choose Back to return to the previous step, Ctrl+C to exit.\n', + '│ Use Up/Down to move, Enter to confirm the current selection, choose Back to return to the previous step, Ctrl+C to exit.\n', ); } while (true) { @@ -272,7 +272,7 @@ async function chooseCredentialRef( } if (choice === 'paste') { io.stdout.write( - 'KTX will save the key in .ktx/secrets/anthropic-api-key with local file permissions, then write a file: reference in ktx.yaml.\n', + '│ KTX will save the key in .ktx/secrets/anthropic-api-key with local file permissions, then write a file: reference in ktx.yaml.\n', ); const value = await prompts.password({ message: withTextInputNavigation('Anthropic API key') }); if (value === undefined) { @@ -394,7 +394,7 @@ export async function runKtxSetupAnthropicModelStep( deps: KtxSetupModelDeps = {}, ): Promise { if (args.skipLlm) { - io.stdout.write('LLM setup skipped.\n'); + io.stdout.write('│ LLM setup skipped.\n'); return { status: 'skipped', projectDir: args.projectDir }; } @@ -406,7 +406,7 @@ export async function runKtxSetupAnthropicModelStep( !args.anthropicApiKeyFile && !args.anthropicModel ) { - io.stdout.write(`LLM ready: yes (${project.config.llm.models.default})\n`); + io.stdout.write(`│ LLM ready: yes (${project.config.llm.models.default})\n`); return { status: 'ready', projectDir: args.projectDir }; } @@ -439,7 +439,7 @@ export async function runKtxSetupAnthropicModelStep( const health = await healthCheck(buildHealthConfig(credential.value, model.model)); if (health.ok) { await persistLlmConfig(args.projectDir, credential.ref, model.model); - io.stdout.write(`LLM ready: yes (${model.model})\n`); + io.stdout.write(`│ LLM ready: yes (${model.model})\n`); return { status: 'ready', projectDir: args.projectDir }; } diff --git a/packages/cli/src/setup-project.test.ts b/packages/cli/src/setup-project.test.ts index 3dd1b0cd..9c01402c 100644 --- a/packages/cli/src/setup-project.test.ts +++ b/packages/cli/src/setup-project.test.ts @@ -142,10 +142,11 @@ describe('setup project step', () => { expect(result.projectDir).toBe(projectDir); expect(prompts.select).toHaveBeenCalledWith( expect.objectContaining({ - message: 'Which KTX project should setup use?', + message: 'Where should KTX create the project?', options: [ - expect.objectContaining({ value: 'current', label: 'Use current directory' }), - expect.objectContaining({ value: 'new', label: 'Create a new project folder' }), + expect.objectContaining({ value: 'current', label: 'Current directory' }), + expect.objectContaining({ value: 'new-default', label: 'New subfolder (./ktx-project)' }), + expect.objectContaining({ value: 'new-custom', label: 'Custom path' }), expect.objectContaining({ value: 'exit', label: 'Exit' }), ], }), @@ -159,7 +160,7 @@ describe('setup project step', () => { it('offers an absolute default destination for a new project folder', async () => { const startDir = join(tempDir, 'start'); const projectDir = join(startDir, 'ktx-project'); - const prompts = makePromptAdapter({ choices: ['new', 'default', 'create'] }); + const prompts = makePromptAdapter({ choices: ['new-default', 'create'] }); const testIo = makeIo({ stdoutIsTty: true }); const result = await runKtxSetupProjectStep( @@ -171,33 +172,28 @@ describe('setup project step', () => { expect(result.status).toBe('ready'); expect(result.projectDir).toBe(projectDir); expect(prompts.select).toHaveBeenNthCalledWith( - 2, + 1, expect.objectContaining({ message: 'Where should KTX create the project?', - options: [ - expect.objectContaining({ - value: 'default', - label: `Create the default project folder: ${projectDir}`, - }), - expect.objectContaining({ value: 'custom', label: 'Enter a custom path' }), - expect.objectContaining({ value: 'back', label: 'Back' }), - ], + options: expect.arrayContaining([ + expect.objectContaining({ value: 'new-default', label: 'New subfolder (./ktx-project)' }), + ]), }), ); expect(prompts.select).toHaveBeenNthCalledWith( - 3, + 2, expect.objectContaining({ message: `Create KTX project at ${projectDir}?` }), ); expect(prompts.text).not.toHaveBeenCalled(); expect(result.status === 'ready' ? result.project.config.project : '').toBe('ktx-project'); - expect(testIo.stdout()).toContain(`KTX will create:\n ${projectDir}`); + expect(testIo.stdout()).toContain(`│ KTX will create:\n│ ${projectDir}`); await expect(stat(join(projectDir, 'ktx.yaml'))).resolves.toBeDefined(); }); it('prompts for a custom path and resolves it inside the current setup directory', async () => { const startDir = join(tempDir, 'start'); const projectDir = join(startDir, 'analytics-ktx'); - const prompts = makePromptAdapter({ choices: ['new', 'custom', 'create'], textValue: 'analytics-ktx' }); + const prompts = makePromptAdapter({ choices: ['new-custom', 'create'], textValue: 'analytics-ktx' }); const result = await runKtxSetupProjectStep( { projectDir: startDir, mode: 'auto', inputMode: 'auto', yes: false }, @@ -209,7 +205,7 @@ describe('setup project step', () => { expect(result.projectDir).toBe(projectDir); expect(prompts.text).toHaveBeenCalledWith( expect.objectContaining({ - message: 'Project folder path\nPress Escape to go back.\n', + message: 'Project folder path\n│ Press Escape to go back.\n│', placeholder: './analytics-ktx, ~/analytics-ktx, or /Users/you/projects/analytics-ktx', }), ); @@ -220,7 +216,7 @@ describe('setup project step', () => { const startDir = join(tempDir, 'start'); const homeDir = join(tempDir, 'home'); const projectDir = join(homeDir, 'analytics-ktx'); - const prompts = makePromptAdapter({ choices: ['new', 'custom', 'create'], textValue: '~/analytics-ktx' }); + const prompts = makePromptAdapter({ choices: ['new-custom', 'create'], textValue: '~/analytics-ktx' }); const result = await runKtxSetupProjectStep( { projectDir: startDir, mode: 'auto', inputMode: 'auto', yes: false }, @@ -238,7 +234,7 @@ describe('setup project step', () => { const homeDir = join(tempDir, 'home'); const customProjectDir = join(homeDir, 'analytics-ktx'); const prompts = makePromptAdapter({ - choices: ['new', 'custom', 'back', 'exit'], + choices: ['new-custom', 'back', 'exit'], textValue: '~/analytics-ktx', }); @@ -251,7 +247,7 @@ describe('setup project step', () => { expect(result.status).toBe('cancelled'); expect(result.projectDir).toBe(startDir); expect(prompts.select).toHaveBeenNthCalledWith( - 3, + 2, expect.objectContaining({ message: `Create KTX project at ${customProjectDir}?`, options: [ @@ -262,15 +258,15 @@ describe('setup project step', () => { }), ); expect(prompts.select).toHaveBeenNthCalledWith( - 4, - expect.objectContaining({ message: 'Which KTX project should setup use?' }), + 3, + expect.objectContaining({ message: 'Where should KTX create the project?' }), ); await expect(stat(join(customProjectDir, 'ktx.yaml'))).rejects.toThrow(); }); it('rejects an empty new folder path without creating a project in the process cwd', async () => { const startDir = join(tempDir, 'start'); - const prompts = makePromptAdapter({ choices: ['new', 'custom'], textValue: ' ' }); + const prompts = makePromptAdapter({ choices: ['new-custom'], textValue: ' ' }); const initProject = vi.fn(async () => { throw new Error('initProject should not run for an empty path'); }); @@ -295,7 +291,7 @@ describe('setup project step', () => { const projectDir = join(startDir, 'analytics-ktx'); await mkdir(projectDir, { recursive: true }); await writeFile(join(projectDir, 'README.md'), 'Existing project notes\n', 'utf-8'); - const prompts = makePromptAdapter({ choices: ['new', 'custom', 'use-existing'], textValue: 'analytics-ktx' }); + const prompts = makePromptAdapter({ choices: ['new-custom', 'use-existing'], textValue: 'analytics-ktx' }); const result = await runKtxSetupProjectStep( { projectDir: startDir, mode: 'auto', inputMode: 'auto', yes: false }, @@ -306,7 +302,7 @@ describe('setup project step', () => { expect(result.status).toBe('ready'); expect(result.projectDir).toBe(projectDir); expect(prompts.select).toHaveBeenNthCalledWith( - 3, + 2, expect.objectContaining({ message: `That folder already exists and is not empty: ${projectDir}`, options: expect.arrayContaining([ diff --git a/packages/cli/src/setup-project.ts b/packages/cli/src/setup-project.ts index d164e41d..18512b03 100644 --- a/packages/cli/src/setup-project.ts +++ b/packages/cli/src/setup-project.ts @@ -113,6 +113,55 @@ async function existingFolderState( } } +type ConfirmProjectDirResult = + | { status: 'confirmed'; confirmedCreation: boolean } + | { status: 'choose-another' } + | { status: 'back' } + | { status: 'cancelled' } + | { status: 'not-directory' }; + +async function confirmProjectDir( + selectedDir: string, + io: KtxCliIo, + prompts: KtxSetupProjectPromptAdapter, +): Promise { + const state = await existingFolderState(selectedDir); + + if (state === 'not-directory') { + io.stderr.write(`Project folder path exists and is not a directory: ${selectedDir}\n`); + return { status: 'not-directory' }; + } + + if (state === 'non-empty-directory') { + const action = await prompts.select({ + message: `That folder already exists and is not empty: ${selectedDir}`, + options: [ + { value: 'use-existing', label: 'Yes, create KTX files there' }, + { value: 'choose-another', label: 'Choose another folder' }, + { value: 'back', label: 'Back' }, + ], + }); + if (action === 'choose-another') return { status: 'choose-another' }; + if (action === 'back') return { status: 'back' }; + if (action !== 'use-existing') return { status: 'cancelled' }; + return { status: 'confirmed', confirmedCreation: true }; + } + + io.stdout.write(`│ KTX will create:\n│ ${selectedDir}\n`); + const action = await prompts.select({ + message: `Create KTX project at ${selectedDir}?`, + options: [ + { value: 'create', label: 'Create project' }, + { value: 'choose-another', label: 'Choose another folder' }, + { value: 'back', label: 'Back' }, + ], + }); + if (action === 'choose-another') return { status: 'choose-another' }; + if (action === 'back') return { status: 'back' }; + if (action !== 'create') return { status: 'cancelled' }; + return { status: 'confirmed', confirmedCreation: true }; +} + async function normalizeSetupGitignore(projectDir: string): Promise { const gitignorePath = join(projectDir, '.ktx/.gitignore'); await mkdir(join(projectDir, '.ktx'), { recursive: true }); @@ -143,7 +192,7 @@ async function loadExistingProject(projectDir: string, deps: KtxSetupProjectDeps } function printProjectSummary(io: KtxCliIo, projectDir: string): void { - io.stdout.write(`Project: ${projectDir}\n`); + io.stdout.write(`│ Project: ${projectDir}\n`); } async function promptForNewProjectDir( @@ -155,8 +204,6 @@ async function promptForNewProjectDir( const defaultProjectDir = join(projectDir, DEFAULT_NEW_PROJECT_FOLDER_NAME); while (true) { - io.stdout.write(`Relative paths are resolved from:\n ${projectDir}\n`); - io.stdout.write(`Home paths are resolved from:\n ${homeDir}\n`); const destinationChoice = await prompts.select({ message: 'Where should KTX create the project?', options: [ @@ -193,55 +240,12 @@ async function promptForNewProjectDir( return { status: 'cancelled', projectDir }; } - const state = await existingFolderState(selectedDir); - let confirmedCreation = false; - if (state === 'not-directory') { - io.stderr.write(`Project folder path exists and is not a directory: ${selectedDir}\n`); - return { status: 'missing-input', projectDir }; - } - if (state === 'non-empty-directory') { - const existingAction = await prompts.select({ - message: `That folder already exists and is not empty: ${selectedDir}`, - options: [ - { value: 'use-existing', label: 'Yes, create KTX files there' }, - { value: 'choose-another', label: 'Choose another folder' }, - { value: 'back', label: 'Back' }, - ], - }); - if (existingAction === 'choose-another') { - continue; - } - if (existingAction === 'back') { - return { status: 'back', projectDir }; - } - if (existingAction !== 'use-existing') { - return { status: 'cancelled', projectDir }; - } - confirmedCreation = true; - } - - io.stdout.write(`KTX will create:\n ${selectedDir}\n`); - if (state !== 'non-empty-directory') { - const createAction = await prompts.select({ - message: `Create KTX project at ${selectedDir}?`, - options: [ - { value: 'create', label: 'Create project' }, - { value: 'choose-another', label: 'Choose another folder' }, - { value: 'back', label: 'Back' }, - ], - }); - if (createAction === 'choose-another') { - continue; - } - if (createAction === 'back') { - return { status: 'back', projectDir }; - } - if (createAction !== 'create') { - return { status: 'cancelled', projectDir }; - } - confirmedCreation = true; - } - return { status: 'selected', projectDir: selectedDir, confirmedCreation }; + const confirmed = await confirmProjectDir(selectedDir, io, prompts); + if (confirmed.status === 'not-directory') return { status: 'missing-input', projectDir }; + if (confirmed.status === 'choose-another') continue; + if (confirmed.status === 'back') return { status: 'back', projectDir }; + if (confirmed.status === 'cancelled') return { status: 'cancelled', projectDir }; + return { status: 'selected', projectDir: selectedDir, confirmedCreation: confirmed.confirmedCreation }; } } @@ -323,15 +327,17 @@ export async function runKtxSetupProjectStep( } const prompts = deps.prompts ?? createClackSetupProjectPromptAdapter(); + const defaultProjectDir = join(projectDir, DEFAULT_NEW_PROJECT_FOLDER_NAME); io.stdout.write( - 'Use Up/Down to move, Enter to confirm the current selection, choose Back to return to the previous step, Ctrl+C to exit.\n', + '│ Use Up/Down to move, Enter to confirm the current selection, choose Back to return to the previous step, Ctrl+C to exit.\n', ); while (true) { const choice = await prompts.select({ - message: 'Which KTX project should setup use?', + message: 'Where should KTX create the project?', options: [ - { value: 'current', label: 'Use current directory' }, - { value: 'new', label: 'Create a new project folder' }, + { value: 'current', label: 'Current directory' }, + { value: 'new-default', label: 'New subfolder (./ktx-project)' }, + { value: 'new-custom', label: 'Custom path' }, ...(args.allowBack ? [{ value: 'back', label: 'Back' }] : []), ...(args.allowBack ? [] : [{ value: 'exit', label: 'Exit' }]), ], @@ -346,27 +352,51 @@ export async function runKtxSetupProjectStep( return { status: 'cancelled', projectDir }; } - let selectedDir = projectDir; - let confirmedCreation = false; - if (choice === 'new') { - const selected = await promptForNewProjectDir(projectDir, homeDir, io, prompts); - if (selected.status === 'back') { - continue; - } - if (selected.status !== 'selected') { - return selected; - } - selectedDir = selected.projectDir; - confirmedCreation = selected.confirmedCreation; + if (choice === 'current') { + const project = await createProject(projectDir, deps); + printProjectSummary(io, projectDir); + return { status: 'ready', projectDir, project }; } - if (choice !== 'current' && choice !== 'new') { - prompts.cancel('Setup cancelled.'); - return { status: 'cancelled', projectDir }; + if (choice === 'new-default') { + const confirmed = await confirmProjectDir(defaultProjectDir, io, prompts); + if (confirmed.status === 'choose-another' || confirmed.status === 'back') continue; + if (confirmed.status === 'not-directory') return { status: 'missing-input', projectDir }; + if (confirmed.status === 'cancelled') return { status: 'cancelled', projectDir }; + const project = await createProject(defaultProjectDir, deps); + printProjectSummary(io, defaultProjectDir); + return { + status: 'ready', + projectDir: defaultProjectDir, + project, + confirmedCreation: confirmed.confirmedCreation, + }; } - const project = await createProject(selectedDir, deps); - printProjectSummary(io, selectedDir); - return { status: 'ready', projectDir: selectedDir, project, confirmedCreation }; + if (choice === 'new-custom') { + const rawPath = await prompts.text({ + message: withTextInputNavigation('Project folder path'), + placeholder: './analytics-ktx, ~/analytics-ktx, or /Users/you/projects/analytics-ktx', + }); + if (rawPath === undefined) continue; + const trimmed = rawPath.trim(); + if (trimmed.length === 0) { + io.stderr.write( + 'Enter a relative path like ./analytics-ktx, a home path like ~/analytics-ktx, or an absolute path.\n', + ); + return { status: 'missing-input', projectDir }; + } + const customDir = resolveFromProjectDir(projectDir, trimmed, homeDir); + const confirmed = await confirmProjectDir(customDir, io, prompts); + if (confirmed.status === 'choose-another' || confirmed.status === 'back') continue; + if (confirmed.status === 'not-directory') return { status: 'missing-input', projectDir }; + if (confirmed.status === 'cancelled') return { status: 'cancelled', projectDir }; + const project = await createProject(customDir, deps); + printProjectSummary(io, customDir); + return { status: 'ready', projectDir: customDir, project, confirmedCreation: confirmed.confirmedCreation }; + } + + prompts.cancel('Setup cancelled.'); + return { status: 'cancelled', projectDir }; } } diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index b75e2f65..76ba5d0f 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -66,10 +66,10 @@ function connectionNamePrompt(label: string): string { function textInputPrompt(message: string): string { const normalized = message.replace(/\n+$/, ''); if (!normalized.includes('\n')) { - return `${normalized}\nPress Escape to go back.\n`; + return `${normalized}\n│ Press Escape to go back.\n│`; } const [title, ...bodyLines] = normalized.split('\n'); - return `${title}\n\n${bodyLines.join('\n')}\nPress Escape to go back.\n`; + return `${title}\n│\n│ ${bodyLines.join('\n│ ')}\n│ Press Escape to go back.\n│`; } describe('setup sources step', () => { @@ -664,7 +664,7 @@ describe('setup sources step', () => { expect(runInitialIngest).toHaveBeenCalledTimes(1); expect((await readConfig()).connections['dbt-main']).toMatchObject({ driver: 'dbt', source_dir: '/repo/dbt' }); expect(io.stdout()).toContain('Context source saved without a completed context build for dbt-main.'); - expect(io.stdout()).toContain('Run later: ktx ingest dbt-main'); + expect(io.stdout()).toContain('Run later: ktx ingest run --connection-id dbt-main --adapter '); }); it('retries initial source ingest from the failure menu', async () => { diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index 2237e253..6674ef75 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -2,7 +2,7 @@ import { mkdtemp, readdir, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { cancel, isCancel, log, multiselect, password, select, text } from '@clack/prompts'; +import { cancel, confirm, isCancel, log, multiselect, password, select, text } from '@clack/prompts'; import { localConnectionTypeForConfig, resolveNotionAuthToken } from '@ktx/context/connections'; import { resolveKtxConfigReference } from '@ktx/context/core'; import { @@ -136,12 +136,23 @@ const PRIMARY_SOURCE_DRIVERS = new Set([ function createPromptAdapter(): KtxSetupSourcesPromptAdapter { return { async multiselect(options) { - const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options))); - if (isCancel(value)) { - cancel('Setup cancelled.'); - return ['back']; + while (true) { + const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options))); + if (isCancel(value)) { + cancel('Setup cancelled.'); + return ['back']; + } + const selected = [...value] as string[]; + if (selected.length === 0 && !options.required) { + const skipConfirmed = await confirm({ message: 'Nothing selected. Skip this step?', initialValue: false }); + if (isCancel(skipConfirmed)) { + cancel('Setup cancelled.'); + return ['back']; + } + if (!skipConfirmed) continue; + } + return selected; } - return [...value] as string[]; }, async select(options) { const value = await withSetupInterruptConfirmation(() => select(withMenuOptionsSpacing(options))); @@ -699,7 +710,7 @@ async function runInitialSourceIngestWithRecovery(input: { deps: KtxSetupSourcesDeps; }): Promise<'ready' | 'continue' | 'back' | 'failed'> { while (true) { - input.io.stdout.write(`Building context from ${input.connectionId}. Large sources can take a while.\n`); + input.io.stdout.write(`│ Building context from ${input.connectionId}. Large sources can take a while.\n`); const ingestCode = await (input.deps.runInitialIngest ?? defaultRunInitialIngest)( input.args.projectDir, input.connectionId, @@ -727,8 +738,8 @@ async function runInitialSourceIngestWithRecovery(input: { continue; } if (action === 'continue') { - input.io.stdout.write(`Context source saved without a completed context build for ${input.connectionId}.\n`); - input.io.stdout.write(`Run later: ktx ingest run --connection-id ${input.connectionId} --adapter \n`); + input.io.stdout.write(`│ Context source saved without a completed context build for ${input.connectionId}.\n`); + input.io.stdout.write(`│ Run later: ktx ingest run --connection-id ${input.connectionId} --adapter \n`); return 'continue'; } return 'back'; @@ -1355,7 +1366,7 @@ export async function runKtxSetupSourcesStep( try { if (args.skipSources) { await markSourcesComplete(args.projectDir); - io.stdout.write('Context source setup skipped.\n'); + io.stdout.write('│ Context source setup skipped.\n'); return { status: 'skipped', projectDir: args.projectDir }; } @@ -1368,7 +1379,7 @@ export async function runKtxSetupSourcesStep( return { status: 'failed', projectDir: args.projectDir }; } if (args.inputMode !== 'disabled') { - io.stdout.write(`${message}\n`); + io.stdout.write(`│ ${message}\n`); return { status: 'skipped', projectDir: args.projectDir }; } } @@ -1392,7 +1403,7 @@ export async function runKtxSetupSourcesStep( return { status: 'missing-input', projectDir: args.projectDir }; } await markSourcesComplete(args.projectDir); - io.stdout.write('No context sources selected.\n'); + io.stdout.write('│ No context sources selected.\n'); return { status: 'skipped', projectDir: args.projectDir }; } @@ -1465,7 +1476,7 @@ export async function runKtxSetupSourcesStep( break; } } else { - io.stdout.write(`Context source ${connectionId} saved. It will be built during the context build step.\n`); + io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`); } readyConnectionIds.push(connectionId); } diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index b36b0923..e74dca5d 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -715,7 +715,7 @@ describe('setup status', () => { expect(projectPrompts.text).toHaveBeenCalledWith( expect.objectContaining({ - message: 'Project folder path\nPress Escape to go back.\n', + message: 'Project folder path\n│ Press Escape to go back.\n│', placeholder: './analytics-ktx, ~/analytics-ktx, or /Users/you/projects/analytics-ktx', }), ); diff --git a/packages/connector-bigquery/src/connector.ts b/packages/connector-bigquery/src/connector.ts index 1da246b3..a994912e 100644 --- a/packages/connector-bigquery/src/connector.ts +++ b/packages/connector-bigquery/src/connector.ts @@ -14,6 +14,7 @@ import { type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, + type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult, @@ -63,6 +64,7 @@ export interface KtxBigQueryQueryJob { export interface KtxBigQueryTableRef { id?: string; + metadata?: { type?: string }; get(): Promise< [ { @@ -369,6 +371,25 @@ export class KtxBigQueryScanConnector implements KtxScanConnector { return datasets.map((dataset) => dataset.id).filter((id): id is string => Boolean(id)); } + async listTables(datasetIds?: string[]): Promise { + const filterDatasets = datasetIds ?? (await this.listDatasets()); + const entries: KtxTableListEntry[] = []; + for (const datasetId of filterDatasets) { + const dataset = this.getClient().dataset(datasetId); + const [tables] = await dataset.getTables(); + for (const table of tables) { + if (!table.id) continue; + entries.push({ + schema: datasetId, + name: table.id, + kind: table.metadata?.type === 'VIEW' ? 'view' : 'table', + }); + } + } + entries.sort((a, b) => a.schema.localeCompare(b.schema) || a.name.localeCompare(b.name)); + return entries; + } + async cleanup(): Promise { this.client = null; } diff --git a/packages/connector-clickhouse/src/connector.ts b/packages/connector-clickhouse/src/connector.ts index ae66443e..0273a62b 100644 --- a/packages/connector-clickhouse/src/connector.ts +++ b/packages/connector-clickhouse/src/connector.ts @@ -16,6 +16,7 @@ import { type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, + type KtxTableListEntry, type KtxTableSampleResult, } from '@ktx/context/scan'; import { readFileSync } from 'node:fs'; @@ -128,6 +129,12 @@ interface ClickHouseDatabaseRow { name: string; } +interface ClickHouseTableListRow { + database: string; + name: string; + engine: string; +} + interface ClickHouseCompactResponse { meta?: Array<{ name: string; type: string }>; data?: unknown[][]; @@ -417,6 +424,25 @@ export class KtxClickHouseScanConnector implements KtxScanConnector { return rows.map((row) => row.name); } + async listTables(schemas?: string[]): Promise { + const filterSchemas = schemas ?? (await this.listSchemas()); + if (filterSchemas.length === 0) return []; + const rows = await this.queryEachRow( + ` + SELECT database, name, engine + FROM system.tables + WHERE database IN ({schemas:Array(String)}) + ORDER BY database, name + `, + { schemas: filterSchemas }, + ); + return rows.map((row) => ({ + schema: row.database, + name: row.name, + kind: row.engine === 'View' || row.engine === 'MaterializedView' ? ('view' as const) : ('table' as const), + })); + } + async cleanup(): Promise { if (this.client) { await this.client.close(); diff --git a/packages/connector-mysql/src/connector.ts b/packages/connector-mysql/src/connector.ts index 5b96da4d..69a09272 100644 --- a/packages/connector-mysql/src/connector.ts +++ b/packages/connector-mysql/src/connector.ts @@ -15,6 +15,7 @@ import { type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, + type KtxTableListEntry, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, @@ -129,6 +130,12 @@ interface MysqlSchemaRow extends RowDataPacket { SCHEMA_NAME: string; } +interface MysqlTableListRow extends RowDataPacket { + TABLE_SCHEMA: string; + TABLE_NAME: string; + TABLE_TYPE: string; +} + interface MysqlCountRow extends RowDataPacket { count?: unknown; cardinality?: unknown; @@ -466,6 +473,27 @@ export class KtxMysqlScanConnector implements KtxScanConnector { return rows.map((row) => row.SCHEMA_NAME); } + async listTables(schemas?: string[]): Promise { + const filterSchemas = schemas ?? (await this.listSchemas()); + if (filterSchemas.length === 0) return []; + const placeholders = filterSchemas.map(() => '?').join(', '); + const rows = await this.queryRaw( + ` + SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA IN (${placeholders}) + AND TABLE_TYPE IN ('BASE TABLE', 'VIEW') + ORDER BY TABLE_SCHEMA, TABLE_NAME + `, + filterSchemas, + ); + return rows.map((row) => ({ + schema: row.TABLE_SCHEMA, + name: row.TABLE_NAME, + kind: row.TABLE_TYPE === 'VIEW' ? ('view' as const) : ('table' as const), + })); + } + async cleanup(): Promise { if (this.pool) { await this.pool.end(); diff --git a/packages/connector-postgres/src/connector.ts b/packages/connector-postgres/src/connector.ts index f4aa2f86..65490040 100644 --- a/packages/connector-postgres/src/connector.ts +++ b/packages/connector-postgres/src/connector.ts @@ -17,6 +17,7 @@ import { type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, + type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult, @@ -179,6 +180,12 @@ interface PostgresSchemaRow { schema_name: string; } +interface PostgresTableListRow { + schema_name: string; + table_name: string; + table_kind: string; +} + interface PostgresCountRow { count?: unknown; cardinality?: unknown; @@ -523,6 +530,27 @@ export class KtxPostgresScanConnector implements KtxScanConnector { return rows.map((row) => row.schema_name); } + async listTables(schemas?: string[]): Promise { + const filterSchemas = schemas ?? (await this.listSchemas()); + if (filterSchemas.length === 0) return []; + const rows = await this.queryRaw( + ` + SELECT n.nspname AS schema_name, c.relname AS table_name, c.relkind AS table_kind + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid + WHERE n.nspname = ANY($1) + AND c.relkind IN ('r', 'v') + ORDER BY n.nspname, c.relname + `, + [filterSchemas], + ); + return rows.map((row) => ({ + schema: row.schema_name, + name: row.table_name, + kind: row.table_kind === 'v' ? ('view' as const) : ('table' as const), + })); + } + async cleanup(): Promise { if (this.pool) { await this.pool.end(); diff --git a/packages/connector-snowflake/src/connector.test.ts b/packages/connector-snowflake/src/connector.test.ts index 282b32bc..91bb33d4 100644 --- a/packages/connector-snowflake/src/connector.test.ts +++ b/packages/connector-snowflake/src/connector.test.ts @@ -60,6 +60,10 @@ function fakeDriverFactory(): KtxSnowflakeDriverFactory { }, ]), listSchemas: vi.fn(async () => ['PUBLIC', 'MART']), + listTables: vi.fn(async () => [ + { schema: 'PUBLIC', name: 'ORDERS', kind: 'table' as const }, + { schema: 'PUBLIC', name: 'ORDER_SUMMARY', kind: 'view' as const }, + ]), cleanup: vi.fn(async () => undefined), }; return { createDriver: vi.fn(() => driver) }; diff --git a/packages/connector-snowflake/src/connector.ts b/packages/connector-snowflake/src/connector.ts index 31dd18a0..063976f7 100644 --- a/packages/connector-snowflake/src/connector.ts +++ b/packages/connector-snowflake/src/connector.ts @@ -19,6 +19,7 @@ import { type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, + type KtxTableListEntry, type KtxTableSampleResult, } from '@ktx/context/scan'; import * as snowflake from 'snowflake-sdk'; @@ -75,6 +76,7 @@ export interface KtxSnowflakeDriver { query(sql: string, params?: unknown): Promise; getSchemaMetadata(schemaName?: string): Promise; listSchemas(): Promise; + listTables(schemas?: string[]): Promise; cleanup(): Promise; } @@ -344,6 +346,31 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver { return result.rows.map((row) => String(row[1])).filter((name) => name !== 'INFORMATION_SCHEMA'); } + async listTables(schemas?: string[]): Promise { + const filterSchemas = schemas ?? (await this.listSchemas()); + if (filterSchemas.length === 0) return []; + const entries: KtxTableListEntry[] = []; + for (const schemaName of filterSchemas) { + const result = await this.query( + ` + SELECT TABLE_NAME, TABLE_TYPE + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = ? AND TABLE_CATALOG = ? + ORDER BY TABLE_NAME + `, + [schemaName, this.resolved.database], + ); + for (const row of result.rows) { + entries.push({ + schema: schemaName, + name: String(row[0]), + kind: String(row[1]) === 'VIEW' ? 'view' : 'table', + }); + } + } + return entries; + } + async cleanup(): Promise { const closers = this.closeSdkOptions; this.closeSdkOptions = []; @@ -594,6 +621,10 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector { return this.getDriver().listSchemas(); } + listTables(schemas?: string[]): Promise { + return this.getDriver().listTables(schemas); + } + async cleanup(): Promise { if (this.driverInstance) { await this.driverInstance.cleanup(); diff --git a/packages/connector-sqlserver/src/connector.ts b/packages/connector-sqlserver/src/connector.ts index 1f31286d..189ff98b 100644 --- a/packages/connector-sqlserver/src/connector.ts +++ b/packages/connector-sqlserver/src/connector.ts @@ -14,6 +14,7 @@ import { type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, + type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult, @@ -441,6 +442,32 @@ export class KtxSqlServerScanConnector implements KtxScanConnector { return rows.map((row) => row.schema_name); } + async listTables(schemas?: string[]): Promise { + const filterSchemas = schemas ?? (await this.listSchemas()); + if (filterSchemas.length === 0) return []; + const params: Record = {}; + const placeholders = filterSchemas.map((s, i) => { + params[`schema${i}`] = s; + return `@schema${i}`; + }); + const rows = await this.queryRaw<{ schema_name: string; table_name: string; table_type: string }>( + ` + SELECT s.name AS schema_name, o.name AS table_name, o.type_desc AS table_type + FROM sys.objects o + JOIN sys.schemas s ON o.schema_id = s.schema_id + WHERE o.type IN ('U', 'V') + AND s.name IN (${placeholders.join(', ')}) + ORDER BY s.name, o.name + `, + params, + ); + return rows.map((row) => ({ + schema: row.schema_name, + name: row.table_name, + kind: row.table_type === 'VIEW' ? ('view' as const) : ('table' as const), + })); + } + async cleanup(): Promise { if (this.pool) { await this.pool.close(); diff --git a/packages/context/src/scan/index.ts b/packages/context/src/scan/index.ts index c1478993..94450891 100644 --- a/packages/context/src/scan/index.ts +++ b/packages/context/src/scan/index.ts @@ -105,7 +105,7 @@ export type { LocalScanStatusResponse, RunLocalScanOptions, } from './local-scan.js'; -export { getLocalScanReport, getLocalScanStatus, runLocalScan } from './local-scan.js'; +export { filterSnapshotTables, getLocalScanReport, getLocalScanStatus, resolveEnabledTables, runLocalScan } from './local-scan.js'; export type { ReadLocalScanStructuralSnapshotInput } from './local-structural-artifacts.js'; export { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js'; export type { @@ -393,6 +393,7 @@ export type { KtxSchemaTable, KtxSchemaTableKind, KtxStructuralSyncStats, + KtxTableListEntry, KtxTableRef, KtxTableSampleInput, KtxTableSampleResult, diff --git a/packages/context/src/scan/local-scan.test.ts b/packages/context/src/scan/local-scan.test.ts index c74aad37..6c3e877f 100644 --- a/packages/context/src/scan/local-scan.test.ts +++ b/packages/context/src/scan/local-scan.test.ts @@ -6,8 +6,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import YAML from 'yaml'; import type { SourceAdapter } from '../ingest/index.js'; import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; -import { getLocalScanReport, getLocalScanStatus, runLocalScan } from './local-scan.js'; -import type { KtxQueryResult, KtxReadOnlyQueryInput } from './types.js'; +import { filterSnapshotTables, getLocalScanReport, getLocalScanStatus, resolveEnabledTables, runLocalScan } from './local-scan.js'; +import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxSchemaSnapshot, KtxSchemaTable } from './types.js'; function relationshipSqlResult( input: KtxReadOnlyQueryInput, @@ -1492,3 +1492,79 @@ describe('local scan', () => { ); }); }); + +describe('resolveEnabledTables', () => { + it('returns null when no enabled_tables field', () => { + expect(resolveEnabledTables({ driver: 'postgres' })).toBeNull(); + }); + + it('returns null for empty array', () => { + expect(resolveEnabledTables({ driver: 'postgres', enabled_tables: [] })).toBeNull(); + }); + + it('returns Set of enabled table names', () => { + const result = resolveEnabledTables({ + driver: 'postgres', + enabled_tables: ['public.users', 'public.orders'], + }); + expect(result).toBeInstanceOf(Set); + expect(result!.size).toBe(2); + expect(result!.has('public.users')).toBe(true); + expect(result!.has('public.orders')).toBe(true); + }); + + it('returns null for undefined connection', () => { + expect(resolveEnabledTables(undefined)).toBeNull(); + }); +}); + +describe('filterSnapshotTables', () => { + function makeSnapshot(tables: Array<{ db: string; name: string }>): KtxSchemaSnapshot { + return { + connectionId: 'test', + driver: 'postgres', + extractedAt: '2026-01-01T00:00:00Z', + scope: {}, + metadata: {}, + tables: tables.map( + (t): KtxSchemaTable => ({ + catalog: null, + db: t.db, + name: t.name, + kind: 'table', + comment: null, + estimatedRows: null, + columns: [], + foreignKeys: [], + }), + ), + }; + } + + it('keeps only enabled tables', () => { + const snapshot = makeSnapshot([ + { db: 'public', name: 'users' }, + { db: 'public', name: 'orders' }, + { db: 'public', name: 'logs' }, + ]); + const enabled = new Set(['public.users', 'public.orders']); + const filtered = filterSnapshotTables(snapshot, enabled); + expect(filtered.tables).toHaveLength(2); + expect(filtered.tables.map((t) => t.name)).toEqual(['users', 'orders']); + }); + + it('returns empty tables when none match', () => { + const snapshot = makeSnapshot([{ db: 'public', name: 'users' }]); + const enabled = new Set(['public.orders']); + const filtered = filterSnapshotTables(snapshot, enabled); + expect(filtered.tables).toHaveLength(0); + }); + + it('preserves other snapshot fields', () => { + const snapshot = makeSnapshot([{ db: 'public', name: 'users' }]); + const enabled = new Set(['public.users']); + const filtered = filterSnapshotTables(snapshot, enabled); + expect(filtered.connectionId).toBe('test'); + expect(filtered.driver).toBe('postgres'); + }); +}); diff --git a/packages/context/src/scan/local-scan.ts b/packages/context/src/scan/local-scan.ts index 15fdf6f3..7f3c00a0 100644 --- a/packages/context/src/scan/local-scan.ts +++ b/packages/context/src/scan/local-scan.ts @@ -29,10 +29,13 @@ import type { KtxConnectionDriver, KtxProgressPort, KtxScanConnector, + KtxScanContext, KtxScanEnrichmentStateSummary, + KtxScanInput, KtxScanMode, KtxScanReport, KtxScanTrigger, + KtxSchemaSnapshot, } from './types.js'; export interface RunLocalScanOptions { @@ -313,17 +316,45 @@ async function readScanReport( } } +export function resolveEnabledTables(connection: Record | undefined): Set | null { + const raw = connection?.enabled_tables; + if (!Array.isArray(raw) || raw.length === 0) return null; + return new Set(raw.filter((v): v is string => typeof v === 'string')); +} + +export function filterSnapshotTables(snapshot: KtxSchemaSnapshot, enabledTables: Set): KtxSchemaSnapshot { + return { + ...snapshot, + tables: snapshot.tables.filter((table) => { + const key = table.db ? `${table.db}.${table.name}` : table.name; + return enabledTables.has(key); + }), + }; +} + +function createFilteredConnector(connector: KtxScanConnector, enabledTables: Set): KtxScanConnector { + return { + ...connector, + async introspect(input: KtxScanInput, ctx: KtxScanContext): Promise { + const snapshot = await connector.introspect(input, ctx); + return filterSnapshotTables(snapshot, enabledTables); + }, + }; +} + export async function runLocalScan(options: RunLocalScanOptions): Promise { const mode = options.mode ?? 'structural'; assertSupportedMode(mode); await options.progress?.update(0.05, 'Preparing scan'); - const connector = await resolveScanConnector(options, mode); + const rawConnector = await resolveScanConnector(options, mode); const connection = options.project.config.connections[options.connectionId]; if (!connection) { throw new Error(`Connection "${options.connectionId}" is not configured in ktx.yaml`); } const driver = normalizeDriver(connection.driver); + const enabledTables = resolveEnabledTables(connection); + const connector = rawConnector && enabledTables ? createFilteredConnector(rawConnector, enabledTables) : rawConnector; const adapters = options.adapters ?? createDefaultLocalIngestAdapters(options.project, { databaseIntrospectionUrl: options.databaseIntrospectionUrl }); @@ -372,13 +403,28 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise { + const take = Math.min(remaining, ds[field]); + ds[field] -= take; + remaining -= take; + }; + subFrom('tablesAdded'); + subFrom('tablesUnchanged'); + subFrom('tablesModified'); + await options.progress?.update(0.6, scanChangeSummary(report.diffSummary)); + } const manifestArtifacts = await writeLocalScanManifestShards({ project: options.project, connectionId: options.connectionId, diff --git a/packages/context/src/scan/types.ts b/packages/context/src/scan/types.ts index 71bb3fb3..2a9cad00 100644 --- a/packages/context/src/scan/types.ts +++ b/packages/context/src/scan/types.ts @@ -277,6 +277,12 @@ export interface KtxQueryResult { rowCount: number | null; } +export interface KtxTableListEntry { + schema: string; + name: string; + kind: 'table' | 'view'; +} + export interface KtxScanConnector { id: string; driver: KtxConnectionDriver;