From 1b5a9fe120af8d3af830dacdb0579a6f1d33031f Mon Sep 17 00:00:00 2001 From: Luca Martial Date: Sun, 10 May 2026 16:12:51 -0700 Subject: [PATCH] Improve connector credential setup UX --- packages/cli/src/context-build-view.test.ts | 130 +++++- packages/cli/src/context-build-view.ts | 80 +++- packages/cli/src/setup-sources.test.ts | 329 +++++++++++++- packages/cli/src/setup-sources.ts | 417 +++++++++++++++--- .../metabase/local-metabase.adapter.test.ts | 30 +- .../metabase/local-metabase.adapter.ts | 11 +- 6 files changed, 882 insertions(+), 115 deletions(-) diff --git a/packages/cli/src/context-build-view.test.ts b/packages/cli/src/context-build-view.test.ts index c14102ec..1c6965d8 100644 --- a/packages/cli/src/context-build-view.test.ts +++ b/packages/cli/src/context-build-view.test.ts @@ -98,11 +98,11 @@ describe('parseScanSummary', () => { describe('parseIngestSummary', () => { it('extracts work units and saved memory', () => { - expect(parseIngestSummary('Work units: 5\nSaved memory: 3 wiki, 2 SL')).toBe('5 work units · 3 wiki, 2 SL'); + expect(parseIngestSummary('Work units: 5\nSaved memory: 3 wiki, 2 SL')).toBe('5 items indexed · 3 wiki, 2 SL'); }); it('extracts work units alone when no saved memory', () => { - expect(parseIngestSummary('Work units: 5\nStatus: done')).toBe('5 work units'); + expect(parseIngestSummary('Work units: 5\nStatus: done')).toBe('5 items indexed'); }); it('extracts saved memory alone when no work units', () => { @@ -127,10 +127,18 @@ describe('initViewState', () => { expect(state.contextSources[0].target.connectionId).toBe('dbt-main'); expect(state.frame).toBe(0); }); + + it('initializes global timing fields', () => { + const state = initViewState([ + { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, + ]); + expect(state.startedAt).toBeNull(); + expect(state.totalElapsedMs).toBe(0); + }); }); describe('renderContextBuildView', () => { - it('renders all-queued state', () => { + it('renders all-queued state with ○ icon and progress counter', () => { const state = initViewState([ { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, { connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] }, @@ -138,6 +146,8 @@ describe('renderContextBuildView', () => { const output = renderContextBuildView(state, { styled: false }); expect(output).toContain('Building KTX context'); + expect(output).toContain('(0/2)'); + expect(output).toContain('○'); expect(output).toContain('Primary sources:'); expect(output).toContain('warehouse'); expect(output).toContain('queued'); @@ -145,6 +155,29 @@ describe('renderContextBuildView', () => { expect(output).toContain('dbt-main'); }); + it('renders header with total elapsed time when set', () => { + const state = initViewState([ + { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, + ]); + state.totalElapsedMs = 65000; + + const output = renderContextBuildView(state, { styled: false }); + expect(output).toContain('(0/1 · 1m05s)'); + }); + + it('renders dynamic separator matching header width', () => { + const state = initViewState([ + { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, + ]); + state.totalElapsedMs = 120000; + + const output = renderContextBuildView(state, { styled: false }); + const lines = output.split('\n'); + const headerLine = lines.find((l) => l.includes('Building KTX context'))!; + const separatorLine = lines.find((l) => /^─+$/.test(l))!; + expect(separatorLine.length).toBeGreaterThanOrEqual(headerLine.length); + }); + it('renders completed state with summary', () => { const state = initViewState([ { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, @@ -156,6 +189,74 @@ describe('renderContextBuildView', () => { const output = renderContextBuildView(state, { styled: false }); expect(output).toContain('42 tables'); expect(output).toContain('1m12s'); + expect(output).toContain('(1/1)'); + }); + + it('renders running target with elapsed time', () => { + const state = initViewState([ + { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, + ]); + state.primarySources[0].status = 'running'; + state.primarySources[0].elapsedMs = 30000; + + const output = renderContextBuildView(state, { styled: false }); + expect(output).toContain('scanning...'); + expect(output).toContain('30s'); + }); + + it('renders running target with progress bar when percentage is available', () => { + const state = initViewState([ + { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, + ]); + state.primarySources[0].status = 'running'; + state.primarySources[0].detailLine = '[50%] Scanning tables...'; + state.primarySources[0].elapsedMs = 15000; + + const output = renderContextBuildView(state, { styled: false }); + expect(output).toContain('██████░░░░░░'); + expect(output).toContain('50%'); + expect(output).toContain('Scanning tables...'); + expect(output).toContain('15s'); + }); + + it('renders completion summary when all targets are done', () => { + const state = initViewState([ + { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, + { connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] }, + ]); + state.primarySources[0].status = 'done'; + state.primarySources[0].elapsedMs = 72000; + state.contextSources[0].status = 'done'; + state.contextSources[0].elapsedMs = 34000; + state.totalElapsedMs = 106000; + + const output = renderContextBuildView(state, { styled: false }); + expect(output).toContain('Done in 1m46s · 2 sources processed'); + }); + + it('renders singular source label in completion summary', () => { + const state = initViewState([ + { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, + ]); + state.primarySources[0].status = 'done'; + state.primarySources[0].elapsedMs = 5000; + state.totalElapsedMs = 5000; + + const output = renderContextBuildView(state, { styled: false }); + expect(output).toContain('Done in 5s · 1 source processed'); + }); + + it('does not render completion summary while targets are still active', () => { + const state = initViewState([ + { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, + { connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] }, + ]); + state.primarySources[0].status = 'done'; + state.contextSources[0].status = 'running'; + state.totalElapsedMs = 30000; + + const output = renderContextBuildView(state, { styled: false }); + expect(output).not.toContain('Done in'); }); it('renders failed state', () => { @@ -178,6 +279,29 @@ describe('renderContextBuildView', () => { expect(output).not.toContain('Primary sources:'); expect(output).toContain('Context sources:'); }); + + it('preserves detach hint while targets are active', () => { + const state = initViewState([ + { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, + ]); + state.primarySources[0].status = 'running'; + + const output = renderContextBuildView(state, { styled: false, showHint: true, projectDir: '/tmp/project' }); + expect(output).toContain('d to detach'); + expect(output).toContain('ktx setup --project-dir /tmp/project'); + expect(output).toContain('to resume'); + }); + + it('omits detach hint when all targets are done', () => { + const state = initViewState([ + { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, + ]); + state.primarySources[0].status = 'done'; + state.totalElapsedMs = 5000; + + const output = renderContextBuildView(state, { styled: false, showHint: true }); + expect(output).not.toContain('d to detach'); + }); }); describe('runContextBuild', () => { diff --git a/packages/cli/src/context-build-view.ts b/packages/cli/src/context-build-view.ts index 2e39537c..96a8aa57 100644 --- a/packages/cli/src/context-build-view.ts +++ b/packages/cli/src/context-build-view.ts @@ -30,6 +30,8 @@ export interface ContextBuildViewState { primarySources: ContextBuildTargetState[]; contextSources: ContextBuildTargetState[]; frame: number; + startedAt: number | null; + totalElapsedMs: number; } export interface ContextBuildArgs { @@ -79,7 +81,7 @@ function statusIcon(status: ContextBuildTargetState['status'], frame: number, st case 'running': return SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋'; default: - return '·'; + return '○'; } } switch (status) { @@ -90,10 +92,27 @@ function statusIcon(status: ContextBuildTargetState['status'], frame: number, st case 'running': return cyan(SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋'); default: - return dim('·'); + return dim('○'); } } +function extractPercent(detailLine: string | null): number | null { + if (!detailLine) return null; + const match = detailLine.match(/^\[(\d+)%\]/); + return match ? Number(match[1]) : null; +} + +const BAR_WIDTH = 12; +const BAR_FILLED = '█'; +const BAR_EMPTY = '░'; + +function renderProgressBar(percent: number, styled: boolean): string { + const filled = Math.round((percent / 100) * BAR_WIDTH); + const empty = BAR_WIDTH - filled; + const bar = `${BAR_FILLED.repeat(filled)}${BAR_EMPTY.repeat(empty)}`; + return styled ? cyan(bar) : bar; +} + function targetDetail(target: ContextBuildTargetState, styled: boolean): string { if (target.status === 'done') { const parts: string[] = []; @@ -105,7 +124,17 @@ function targetDetail(target: ContextBuildTargetState, styled: boolean): string return styled ? red('failed') : 'failed'; } if (target.status === 'running') { - return target.detailLine ?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...'); + const percent = extractPercent(target.detailLine); + const progressText = target.detailLine?.replace(/^\[\d+%\]\s*/, '') + ?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...'); + const elapsed = target.elapsedMs > 0 ? formatDuration(target.elapsedMs) : null; + const parts: string[] = []; + if (percent !== null) { + parts.push(`${renderProgressBar(percent, styled)} ${percent}%`); + } + parts.push(progressText); + if (elapsed) parts.push(styled ? dim(elapsed) : elapsed); + return parts.join(' '); } return styled ? dim('queued') : 'queued'; } @@ -140,17 +169,39 @@ export function renderContextBuildView( ): string { const styled = options.styled ?? true; const width = columnWidth(state); + const allTargets = [...state.primarySources, ...state.contextSources]; + const doneCount = allTargets.filter((t) => t.status === 'done' || t.status === 'failed').length; + const totalCount = allTargets.length; + const hasActive = allTargets.some((t) => t.status === 'running' || t.status === 'queued'); + const allDone = totalCount > 0 && !hasActive; + + const headerParts = ['Building KTX context']; + if (totalCount > 0) { + const progressParts: string[] = [`${doneCount}/${totalCount}`]; + if (state.totalElapsedMs > 0) progressParts.push(formatDuration(state.totalElapsedMs)); + const progress = `(${progressParts.join(' · ')})`; + headerParts.push(styled ? dim(progress) : progress); + } + const header = headerParts.join(' '); + const headerPlainLength = header.replace(/\x1b\[[0-9;]*m/g, '').length; + const separator = '─'.repeat(Math.max(21, headerPlainLength)); + const lines: string[] = [ '', - 'Building KTX context', - '─────────────────────', + header, + separator, ...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width), ...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width), '', ]; - const hasActive = [...state.primarySources, ...state.contextSources].some( - (t) => t.status === 'running' || t.status === 'queued', - ); + + if (allDone && state.totalElapsedMs > 0) { + const sourcesLabel = totalCount === 1 ? '1 source' : `${totalCount} sources`; + const summary = ` Done in ${formatDuration(state.totalElapsedMs)} · ${sourcesLabel} processed`; + lines.push(styled ? green(summary) : summary); + lines.push(''); + } + if (options.showHint && hasActive) { const hint = ` d to detach · ${resumeCommand(options.projectDir)} to resume`; lines.push(styled ? dim(hint) : hint); @@ -177,7 +228,7 @@ export function parseScanSummary(output: string): string | null { export function parseIngestSummary(output: string): string | null { const parts: string[] = []; const workUnits = output.match(/Work units: (\d+)/); - if (workUnits) parts.push(`${workUnits[1]} work units`); + if (workUnits) parts.push(`${workUnits[1]} items indexed`); const savedMemory = output.match(/Saved memory: (.+)/); if (savedMemory) parts.push(savedMemory[1]); return parts.length > 0 ? parts.join(' · ') : null; @@ -289,6 +340,8 @@ export function initViewState(targets: KtxPublicIngestPlanTarget[]): ContextBuil primarySources: targets.filter((t) => t.operation === 'scan').map(makeTargetState), contextSources: targets.filter((t) => t.operation === 'source-ingest').map(makeTargetState), frame: 0, + startedAt: null, + totalElapsedMs: 0, }; } @@ -303,6 +356,8 @@ export async function runContextBuild( const isTTY = io.stdout.isTTY === true; const nowFn = deps.now ?? (() => Date.now()); + state.startedAt = nowFn(); + const repainter = isTTY ? createRepainter(io) : null; const viewOpts = { styled: true, projectDir: args.projectDir }; const paint = (hint: boolean) => repainter?.paint(renderContextBuildView(state, { ...viewOpts, showHint: hint })); @@ -312,6 +367,9 @@ export async function runContextBuild( if (repainter) { spinnerInterval = setInterval(() => { state.frame++; + if (state.startedAt !== null) { + state.totalElapsedMs = nowFn() - state.startedAt; + } for (const t of [...state.primarySources, ...state.contextSources]) { if (t.status === 'running' && t.startedAt !== null) { t.elapsedMs = nowFn() - t.startedAt; @@ -400,6 +458,10 @@ export async function runContextBuild( cleanupKeystroke?.(); } + if (state.startedAt !== null) { + state.totalElapsedMs = nowFn() - state.startedAt; + } + if (detached) { return { exitCode: 0, detached: true }; } diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index b8ff4eed..1ef973c9 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { @@ -8,6 +8,7 @@ import { serializeKtxProjectConfig, } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { KtxCliIo } from './cli-runtime.js'; import { runKtxSetupSourcesStep, type KtxSetupSourcesDeps, @@ -41,14 +42,17 @@ function prompts(values: { multiselect?: string[][]; select?: string[]; text?: Array; + password?: Array; }): KtxSetupSourcesPromptAdapter { const multiselectValues = [...(values.multiselect ?? [])]; const selectValues = [...(values.select ?? [])]; const textValues = [...(values.text ?? [])]; + const passwordValues = [...(values.password ?? [])]; return { multiselect: vi.fn(async () => multiselectValues.shift() ?? []), select: vi.fn(async () => selectValues.shift() ?? 'skip'), text: vi.fn(async () => (textValues.length > 0 ? textValues.shift() : '')), + password: vi.fn(async () => (passwordValues.length > 0 ? passwordValues.shift() : undefined)), cancel: vi.fn(), log: vi.fn(), }; @@ -207,6 +211,193 @@ describe('setup sources step', () => { expect(runMapping).toHaveBeenCalledWith(projectDir, 'prod_metabase', io.io); }); + it('defaults interactive Metabase and Looker source setup to the only warehouse connection', async () => { + await addPrimarySource(); + const cases: Array<{ + source: 'metabase' | 'looker'; + text: string[]; + deps: KtxSetupSourcesDeps; + expectedConnection: Record; + }> = [ + { + source: 'metabase', + text: ['metabase-main', 'https://metabase.example.com'], + deps: { + discoverMetabaseDatabases: vi.fn(async () => [ + { id: 1, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' }, + ]), + validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })), + runMapping: vi.fn(async () => 0), + }, + expectedConnection: { + driver: 'metabase', + mappings: { databaseMappings: { '1': 'warehouse' } }, + }, + }, + { + source: 'looker', + text: ['looker-main', 'https://looker.example.com', 'client-id', ''], + deps: { + validateLooker: vi.fn(async () => ({ ok: true as const, detail: 'mapping refreshed' })), + runMapping: vi.fn(async () => 0), + }, + expectedConnection: { + driver: 'looker', + mappings: { connectionMappings: { warehouse: 'warehouse' } }, + }, + }, + ]; + + for (const testCase of cases) { + const testPrompts = prompts({ + multiselect: [[testCase.source]], + select: ['env', 'done'], + text: testCase.text, + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + ...testCase.deps, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: [`${testCase.source}-main`] }); + + expect( + vi.mocked(testPrompts.text).mock.calls.some(([options]) => options.message.includes('Mapped warehouse')), + ).toBe(false); + if (testCase.source === 'metabase') { + expect( + vi.mocked(testPrompts.text).mock.calls.some(([options]) => options.message.includes('Metabase database id')), + ).toBe(false); + } + expect((await readConfig()).connections[`${testCase.source}-main`]).toMatchObject(testCase.expectedConnection); + } + }); + + it('prompts for the mapped warehouse when interactive Metabase and Looker source setup has multiple choices', async () => { + await addPrimarySource(); + await addConnection('analytics_warehouse', { + driver: 'snowflake', + account: 'acme', + database: 'analytics', + readonly: true, + }); + + const cases: Array<{ + source: 'metabase' | 'looker'; + text: string[]; + deps: KtxSetupSourcesDeps; + expectedConnection: Record; + }> = [ + { + source: 'metabase', + text: ['metabase-main', 'https://metabase.example.com'], + deps: { + discoverMetabaseDatabases: vi.fn(async () => [ + { id: 1, name: 'Finance', engine: 'postgres', host: 'db.example.com', dbName: 'finance' }, + { id: 2, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' }, + ]), + validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })), + runMapping: vi.fn(async () => 0), + }, + expectedConnection: { + driver: 'metabase', + mappings: { databaseMappings: { '2': 'analytics_warehouse' } }, + }, + }, + { + source: 'looker', + text: ['looker-main', 'https://looker.example.com', 'client-id', 'analytics'], + deps: { + validateLooker: vi.fn(async () => ({ ok: true as const, detail: 'mapping refreshed' })), + runMapping: vi.fn(async () => 0), + }, + expectedConnection: { + driver: 'looker', + mappings: { connectionMappings: { analytics: 'analytics_warehouse' } }, + }, + }, + ]; + + for (const testCase of cases) { + const testPrompts = prompts({ + multiselect: [[testCase.source]], + select: testCase.source === 'metabase' ? ['env', 'analytics_warehouse', '2', 'done'] : ['env', 'analytics_warehouse', 'done'], + text: testCase.text, + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + ...testCase.deps, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: [`${testCase.source}-main`] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'Mapped warehouse connection', + options: [ + { value: 'analytics_warehouse', label: 'analytics_warehouse (SNOWFLAKE)' }, + { value: 'warehouse', label: 'warehouse (POSTGRESQL)' }, + { value: 'back', label: 'Back' }, + ], + }); + if (testCase.source === 'metabase') { + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'Metabase database', + options: [ + { value: '1', label: '1: Finance (postgres)' }, + { value: '2', label: '2: Analytics (postgres)' }, + { value: 'back', label: 'Back' }, + ], + }); + expect( + vi.mocked(testPrompts.text).mock.calls.some(([options]) => options.message.includes('Metabase database id')), + ).toBe(false); + } + expect((await readConfig()).connections[`${testCase.source}-main`]).toMatchObject(testCase.expectedConnection); + } + }); + + it('lets visible Metabase mapping surface refresh and validation failures', async () => { + await addPrimarySource(); + const runMapping = vi.fn(async (_projectDir: string, _connectionId: string, io: KtxCliIo) => { + io.stderr.write('1: Metabase database does not match KTX connection database\n'); + return 1; + }); + const io = makeIo(); + const testPrompts = prompts({ + multiselect: [['metabase']], + select: ['env'], + text: ['metabase-main', 'https://metabase.example.com'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + io.io, + { + prompts: testPrompts, + discoverMetabaseDatabases: vi.fn(async () => [ + { id: 1, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' }, + ]), + runMapping, + }, + ), + ).resolves.toEqual({ status: 'failed', projectDir }); + + expect(runMapping).toHaveBeenCalledWith(projectDir, 'metabase-main', io.io); + expect(io.stderr()).toContain('1: Metabase database does not match KTX connection database'); + expect(io.stderr()).not.toContain('Metabase mapping validation failed'); + }); + it('does not mark sources complete when validation fails', async () => { await addPrimarySource(); const io = makeIo(); @@ -333,8 +524,8 @@ describe('setup sources step', () => { const io = makeIo(); const testPrompts = prompts({ multiselect: [['dbt']], - select: ['git'], - text: ['dbt-main', 'https://github.com/acme-org/private-repo', 'main', '', 'env:GITHUB_TOKEN'], + select: ['git', 'env'], + text: ['dbt-main', 'https://github.com/acme-org/private-repo', 'main', ''], }); await expect( @@ -350,19 +541,16 @@ describe('setup sources step', () => { ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); expect(testGitRepo).toHaveBeenCalledWith({ repoUrl: 'https://github.com/acme-org/private-repo' }); - expect(testPrompts.text).toHaveBeenNthCalledWith(5, { - message: textInputPrompt( - [ - 'This repo requires authentication.', - 'Generate a token at: https://github.com/settings/tokens/new', - 'Store it in an env var, then enter env:VARIABLE_NAME here (e.g. env:GITHUB_TOKEN).', - 'Or use file:/absolute/path if the token is stored in a file.', - 'Press Enter to skip and try without authentication anyway.', - ].join('\n'), - ), - placeholder: 'env:GITHUB_TOKEN', + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'This repo requires authentication.', + options: [ + { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, + { value: 'paste', label: 'Paste a token and save it as a local secret file' }, + { value: 'skip', label: 'Skip — try without authentication' }, + { value: 'back', label: 'Back' }, + ], }); - expect(testPrompts.text).toHaveBeenCalledTimes(5); + expect(testPrompts.text).toHaveBeenCalledTimes(4); }); it('enables the dbt adapter when adding a dbt source connection', async () => { @@ -692,13 +880,11 @@ describe('setup sources step', () => { }, { source: 'metabase', + select: ['back', 'env'], text: [ 'metabase-main', 'https://old-metabase.example.com', - undefined, 'https://metabase.example.com', - 'env:METABASE_API_KEY', - 'warehouse', '1', ], deps: { @@ -709,14 +895,13 @@ describe('setup sources step', () => { }, { source: 'looker', + select: ['env'], text: [ 'looker-main', 'https://old-looker.example.com', undefined, 'https://looker.example.com', 'client-id', - 'env:LOOKER_CLIENT_SECRET', - 'warehouse', '', ], deps: { @@ -727,10 +912,10 @@ describe('setup sources step', () => { }, { source: 'notion', - select: ['back', 'all_accessible'], - text: ['notion-main', 'env:NOTION_TOKEN', 'env:NOTION_TOKEN'], + select: ['env', 'back', 'env', 'all_accessible'], + text: ['notion-main'], deps: { validateNotion: vi.fn(async () => ({ ok: true as const, detail: 'roots=0' })) }, - repeatedTextMessage: textInputPrompt('Notion token ref'), + repeatedSelectMessage: 'How should KTX find your Notion integration token?', }, ]; @@ -787,4 +972,102 @@ describe('setup sources step', () => { expect(io.stdout()).toContain('Connect a primary source before adding context sources.'); expect((await readConfig()).setup?.completed_steps ?? []).not.toContain('sources'); }); + + it('auto-detects dbt_project.yml at the root of a local path', async () => { + await addPrimarySource(); + const dbtDir = join(tempDir, 'dbt-repo'); + await mkdir(dbtDir, { recursive: true }); + await writeFile(join(dbtDir, 'dbt_project.yml'), 'name: analytics\n'); + + const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); + const io = makeIo(); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['path'], + text: ['dbt-main', dbtDir], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + io.io, + { prompts: testPrompts, validateDbt }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); + + expect(testPrompts.text).toHaveBeenCalledTimes(2); + const config = await readConfig(); + expect(config.connections['dbt-main']).toMatchObject({ driver: 'dbt', source_dir: dbtDir }); + expect(config.connections['dbt-main']).not.toHaveProperty('path'); + }); + + it('auto-detects dbt_project.yml in a subdirectory of a local path', async () => { + await addPrimarySource(); + const dbtDir = join(tempDir, 'monorepo'); + await mkdir(join(dbtDir, 'analytics', 'dbt'), { recursive: true }); + await writeFile(join(dbtDir, 'analytics', 'dbt', 'dbt_project.yml'), 'name: analytics\n'); + + const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); + const io = makeIo(); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['path'], + text: ['dbt-main', dbtDir], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + io.io, + { prompts: testPrompts, validateDbt }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); + + expect(testPrompts.text).toHaveBeenCalledTimes(2); + expect(testPrompts.log).toHaveBeenCalledWith('Found dbt_project.yml in analytics/dbt/'); + const config = await readConfig(); + expect(config.connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: dbtDir, + path: 'analytics/dbt', + }); + }); + + it('shows a picker when multiple dbt projects are found in a local path', async () => { + await addPrimarySource(); + const dbtDir = join(tempDir, 'multi-dbt'); + await mkdir(join(dbtDir, 'analytics'), { recursive: true }); + await mkdir(join(dbtDir, 'staging'), { recursive: true }); + await writeFile(join(dbtDir, 'analytics', 'dbt_project.yml'), 'name: analytics\n'); + await writeFile(join(dbtDir, 'staging', 'dbt_project.yml'), 'name: staging\n'); + + const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); + const io = makeIo(); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['path', 'staging'], + text: ['dbt-main', dbtDir], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + io.io, + { prompts: testPrompts, validateDbt }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: 'Multiple dbt projects found — which one should KTX use?', + }), + ); + expect(testPrompts.text).toHaveBeenCalledTimes(2); + const config = await readConfig(); + expect(config.connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: dbtDir, + path: 'staging', + }); + }); }); diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index b0e0fe2e..4690f29c 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -1,14 +1,18 @@ import { mkdtemp, readdir, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join, resolve } from 'node:path'; +import { join, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { cancel, isCancel, log, multiselect, select, text } from '@clack/prompts'; -import { resolveNotionAuthToken } from '@ktx/context/connections'; +import { cancel, isCancel, log, multiselect, password, select, text } from '@clack/prompts'; +import { localConnectionTypeForConfig, resolveNotionAuthToken } from '@ktx/context/connections'; import { resolveKtxConfigReference } from '@ktx/context/core'; import { cloneOrPull, + DEFAULT_METABASE_CLIENT_CONFIG, + discoverMetabaseDatabases, + type DiscoveredMetabaseDatabase, loadDbtSchemaFiles, loadProjectInfo, + MetabaseClient, type NotionApi, NotionClient, parseLookmlStagedDir, @@ -28,6 +32,7 @@ import { runKtxConnection } from './connection.js'; import { withMenuOptionsSpacing, withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js'; import { runKtxPublicIngest } from './public-ingest.js'; import { withSetupInterruptConfirmation } from './setup-interrupt.js'; +import { writeProjectLocalSecretReference } from './setup-secrets.js'; export type KtxSetupSourceType = 'dbt' | 'metricflow' | 'metabase' | 'looker' | 'lookml' | 'notion'; @@ -71,6 +76,7 @@ export interface KtxSetupSourcesPromptAdapter { }): Promise; select(options: { message: string; options: Array<{ value: string; label: string }> }): Promise; text(options: { message: string; placeholder?: string; initialValue?: string }): Promise; + password(options: { message: string }): Promise; cancel(message: string): void; log?(message: string): void; } @@ -86,6 +92,11 @@ export interface KtxSetupSourcesDeps { validateLooker?: (projectDir: string, connectionId: string) => Promise; validateLookml?: (connection: KtxProjectConnectionConfig) => Promise; validateNotion?: (connection: KtxProjectConnectionConfig) => Promise; + discoverMetabaseDatabases?: (args: { + sourceUrl: string; + sourceApiKeyRef: string; + sourceConnectionId: string; + }) => Promise; runMapping?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise; runInitialIngest?: ( projectDir: string, @@ -143,6 +154,12 @@ function createPromptAdapter(): KtxSetupSourcesPromptAdapter { ); return isCancel(value) ? undefined : String(value); }, + async password(options) { + const value = await withSetupInterruptConfirmation(() => + password({ ...options, message: withTextInputNavigation(options.message) }), + ); + return isCancel(value) ? undefined : String(value); + }, cancel(message) { cancel(message); }, @@ -172,17 +189,6 @@ function connectionNamePrompt(label: string): string { return `Name this ${label} connection\nKTX will use this short name in commands and config. You can rename it now.`; } -function gitAuthAfterFailurePrompt(source: KtxSetupSourceType): string { - const label = source === 'dbt' ? 'This' : `This ${sourceLabel(source)}`; - return [ - `${label} repo requires authentication.`, - 'Generate a token at: https://github.com/settings/tokens/new', - 'Store it in an env var, then enter env:VARIABLE_NAME here (e.g. env:GITHUB_TOKEN).', - 'Or use file:/absolute/path if the token is stored in a file.', - 'Press Enter to skip and try without authentication anyway.', - ].join('\n'); -} - function sourceSubpathPrompt(source: KtxSetupSourceType): string { if (source === 'dbt') { return [ @@ -198,6 +204,21 @@ function sourceSubpathPrompt(source: KtxSetupSourceType): string { ].join('\n'); } +const SCAN_SKIP_DIRS = new Set(['.git', 'node_modules', '.venv', 'target', 'dbt_packages', 'dbt_modules', '__pycache__']); + +async function findDbtProjectSubpaths(rootDir: string): Promise { + const entries = await readdir(rootDir, { withFileTypes: true, recursive: true }); + const subpaths: string[] = []; + for (const entry of entries) { + if (!entry.isFile()) continue; + if (entry.name !== 'dbt_project.yml' && entry.name !== 'dbt_project.yaml') continue; + const relDir = relative(rootDir, entry.parentPath); + if (relDir.split('/').some((part) => SCAN_SKIP_DIRS.has(part))) continue; + subpaths.push(relDir); + } + return subpaths; +} + async function promptText( prompts: KtxSetupSourcesPromptAdapter, options: { message: string; placeholder?: string; initialValue?: string }, @@ -222,6 +243,70 @@ function credentialRef(value: string | undefined, label: string): string { return ref; } +async function chooseSourceCredentialRef(input: { + prompts: KtxSetupSourcesPromptAdapter; + projectDir: string; + label: string; + envName: string; + secretFileName: string; +}): Promise { + while (true) { + const choice = await input.prompts.select({ + message: `How should KTX find your ${input.label}?`, + options: [ + { value: 'env', label: `Use ${input.envName} from the environment` }, + { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'back', label: 'Back' }, + ], + }); + if (choice === 'back') return 'back'; + if (choice === 'paste') { + const value = await input.prompts.password({ message: input.label }); + if (value === undefined) continue; + if (!value.trim()) continue; + return await writeProjectLocalSecretReference({ + projectDir: input.projectDir, + fileName: input.secretFileName, + value, + }); + } + return `env:${input.envName}`; + } +} + +async function chooseGitAuthCredentialRef(input: { + prompts: KtxSetupSourcesPromptAdapter; + projectDir: string; + source: KtxSetupSourceType; + connectionId: string; +}): Promise { + const label = input.source === 'dbt' ? 'This' : `This ${sourceLabel(input.source)}`; + while (true) { + const choice = await input.prompts.select({ + message: `${label} repo requires authentication.`, + options: [ + { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, + { value: 'paste', label: 'Paste a token and save it as a local secret file' }, + { value: 'skip', label: 'Skip — try without authentication' }, + { value: 'back', label: 'Back' }, + ], + }); + if (choice === 'back') return 'back'; + if (choice === 'skip') return undefined; + if (choice === 'paste') { + const value = await input.prompts.password({ message: 'Git access token' }); + if (value === undefined) continue; + if (!value.trim()) continue; + return await writeProjectLocalSecretReference({ + projectDir: input.projectDir, + fileName: `${input.connectionId}-auth-token`, + value, + }); + } + return 'env:GITHUB_TOKEN'; + } +} + function repoOrLocalSource(args: KtxSetupSourcesArgs): { sourceDir?: string; repoUrl?: string } { if (args.sourcePath && args.sourceGitUrl) { throw new Error('Choose only one source location: --source-path or --source-git-url.'); @@ -512,16 +597,6 @@ async function defaultValidateMetricflow(connection: KtxProjectConnectionConfig) }; } -async function defaultValidateMetabase(projectDir: string, connectionId: string): Promise { - const code = await runKtxConnection( - { command: 'map', projectDir, sourceConnectionId: connectionId, json: true }, - { stdout: { write() {} }, stderr: { write() {} } }, - ); - return code === 0 - ? { ok: true, detail: 'mapping validated' } - : { ok: false, message: 'Metabase mapping validation failed' }; -} - async function defaultValidateLooker(projectDir: string, connectionId: string): Promise { const code = await runKtxConnectionMapping( { command: 'refresh', projectDir, connectionId, autoAccept: true }, @@ -634,6 +709,11 @@ type SourcePromptState = KtxSetupSourcesArgs & { type SourcePromptStep = (state: SourcePromptState) => Promise<'next' | 'back'>; +interface WarehouseConnectionChoice { + id: string; + connectionType: string; +} + type InteractiveSourceConnectionChoice = | { kind: 'existing'; connectionId: string; connection: KtxProjectConnectionConfig } | { kind: 'new'; args: KtxSetupSourcesArgs } @@ -672,6 +752,107 @@ function resetRepoLocationFields(state: SourcePromptState): void { delete state.sourceProjectName; } +function warehouseConnectionChoices(config: KtxProjectConfig): WarehouseConnectionChoice[] { + return Object.entries(config.connections) + .filter(([, connection]) => PRIMARY_SOURCE_DRIVERS.has(String(connection.driver ?? '').toLowerCase())) + .map(([id, connection]) => ({ id, connectionType: localConnectionTypeForConfig(id, connection) })) + .sort((left, right) => left.id.localeCompare(right.id)); +} + +async function chooseMappedWarehouseConnectionId(input: { + projectDir: string; + prompts: KtxSetupSourcesPromptAdapter; +}): Promise { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const choices = warehouseConnectionChoices(project.config); + if (choices.length === 1) { + return choices[0].id; + } + if (choices.length === 0) { + const entered = await promptText(input.prompts, { message: 'Mapped warehouse connection id' }); + return entered === undefined ? 'back' : entered; + } + + const selected = await input.prompts.select({ + message: 'Mapped warehouse connection', + options: [ + ...choices.map((choice) => ({ + value: choice.id, + label: `${choice.id} (${choice.connectionType})`, + })), + { value: 'back', label: 'Back' }, + ], + }); + return selected === 'back' ? 'back' : selected; +} + +async function defaultDiscoverMetabaseDatabases(input: { + sourceUrl: string; + sourceApiKeyRef: string; +}): Promise { + const apiKey = resolveKtxConfigReference(input.sourceApiKeyRef, process.env); + if (!apiKey) { + throw new Error('Metabase API key ref could not be resolved'); + } + const client = new MetabaseClient( + { apiUrl: input.sourceUrl, apiKey }, + DEFAULT_METABASE_CLIENT_CONFIG, + ); + try { + return await discoverMetabaseDatabases(client); + } finally { + await client.cleanup(); + } +} + +function metabaseDatabaseLabel(database: DiscoveredMetabaseDatabase): string { + const detail = [database.engine].filter(Boolean).join(', '); + return detail ? `${database.id}: ${database.name} (${detail})` : `${database.id}: ${database.name}`; +} + +async function chooseMetabaseDatabaseId(input: { + state: SourcePromptState; + prompts: KtxSetupSourcesPromptAdapter; + deps: KtxSetupSourcesDeps; +}): Promise { + const sourceUrl = input.state.sourceUrl; + const sourceApiKeyRef = input.state.sourceApiKeyRef; + if (sourceUrl && sourceApiKeyRef) { + try { + const discovered = await (input.deps.discoverMetabaseDatabases ?? defaultDiscoverMetabaseDatabases)({ + sourceUrl, + sourceApiKeyRef, + sourceConnectionId: input.state.sourceConnectionId ?? 'metabase-main', + }); + if (discovered.length === 1) { + return discovered[0].id; + } + if (discovered.length > 1) { + const selected = await input.prompts.select({ + message: 'Metabase database', + options: [ + ...discovered + .slice() + .sort((left, right) => left.id - right.id) + .map((database) => ({ + value: String(database.id), + label: metabaseDatabaseLabel(database), + })), + { value: 'back', label: 'Back' }, + ], + }); + return selected === 'back' ? 'back' : Number.parseInt(selected, 10); + } + } catch { + // Discovery is a convenience. Fall back to the raw id prompt when credentials + // are unavailable locally or the Metabase API cannot be reached yet. + } + } + + const databaseId = await promptText(input.prompts, { message: 'Metabase database id' }); + return databaseId === undefined ? 'back' : Number.parseInt(databaseId, 10); +} + function connectionIdPromptSteps( args: KtxSetupSourcesArgs, source: KtxSetupSourceType, @@ -703,6 +884,7 @@ async function promptForInteractiveSource( prompts: KtxSetupSourcesPromptAdapter, defaultConnectionId = `${source}-main`, testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection, + discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'], ): Promise { const initialState: SourcePromptState = { ...args, source }; if (args.sourceConnectionId) { @@ -757,23 +939,6 @@ async function promptForInteractiveSource( }, ] : []), - ...(state.sourceLocation - ? [ - async (currentState: SourcePromptState) => { - const subpath = await promptText(prompts, { - message: sourceSubpathPrompt(source), - placeholder: 'optional', - }); - if (subpath === undefined) return 'back'; - if (subpath) { - currentState.sourceSubpath = subpath; - } else { - delete currentState.sourceSubpath; - } - return 'next'; - }, - ] - : []), ...(state.sourceLocation === 'git' ? [ async (currentState: SourcePromptState) => { @@ -783,11 +948,13 @@ async function promptForInteractiveSource( prompts.log?.('Repository connected.'); return 'next'; } - const authRef = await promptText(prompts, { - message: gitAuthAfterFailurePrompt(source), - placeholder: 'env:GITHUB_TOKEN', + const authRef = await chooseGitAuthCredentialRef({ + prompts, + projectDir: args.projectDir, + source, + connectionId: currentState.sourceConnectionId ?? `${source}-main`, }); - if (authRef === undefined) return 'back'; + if (authRef === 'back') return 'back'; if (authRef) { currentState.sourceAuthTokenRef = authRef; } else { @@ -797,6 +964,79 @@ async function promptForInteractiveSource( }, ] : []), + ...(state.sourceLocation + ? [ + async (currentState: SourcePromptState) => { + if (source === 'dbt') { + let scanDir: string | undefined; + if (currentState.sourceLocation === 'path' && currentState.sourcePath) { + scanDir = currentState.sourcePath; + } else if (currentState.sourceLocation === 'git' && currentState.sourceGitUrl) { + try { + const cacheDir = await mkdtemp(join(tmpdir(), 'ktx-setup-dbt-scan-')); + const authToken = currentState.sourceAuthTokenRef + ? resolveKtxConfigReference(currentState.sourceAuthTokenRef, process.env) + : null; + await cloneOrPull({ + repoUrl: currentState.sourceGitUrl, + authToken, + cacheDir, + branch: currentState.sourceBranch ?? 'main', + }); + scanDir = cacheDir; + } catch { + // Clone failed — fall through to manual prompt + } + } + if (scanDir) { + try { + const subpaths = await findDbtProjectSubpaths(scanDir); + if (subpaths.length === 1) { + const found = subpaths[0]!; + if (found) { + currentState.sourceSubpath = found; + prompts.log?.(`Found dbt_project.yml in ${found}/`); + } else { + delete currentState.sourceSubpath; + } + return 'next'; + } + if (subpaths.length > 1) { + const selected = await prompts.select({ + message: 'Multiple dbt projects found — which one should KTX use?', + options: [ + ...subpaths.map((p) => ({ value: p || '.', label: p || '(project root)' })), + { value: 'back', label: 'Back' }, + ], + }); + if (selected === 'back') return 'back'; + const subpath = selected === '.' ? '' : selected; + if (subpath) { + currentState.sourceSubpath = subpath; + } else { + delete currentState.sourceSubpath; + } + return 'next'; + } + } catch { + // Directory unreadable — fall through to manual prompt + } + } + } + const subpath = await promptText(prompts, { + message: sourceSubpathPrompt(source), + placeholder: 'optional', + }); + if (subpath === undefined) return 'back'; + if (subpath) { + currentState.sourceSubpath = subpath; + } else { + delete currentState.sourceSubpath; + } + return 'next'; + }, + ] + : []), ]); } @@ -810,24 +1050,34 @@ async function promptForInteractiveSource( return 'next'; }, async (state) => { - const sourceApiKeyRef = await promptText(prompts, { - message: 'Metabase API key ref', - placeholder: 'env:METABASE_API_KEY', + const ref = await chooseSourceCredentialRef({ + prompts, + projectDir: args.projectDir, + label: 'Metabase API key', + envName: 'METABASE_API_KEY', + secretFileName: `${state.sourceConnectionId ?? 'metabase-main'}-api-key`, }); - if (sourceApiKeyRef === undefined) return 'back'; - state.sourceApiKeyRef = sourceApiKeyRef; + if (ref === 'back') return 'back'; + state.sourceApiKeyRef = ref; return 'next'; }, async (state) => { - const sourceWarehouseConnectionId = await promptText(prompts, { message: 'Mapped warehouse connection id' }); - if (sourceWarehouseConnectionId === undefined) return 'back'; + const sourceWarehouseConnectionId = await chooseMappedWarehouseConnectionId({ + projectDir: args.projectDir, + prompts, + }); + if (sourceWarehouseConnectionId === 'back') return 'back'; state.sourceWarehouseConnectionId = sourceWarehouseConnectionId; return 'next'; }, async (state) => { - const databaseId = await promptText(prompts, { message: 'Metabase database id' }); - if (databaseId === undefined) return 'back'; - state.metabaseDatabaseId = Number.parseInt(databaseId, 10); + const databaseId = await chooseMetabaseDatabaseId({ + state, + prompts, + deps: { discoverMetabaseDatabases: discoverMetabaseDatabaseList }, + }); + if (databaseId === 'back') return 'back'; + state.metabaseDatabaseId = databaseId; return 'next'; }, ]); @@ -849,17 +1099,23 @@ async function promptForInteractiveSource( return 'next'; }, async (state) => { - const sourceClientSecretRef = await promptText(prompts, { - message: 'Looker client secret ref', - placeholder: 'env:LOOKER_CLIENT_SECRET', + const ref = await chooseSourceCredentialRef({ + prompts, + projectDir: args.projectDir, + label: 'Looker client secret', + envName: 'LOOKER_CLIENT_SECRET', + secretFileName: `${state.sourceConnectionId ?? 'looker-main'}-client-secret`, }); - if (sourceClientSecretRef === undefined) return 'back'; - state.sourceClientSecretRef = sourceClientSecretRef; + if (ref === 'back') return 'back'; + state.sourceClientSecretRef = ref; return 'next'; }, async (state) => { - const sourceWarehouseConnectionId = await promptText(prompts, { message: 'Mapped warehouse connection id' }); - if (sourceWarehouseConnectionId === undefined) return 'back'; + const sourceWarehouseConnectionId = await chooseMappedWarehouseConnectionId({ + projectDir: args.projectDir, + prompts, + }); + if (sourceWarehouseConnectionId === 'back') return 'back'; state.sourceWarehouseConnectionId = sourceWarehouseConnectionId; return 'next'; }, @@ -882,12 +1138,15 @@ async function promptForInteractiveSource( return await runSourcePromptSteps(initialState, (state) => [ ...connectionSteps, async (currentState) => { - const sourceApiKeyRef = await promptText(prompts, { - message: 'Notion token ref', - placeholder: 'env:NOTION_TOKEN', + const ref = await chooseSourceCredentialRef({ + prompts, + projectDir: args.projectDir, + label: 'Notion integration token', + envName: 'NOTION_TOKEN', + secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`, }); - if (sourceApiKeyRef === undefined) return 'back'; - currentState.sourceApiKeyRef = sourceApiKeyRef; + if (ref === 'back') return 'back'; + currentState.sourceApiKeyRef = ref; return 'next'; }, async (currentState) => { @@ -956,13 +1215,21 @@ async function chooseInteractiveSourceConnection(input: { connections: Record; prompts: KtxSetupSourcesPromptAdapter; testGitRepo?: KtxSetupSourcesDeps['testGitRepo']; + discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases']; }): Promise { const existingIds = existingConnectionIdsBySource(input.connections, input.source); const defaultConnectionId = defaultConnectionIdForSource(input.connections, input.source); const label = sourceLabel(input.source); if (existingIds.length === 0) { - const sourceArgs = await promptForInteractiveSource(input.args, input.source, input.prompts, defaultConnectionId, input.testGitRepo); + const sourceArgs = await promptForInteractiveSource( + input.args, + input.source, + input.prompts, + defaultConnectionId, + input.testGitRepo, + input.discoverMetabaseDatabases, + ); return sourceArgs === 'back' ? 'back' : { kind: 'new', args: sourceArgs }; } @@ -987,7 +1254,14 @@ async function chooseInteractiveSourceConnection(input: { } continue; } - const sourceArgs = await promptForInteractiveSource(input.args, input.source, input.prompts, defaultConnectionId, input.testGitRepo); + const sourceArgs = await promptForInteractiveSource( + input.args, + input.source, + input.prompts, + defaultConnectionId, + input.testGitRepo, + input.discoverMetabaseDatabases, + ); if (sourceArgs === 'back') { continue; } @@ -1026,7 +1300,9 @@ async function validateSource( return await (deps.validateMetricflow ?? defaultValidateMetricflow)(args.connection); } if (source === 'metabase') { - return await (deps.validateMetabase ?? defaultValidateMetabase)(args.projectDir, args.connectionId); + return deps.validateMetabase + ? await deps.validateMetabase(args.projectDir, args.connectionId) + : { ok: true, detail: 'mapping validation runs after the connection is saved' }; } if (source === 'looker') { return await (deps.validateLooker ?? defaultValidateLooker)(args.projectDir, args.connectionId); @@ -1097,6 +1373,7 @@ export async function runKtxSetupSourcesStep( connections: (await loadKtxProject({ projectDir: args.projectDir })).config.connections, prompts, testGitRepo: deps.testGitRepo, + discoverMetabaseDatabases: deps.discoverMetabaseDatabases, }); if (sourceChoice === 'back') { if (args.source) { diff --git a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts index 2e492f07..0c854f6d 100644 --- a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts +++ b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts @@ -1,8 +1,21 @@ -import { describe, expect, it } from 'vitest'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import type { KtxProjectConnectionConfig } from '../../../project/index.js'; import { metabaseRuntimeConfigFromLocalConnection } from './local-metabase.adapter.js'; describe('metabaseRuntimeConfigFromLocalConnection', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-runtime-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + it('resolves api_url and env-backed api_key_ref from a flat ktx.yaml connection', () => { const connection: KtxProjectConnectionConfig = { driver: 'metabase', @@ -20,6 +33,21 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => { }); }); + it('resolves file-backed api_key_ref from pasted setup secrets', async () => { + const keyPath = join(tempDir, 'metabase-main-api-key'); + await writeFile(keyPath, 'mb_file_key\n', 'utf-8'); // pragma: allowlist secret + const connection: KtxProjectConnectionConfig = { + driver: 'metabase', + api_url: 'https://metabase.example.com', + api_key_ref: `file:${keyPath}`, + }; + + expect(metabaseRuntimeConfigFromLocalConnection('prod-metabase', connection)).toEqual({ + apiUrl: 'https://metabase.example.com', + apiKey: 'mb_file_key', // pragma: allowlist secret + }); + }); + it('accepts url as the local api URL alias', () => { const connection: KtxProjectConnectionConfig = { driver: 'metabase', diff --git a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts index a13b3923..bd81413f 100644 --- a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts +++ b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts @@ -1,5 +1,6 @@ import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js'; import { ktxLocalStateDbPath } from '../../../project/index.js'; +import { resolveKtxConfigReference } from '../../../core/config-reference.js'; import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './client.js'; import { IngestMetabaseClientFactory, @@ -13,14 +14,6 @@ function stringField(value: unknown): string | null { return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; } -function resolveEnvReference(ref: string, env: NodeJS.ProcessEnv): string | null { - if (!ref.startsWith('env:')) { - return null; - } - const name = ref.slice('env:'.length); - return stringField(env[name]); -} - function hasNetworkProxy(connection: KtxProjectConnectionConfig): boolean { return connection.networkProxy != null || connection.network_proxy != null; } @@ -42,7 +35,7 @@ export function metabaseRuntimeConfigFromLocalConnection( const apiUrl = stringField(connection.api_url) ?? stringField(connection.apiUrl) ?? stringField(connection.url); const literalApiKey = stringField(connection.api_key) ?? stringField(connection.apiKey); const apiKeyRef = stringField(connection.api_key_ref) ?? stringField(connection.apiKeyRef); - const apiKey = literalApiKey ?? (apiKeyRef ? resolveEnvReference(apiKeyRef, env) : null); + const apiKey = literalApiKey ?? (apiKeyRef ? resolveKtxConfigReference(apiKeyRef, env) : null); if (!apiUrl) { throw new Error(`Connection "${connectionId}" is missing metabase api_url`);