mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-07 07:55:13 +02:00
Improve connector credential setup UX
This commit is contained in:
Luca Martial
parent
d89be2390f
commit
1b5a9fe120
6 changed files with 882 additions and 115 deletions
|
|
@ -98,11 +98,11 @@ describe('parseScanSummary', () => {
|
|||
|
||||
describe('parseIngestSummary', () => {
|
||||
it('extracts work units and saved memory', () => {
|
||||
expect(parseIngestSummary('Work units: 5\nSaved memory: 3 wiki, 2 SL')).toBe('5 work units · 3 wiki, 2 SL');
|
||||
expect(parseIngestSummary('Work units: 5\nSaved memory: 3 wiki, 2 SL')).toBe('5 items indexed · 3 wiki, 2 SL');
|
||||
});
|
||||
|
||||
it('extracts work units alone when no saved memory', () => {
|
||||
expect(parseIngestSummary('Work units: 5\nStatus: done')).toBe('5 work units');
|
||||
expect(parseIngestSummary('Work units: 5\nStatus: done')).toBe('5 items indexed');
|
||||
});
|
||||
|
||||
it('extracts saved memory alone when no work units', () => {
|
||||
|
|
@ -127,10 +127,18 @@ describe('initViewState', () => {
|
|||
expect(state.contextSources[0].target.connectionId).toBe('dbt-main');
|
||||
expect(state.frame).toBe(0);
|
||||
});
|
||||
|
||||
it('initializes global timing fields', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
expect(state.startedAt).toBeNull();
|
||||
expect(state.totalElapsedMs).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderContextBuildView', () => {
|
||||
it('renders all-queued state', () => {
|
||||
it('renders all-queued state with ○ icon and progress counter', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
|
||||
|
|
@ -138,6 +146,8 @@ describe('renderContextBuildView', () => {
|
|||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('Building KTX context');
|
||||
expect(output).toContain('(0/2)');
|
||||
expect(output).toContain('○');
|
||||
expect(output).toContain('Primary sources:');
|
||||
expect(output).toContain('warehouse');
|
||||
expect(output).toContain('queued');
|
||||
|
|
@ -145,6 +155,29 @@ describe('renderContextBuildView', () => {
|
|||
expect(output).toContain('dbt-main');
|
||||
});
|
||||
|
||||
it('renders header with total elapsed time when set', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.totalElapsedMs = 65000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('(0/1 · 1m05s)');
|
||||
});
|
||||
|
||||
it('renders dynamic separator matching header width', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.totalElapsedMs = 120000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
const lines = output.split('\n');
|
||||
const headerLine = lines.find((l) => l.includes('Building KTX context'))!;
|
||||
const separatorLine = lines.find((l) => /^─+$/.test(l))!;
|
||||
expect(separatorLine.length).toBeGreaterThanOrEqual(headerLine.length);
|
||||
});
|
||||
|
||||
it('renders completed state with summary', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
|
|
@ -156,6 +189,74 @@ describe('renderContextBuildView', () => {
|
|||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('42 tables');
|
||||
expect(output).toContain('1m12s');
|
||||
expect(output).toContain('(1/1)');
|
||||
});
|
||||
|
||||
it('renders running target with elapsed time', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'running';
|
||||
state.primarySources[0].elapsedMs = 30000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('scanning...');
|
||||
expect(output).toContain('30s');
|
||||
});
|
||||
|
||||
it('renders running target with progress bar when percentage is available', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'running';
|
||||
state.primarySources[0].detailLine = '[50%] Scanning tables...';
|
||||
state.primarySources[0].elapsedMs = 15000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('██████░░░░░░');
|
||||
expect(output).toContain('50%');
|
||||
expect(output).toContain('Scanning tables...');
|
||||
expect(output).toContain('15s');
|
||||
});
|
||||
|
||||
it('renders completion summary when all targets are done', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'done';
|
||||
state.primarySources[0].elapsedMs = 72000;
|
||||
state.contextSources[0].status = 'done';
|
||||
state.contextSources[0].elapsedMs = 34000;
|
||||
state.totalElapsedMs = 106000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('Done in 1m46s · 2 sources processed');
|
||||
});
|
||||
|
||||
it('renders singular source label in completion summary', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'done';
|
||||
state.primarySources[0].elapsedMs = 5000;
|
||||
state.totalElapsedMs = 5000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('Done in 5s · 1 source processed');
|
||||
});
|
||||
|
||||
it('does not render completion summary while targets are still active', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'done';
|
||||
state.contextSources[0].status = 'running';
|
||||
state.totalElapsedMs = 30000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).not.toContain('Done in');
|
||||
});
|
||||
|
||||
it('renders failed state', () => {
|
||||
|
|
@ -178,6 +279,29 @@ describe('renderContextBuildView', () => {
|
|||
expect(output).not.toContain('Primary sources:');
|
||||
expect(output).toContain('Context sources:');
|
||||
});
|
||||
|
||||
it('preserves detach hint while targets are active', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'running';
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false, showHint: true, projectDir: '/tmp/project' });
|
||||
expect(output).toContain('d to detach');
|
||||
expect(output).toContain('ktx setup --project-dir /tmp/project');
|
||||
expect(output).toContain('to resume');
|
||||
});
|
||||
|
||||
it('omits detach hint when all targets are done', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'done';
|
||||
state.totalElapsedMs = 5000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false, showHint: true });
|
||||
expect(output).not.toContain('d to detach');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runContextBuild', () => {
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ export interface ContextBuildViewState {
|
|||
primarySources: ContextBuildTargetState[];
|
||||
contextSources: ContextBuildTargetState[];
|
||||
frame: number;
|
||||
startedAt: number | null;
|
||||
totalElapsedMs: number;
|
||||
}
|
||||
|
||||
export interface ContextBuildArgs {
|
||||
|
|
@ -79,7 +81,7 @@ function statusIcon(status: ContextBuildTargetState['status'], frame: number, st
|
|||
case 'running':
|
||||
return SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋';
|
||||
default:
|
||||
return '·';
|
||||
return '○';
|
||||
}
|
||||
}
|
||||
switch (status) {
|
||||
|
|
@ -90,10 +92,27 @@ function statusIcon(status: ContextBuildTargetState['status'], frame: number, st
|
|||
case 'running':
|
||||
return cyan(SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋');
|
||||
default:
|
||||
return dim('·');
|
||||
return dim('○');
|
||||
}
|
||||
}
|
||||
|
||||
function extractPercent(detailLine: string | null): number | null {
|
||||
if (!detailLine) return null;
|
||||
const match = detailLine.match(/^\[(\d+)%\]/);
|
||||
return match ? Number(match[1]) : null;
|
||||
}
|
||||
|
||||
const BAR_WIDTH = 12;
|
||||
const BAR_FILLED = '█';
|
||||
const BAR_EMPTY = '░';
|
||||
|
||||
function renderProgressBar(percent: number, styled: boolean): string {
|
||||
const filled = Math.round((percent / 100) * BAR_WIDTH);
|
||||
const empty = BAR_WIDTH - filled;
|
||||
const bar = `${BAR_FILLED.repeat(filled)}${BAR_EMPTY.repeat(empty)}`;
|
||||
return styled ? cyan(bar) : bar;
|
||||
}
|
||||
|
||||
function targetDetail(target: ContextBuildTargetState, styled: boolean): string {
|
||||
if (target.status === 'done') {
|
||||
const parts: string[] = [];
|
||||
|
|
@ -105,7 +124,17 @@ function targetDetail(target: ContextBuildTargetState, styled: boolean): string
|
|||
return styled ? red('failed') : 'failed';
|
||||
}
|
||||
if (target.status === 'running') {
|
||||
return target.detailLine ?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...');
|
||||
const percent = extractPercent(target.detailLine);
|
||||
const progressText = target.detailLine?.replace(/^\[\d+%\]\s*/, '')
|
||||
?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...');
|
||||
const elapsed = target.elapsedMs > 0 ? formatDuration(target.elapsedMs) : null;
|
||||
const parts: string[] = [];
|
||||
if (percent !== null) {
|
||||
parts.push(`${renderProgressBar(percent, styled)} ${percent}%`);
|
||||
}
|
||||
parts.push(progressText);
|
||||
if (elapsed) parts.push(styled ? dim(elapsed) : elapsed);
|
||||
return parts.join(' ');
|
||||
}
|
||||
return styled ? dim('queued') : 'queued';
|
||||
}
|
||||
|
|
@ -140,17 +169,39 @@ export function renderContextBuildView(
|
|||
): string {
|
||||
const styled = options.styled ?? true;
|
||||
const width = columnWidth(state);
|
||||
const allTargets = [...state.primarySources, ...state.contextSources];
|
||||
const doneCount = allTargets.filter((t) => t.status === 'done' || t.status === 'failed').length;
|
||||
const totalCount = allTargets.length;
|
||||
const hasActive = allTargets.some((t) => t.status === 'running' || t.status === 'queued');
|
||||
const allDone = totalCount > 0 && !hasActive;
|
||||
|
||||
const headerParts = ['Building KTX context'];
|
||||
if (totalCount > 0) {
|
||||
const progressParts: string[] = [`${doneCount}/${totalCount}`];
|
||||
if (state.totalElapsedMs > 0) progressParts.push(formatDuration(state.totalElapsedMs));
|
||||
const progress = `(${progressParts.join(' · ')})`;
|
||||
headerParts.push(styled ? dim(progress) : progress);
|
||||
}
|
||||
const header = headerParts.join(' ');
|
||||
const headerPlainLength = header.replace(/\x1b\[[0-9;]*m/g, '').length;
|
||||
const separator = '─'.repeat(Math.max(21, headerPlainLength));
|
||||
|
||||
const lines: string[] = [
|
||||
'',
|
||||
'Building KTX context',
|
||||
'─────────────────────',
|
||||
header,
|
||||
separator,
|
||||
...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width),
|
||||
...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width),
|
||||
'',
|
||||
];
|
||||
const hasActive = [...state.primarySources, ...state.contextSources].some(
|
||||
(t) => t.status === 'running' || t.status === 'queued',
|
||||
);
|
||||
|
||||
if (allDone && state.totalElapsedMs > 0) {
|
||||
const sourcesLabel = totalCount === 1 ? '1 source' : `${totalCount} sources`;
|
||||
const summary = ` Done in ${formatDuration(state.totalElapsedMs)} · ${sourcesLabel} processed`;
|
||||
lines.push(styled ? green(summary) : summary);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
if (options.showHint && hasActive) {
|
||||
const hint = ` d to detach · ${resumeCommand(options.projectDir)} to resume`;
|
||||
lines.push(styled ? dim(hint) : hint);
|
||||
|
|
@ -177,7 +228,7 @@ export function parseScanSummary(output: string): string | null {
|
|||
export function parseIngestSummary(output: string): string | null {
|
||||
const parts: string[] = [];
|
||||
const workUnits = output.match(/Work units: (\d+)/);
|
||||
if (workUnits) parts.push(`${workUnits[1]} work units`);
|
||||
if (workUnits) parts.push(`${workUnits[1]} items indexed`);
|
||||
const savedMemory = output.match(/Saved memory: (.+)/);
|
||||
if (savedMemory) parts.push(savedMemory[1]);
|
||||
return parts.length > 0 ? parts.join(' · ') : null;
|
||||
|
|
@ -289,6 +340,8 @@ export function initViewState(targets: KtxPublicIngestPlanTarget[]): ContextBuil
|
|||
primarySources: targets.filter((t) => t.operation === 'scan').map(makeTargetState),
|
||||
contextSources: targets.filter((t) => t.operation === 'source-ingest').map(makeTargetState),
|
||||
frame: 0,
|
||||
startedAt: null,
|
||||
totalElapsedMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -303,6 +356,8 @@ export async function runContextBuild(
|
|||
const isTTY = io.stdout.isTTY === true;
|
||||
const nowFn = deps.now ?? (() => Date.now());
|
||||
|
||||
state.startedAt = nowFn();
|
||||
|
||||
const repainter = isTTY ? createRepainter(io) : null;
|
||||
const viewOpts = { styled: true, projectDir: args.projectDir };
|
||||
const paint = (hint: boolean) => repainter?.paint(renderContextBuildView(state, { ...viewOpts, showHint: hint }));
|
||||
|
|
@ -312,6 +367,9 @@ export async function runContextBuild(
|
|||
if (repainter) {
|
||||
spinnerInterval = setInterval(() => {
|
||||
state.frame++;
|
||||
if (state.startedAt !== null) {
|
||||
state.totalElapsedMs = nowFn() - state.startedAt;
|
||||
}
|
||||
for (const t of [...state.primarySources, ...state.contextSources]) {
|
||||
if (t.status === 'running' && t.startedAt !== null) {
|
||||
t.elapsedMs = nowFn() - t.startedAt;
|
||||
|
|
@ -400,6 +458,10 @@ export async function runContextBuild(
|
|||
cleanupKeystroke?.();
|
||||
}
|
||||
|
||||
if (state.startedAt !== null) {
|
||||
state.totalElapsedMs = nowFn() - state.startedAt;
|
||||
}
|
||||
|
||||
if (detached) {
|
||||
return { exitCode: 0, detached: true };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
|
|
@ -8,6 +8,7 @@ import {
|
|||
serializeKtxProjectConfig,
|
||||
} from '@ktx/context/project';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import {
|
||||
runKtxSetupSourcesStep,
|
||||
type KtxSetupSourcesDeps,
|
||||
|
|
@ -41,14 +42,17 @@ function prompts(values: {
|
|||
multiselect?: string[][];
|
||||
select?: string[];
|
||||
text?: Array<string | undefined>;
|
||||
password?: Array<string | undefined>;
|
||||
}): KtxSetupSourcesPromptAdapter {
|
||||
const multiselectValues = [...(values.multiselect ?? [])];
|
||||
const selectValues = [...(values.select ?? [])];
|
||||
const textValues = [...(values.text ?? [])];
|
||||
const passwordValues = [...(values.password ?? [])];
|
||||
return {
|
||||
multiselect: vi.fn(async () => multiselectValues.shift() ?? []),
|
||||
select: vi.fn(async () => selectValues.shift() ?? 'skip'),
|
||||
text: vi.fn(async () => (textValues.length > 0 ? textValues.shift() : '')),
|
||||
password: vi.fn(async () => (passwordValues.length > 0 ? passwordValues.shift() : undefined)),
|
||||
cancel: vi.fn(),
|
||||
log: vi.fn(),
|
||||
};
|
||||
|
|
@ -207,6 +211,193 @@ describe('setup sources step', () => {
|
|||
expect(runMapping).toHaveBeenCalledWith(projectDir, 'prod_metabase', io.io);
|
||||
});
|
||||
|
||||
it('defaults interactive Metabase and Looker source setup to the only warehouse connection', async () => {
|
||||
await addPrimarySource();
|
||||
const cases: Array<{
|
||||
source: 'metabase' | 'looker';
|
||||
text: string[];
|
||||
deps: KtxSetupSourcesDeps;
|
||||
expectedConnection: Record<string, unknown>;
|
||||
}> = [
|
||||
{
|
||||
source: 'metabase',
|
||||
text: ['metabase-main', 'https://metabase.example.com'],
|
||||
deps: {
|
||||
discoverMetabaseDatabases: vi.fn(async () => [
|
||||
{ id: 1, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' },
|
||||
]),
|
||||
validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })),
|
||||
runMapping: vi.fn(async () => 0),
|
||||
},
|
||||
expectedConnection: {
|
||||
driver: 'metabase',
|
||||
mappings: { databaseMappings: { '1': 'warehouse' } },
|
||||
},
|
||||
},
|
||||
{
|
||||
source: 'looker',
|
||||
text: ['looker-main', 'https://looker.example.com', 'client-id', ''],
|
||||
deps: {
|
||||
validateLooker: vi.fn(async () => ({ ok: true as const, detail: 'mapping refreshed' })),
|
||||
runMapping: vi.fn(async () => 0),
|
||||
},
|
||||
expectedConnection: {
|
||||
driver: 'looker',
|
||||
mappings: { connectionMappings: { warehouse: 'warehouse' } },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
const testPrompts = prompts({
|
||||
multiselect: [[testCase.source]],
|
||||
select: ['env', 'done'],
|
||||
text: testCase.text,
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
makeIo().io,
|
||||
{
|
||||
prompts: testPrompts,
|
||||
...testCase.deps,
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: [`${testCase.source}-main`] });
|
||||
|
||||
expect(
|
||||
vi.mocked(testPrompts.text).mock.calls.some(([options]) => options.message.includes('Mapped warehouse')),
|
||||
).toBe(false);
|
||||
if (testCase.source === 'metabase') {
|
||||
expect(
|
||||
vi.mocked(testPrompts.text).mock.calls.some(([options]) => options.message.includes('Metabase database id')),
|
||||
).toBe(false);
|
||||
}
|
||||
expect((await readConfig()).connections[`${testCase.source}-main`]).toMatchObject(testCase.expectedConnection);
|
||||
}
|
||||
});
|
||||
|
||||
it('prompts for the mapped warehouse when interactive Metabase and Looker source setup has multiple choices', async () => {
|
||||
await addPrimarySource();
|
||||
await addConnection('analytics_warehouse', {
|
||||
driver: 'snowflake',
|
||||
account: 'acme',
|
||||
database: 'analytics',
|
||||
readonly: true,
|
||||
});
|
||||
|
||||
const cases: Array<{
|
||||
source: 'metabase' | 'looker';
|
||||
text: string[];
|
||||
deps: KtxSetupSourcesDeps;
|
||||
expectedConnection: Record<string, unknown>;
|
||||
}> = [
|
||||
{
|
||||
source: 'metabase',
|
||||
text: ['metabase-main', 'https://metabase.example.com'],
|
||||
deps: {
|
||||
discoverMetabaseDatabases: vi.fn(async () => [
|
||||
{ id: 1, name: 'Finance', engine: 'postgres', host: 'db.example.com', dbName: 'finance' },
|
||||
{ id: 2, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' },
|
||||
]),
|
||||
validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })),
|
||||
runMapping: vi.fn(async () => 0),
|
||||
},
|
||||
expectedConnection: {
|
||||
driver: 'metabase',
|
||||
mappings: { databaseMappings: { '2': 'analytics_warehouse' } },
|
||||
},
|
||||
},
|
||||
{
|
||||
source: 'looker',
|
||||
text: ['looker-main', 'https://looker.example.com', 'client-id', 'analytics'],
|
||||
deps: {
|
||||
validateLooker: vi.fn(async () => ({ ok: true as const, detail: 'mapping refreshed' })),
|
||||
runMapping: vi.fn(async () => 0),
|
||||
},
|
||||
expectedConnection: {
|
||||
driver: 'looker',
|
||||
mappings: { connectionMappings: { analytics: 'analytics_warehouse' } },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
const testPrompts = prompts({
|
||||
multiselect: [[testCase.source]],
|
||||
select: testCase.source === 'metabase' ? ['env', 'analytics_warehouse', '2', 'done'] : ['env', 'analytics_warehouse', 'done'],
|
||||
text: testCase.text,
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
makeIo().io,
|
||||
{
|
||||
prompts: testPrompts,
|
||||
...testCase.deps,
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: [`${testCase.source}-main`] });
|
||||
|
||||
expect(testPrompts.select).toHaveBeenCalledWith({
|
||||
message: 'Mapped warehouse connection',
|
||||
options: [
|
||||
{ value: 'analytics_warehouse', label: 'analytics_warehouse (SNOWFLAKE)' },
|
||||
{ value: 'warehouse', label: 'warehouse (POSTGRESQL)' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
if (testCase.source === 'metabase') {
|
||||
expect(testPrompts.select).toHaveBeenCalledWith({
|
||||
message: 'Metabase database',
|
||||
options: [
|
||||
{ value: '1', label: '1: Finance (postgres)' },
|
||||
{ value: '2', label: '2: Analytics (postgres)' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
expect(
|
||||
vi.mocked(testPrompts.text).mock.calls.some(([options]) => options.message.includes('Metabase database id')),
|
||||
).toBe(false);
|
||||
}
|
||||
expect((await readConfig()).connections[`${testCase.source}-main`]).toMatchObject(testCase.expectedConnection);
|
||||
}
|
||||
});
|
||||
|
||||
it('lets visible Metabase mapping surface refresh and validation failures', async () => {
|
||||
await addPrimarySource();
|
||||
const runMapping = vi.fn(async (_projectDir: string, _connectionId: string, io: KtxCliIo) => {
|
||||
io.stderr.write('1: Metabase database does not match KTX connection database\n');
|
||||
return 1;
|
||||
});
|
||||
const io = makeIo();
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['metabase']],
|
||||
select: ['env'],
|
||||
text: ['metabase-main', 'https://metabase.example.com'],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
io.io,
|
||||
{
|
||||
prompts: testPrompts,
|
||||
discoverMetabaseDatabases: vi.fn(async () => [
|
||||
{ id: 1, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' },
|
||||
]),
|
||||
runMapping,
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ status: 'failed', projectDir });
|
||||
|
||||
expect(runMapping).toHaveBeenCalledWith(projectDir, 'metabase-main', io.io);
|
||||
expect(io.stderr()).toContain('1: Metabase database does not match KTX connection database');
|
||||
expect(io.stderr()).not.toContain('Metabase mapping validation failed');
|
||||
});
|
||||
|
||||
it('does not mark sources complete when validation fails', async () => {
|
||||
await addPrimarySource();
|
||||
const io = makeIo();
|
||||
|
|
@ -333,8 +524,8 @@ describe('setup sources step', () => {
|
|||
const io = makeIo();
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['dbt']],
|
||||
select: ['git'],
|
||||
text: ['dbt-main', 'https://github.com/acme-org/private-repo', 'main', '', 'env:GITHUB_TOKEN'],
|
||||
select: ['git', 'env'],
|
||||
text: ['dbt-main', 'https://github.com/acme-org/private-repo', 'main', ''],
|
||||
});
|
||||
|
||||
await expect(
|
||||
|
|
@ -350,19 +541,16 @@ describe('setup sources step', () => {
|
|||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
|
||||
|
||||
expect(testGitRepo).toHaveBeenCalledWith({ repoUrl: 'https://github.com/acme-org/private-repo' });
|
||||
expect(testPrompts.text).toHaveBeenNthCalledWith(5, {
|
||||
message: textInputPrompt(
|
||||
[
|
||||
'This repo requires authentication.',
|
||||
'Generate a token at: https://github.com/settings/tokens/new',
|
||||
'Store it in an env var, then enter env:VARIABLE_NAME here (e.g. env:GITHUB_TOKEN).',
|
||||
'Or use file:/absolute/path if the token is stored in a file.',
|
||||
'Press Enter to skip and try without authentication anyway.',
|
||||
].join('\n'),
|
||||
),
|
||||
placeholder: 'env:GITHUB_TOKEN',
|
||||
expect(testPrompts.select).toHaveBeenCalledWith({
|
||||
message: 'This repo requires authentication.',
|
||||
options: [
|
||||
{ value: 'env', label: 'Use GITHUB_TOKEN from the environment' },
|
||||
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
|
||||
{ value: 'skip', label: 'Skip — try without authentication' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
expect(testPrompts.text).toHaveBeenCalledTimes(5);
|
||||
expect(testPrompts.text).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it('enables the dbt adapter when adding a dbt source connection', async () => {
|
||||
|
|
@ -692,13 +880,11 @@ describe('setup sources step', () => {
|
|||
},
|
||||
{
|
||||
source: 'metabase',
|
||||
select: ['back', 'env'],
|
||||
text: [
|
||||
'metabase-main',
|
||||
'https://old-metabase.example.com',
|
||||
undefined,
|
||||
'https://metabase.example.com',
|
||||
'env:METABASE_API_KEY',
|
||||
'warehouse',
|
||||
'1',
|
||||
],
|
||||
deps: {
|
||||
|
|
@ -709,14 +895,13 @@ describe('setup sources step', () => {
|
|||
},
|
||||
{
|
||||
source: 'looker',
|
||||
select: ['env'],
|
||||
text: [
|
||||
'looker-main',
|
||||
'https://old-looker.example.com',
|
||||
undefined,
|
||||
'https://looker.example.com',
|
||||
'client-id',
|
||||
'env:LOOKER_CLIENT_SECRET',
|
||||
'warehouse',
|
||||
'',
|
||||
],
|
||||
deps: {
|
||||
|
|
@ -727,10 +912,10 @@ describe('setup sources step', () => {
|
|||
},
|
||||
{
|
||||
source: 'notion',
|
||||
select: ['back', 'all_accessible'],
|
||||
text: ['notion-main', 'env:NOTION_TOKEN', 'env:NOTION_TOKEN'],
|
||||
select: ['env', 'back', 'env', 'all_accessible'],
|
||||
text: ['notion-main'],
|
||||
deps: { validateNotion: vi.fn(async () => ({ ok: true as const, detail: 'roots=0' })) },
|
||||
repeatedTextMessage: textInputPrompt('Notion token ref'),
|
||||
repeatedSelectMessage: 'How should KTX find your Notion integration token?',
|
||||
},
|
||||
];
|
||||
|
||||
|
|
@ -787,4 +972,102 @@ describe('setup sources step', () => {
|
|||
expect(io.stdout()).toContain('Connect a primary source before adding context sources.');
|
||||
expect((await readConfig()).setup?.completed_steps ?? []).not.toContain('sources');
|
||||
});
|
||||
|
||||
it('auto-detects dbt_project.yml at the root of a local path', async () => {
|
||||
await addPrimarySource();
|
||||
const dbtDir = join(tempDir, 'dbt-repo');
|
||||
await mkdir(dbtDir, { recursive: true });
|
||||
await writeFile(join(dbtDir, 'dbt_project.yml'), 'name: analytics\n');
|
||||
|
||||
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
|
||||
const io = makeIo();
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['dbt']],
|
||||
select: ['path'],
|
||||
text: ['dbt-main', dbtDir],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
io.io,
|
||||
{ prompts: testPrompts, validateDbt },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
|
||||
|
||||
expect(testPrompts.text).toHaveBeenCalledTimes(2);
|
||||
const config = await readConfig();
|
||||
expect(config.connections['dbt-main']).toMatchObject({ driver: 'dbt', source_dir: dbtDir });
|
||||
expect(config.connections['dbt-main']).not.toHaveProperty('path');
|
||||
});
|
||||
|
||||
it('auto-detects dbt_project.yml in a subdirectory of a local path', async () => {
|
||||
await addPrimarySource();
|
||||
const dbtDir = join(tempDir, 'monorepo');
|
||||
await mkdir(join(dbtDir, 'analytics', 'dbt'), { recursive: true });
|
||||
await writeFile(join(dbtDir, 'analytics', 'dbt', 'dbt_project.yml'), 'name: analytics\n');
|
||||
|
||||
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
|
||||
const io = makeIo();
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['dbt']],
|
||||
select: ['path'],
|
||||
text: ['dbt-main', dbtDir],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
io.io,
|
||||
{ prompts: testPrompts, validateDbt },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
|
||||
|
||||
expect(testPrompts.text).toHaveBeenCalledTimes(2);
|
||||
expect(testPrompts.log).toHaveBeenCalledWith('Found dbt_project.yml in analytics/dbt/');
|
||||
const config = await readConfig();
|
||||
expect(config.connections['dbt-main']).toMatchObject({
|
||||
driver: 'dbt',
|
||||
source_dir: dbtDir,
|
||||
path: 'analytics/dbt',
|
||||
});
|
||||
});
|
||||
|
||||
it('shows a picker when multiple dbt projects are found in a local path', async () => {
|
||||
await addPrimarySource();
|
||||
const dbtDir = join(tempDir, 'multi-dbt');
|
||||
await mkdir(join(dbtDir, 'analytics'), { recursive: true });
|
||||
await mkdir(join(dbtDir, 'staging'), { recursive: true });
|
||||
await writeFile(join(dbtDir, 'analytics', 'dbt_project.yml'), 'name: analytics\n');
|
||||
await writeFile(join(dbtDir, 'staging', 'dbt_project.yml'), 'name: staging\n');
|
||||
|
||||
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
|
||||
const io = makeIo();
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['dbt']],
|
||||
select: ['path', 'staging'],
|
||||
text: ['dbt-main', dbtDir],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
io.io,
|
||||
{ prompts: testPrompts, validateDbt },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
|
||||
|
||||
expect(testPrompts.select).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: 'Multiple dbt projects found — which one should KTX use?',
|
||||
}),
|
||||
);
|
||||
expect(testPrompts.text).toHaveBeenCalledTimes(2);
|
||||
const config = await readConfig();
|
||||
expect(config.connections['dbt-main']).toMatchObject({
|
||||
driver: 'dbt',
|
||||
source_dir: dbtDir,
|
||||
path: 'staging',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
import { mkdtemp, readdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { join, relative, resolve } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { cancel, isCancel, log, multiselect, select, text } from '@clack/prompts';
|
||||
import { resolveNotionAuthToken } from '@ktx/context/connections';
|
||||
import { cancel, isCancel, log, multiselect, password, select, text } from '@clack/prompts';
|
||||
import { localConnectionTypeForConfig, resolveNotionAuthToken } from '@ktx/context/connections';
|
||||
import { resolveKtxConfigReference } from '@ktx/context/core';
|
||||
import {
|
||||
cloneOrPull,
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
discoverMetabaseDatabases,
|
||||
type DiscoveredMetabaseDatabase,
|
||||
loadDbtSchemaFiles,
|
||||
loadProjectInfo,
|
||||
MetabaseClient,
|
||||
type NotionApi,
|
||||
NotionClient,
|
||||
parseLookmlStagedDir,
|
||||
|
|
@ -28,6 +32,7 @@ import { runKtxConnection } from './connection.js';
|
|||
import { withMenuOptionsSpacing, withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
|
||||
import { runKtxPublicIngest } from './public-ingest.js';
|
||||
import { withSetupInterruptConfirmation } from './setup-interrupt.js';
|
||||
import { writeProjectLocalSecretReference } from './setup-secrets.js';
|
||||
|
||||
export type KtxSetupSourceType = 'dbt' | 'metricflow' | 'metabase' | 'looker' | 'lookml' | 'notion';
|
||||
|
||||
|
|
@ -71,6 +76,7 @@ export interface KtxSetupSourcesPromptAdapter {
|
|||
}): Promise<string[]>;
|
||||
select(options: { message: string; options: Array<{ value: string; label: string }> }): Promise<string>;
|
||||
text(options: { message: string; placeholder?: string; initialValue?: string }): Promise<string | undefined>;
|
||||
password(options: { message: string }): Promise<string | undefined>;
|
||||
cancel(message: string): void;
|
||||
log?(message: string): void;
|
||||
}
|
||||
|
|
@ -86,6 +92,11 @@ export interface KtxSetupSourcesDeps {
|
|||
validateLooker?: (projectDir: string, connectionId: string) => Promise<SourceValidationResult>;
|
||||
validateLookml?: (connection: KtxProjectConnectionConfig) => Promise<SourceValidationResult>;
|
||||
validateNotion?: (connection: KtxProjectConnectionConfig) => Promise<SourceValidationResult>;
|
||||
discoverMetabaseDatabases?: (args: {
|
||||
sourceUrl: string;
|
||||
sourceApiKeyRef: string;
|
||||
sourceConnectionId: string;
|
||||
}) => Promise<DiscoveredMetabaseDatabase[]>;
|
||||
runMapping?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise<number>;
|
||||
runInitialIngest?: (
|
||||
projectDir: string,
|
||||
|
|
@ -143,6 +154,12 @@ function createPromptAdapter(): KtxSetupSourcesPromptAdapter {
|
|||
);
|
||||
return isCancel(value) ? undefined : String(value);
|
||||
},
|
||||
async password(options) {
|
||||
const value = await withSetupInterruptConfirmation(() =>
|
||||
password({ ...options, message: withTextInputNavigation(options.message) }),
|
||||
);
|
||||
return isCancel(value) ? undefined : String(value);
|
||||
},
|
||||
cancel(message) {
|
||||
cancel(message);
|
||||
},
|
||||
|
|
@ -172,17 +189,6 @@ function connectionNamePrompt(label: string): string {
|
|||
return `Name this ${label} connection\nKTX will use this short name in commands and config. You can rename it now.`;
|
||||
}
|
||||
|
||||
function gitAuthAfterFailurePrompt(source: KtxSetupSourceType): string {
|
||||
const label = source === 'dbt' ? 'This' : `This ${sourceLabel(source)}`;
|
||||
return [
|
||||
`${label} repo requires authentication.`,
|
||||
'Generate a token at: https://github.com/settings/tokens/new',
|
||||
'Store it in an env var, then enter env:VARIABLE_NAME here (e.g. env:GITHUB_TOKEN).',
|
||||
'Or use file:/absolute/path if the token is stored in a file.',
|
||||
'Press Enter to skip and try without authentication anyway.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function sourceSubpathPrompt(source: KtxSetupSourceType): string {
|
||||
if (source === 'dbt') {
|
||||
return [
|
||||
|
|
@ -198,6 +204,21 @@ function sourceSubpathPrompt(source: KtxSetupSourceType): string {
|
|||
].join('\n');
|
||||
}
|
||||
|
||||
const SCAN_SKIP_DIRS = new Set(['.git', 'node_modules', '.venv', 'target', 'dbt_packages', 'dbt_modules', '__pycache__']);
|
||||
|
||||
async function findDbtProjectSubpaths(rootDir: string): Promise<string[]> {
|
||||
const entries = await readdir(rootDir, { withFileTypes: true, recursive: true });
|
||||
const subpaths: string[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
if (entry.name !== 'dbt_project.yml' && entry.name !== 'dbt_project.yaml') continue;
|
||||
const relDir = relative(rootDir, entry.parentPath);
|
||||
if (relDir.split('/').some((part) => SCAN_SKIP_DIRS.has(part))) continue;
|
||||
subpaths.push(relDir);
|
||||
}
|
||||
return subpaths;
|
||||
}
|
||||
|
||||
async function promptText(
|
||||
prompts: KtxSetupSourcesPromptAdapter,
|
||||
options: { message: string; placeholder?: string; initialValue?: string },
|
||||
|
|
@ -222,6 +243,70 @@ function credentialRef(value: string | undefined, label: string): string {
|
|||
return ref;
|
||||
}
|
||||
|
||||
async function chooseSourceCredentialRef(input: {
|
||||
prompts: KtxSetupSourcesPromptAdapter;
|
||||
projectDir: string;
|
||||
label: string;
|
||||
envName: string;
|
||||
secretFileName: string;
|
||||
}): Promise<string | 'back'> {
|
||||
while (true) {
|
||||
const choice = await input.prompts.select({
|
||||
message: `How should KTX find your ${input.label}?`,
|
||||
options: [
|
||||
{ value: 'env', label: `Use ${input.envName} from the environment` },
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
if (choice === 'back') return 'back';
|
||||
if (choice === 'paste') {
|
||||
const value = await input.prompts.password({ message: input.label });
|
||||
if (value === undefined) continue;
|
||||
if (!value.trim()) continue;
|
||||
return await writeProjectLocalSecretReference({
|
||||
projectDir: input.projectDir,
|
||||
fileName: input.secretFileName,
|
||||
value,
|
||||
});
|
||||
}
|
||||
return `env:${input.envName}`;
|
||||
}
|
||||
}
|
||||
|
||||
async function chooseGitAuthCredentialRef(input: {
|
||||
prompts: KtxSetupSourcesPromptAdapter;
|
||||
projectDir: string;
|
||||
source: KtxSetupSourceType;
|
||||
connectionId: string;
|
||||
}): Promise<string | undefined | 'back'> {
|
||||
const label = input.source === 'dbt' ? 'This' : `This ${sourceLabel(input.source)}`;
|
||||
while (true) {
|
||||
const choice = await input.prompts.select({
|
||||
message: `${label} repo requires authentication.`,
|
||||
options: [
|
||||
{ value: 'env', label: 'Use GITHUB_TOKEN from the environment' },
|
||||
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
|
||||
{ value: 'skip', label: 'Skip — try without authentication' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
if (choice === 'back') return 'back';
|
||||
if (choice === 'skip') return undefined;
|
||||
if (choice === 'paste') {
|
||||
const value = await input.prompts.password({ message: 'Git access token' });
|
||||
if (value === undefined) continue;
|
||||
if (!value.trim()) continue;
|
||||
return await writeProjectLocalSecretReference({
|
||||
projectDir: input.projectDir,
|
||||
fileName: `${input.connectionId}-auth-token`,
|
||||
value,
|
||||
});
|
||||
}
|
||||
return 'env:GITHUB_TOKEN';
|
||||
}
|
||||
}
|
||||
|
||||
function repoOrLocalSource(args: KtxSetupSourcesArgs): { sourceDir?: string; repoUrl?: string } {
|
||||
if (args.sourcePath && args.sourceGitUrl) {
|
||||
throw new Error('Choose only one source location: --source-path or --source-git-url.');
|
||||
|
|
@ -512,16 +597,6 @@ async function defaultValidateMetricflow(connection: KtxProjectConnectionConfig)
|
|||
};
|
||||
}
|
||||
|
||||
async function defaultValidateMetabase(projectDir: string, connectionId: string): Promise<SourceValidationResult> {
|
||||
const code = await runKtxConnection(
|
||||
{ command: 'map', projectDir, sourceConnectionId: connectionId, json: true },
|
||||
{ stdout: { write() {} }, stderr: { write() {} } },
|
||||
);
|
||||
return code === 0
|
||||
? { ok: true, detail: 'mapping validated' }
|
||||
: { ok: false, message: 'Metabase mapping validation failed' };
|
||||
}
|
||||
|
||||
async function defaultValidateLooker(projectDir: string, connectionId: string): Promise<SourceValidationResult> {
|
||||
const code = await runKtxConnectionMapping(
|
||||
{ command: 'refresh', projectDir, connectionId, autoAccept: true },
|
||||
|
|
@ -634,6 +709,11 @@ type SourcePromptState = KtxSetupSourcesArgs & {
|
|||
|
||||
type SourcePromptStep = (state: SourcePromptState) => Promise<'next' | 'back'>;
|
||||
|
||||
interface WarehouseConnectionChoice {
|
||||
id: string;
|
||||
connectionType: string;
|
||||
}
|
||||
|
||||
type InteractiveSourceConnectionChoice =
|
||||
| { kind: 'existing'; connectionId: string; connection: KtxProjectConnectionConfig }
|
||||
| { kind: 'new'; args: KtxSetupSourcesArgs }
|
||||
|
|
@ -672,6 +752,107 @@ function resetRepoLocationFields(state: SourcePromptState): void {
|
|||
delete state.sourceProjectName;
|
||||
}
|
||||
|
||||
function warehouseConnectionChoices(config: KtxProjectConfig): WarehouseConnectionChoice[] {
|
||||
return Object.entries(config.connections)
|
||||
.filter(([, connection]) => PRIMARY_SOURCE_DRIVERS.has(String(connection.driver ?? '').toLowerCase()))
|
||||
.map(([id, connection]) => ({ id, connectionType: localConnectionTypeForConfig(id, connection) }))
|
||||
.sort((left, right) => left.id.localeCompare(right.id));
|
||||
}
|
||||
|
||||
async function chooseMappedWarehouseConnectionId(input: {
|
||||
projectDir: string;
|
||||
prompts: KtxSetupSourcesPromptAdapter;
|
||||
}): Promise<string | 'back'> {
|
||||
const project = await loadKtxProject({ projectDir: input.projectDir });
|
||||
const choices = warehouseConnectionChoices(project.config);
|
||||
if (choices.length === 1) {
|
||||
return choices[0].id;
|
||||
}
|
||||
if (choices.length === 0) {
|
||||
const entered = await promptText(input.prompts, { message: 'Mapped warehouse connection id' });
|
||||
return entered === undefined ? 'back' : entered;
|
||||
}
|
||||
|
||||
const selected = await input.prompts.select({
|
||||
message: 'Mapped warehouse connection',
|
||||
options: [
|
||||
...choices.map((choice) => ({
|
||||
value: choice.id,
|
||||
label: `${choice.id} (${choice.connectionType})`,
|
||||
})),
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
return selected === 'back' ? 'back' : selected;
|
||||
}
|
||||
|
||||
async function defaultDiscoverMetabaseDatabases(input: {
|
||||
sourceUrl: string;
|
||||
sourceApiKeyRef: string;
|
||||
}): Promise<DiscoveredMetabaseDatabase[]> {
|
||||
const apiKey = resolveKtxConfigReference(input.sourceApiKeyRef, process.env);
|
||||
if (!apiKey) {
|
||||
throw new Error('Metabase API key ref could not be resolved');
|
||||
}
|
||||
const client = new MetabaseClient(
|
||||
{ apiUrl: input.sourceUrl, apiKey },
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
);
|
||||
try {
|
||||
return await discoverMetabaseDatabases(client);
|
||||
} finally {
|
||||
await client.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
function metabaseDatabaseLabel(database: DiscoveredMetabaseDatabase): string {
|
||||
const detail = [database.engine].filter(Boolean).join(', ');
|
||||
return detail ? `${database.id}: ${database.name} (${detail})` : `${database.id}: ${database.name}`;
|
||||
}
|
||||
|
||||
async function chooseMetabaseDatabaseId(input: {
|
||||
state: SourcePromptState;
|
||||
prompts: KtxSetupSourcesPromptAdapter;
|
||||
deps: KtxSetupSourcesDeps;
|
||||
}): Promise<number | 'back'> {
|
||||
const sourceUrl = input.state.sourceUrl;
|
||||
const sourceApiKeyRef = input.state.sourceApiKeyRef;
|
||||
if (sourceUrl && sourceApiKeyRef) {
|
||||
try {
|
||||
const discovered = await (input.deps.discoverMetabaseDatabases ?? defaultDiscoverMetabaseDatabases)({
|
||||
sourceUrl,
|
||||
sourceApiKeyRef,
|
||||
sourceConnectionId: input.state.sourceConnectionId ?? 'metabase-main',
|
||||
});
|
||||
if (discovered.length === 1) {
|
||||
return discovered[0].id;
|
||||
}
|
||||
if (discovered.length > 1) {
|
||||
const selected = await input.prompts.select({
|
||||
message: 'Metabase database',
|
||||
options: [
|
||||
...discovered
|
||||
.slice()
|
||||
.sort((left, right) => left.id - right.id)
|
||||
.map((database) => ({
|
||||
value: String(database.id),
|
||||
label: metabaseDatabaseLabel(database),
|
||||
})),
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
return selected === 'back' ? 'back' : Number.parseInt(selected, 10);
|
||||
}
|
||||
} catch {
|
||||
// Discovery is a convenience. Fall back to the raw id prompt when credentials
|
||||
// are unavailable locally or the Metabase API cannot be reached yet.
|
||||
}
|
||||
}
|
||||
|
||||
const databaseId = await promptText(input.prompts, { message: 'Metabase database id' });
|
||||
return databaseId === undefined ? 'back' : Number.parseInt(databaseId, 10);
|
||||
}
|
||||
|
||||
function connectionIdPromptSteps(
|
||||
args: KtxSetupSourcesArgs,
|
||||
source: KtxSetupSourceType,
|
||||
|
|
@ -703,6 +884,7 @@ async function promptForInteractiveSource(
|
|||
prompts: KtxSetupSourcesPromptAdapter,
|
||||
defaultConnectionId = `${source}-main`,
|
||||
testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection,
|
||||
discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'],
|
||||
): Promise<KtxSetupSourcesArgs | 'back'> {
|
||||
const initialState: SourcePromptState = { ...args, source };
|
||||
if (args.sourceConnectionId) {
|
||||
|
|
@ -757,23 +939,6 @@ async function promptForInteractiveSource(
|
|||
},
|
||||
]
|
||||
: []),
|
||||
...(state.sourceLocation
|
||||
? [
|
||||
async (currentState: SourcePromptState) => {
|
||||
const subpath = await promptText(prompts, {
|
||||
message: sourceSubpathPrompt(source),
|
||||
placeholder: 'optional',
|
||||
});
|
||||
if (subpath === undefined) return 'back';
|
||||
if (subpath) {
|
||||
currentState.sourceSubpath = subpath;
|
||||
} else {
|
||||
delete currentState.sourceSubpath;
|
||||
}
|
||||
return 'next';
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(state.sourceLocation === 'git'
|
||||
? [
|
||||
async (currentState: SourcePromptState) => {
|
||||
|
|
@ -783,11 +948,13 @@ async function promptForInteractiveSource(
|
|||
prompts.log?.('Repository connected.');
|
||||
return 'next';
|
||||
}
|
||||
const authRef = await promptText(prompts, {
|
||||
message: gitAuthAfterFailurePrompt(source),
|
||||
placeholder: 'env:GITHUB_TOKEN',
|
||||
const authRef = await chooseGitAuthCredentialRef({
|
||||
prompts,
|
||||
projectDir: args.projectDir,
|
||||
source,
|
||||
connectionId: currentState.sourceConnectionId ?? `${source}-main`,
|
||||
});
|
||||
if (authRef === undefined) return 'back';
|
||||
if (authRef === 'back') return 'back';
|
||||
if (authRef) {
|
||||
currentState.sourceAuthTokenRef = authRef;
|
||||
} else {
|
||||
|
|
@ -797,6 +964,79 @@ async function promptForInteractiveSource(
|
|||
},
|
||||
]
|
||||
: []),
|
||||
...(state.sourceLocation
|
||||
? [
|
||||
async (currentState: SourcePromptState) => {
|
||||
if (source === 'dbt') {
|
||||
let scanDir: string | undefined;
|
||||
if (currentState.sourceLocation === 'path' && currentState.sourcePath) {
|
||||
scanDir = currentState.sourcePath;
|
||||
} else if (currentState.sourceLocation === 'git' && currentState.sourceGitUrl) {
|
||||
try {
|
||||
const cacheDir = await mkdtemp(join(tmpdir(), 'ktx-setup-dbt-scan-'));
|
||||
const authToken = currentState.sourceAuthTokenRef
|
||||
? resolveKtxConfigReference(currentState.sourceAuthTokenRef, process.env)
|
||||
: null;
|
||||
await cloneOrPull({
|
||||
repoUrl: currentState.sourceGitUrl,
|
||||
authToken,
|
||||
cacheDir,
|
||||
branch: currentState.sourceBranch ?? 'main',
|
||||
});
|
||||
scanDir = cacheDir;
|
||||
} catch {
|
||||
// Clone failed — fall through to manual prompt
|
||||
}
|
||||
}
|
||||
if (scanDir) {
|
||||
try {
|
||||
const subpaths = await findDbtProjectSubpaths(scanDir);
|
||||
if (subpaths.length === 1) {
|
||||
const found = subpaths[0]!;
|
||||
if (found) {
|
||||
currentState.sourceSubpath = found;
|
||||
prompts.log?.(`Found dbt_project.yml in ${found}/`);
|
||||
} else {
|
||||
delete currentState.sourceSubpath;
|
||||
}
|
||||
return 'next';
|
||||
}
|
||||
if (subpaths.length > 1) {
|
||||
const selected = await prompts.select({
|
||||
message: 'Multiple dbt projects found — which one should KTX use?',
|
||||
options: [
|
||||
...subpaths.map((p) => ({ value: p || '.', label: p || '(project root)' })),
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
if (selected === 'back') return 'back';
|
||||
const subpath = selected === '.' ? '' : selected;
|
||||
if (subpath) {
|
||||
currentState.sourceSubpath = subpath;
|
||||
} else {
|
||||
delete currentState.sourceSubpath;
|
||||
}
|
||||
return 'next';
|
||||
}
|
||||
} catch {
|
||||
// Directory unreadable — fall through to manual prompt
|
||||
}
|
||||
}
|
||||
}
|
||||
const subpath = await promptText(prompts, {
|
||||
message: sourceSubpathPrompt(source),
|
||||
placeholder: 'optional',
|
||||
});
|
||||
if (subpath === undefined) return 'back';
|
||||
if (subpath) {
|
||||
currentState.sourceSubpath = subpath;
|
||||
} else {
|
||||
delete currentState.sourceSubpath;
|
||||
}
|
||||
return 'next';
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]);
|
||||
}
|
||||
|
||||
|
|
@ -810,24 +1050,34 @@ async function promptForInteractiveSource(
|
|||
return 'next';
|
||||
},
|
||||
async (state) => {
|
||||
const sourceApiKeyRef = await promptText(prompts, {
|
||||
message: 'Metabase API key ref',
|
||||
placeholder: 'env:METABASE_API_KEY',
|
||||
const ref = await chooseSourceCredentialRef({
|
||||
prompts,
|
||||
projectDir: args.projectDir,
|
||||
label: 'Metabase API key',
|
||||
envName: 'METABASE_API_KEY',
|
||||
secretFileName: `${state.sourceConnectionId ?? 'metabase-main'}-api-key`,
|
||||
});
|
||||
if (sourceApiKeyRef === undefined) return 'back';
|
||||
state.sourceApiKeyRef = sourceApiKeyRef;
|
||||
if (ref === 'back') return 'back';
|
||||
state.sourceApiKeyRef = ref;
|
||||
return 'next';
|
||||
},
|
||||
async (state) => {
|
||||
const sourceWarehouseConnectionId = await promptText(prompts, { message: 'Mapped warehouse connection id' });
|
||||
if (sourceWarehouseConnectionId === undefined) return 'back';
|
||||
const sourceWarehouseConnectionId = await chooseMappedWarehouseConnectionId({
|
||||
projectDir: args.projectDir,
|
||||
prompts,
|
||||
});
|
||||
if (sourceWarehouseConnectionId === 'back') return 'back';
|
||||
state.sourceWarehouseConnectionId = sourceWarehouseConnectionId;
|
||||
return 'next';
|
||||
},
|
||||
async (state) => {
|
||||
const databaseId = await promptText(prompts, { message: 'Metabase database id' });
|
||||
if (databaseId === undefined) return 'back';
|
||||
state.metabaseDatabaseId = Number.parseInt(databaseId, 10);
|
||||
const databaseId = await chooseMetabaseDatabaseId({
|
||||
state,
|
||||
prompts,
|
||||
deps: { discoverMetabaseDatabases: discoverMetabaseDatabaseList },
|
||||
});
|
||||
if (databaseId === 'back') return 'back';
|
||||
state.metabaseDatabaseId = databaseId;
|
||||
return 'next';
|
||||
},
|
||||
]);
|
||||
|
|
@ -849,17 +1099,23 @@ async function promptForInteractiveSource(
|
|||
return 'next';
|
||||
},
|
||||
async (state) => {
|
||||
const sourceClientSecretRef = await promptText(prompts, {
|
||||
message: 'Looker client secret ref',
|
||||
placeholder: 'env:LOOKER_CLIENT_SECRET',
|
||||
const ref = await chooseSourceCredentialRef({
|
||||
prompts,
|
||||
projectDir: args.projectDir,
|
||||
label: 'Looker client secret',
|
||||
envName: 'LOOKER_CLIENT_SECRET',
|
||||
secretFileName: `${state.sourceConnectionId ?? 'looker-main'}-client-secret`,
|
||||
});
|
||||
if (sourceClientSecretRef === undefined) return 'back';
|
||||
state.sourceClientSecretRef = sourceClientSecretRef;
|
||||
if (ref === 'back') return 'back';
|
||||
state.sourceClientSecretRef = ref;
|
||||
return 'next';
|
||||
},
|
||||
async (state) => {
|
||||
const sourceWarehouseConnectionId = await promptText(prompts, { message: 'Mapped warehouse connection id' });
|
||||
if (sourceWarehouseConnectionId === undefined) return 'back';
|
||||
const sourceWarehouseConnectionId = await chooseMappedWarehouseConnectionId({
|
||||
projectDir: args.projectDir,
|
||||
prompts,
|
||||
});
|
||||
if (sourceWarehouseConnectionId === 'back') return 'back';
|
||||
state.sourceWarehouseConnectionId = sourceWarehouseConnectionId;
|
||||
return 'next';
|
||||
},
|
||||
|
|
@ -882,12 +1138,15 @@ async function promptForInteractiveSource(
|
|||
return await runSourcePromptSteps(initialState, (state) => [
|
||||
...connectionSteps,
|
||||
async (currentState) => {
|
||||
const sourceApiKeyRef = await promptText(prompts, {
|
||||
message: 'Notion token ref',
|
||||
placeholder: 'env:NOTION_TOKEN',
|
||||
const ref = await chooseSourceCredentialRef({
|
||||
prompts,
|
||||
projectDir: args.projectDir,
|
||||
label: 'Notion integration token',
|
||||
envName: 'NOTION_TOKEN',
|
||||
secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`,
|
||||
});
|
||||
if (sourceApiKeyRef === undefined) return 'back';
|
||||
currentState.sourceApiKeyRef = sourceApiKeyRef;
|
||||
if (ref === 'back') return 'back';
|
||||
currentState.sourceApiKeyRef = ref;
|
||||
return 'next';
|
||||
},
|
||||
async (currentState) => {
|
||||
|
|
@ -956,13 +1215,21 @@ async function chooseInteractiveSourceConnection(input: {
|
|||
connections: Record<string, KtxProjectConnectionConfig>;
|
||||
prompts: KtxSetupSourcesPromptAdapter;
|
||||
testGitRepo?: KtxSetupSourcesDeps['testGitRepo'];
|
||||
discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases'];
|
||||
}): Promise<InteractiveSourceConnectionChoice> {
|
||||
const existingIds = existingConnectionIdsBySource(input.connections, input.source);
|
||||
const defaultConnectionId = defaultConnectionIdForSource(input.connections, input.source);
|
||||
const label = sourceLabel(input.source);
|
||||
|
||||
if (existingIds.length === 0) {
|
||||
const sourceArgs = await promptForInteractiveSource(input.args, input.source, input.prompts, defaultConnectionId, input.testGitRepo);
|
||||
const sourceArgs = await promptForInteractiveSource(
|
||||
input.args,
|
||||
input.source,
|
||||
input.prompts,
|
||||
defaultConnectionId,
|
||||
input.testGitRepo,
|
||||
input.discoverMetabaseDatabases,
|
||||
);
|
||||
return sourceArgs === 'back' ? 'back' : { kind: 'new', args: sourceArgs };
|
||||
}
|
||||
|
||||
|
|
@ -987,7 +1254,14 @@ async function chooseInteractiveSourceConnection(input: {
|
|||
}
|
||||
continue;
|
||||
}
|
||||
const sourceArgs = await promptForInteractiveSource(input.args, input.source, input.prompts, defaultConnectionId, input.testGitRepo);
|
||||
const sourceArgs = await promptForInteractiveSource(
|
||||
input.args,
|
||||
input.source,
|
||||
input.prompts,
|
||||
defaultConnectionId,
|
||||
input.testGitRepo,
|
||||
input.discoverMetabaseDatabases,
|
||||
);
|
||||
if (sourceArgs === 'back') {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -1026,7 +1300,9 @@ async function validateSource(
|
|||
return await (deps.validateMetricflow ?? defaultValidateMetricflow)(args.connection);
|
||||
}
|
||||
if (source === 'metabase') {
|
||||
return await (deps.validateMetabase ?? defaultValidateMetabase)(args.projectDir, args.connectionId);
|
||||
return deps.validateMetabase
|
||||
? await deps.validateMetabase(args.projectDir, args.connectionId)
|
||||
: { ok: true, detail: 'mapping validation runs after the connection is saved' };
|
||||
}
|
||||
if (source === 'looker') {
|
||||
return await (deps.validateLooker ?? defaultValidateLooker)(args.projectDir, args.connectionId);
|
||||
|
|
@ -1097,6 +1373,7 @@ export async function runKtxSetupSourcesStep(
|
|||
connections: (await loadKtxProject({ projectDir: args.projectDir })).config.connections,
|
||||
prompts,
|
||||
testGitRepo: deps.testGitRepo,
|
||||
discoverMetabaseDatabases: deps.discoverMetabaseDatabases,
|
||||
});
|
||||
if (sourceChoice === 'back') {
|
||||
if (args.source) {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,21 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import type { KtxProjectConnectionConfig } from '../../../project/index.js';
|
||||
import { metabaseRuntimeConfigFromLocalConnection } from './local-metabase.adapter.js';
|
||||
|
||||
describe('metabaseRuntimeConfigFromLocalConnection', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-runtime-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('resolves api_url and env-backed api_key_ref from a flat ktx.yaml connection', () => {
|
||||
const connection: KtxProjectConnectionConfig = {
|
||||
driver: 'metabase',
|
||||
|
|
@ -20,6 +33,21 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('resolves file-backed api_key_ref from pasted setup secrets', async () => {
|
||||
const keyPath = join(tempDir, 'metabase-main-api-key');
|
||||
await writeFile(keyPath, 'mb_file_key\n', 'utf-8'); // pragma: allowlist secret
|
||||
const connection: KtxProjectConnectionConfig = {
|
||||
driver: 'metabase',
|
||||
api_url: 'https://metabase.example.com',
|
||||
api_key_ref: `file:${keyPath}`,
|
||||
};
|
||||
|
||||
expect(metabaseRuntimeConfigFromLocalConnection('prod-metabase', connection)).toEqual({
|
||||
apiUrl: 'https://metabase.example.com',
|
||||
apiKey: 'mb_file_key', // pragma: allowlist secret
|
||||
});
|
||||
});
|
||||
|
||||
it('accepts url as the local api URL alias', () => {
|
||||
const connection: KtxProjectConnectionConfig = {
|
||||
driver: 'metabase',
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js';
|
||||
import { ktxLocalStateDbPath } from '../../../project/index.js';
|
||||
import { resolveKtxConfigReference } from '../../../core/config-reference.js';
|
||||
import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './client.js';
|
||||
import {
|
||||
IngestMetabaseClientFactory,
|
||||
|
|
@ -13,14 +14,6 @@ function stringField(value: unknown): string | null {
|
|||
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function resolveEnvReference(ref: string, env: NodeJS.ProcessEnv): string | null {
|
||||
if (!ref.startsWith('env:')) {
|
||||
return null;
|
||||
}
|
||||
const name = ref.slice('env:'.length);
|
||||
return stringField(env[name]);
|
||||
}
|
||||
|
||||
function hasNetworkProxy(connection: KtxProjectConnectionConfig): boolean {
|
||||
return connection.networkProxy != null || connection.network_proxy != null;
|
||||
}
|
||||
|
|
@ -42,7 +35,7 @@ export function metabaseRuntimeConfigFromLocalConnection(
|
|||
const apiUrl = stringField(connection.api_url) ?? stringField(connection.apiUrl) ?? stringField(connection.url);
|
||||
const literalApiKey = stringField(connection.api_key) ?? stringField(connection.apiKey);
|
||||
const apiKeyRef = stringField(connection.api_key_ref) ?? stringField(connection.apiKeyRef);
|
||||
const apiKey = literalApiKey ?? (apiKeyRef ? resolveEnvReference(apiKeyRef, env) : null);
|
||||
const apiKey = literalApiKey ?? (apiKeyRef ? resolveKtxConfigReference(apiKeyRef, env) : null);
|
||||
|
||||
if (!apiUrl) {
|
||||
throw new Error(`Connection "${connectionId}" is missing metabase api_url`);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue