mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-10 08:05:14 +02:00
Merge origin/main into npx-ktx-python-daemon
This commit is contained in:
commit
88a65bbdc7
148 changed files with 14743 additions and 3508 deletions
|
|
@ -28,7 +28,8 @@
|
|||
"assets:demo": "node scripts/build-demo-assets.mjs",
|
||||
"build": "node -e \"fs.rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json && node ../../scripts/prepare-cli-bin.mjs",
|
||||
"smoke": "vitest run src/standalone-smoke.test.ts src/example-smoke.test.ts --testTimeout 30000",
|
||||
"test": "vitest run",
|
||||
"test": "vitest run --exclude src/standalone-smoke.test.ts --exclude src/example-smoke.test.ts --exclude src/setup-databases.test.ts --exclude src/scan.test.ts --exclude src/commands/connection-metabase-setup.test.ts --exclude src/setup-models.test.ts --exclude src/setup-sources.test.ts --exclude src/setup.test.ts --exclude src/connection.test.ts --exclude src/setup-embeddings.test.ts --exclude src/ingest.test.ts --exclude src/commands/connection-mapping.test.ts --exclude src/ingest-viz.test.ts --exclude src/demo.test.ts --exclude src/setup-project.test.ts --exclude src/sl.test.ts --exclude src/local-scan-connectors.test.ts --exclude src/commands/connection-notion.test.ts",
|
||||
"test:slow": "vitest run src/setup-databases.test.ts src/scan.test.ts src/commands/connection-metabase-setup.test.ts src/setup-models.test.ts src/setup-sources.test.ts src/setup.test.ts src/connection.test.ts src/setup-embeddings.test.ts src/ingest.test.ts src/commands/connection-mapping.test.ts src/ingest-viz.test.ts src/demo.test.ts src/setup-project.test.ts src/sl.test.ts src/local-scan-connectors.test.ts src/commands/connection-notion.test.ts --testTimeout 30000",
|
||||
"type-check": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
|
|
@ -38,7 +39,6 @@
|
|||
"@ktx/connector-clickhouse": "workspace:*",
|
||||
"@ktx/connector-mysql": "workspace:*",
|
||||
"@ktx/connector-postgres": "workspace:*",
|
||||
"@ktx/connector-posthog": "workspace:*",
|
||||
"@ktx/connector-snowflake": "workspace:*",
|
||||
"@ktx/connector-sqlite": "workspace:*",
|
||||
"@ktx/connector-sqlserver": "workspace:*",
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ export interface KtxCliPackageInfo {
|
|||
}
|
||||
|
||||
export interface KtxCliIo {
|
||||
stdout: { isTTY?: boolean; write(chunk: string): void };
|
||||
stdout: { isTTY?: boolean; columns?: number; write(chunk: string): void };
|
||||
stderr: { write(chunk: string): void };
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,20 +3,23 @@ import { describe, expect, it, vi } from 'vitest';
|
|||
import type { KtxPublicIngestProject, KtxPublicIngestTargetResult } from './public-ingest.js';
|
||||
import {
|
||||
extractProgressMessage,
|
||||
createRepainter,
|
||||
initViewState,
|
||||
parseIngestSummary,
|
||||
parseScanSummary,
|
||||
renderContextBuildView,
|
||||
runContextBuild,
|
||||
viewStateFromSourceProgress,
|
||||
} from './context-build-view.js';
|
||||
|
||||
function makeIo(options: { isTTY?: boolean } = {}) {
|
||||
function makeIo(options: { isTTY?: boolean; columns?: number } = {}) {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
isTTY: options.isTTY,
|
||||
columns: options.columns,
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
|
|
@ -98,7 +101,7 @@ describe('parseScanSummary', () => {
|
|||
|
||||
describe('parseIngestSummary', () => {
|
||||
it('extracts work units and saved memory', () => {
|
||||
expect(parseIngestSummary('Work units: 5\nSaved memory: 3 wiki, 2 SL')).toBe('5 work units · 3 wiki, 2 SL');
|
||||
expect(parseIngestSummary('Work units: 5\nSaved memory: 3 wiki, 2 SL')).toBe('3 wiki, 2 SL');
|
||||
});
|
||||
|
||||
it('extracts work units alone when no saved memory', () => {
|
||||
|
|
@ -127,10 +130,18 @@ describe('initViewState', () => {
|
|||
expect(state.contextSources[0].target.connectionId).toBe('dbt-main');
|
||||
expect(state.frame).toBe(0);
|
||||
});
|
||||
|
||||
it('initializes global timing fields', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
expect(state.startedAt).toBeNull();
|
||||
expect(state.totalElapsedMs).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderContextBuildView', () => {
|
||||
it('renders all-queued state', () => {
|
||||
it('renders all-queued state with ○ icon and progress counter', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
|
||||
|
|
@ -138,6 +149,8 @@ describe('renderContextBuildView', () => {
|
|||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('Building KTX context');
|
||||
expect(output).toContain('(0/2)');
|
||||
expect(output).toContain('○');
|
||||
expect(output).toContain('Primary sources:');
|
||||
expect(output).toContain('warehouse');
|
||||
expect(output).toContain('queued');
|
||||
|
|
@ -145,6 +158,29 @@ describe('renderContextBuildView', () => {
|
|||
expect(output).toContain('dbt-main');
|
||||
});
|
||||
|
||||
it('renders header with total elapsed time when set', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.totalElapsedMs = 65000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('(0/1 · 1m05s)');
|
||||
});
|
||||
|
||||
it('renders dynamic separator matching header width', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.totalElapsedMs = 120000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
const lines = output.split('\n');
|
||||
const headerLine = lines.find((l) => l.includes('Building KTX context'))!;
|
||||
const separatorLine = lines.find((l) => /^─+$/.test(l))!;
|
||||
expect(separatorLine.length).toBeGreaterThanOrEqual(headerLine.length);
|
||||
});
|
||||
|
||||
it('renders completed state with summary', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
|
|
@ -156,6 +192,74 @@ describe('renderContextBuildView', () => {
|
|||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('42 tables');
|
||||
expect(output).toContain('1m12s');
|
||||
expect(output).toContain('(1/1)');
|
||||
});
|
||||
|
||||
it('renders running target with elapsed time', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'running';
|
||||
state.primarySources[0].elapsedMs = 30000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('scanning...');
|
||||
expect(output).toContain('(30s)');
|
||||
});
|
||||
|
||||
it('renders running target with progress bar when percentage is available', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'running';
|
||||
state.primarySources[0].detailLine = '[50%] Scanning tables...';
|
||||
state.primarySources[0].elapsedMs = 15000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('██████░░░░░░');
|
||||
expect(output).toContain('50%');
|
||||
expect(output).toContain('Scanning tables...');
|
||||
expect(output).toContain('(15s)');
|
||||
});
|
||||
|
||||
it('renders completion summary when all targets are done', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'done';
|
||||
state.primarySources[0].elapsedMs = 72000;
|
||||
state.contextSources[0].status = 'done';
|
||||
state.contextSources[0].elapsedMs = 34000;
|
||||
state.totalElapsedMs = 106000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('Done in 1m46s · 2 sources processed');
|
||||
});
|
||||
|
||||
it('renders singular source label in completion summary', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'done';
|
||||
state.primarySources[0].elapsedMs = 5000;
|
||||
state.totalElapsedMs = 5000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('Done in 5s · 1 source processed');
|
||||
});
|
||||
|
||||
it('does not render completion summary while targets are still active', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'done';
|
||||
state.contextSources[0].status = 'running';
|
||||
state.totalElapsedMs = 30000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).not.toContain('Done in');
|
||||
});
|
||||
|
||||
it('renders failed state', () => {
|
||||
|
|
@ -178,6 +282,54 @@ describe('renderContextBuildView', () => {
|
|||
expect(output).not.toContain('Primary sources:');
|
||||
expect(output).toContain('Context sources:');
|
||||
});
|
||||
|
||||
it('preserves detach hint while targets are active', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'running';
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false, showHint: true, projectDir: '/tmp/project' });
|
||||
expect(output).toContain('d to detach');
|
||||
expect(output).toContain('ktx setup --project-dir /tmp/project');
|
||||
expect(output).toContain('to resume');
|
||||
});
|
||||
|
||||
it('omits detach hint when all targets are done', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'done';
|
||||
state.totalElapsedMs = 5000;
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false, showHint: true });
|
||||
expect(output).not.toContain('d to detach');
|
||||
});
|
||||
});
|
||||
|
||||
describe('createRepainter', () => {
|
||||
it('moves up visual rows, not just newline count, when content wraps', () => {
|
||||
const io = makeIo({ isTTY: true, columns: 5 });
|
||||
const repainter = createRepainter(io.io);
|
||||
|
||||
repainter.paint('abcdefghijk\n');
|
||||
repainter.paint('updated\n');
|
||||
repainter.paint('done\n');
|
||||
|
||||
const cursorMoves = [...io.stdout().matchAll(/\u001b\[(\d+)A\r/g)].map((match) => Number(match[1]));
|
||||
expect(cursorMoves).toEqual([3, 2]);
|
||||
});
|
||||
|
||||
it('returns to the start of a single-line frame without moving up when content has no newline', () => {
|
||||
const io = makeIo({ isTTY: true, columns: 80 });
|
||||
const repainter = createRepainter(io.io);
|
||||
|
||||
repainter.paint('hello');
|
||||
repainter.paint('bye');
|
||||
|
||||
expect(io.stdout()).toContain('\rbye');
|
||||
expect(io.stdout()).not.toContain('\u001b[1A\rbye');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runContextBuild', () => {
|
||||
|
|
@ -298,6 +450,135 @@ describe('runContextBuild', () => {
|
|||
expect(mockExit).toHaveBeenCalledWith(0);
|
||||
expect(io.stdout()).toContain('Context build continuing in the background.');
|
||||
expect(io.stdout()).toContain('Resume: ktx setup --project-dir /tmp/project');
|
||||
expect(io.stdout()).toContain('Status: ktx setup context status --project-dir /tmp/project');
|
||||
mockExit.mockRestore();
|
||||
});
|
||||
|
||||
it('calls onSourceProgress when sources start and finish', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres' },
|
||||
dbt_main: { driver: 'dbt' },
|
||||
});
|
||||
const progressUpdates: Array<Array<{ connectionId: string; status: string }>> = [];
|
||||
const executeTarget = vi.fn(async (target) => successResult(target.connectionId, target.driver, target.operation));
|
||||
|
||||
await runContextBuild(
|
||||
project,
|
||||
{ projectDir: '/tmp/project', inputMode: 'disabled' },
|
||||
io.io,
|
||||
{
|
||||
executeTarget,
|
||||
now: () => 1000,
|
||||
onSourceProgress: (sources) => {
|
||||
progressUpdates.push(sources.map((s) => ({ connectionId: s.connectionId, status: s.status })));
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(progressUpdates).toHaveLength(4);
|
||||
expect(progressUpdates[0]).toEqual([
|
||||
{ connectionId: 'warehouse', status: 'running' },
|
||||
{ connectionId: 'dbt_main', status: 'queued' },
|
||||
]);
|
||||
expect(progressUpdates[1]).toEqual([
|
||||
{ connectionId: 'warehouse', status: 'done' },
|
||||
{ connectionId: 'dbt_main', status: 'queued' },
|
||||
]);
|
||||
expect(progressUpdates[2]).toEqual([
|
||||
{ connectionId: 'warehouse', status: 'done' },
|
||||
{ connectionId: 'dbt_main', status: 'running' },
|
||||
]);
|
||||
expect(progressUpdates[3]).toEqual([
|
||||
{ connectionId: 'warehouse', status: 'done' },
|
||||
{ connectionId: 'dbt_main', status: 'done' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns report IDs and artifact paths parsed from target output', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres' },
|
||||
dbt_main: { driver: 'dbt' },
|
||||
});
|
||||
const executeTarget = vi.fn(async (target, _args, targetIo) => {
|
||||
if (target.operation === 'scan') {
|
||||
targetIo.stdout.write('Report: raw-sources/warehouse/live-database/sync-1/scan-report.json\n');
|
||||
targetIo.stdout.write('Raw sources: raw-sources/warehouse/live-database/sync-1\n');
|
||||
} else {
|
||||
targetIo.stdout.write('Report: report-dbt-1\n');
|
||||
targetIo.stdout.write('Saved memory: 2 wiki, 3 SL\n');
|
||||
}
|
||||
return successResult(target.connectionId, target.driver, target.operation);
|
||||
});
|
||||
|
||||
const result = await runContextBuild(
|
||||
project,
|
||||
{ projectDir: '/tmp/project', inputMode: 'disabled' },
|
||||
io.io,
|
||||
{ executeTarget, now: () => 1000 },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
exitCode: 0,
|
||||
detached: false,
|
||||
reportIds: ['report-dbt-1'],
|
||||
artifactPaths: [
|
||||
'raw-sources/warehouse/live-database/sync-1/scan-report.json',
|
||||
'raw-sources/warehouse/live-database/sync-1',
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('viewStateFromSourceProgress', () => {
|
||||
it('partitions sources into primary and context groups', () => {
|
||||
const state = viewStateFromSourceProgress(
|
||||
[
|
||||
{ connectionId: 'warehouse', operation: 'scan', status: 'running', startedAtMs: 900 },
|
||||
{ connectionId: 'dbt-main', operation: 'source-ingest', status: 'queued' },
|
||||
],
|
||||
1000,
|
||||
500,
|
||||
);
|
||||
|
||||
expect(state.primarySources).toHaveLength(1);
|
||||
expect(state.primarySources[0].target.connectionId).toBe('warehouse');
|
||||
expect(state.primarySources[0].status).toBe('running');
|
||||
expect(state.primarySources[0].elapsedMs).toBe(100);
|
||||
expect(state.contextSources).toHaveLength(1);
|
||||
expect(state.contextSources[0].target.connectionId).toBe('dbt-main');
|
||||
expect(state.contextSources[0].status).toBe('queued');
|
||||
expect(state.totalElapsedMs).toBe(500);
|
||||
});
|
||||
|
||||
it('uses stored elapsedMs for completed sources', () => {
|
||||
const state = viewStateFromSourceProgress(
|
||||
[{ connectionId: 'warehouse', operation: 'scan', status: 'done', elapsedMs: 72000, summaryText: '42 tables' }],
|
||||
99999,
|
||||
);
|
||||
|
||||
expect(state.primarySources[0].elapsedMs).toBe(72000);
|
||||
expect(state.primarySources[0].summaryText).toBe('42 tables');
|
||||
});
|
||||
|
||||
it('renders the same view format as the foreground build', () => {
|
||||
const state = viewStateFromSourceProgress(
|
||||
[
|
||||
{ connectionId: 'warehouse', operation: 'scan', status: 'done', elapsedMs: 72000, summaryText: '42 tables' },
|
||||
{ connectionId: 'dbt-main', operation: 'source-ingest', status: 'running', startedAtMs: 900 },
|
||||
],
|
||||
1000,
|
||||
500,
|
||||
);
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('Building KTX context');
|
||||
expect(output).toContain('Primary sources:');
|
||||
expect(output).toContain('warehouse');
|
||||
expect(output).toContain('42 tables');
|
||||
expect(output).toContain('Context sources:');
|
||||
expect(output).toContain('dbt-main');
|
||||
expect(output).toContain('ingesting...');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ export interface ContextBuildViewState {
|
|||
primarySources: ContextBuildTargetState[];
|
||||
contextSources: ContextBuildTargetState[];
|
||||
frame: number;
|
||||
startedAt: number | null;
|
||||
totalElapsedMs: number;
|
||||
}
|
||||
|
||||
export interface ContextBuildArgs {
|
||||
|
|
@ -42,6 +44,17 @@ export interface ContextBuildArgs {
|
|||
export interface ContextBuildResult {
|
||||
exitCode: number;
|
||||
detached: boolean;
|
||||
reportIds?: string[];
|
||||
artifactPaths?: string[];
|
||||
}
|
||||
|
||||
export interface ContextBuildSourceProgressUpdate {
|
||||
connectionId: string;
|
||||
operation: 'scan' | 'source-ingest';
|
||||
status: 'queued' | 'running' | 'done' | 'failed';
|
||||
startedAtMs?: number;
|
||||
elapsedMs?: number;
|
||||
summaryText?: string;
|
||||
}
|
||||
|
||||
export interface ContextBuildDeps {
|
||||
|
|
@ -49,6 +62,7 @@ export interface ContextBuildDeps {
|
|||
now?: () => number;
|
||||
setupKeystroke?: (onDetach: () => void, onCtrlC: () => void) => (() => void) | null;
|
||||
onDetach?: () => void;
|
||||
onSourceProgress?: (sources: ContextBuildSourceProgressUpdate[]) => void;
|
||||
}
|
||||
|
||||
// --- Rendering ---
|
||||
|
|
@ -79,7 +93,7 @@ function statusIcon(status: ContextBuildTargetState['status'], frame: number, st
|
|||
case 'running':
|
||||
return SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋';
|
||||
default:
|
||||
return '·';
|
||||
return '○';
|
||||
}
|
||||
}
|
||||
switch (status) {
|
||||
|
|
@ -90,10 +104,27 @@ function statusIcon(status: ContextBuildTargetState['status'], frame: number, st
|
|||
case 'running':
|
||||
return cyan(SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋');
|
||||
default:
|
||||
return dim('·');
|
||||
return dim('○');
|
||||
}
|
||||
}
|
||||
|
||||
function extractPercent(detailLine: string | null): number | null {
|
||||
if (!detailLine) return null;
|
||||
const match = detailLine.match(/^\[(\d+)%\]/);
|
||||
return match ? Number(match[1]) : null;
|
||||
}
|
||||
|
||||
const BAR_WIDTH = 12;
|
||||
const BAR_FILLED = '█';
|
||||
const BAR_EMPTY = '░';
|
||||
|
||||
function renderProgressBar(percent: number, styled: boolean): string {
|
||||
const filled = Math.round((percent / 100) * BAR_WIDTH);
|
||||
const empty = BAR_WIDTH - filled;
|
||||
const bar = `${BAR_FILLED.repeat(filled)}${BAR_EMPTY.repeat(empty)}`;
|
||||
return styled ? cyan(bar) : bar;
|
||||
}
|
||||
|
||||
function targetDetail(target: ContextBuildTargetState, styled: boolean): string {
|
||||
if (target.status === 'done') {
|
||||
const parts: string[] = [];
|
||||
|
|
@ -105,7 +136,17 @@ function targetDetail(target: ContextBuildTargetState, styled: boolean): string
|
|||
return styled ? red('failed') : 'failed';
|
||||
}
|
||||
if (target.status === 'running') {
|
||||
return target.detailLine ?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...');
|
||||
const percent = extractPercent(target.detailLine);
|
||||
const progressText = target.detailLine?.replace(/^\[\d+%\]\s*/, '')
|
||||
?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...');
|
||||
const elapsed = target.elapsedMs > 0 ? `(${formatDuration(target.elapsedMs)})` : null;
|
||||
const parts: string[] = [];
|
||||
if (percent !== null) {
|
||||
parts.push(`${renderProgressBar(percent, styled)} ${percent}%`);
|
||||
}
|
||||
parts.push(progressText);
|
||||
if (elapsed) parts.push(styled ? dim(elapsed) : elapsed);
|
||||
return parts.join(' ');
|
||||
}
|
||||
return styled ? dim('queued') : 'queued';
|
||||
}
|
||||
|
|
@ -136,23 +177,46 @@ function resumeCommand(projectDir?: string): string {
|
|||
|
||||
export function renderContextBuildView(
|
||||
state: ContextBuildViewState,
|
||||
options: { styled?: boolean; showHint?: boolean; projectDir?: string } = {},
|
||||
options: { styled?: boolean; showHint?: boolean; hintText?: string; projectDir?: string } = {},
|
||||
): string {
|
||||
const styled = options.styled ?? true;
|
||||
const width = columnWidth(state);
|
||||
const allTargets = [...state.primarySources, ...state.contextSources];
|
||||
const doneCount = allTargets.filter((t) => t.status === 'done' || t.status === 'failed').length;
|
||||
const totalCount = allTargets.length;
|
||||
const hasActive = allTargets.some((t) => t.status === 'running' || t.status === 'queued');
|
||||
const allDone = totalCount > 0 && !hasActive;
|
||||
|
||||
const headerParts = ['Building KTX context'];
|
||||
if (totalCount > 0) {
|
||||
const progressParts: string[] = [`${doneCount}/${totalCount}`];
|
||||
if (state.totalElapsedMs > 0) progressParts.push(formatDuration(state.totalElapsedMs));
|
||||
const progress = `(${progressParts.join(' · ')})`;
|
||||
headerParts.push(styled ? dim(progress) : progress);
|
||||
}
|
||||
const header = headerParts.join(' ');
|
||||
const headerPlainLength = header.replace(/\x1b\[[0-9;]*m/g, '').length;
|
||||
const separator = '─'.repeat(Math.max(21, headerPlainLength));
|
||||
|
||||
const lines: string[] = [
|
||||
'',
|
||||
'Building KTX context',
|
||||
'─────────────────────',
|
||||
header,
|
||||
separator,
|
||||
...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width),
|
||||
...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width),
|
||||
'',
|
||||
];
|
||||
const hasActive = [...state.primarySources, ...state.contextSources].some(
|
||||
(t) => t.status === 'running' || t.status === 'queued',
|
||||
);
|
||||
|
||||
if (allDone && state.totalElapsedMs > 0) {
|
||||
const sourcesLabel = totalCount === 1 ? '1 source' : `${totalCount} sources`;
|
||||
const summary = ` Done in ${formatDuration(state.totalElapsedMs)} · ${sourcesLabel} processed`;
|
||||
lines.push(styled ? green(summary) : summary);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
if (options.showHint && hasActive) {
|
||||
const hint = ` d to detach · ${resumeCommand(options.projectDir)} to resume`;
|
||||
const hintContent = options.hintText ?? `d to detach · ${resumeCommand(options.projectDir)} to resume`;
|
||||
const hint = ` ${hintContent}`;
|
||||
lines.push(styled ? dim(hint) : hint);
|
||||
lines.push('');
|
||||
}
|
||||
|
|
@ -162,6 +226,7 @@ export function renderContextBuildView(
|
|||
// --- IO Capture ---
|
||||
|
||||
const ESC_K_RE = new RegExp(`${ESC.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\[K`, 'g');
|
||||
const ANSI_RE = /\x1b\[[0-9;]*m/g;
|
||||
|
||||
export function extractProgressMessage(chunk: string): string | null {
|
||||
const cleaned = chunk.replace(/^\r/, '').replace(ESC_K_RE, '').replace(/\n$/, '').trim();
|
||||
|
|
@ -175,12 +240,41 @@ export function parseScanSummary(output: string): string | null {
|
|||
}
|
||||
|
||||
export function parseIngestSummary(output: string): string | null {
|
||||
const parts: string[] = [];
|
||||
const workUnits = output.match(/Work units: (\d+)/);
|
||||
if (workUnits) parts.push(`${workUnits[1]} work units`);
|
||||
const savedMemory = output.match(/Saved memory: (.+)/);
|
||||
if (savedMemory) parts.push(savedMemory[1]);
|
||||
return parts.length > 0 ? parts.join(' · ') : null;
|
||||
if (savedMemory) return savedMemory[1];
|
||||
const workUnits = output.match(/Work units: (\d+)/);
|
||||
if (workUnits) return `${workUnits[1]} work units`;
|
||||
return null;
|
||||
}
|
||||
|
||||
function collectOutputMetadata(
|
||||
output: string,
|
||||
operation: KtxPublicIngestPlanTarget['operation'],
|
||||
): { reportIds: string[]; artifactPaths: string[] } {
|
||||
const reportIds = new Set<string>();
|
||||
const artifactPaths = new Set<string>();
|
||||
for (const line of output.split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
const reportLine = trimmed.match(/^Report:\s*(.+)$/);
|
||||
if (reportLine) {
|
||||
const value = reportLine[1].trim();
|
||||
if (value && value !== 'none') {
|
||||
if (operation === 'scan') artifactPaths.add(value);
|
||||
else reportIds.add(value);
|
||||
}
|
||||
}
|
||||
const rawSourcesLine = trimmed.match(/^Raw sources:\s*(.+)$/);
|
||||
if (rawSourcesLine) {
|
||||
const value = rawSourcesLine[1].trim();
|
||||
if (value && value !== 'none') artifactPaths.add(value);
|
||||
}
|
||||
if (operation === 'source-ingest') {
|
||||
for (const match of trimmed.matchAll(/\breport=([^\s]+)/g)) {
|
||||
reportIds.add(match[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { reportIds: [...reportIds], artifactPaths: [...artifactPaths] };
|
||||
}
|
||||
|
||||
interface CapturedIo {
|
||||
|
|
@ -210,19 +304,84 @@ function createCaptureIo(onProgress: (message: string) => void, isTTY: boolean):
|
|||
};
|
||||
}
|
||||
|
||||
// --- Source progress helpers ---
|
||||
|
||||
function collectSourceProgress(targets: ContextBuildTargetState[]): ContextBuildSourceProgressUpdate[] {
|
||||
return targets.map((t) => ({
|
||||
connectionId: t.target.connectionId,
|
||||
operation: t.target.operation,
|
||||
status: t.status,
|
||||
...(t.startedAt !== null ? { startedAtMs: t.startedAt } : {}),
|
||||
...(t.elapsedMs > 0 ? { elapsedMs: t.elapsedMs } : {}),
|
||||
...(t.summaryText ? { summaryText: t.summaryText } : {}),
|
||||
}));
|
||||
}
|
||||
|
||||
export function viewStateFromSourceProgress(
|
||||
sources: ContextBuildSourceProgressUpdate[],
|
||||
now: number,
|
||||
startedAtMs?: number,
|
||||
): ContextBuildViewState {
|
||||
const makeTarget = (s: ContextBuildSourceProgressUpdate): ContextBuildTargetState => ({
|
||||
target: { connectionId: s.connectionId, driver: '', operation: s.operation, debugCommand: '', steps: [] },
|
||||
status: s.status,
|
||||
detailLine: null,
|
||||
summaryText: s.summaryText ?? null,
|
||||
startedAt: s.startedAtMs ?? null,
|
||||
elapsedMs: s.status === 'running' && s.startedAtMs ? now - s.startedAtMs : (s.elapsedMs ?? 0),
|
||||
});
|
||||
|
||||
return {
|
||||
primarySources: sources.filter((s) => s.operation === 'scan').map(makeTarget),
|
||||
contextSources: sources.filter((s) => s.operation === 'source-ingest').map(makeTarget),
|
||||
frame: 0,
|
||||
startedAt: startedAtMs ?? null,
|
||||
totalElapsedMs: startedAtMs ? now - startedAtMs : 0,
|
||||
};
|
||||
}
|
||||
|
||||
// --- Repaint ---
|
||||
|
||||
function createRepainter(io: KtxCliIo) {
|
||||
let lastLineCount = 0;
|
||||
export function createRepainter(io: KtxCliIo) {
|
||||
let hasPainted = false;
|
||||
let lastCursorUpRows = 0;
|
||||
|
||||
const terminalColumns = () => {
|
||||
for (const columns of [io.stdout.columns, process.stdout.columns]) {
|
||||
if (typeof columns === 'number' && Number.isFinite(columns) && columns > 0) return columns;
|
||||
}
|
||||
return 80;
|
||||
};
|
||||
|
||||
const visualRows = (line: string, columns: number) => {
|
||||
const plainLength = line.replace(ANSI_RE, '').length;
|
||||
return Math.max(1, Math.ceil(plainLength / columns));
|
||||
};
|
||||
|
||||
const cursorUpRowsAfterWrite = (content: string) => {
|
||||
const columns = terminalColumns();
|
||||
const endsWithNewline = content.endsWith('\n');
|
||||
const lines = content.split('\n');
|
||||
return lines.reduce((sum, line, index) => {
|
||||
if (index === lines.length - 1) {
|
||||
return endsWithNewline ? sum : sum + Math.max(0, visualRows(line, columns) - 1);
|
||||
}
|
||||
return sum + visualRows(line, columns);
|
||||
}, 0);
|
||||
};
|
||||
|
||||
return {
|
||||
paint(content: string) {
|
||||
if (lastLineCount > 0) {
|
||||
io.stdout.write(`${ESC}[${lastLineCount}A\r`);
|
||||
if (hasPainted) {
|
||||
if (lastCursorUpRows > 0) {
|
||||
io.stdout.write(`${ESC}[${lastCursorUpRows}A`);
|
||||
}
|
||||
io.stdout.write('\r');
|
||||
}
|
||||
io.stdout.write(content);
|
||||
io.stdout.write(content.replaceAll('\n', `${ESC}[K\n`));
|
||||
io.stdout.write(`${ESC}[J`);
|
||||
lastLineCount = (content.match(/\n/g) ?? []).length;
|
||||
hasPainted = true;
|
||||
lastCursorUpRows = cursorUpRowsAfterWrite(content);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -258,7 +417,7 @@ function spawnBackgroundBuild(projectDir: string): { logPath: string } | null {
|
|||
|
||||
// --- Keystroke handling ---
|
||||
|
||||
function defaultSetupKeystroke(onDetach: () => void, onCtrlC: () => void): (() => void) | null {
|
||||
export function defaultSetupKeystroke(onDetach: () => void, onCtrlC: () => void): (() => void) | null {
|
||||
const stdin = process.stdin;
|
||||
if (!stdin.isTTY || typeof stdin.setRawMode !== 'function') {
|
||||
return null;
|
||||
|
|
@ -289,6 +448,8 @@ export function initViewState(targets: KtxPublicIngestPlanTarget[]): ContextBuil
|
|||
primarySources: targets.filter((t) => t.operation === 'scan').map(makeTargetState),
|
||||
contextSources: targets.filter((t) => t.operation === 'source-ingest').map(makeTargetState),
|
||||
frame: 0,
|
||||
startedAt: null,
|
||||
totalElapsedMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -303,6 +464,8 @@ export async function runContextBuild(
|
|||
const isTTY = io.stdout.isTTY === true;
|
||||
const nowFn = deps.now ?? (() => Date.now());
|
||||
|
||||
state.startedAt = nowFn();
|
||||
|
||||
const repainter = isTTY ? createRepainter(io) : null;
|
||||
const viewOpts = { styled: true, projectDir: args.projectDir };
|
||||
const paint = (hint: boolean) => repainter?.paint(renderContextBuildView(state, { ...viewOpts, showHint: hint }));
|
||||
|
|
@ -312,6 +475,9 @@ export async function runContextBuild(
|
|||
if (repainter) {
|
||||
spinnerInterval = setInterval(() => {
|
||||
state.frame++;
|
||||
if (state.startedAt !== null) {
|
||||
state.totalElapsedMs = nowFn() - state.startedAt;
|
||||
}
|
||||
for (const t of [...state.primarySources, ...state.contextSources]) {
|
||||
if (t.status === 'running' && t.startedAt !== null) {
|
||||
t.elapsedMs = nowFn() - t.startedAt;
|
||||
|
|
@ -323,6 +489,8 @@ export async function runContextBuild(
|
|||
|
||||
const orderedTargets = [...state.primarySources, ...state.contextSources];
|
||||
const execTarget = deps.executeTarget ?? executePublicIngestTarget;
|
||||
const reportIds = new Set<string>();
|
||||
const artifactPaths = new Set<string>();
|
||||
|
||||
let detached = false;
|
||||
let cleanupKeystroke: (() => void) | null = null;
|
||||
|
|
@ -339,8 +507,8 @@ export async function runContextBuild(
|
|||
const bg = spawnBackgroundBuild(args.projectDir);
|
||||
io.stdout.write('\n\nContext build continuing in the background.\n');
|
||||
if (bg) io.stdout.write(`Log: ${bg.logPath}\n`);
|
||||
io.stdout.write(`Status: ktx setup context status --project-dir ${resolve(args.projectDir)}\n`);
|
||||
io.stdout.write(`Resume: ${resumeCommand(args.projectDir)}\n`);
|
||||
io.stdout.write(`Status: ktx setup context status --project-dir ${resolve(args.projectDir)}\n`);
|
||||
process.exit(0);
|
||||
},
|
||||
() => {
|
||||
|
|
@ -370,6 +538,7 @@ export async function runContextBuild(
|
|||
targetState.status = 'running';
|
||||
targetState.startedAt = nowFn();
|
||||
paint(true);
|
||||
deps.onSourceProgress?.(collectSourceProgress(orderedTargets));
|
||||
|
||||
const capture = createCaptureIo(
|
||||
(message) => {
|
||||
|
|
@ -386,20 +555,29 @@ export async function runContextBuild(
|
|||
targetState.status = failed ? 'failed' : 'done';
|
||||
targetState.detailLine = null;
|
||||
if (!failed) {
|
||||
const capturedOutput = capture.captured();
|
||||
const metadata = collectOutputMetadata(capturedOutput, targetState.target.operation);
|
||||
for (const reportId of metadata.reportIds) reportIds.add(reportId);
|
||||
for (const artifactPath of metadata.artifactPaths) artifactPaths.add(artifactPath);
|
||||
targetState.summaryText =
|
||||
targetState.target.operation === 'scan'
|
||||
? parseScanSummary(capture.captured())
|
||||
: parseIngestSummary(capture.captured());
|
||||
? parseScanSummary(capturedOutput)
|
||||
: parseIngestSummary(capturedOutput);
|
||||
}
|
||||
if (failed) hasFailure = true;
|
||||
|
||||
paint(true);
|
||||
deps.onSourceProgress?.(collectSourceProgress(orderedTargets));
|
||||
}
|
||||
} finally {
|
||||
if (spinnerInterval) clearInterval(spinnerInterval);
|
||||
cleanupKeystroke?.();
|
||||
}
|
||||
|
||||
if (state.startedAt !== null) {
|
||||
state.totalElapsedMs = nowFn() - state.startedAt;
|
||||
}
|
||||
|
||||
if (detached) {
|
||||
return { exitCode: 0, detached: true };
|
||||
}
|
||||
|
|
@ -410,5 +588,10 @@ export async function runContextBuild(
|
|||
paint(false);
|
||||
}
|
||||
|
||||
return { exitCode: hasFailure ? 1 : 0, detached: false };
|
||||
return {
|
||||
exitCode: hasFailure ? 1 : 0,
|
||||
detached: false,
|
||||
...(reportIds.size > 0 ? { reportIds: [...reportIds] } : {}),
|
||||
...(artifactPaths.size > 0 ? { artifactPaths: [...artifactPaths] } : {}),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
863
packages/cli/src/ingest-viz.test.ts
Normal file
863
packages/cli/src/ingest-viz.test.ts
Normal file
|
|
@ -0,0 +1,863 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
type LocalIngestResult,
|
||||
type MemoryFlowReplayInput,
|
||||
type RunLocalIngestOptions,
|
||||
} from '@ktx/context/ingest';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKtxIngest } from './ingest.js';
|
||||
import {
|
||||
completedLocalBundleRun,
|
||||
emitLiveLocalMemoryFlow,
|
||||
localFakeBundleReport,
|
||||
makeIo,
|
||||
persistLocalBundleReport,
|
||||
writeBundleReportFile,
|
||||
writeWarehouseConfig,
|
||||
} from './ingest.test-utils.js';
|
||||
import { resetVizFallbackWarningsForTest } from './viz-fallback.js';
|
||||
|
||||
describe('runKtxIngest viz and replay', () => {
|
||||
let tempDir: string;
|
||||
let originalTerm: string | undefined;
|
||||
|
||||
beforeEach(async () => {
|
||||
resetVizFallbackWarningsForTest();
|
||||
originalTerm = process.env.TERM;
|
||||
process.env.TERM = 'xterm-256color';
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-ingest-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (originalTerm === undefined) {
|
||||
delete process.env.TERM;
|
||||
} else {
|
||||
process.env.TERM = originalTerm;
|
||||
}
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('renders live memory-flow frames for run --viz when stdout is interactive', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions): Promise<LocalIngestResult> => {
|
||||
input.memoryFlow?.emit({ type: 'source_acquired', adapter: 'fake', trigger: 'manual_resync', fileCount: 1 });
|
||||
input.memoryFlow?.update({ syncId: 'sync-live-1' });
|
||||
input.memoryFlow?.emit({ type: 'raw_snapshot_written', syncId: 'sync-live-1', rawFileCount: 1 });
|
||||
input.memoryFlow?.emit({ type: 'diff_computed', added: 1, modified: 0, deleted: 0, unchanged: 0 });
|
||||
input.memoryFlow?.update({
|
||||
plannedWorkUnits: [
|
||||
{
|
||||
unitKey: 'fake-orders',
|
||||
rawFiles: ['orders/orders.json'],
|
||||
peerFileCount: 0,
|
||||
dependencyCount: 0,
|
||||
},
|
||||
],
|
||||
});
|
||||
input.memoryFlow?.emit({ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 });
|
||||
input.memoryFlow?.emit({ type: 'report_created', runId: 'live-viz-run' });
|
||||
input.memoryFlow?.finish('done');
|
||||
|
||||
return completedLocalBundleRun(input, 'live-viz-run');
|
||||
});
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, columns: 120 });
|
||||
const startLiveMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => null);
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'fake',
|
||||
sourceDir,
|
||||
outputMode: 'viz',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
runLocalIngest: runLocal,
|
||||
startLiveMemoryFlow,
|
||||
jobIdFactory: () => 'live-viz-run',
|
||||
now: () => new Date('2026-04-30T14:00:00.000Z'),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runLocal).toHaveBeenCalledWith(expect.objectContaining({ memoryFlow: expect.any(Object) }));
|
||||
expect(io.stdout()).toContain('\u001b[2J\u001b[H');
|
||||
expect((io.stdout().match(/KTX memory flow/g) ?? []).length).toBeGreaterThan(1);
|
||||
expect(io.stdout()).toContain('KTX memory flow warehouse/fake done');
|
||||
expect(io.stdout()).toContain('fake-orders');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('uses the TUI live session for run --viz when stdin and stdout are interactive', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions): Promise<LocalIngestResult> => {
|
||||
emitLiveLocalMemoryFlow(input.memoryFlow);
|
||||
return completedLocalBundleRun(input, 'live-viz-run');
|
||||
});
|
||||
const liveSession = {
|
||||
update: vi.fn(),
|
||||
close: vi.fn(),
|
||||
isClosed: vi.fn(() => false),
|
||||
};
|
||||
const startLiveMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => liveSession);
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, columns: 120 });
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'fake',
|
||||
sourceDir,
|
||||
outputMode: 'viz',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
runLocalIngest: runLocal,
|
||||
startLiveMemoryFlow,
|
||||
jobIdFactory: () => 'live-viz-run',
|
||||
now: () => new Date('2026-04-30T14:00:00.000Z'),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(startLiveMemoryFlow).toHaveBeenCalledTimes(1);
|
||||
expect(startLiveMemoryFlow.mock.calls[0]?.[0]).toMatchObject({
|
||||
runId: 'live-viz-run',
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'fake',
|
||||
status: 'running',
|
||||
});
|
||||
expect(liveSession.update).toHaveBeenCalled();
|
||||
expect(liveSession.close).toHaveBeenCalledTimes(1);
|
||||
expect(io.stdout()).not.toContain('\u001b[2J\u001b[H');
|
||||
expect(io.stdout()).not.toContain('KTX memory flow');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('prints a final plain summary after live viz completes', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, columns: 120 });
|
||||
const liveSession = {
|
||||
update: vi.fn(),
|
||||
close: vi.fn(),
|
||||
isClosed: vi.fn(() => false),
|
||||
};
|
||||
const startLiveMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => liveSession);
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions) => {
|
||||
emitLiveLocalMemoryFlow(input.memoryFlow);
|
||||
return completedLocalBundleRun(input, 'live-summary');
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'fake',
|
||||
outputMode: 'viz',
|
||||
},
|
||||
io.io,
|
||||
{ runLocalIngest: runLocal, startLiveMemoryFlow },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(liveSession.close).toHaveBeenCalledTimes(1);
|
||||
expect(io.stdout()).toContain('Memory-flow summary: done');
|
||||
expect(io.stdout()).toContain('Connection: warehouse');
|
||||
});
|
||||
|
||||
it('falls back to text live rendering when the TUI live session is unavailable', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions): Promise<LocalIngestResult> => {
|
||||
emitLiveLocalMemoryFlow(input.memoryFlow);
|
||||
return completedLocalBundleRun(input, 'live-viz-run');
|
||||
});
|
||||
const startLiveMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => null);
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, columns: 120 });
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'fake',
|
||||
sourceDir,
|
||||
outputMode: 'viz',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
runLocalIngest: runLocal,
|
||||
startLiveMemoryFlow,
|
||||
jobIdFactory: () => 'live-viz-run',
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(startLiveMemoryFlow).toHaveBeenCalledTimes(1);
|
||||
expect(io.stdout()).toContain('\u001b[2J\u001b[H');
|
||||
expect(io.stdout()).toContain('KTX memory flow warehouse/fake done');
|
||||
});
|
||||
|
||||
it('falls back to text live rendering when TUI startup fails with a redacted warning', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions): Promise<LocalIngestResult> => {
|
||||
emitLiveLocalMemoryFlow(input.memoryFlow);
|
||||
return completedLocalBundleRun(input, 'live-viz-run');
|
||||
});
|
||||
const startLiveMemoryFlow = vi.fn(
|
||||
async (_input: MemoryFlowReplayInput, ioArg: { stderr: { write(chunk: string): void } }) => {
|
||||
ioArg.stderr.write('TUI visualization unavailable: Failed [redacted-url] [redacted]; using text renderer.\n');
|
||||
return null;
|
||||
},
|
||||
);
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, columns: 120 });
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'fake',
|
||||
sourceDir,
|
||||
outputMode: 'viz',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
runLocalIngest: runLocal,
|
||||
startLiveMemoryFlow,
|
||||
jobIdFactory: () => 'live-viz-run',
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stderr()).toContain('TUI visualization unavailable: Failed [redacted-url] [redacted]');
|
||||
expect(io.stdout()).toContain('KTX memory flow warehouse/fake done');
|
||||
expect(io.stdout()).toContain('\u001b[2J\u001b[H');
|
||||
});
|
||||
|
||||
it('does not start live TUI when run --viz disables input', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions): Promise<LocalIngestResult> => {
|
||||
return completedLocalBundleRun(input, 'no-input-live-viz-run');
|
||||
});
|
||||
const startLiveMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => ({
|
||||
update: vi.fn(),
|
||||
close: vi.fn(),
|
||||
isClosed: vi.fn(() => false),
|
||||
}));
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, columns: 120 });
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'fake',
|
||||
sourceDir,
|
||||
outputMode: 'viz',
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io.io,
|
||||
{ runLocalIngest: runLocal, startLiveMemoryFlow },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(startLiveMemoryFlow).not.toHaveBeenCalled();
|
||||
expect(runLocal).toHaveBeenCalledWith(expect.not.objectContaining({ memoryFlow: expect.anything() }));
|
||||
expect(io.stdout()).toContain('KTX memory flow warehouse/fake done');
|
||||
});
|
||||
|
||||
it('does not attach a live memory-flow sink for plain run output', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions) => completedLocalBundleRun(input, 'plain-run'));
|
||||
const io = makeIo({ isTTY: true });
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'fake',
|
||||
sourceDir,
|
||||
outputMode: 'plain',
|
||||
},
|
||||
io.io,
|
||||
{ runLocalIngest: runLocal },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runLocal).toHaveBeenCalledWith(expect.not.objectContaining({ memoryFlow: expect.anything() }));
|
||||
expect(io.stdout()).toContain('Job: plain-run');
|
||||
expect(io.stdout()).not.toContain('KTX memory flow');
|
||||
});
|
||||
|
||||
it('falls back to plain run output for run --viz when stdout is not interactive', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
const io = makeIo({ isTTY: false });
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions) => completedLocalBundleRun(input, 'non-tty-viz-run'));
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'fake',
|
||||
sourceDir,
|
||||
outputMode: 'viz',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
runLocalIngest: runLocal,
|
||||
jobIdFactory: () => 'non-tty-viz-run',
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Job: non-tty-viz-run');
|
||||
expect(io.stdout()).not.toContain('KTX memory flow');
|
||||
expect(io.stderr()).toContain(
|
||||
'Visualization requested but stdout is not an interactive terminal; printing plain output.',
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to plain run output for run --viz when stdin raw mode is unavailable', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, rawMode: false, columns: 120 });
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions) => completedLocalBundleRun(input, 'raw-missing-viz-run'));
|
||||
const startLiveMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => ({
|
||||
update: vi.fn(),
|
||||
close: vi.fn(),
|
||||
isClosed: vi.fn(() => false),
|
||||
}));
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'fake',
|
||||
sourceDir,
|
||||
outputMode: 'viz',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
runLocalIngest: runLocal,
|
||||
startLiveMemoryFlow,
|
||||
jobIdFactory: () => 'raw-missing-viz-run',
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(startLiveMemoryFlow).not.toHaveBeenCalled();
|
||||
expect(runLocal).toHaveBeenCalledWith(expect.not.objectContaining({ memoryFlow: expect.anything() }));
|
||||
expect(io.stdout()).toContain('Job: raw-missing-viz-run');
|
||||
expect(io.stdout()).not.toContain('KTX memory flow');
|
||||
expect(io.stderr()).toContain(
|
||||
'Visualization requested but stdin raw mode is unavailable; printing plain output.',
|
||||
);
|
||||
});
|
||||
|
||||
it('returns an error code for missing status', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxIngest({ command: 'status', projectDir, runId: 'missing-run', outputMode: 'plain' }, io.io),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stderr()).toContain('Local ingest run or report "missing-run" was not found');
|
||||
});
|
||||
|
||||
it('uses the latest local ingest report when status has no run id', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('older-run'));
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('newer-run'));
|
||||
const io = makeIo();
|
||||
|
||||
await expect(runKtxIngest({ command: 'status', projectDir, outputMode: 'plain' }, io.io)).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Run: run-newer-run');
|
||||
expect(io.stdout()).toContain('Job: newer-run');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('renders the latest local ingest report through watch when run id is omitted', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('watch-latest'));
|
||||
const io = makeIo({ isTTY: true });
|
||||
|
||||
await expect(
|
||||
runKtxIngest({ command: 'watch', projectDir, outputMode: 'viz', inputMode: 'disabled' }, io.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('KTX memory flow warehouse/fake done');
|
||||
expect(io.stdout()).toContain('Run: run-watch-latest');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('renders report-file replay through the memory-flow TUI', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const reportFile = await writeBundleReportFile(tempDir);
|
||||
const io = makeIo({ isTTY: true });
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'replay',
|
||||
projectDir,
|
||||
runId: 'job-1',
|
||||
reportFile,
|
||||
outputMode: 'viz',
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('KTX memory flow warehouse/metabase done');
|
||||
expect(io.stdout()).toContain('Saved 2 memories from 2 raw files');
|
||||
expect(io.stdout()).toContain('Commit: abc12345 Run: run-1 Report: report-1');
|
||||
expect(io.stdout()).toContain('SOURCE');
|
||||
expect(io.stdout()).toContain('ACTIONS');
|
||||
expect(io.stdout()).toContain('SAVED');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('prints report-file JSON without looking up local ingest status', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const reportFile = await writeBundleReportFile(tempDir);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxIngest({ command: 'status', projectDir, runId: 'report-1', reportFile, outputMode: 'json' }, io.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const parsed = JSON.parse(io.stdout());
|
||||
expect(parsed).toMatchObject({
|
||||
id: 'report-1',
|
||||
runId: 'run-1',
|
||||
jobId: 'job-1',
|
||||
connectionId: 'warehouse',
|
||||
sourceKey: 'metabase',
|
||||
});
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('routes interactive report-file replay through the stored TUI renderer', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const reportFile = await writeBundleReportFile(tempDir);
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, columns: 120 });
|
||||
const renderStoredMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => true);
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'replay',
|
||||
projectDir,
|
||||
runId: 'run-1',
|
||||
reportFile,
|
||||
outputMode: 'viz',
|
||||
},
|
||||
io.io,
|
||||
{ renderStoredMemoryFlow },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(renderStoredMemoryFlow).toHaveBeenCalledTimes(1);
|
||||
expect(renderStoredMemoryFlow.mock.calls[0]?.[0]).toMatchObject({
|
||||
runId: 'run-1',
|
||||
reportId: 'report-1',
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'metabase',
|
||||
});
|
||||
expect(io.stdout()).toBe('');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('rejects report-file replay when the requested id does not match the report', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const reportFile = await writeBundleReportFile(tempDir);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxIngest({ command: 'replay', projectDir, runId: 'unrelated-id', reportFile, outputMode: 'plain' }, io.io),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stderr()).toContain(
|
||||
`Report file ${reportFile} does not match ingest replay id "unrelated-id"; expected one of report-1, run-1, job-1`,
|
||||
);
|
||||
expect(io.stdout()).toBe('');
|
||||
});
|
||||
|
||||
it('renders memory-flow snapshot for status --viz when stdout is interactive', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('viz-run-1'));
|
||||
|
||||
const io = makeIo({ isTTY: true });
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{ command: 'status', projectDir, runId: 'viz-run-1', outputMode: 'viz', inputMode: 'disabled' },
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('KTX memory flow warehouse/fake done');
|
||||
expect(io.stdout()).toContain('SOURCE');
|
||||
expect(io.stdout()).toContain('CHUNKS');
|
||||
expect(io.stdout()).toContain('WORKUNITS');
|
||||
expect(io.stdout()).toContain('Saved 2 memories from 2 raw files');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('uses the TUI renderer for stored status --viz when stdin and stdout are interactive', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('tui-viz-run'));
|
||||
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, columns: 120 });
|
||||
const renderStoredMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => true);
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'status',
|
||||
projectDir,
|
||||
runId: 'tui-viz-run',
|
||||
outputMode: 'viz',
|
||||
},
|
||||
io.io,
|
||||
{ renderStoredMemoryFlow },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(renderStoredMemoryFlow).toHaveBeenCalledTimes(1);
|
||||
expect(renderStoredMemoryFlow.mock.calls[0]?.[0]).toMatchObject({
|
||||
runId: 'run-tui-viz-run',
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'fake',
|
||||
});
|
||||
expect(io.stdout()).toBe('');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('falls back to the text renderer when TUI declines stored status --viz', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('tui-fallback-run'));
|
||||
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, columns: 120, keypresses: [{ name: 'q' }] });
|
||||
const renderStoredMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => false);
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'status',
|
||||
projectDir,
|
||||
runId: 'tui-fallback-run',
|
||||
outputMode: 'viz',
|
||||
},
|
||||
io.io,
|
||||
{ renderStoredMemoryFlow },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(renderStoredMemoryFlow).toHaveBeenCalledTimes(1);
|
||||
expect(io.stdout()).toContain('KTX memory flow warehouse/fake done');
|
||||
});
|
||||
|
||||
it('does not use TUI for stored --viz when input is disabled', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('tui-no-input-run'));
|
||||
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, columns: 120 });
|
||||
const renderStoredMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => true);
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'replay',
|
||||
projectDir,
|
||||
runId: 'tui-no-input-run',
|
||||
outputMode: 'viz',
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io.io,
|
||||
{ renderStoredMemoryFlow },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(renderStoredMemoryFlow).not.toHaveBeenCalled();
|
||||
expect(io.stdout()).toContain('KTX memory flow warehouse/fake done');
|
||||
});
|
||||
|
||||
it('falls back to plain status for stored --viz when stdin raw mode is unavailable', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('raw-missing-stored-viz-run'));
|
||||
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, rawMode: false, columns: 120 });
|
||||
const renderStoredMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => true);
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'replay',
|
||||
projectDir,
|
||||
runId: 'raw-missing-stored-viz-run',
|
||||
outputMode: 'viz',
|
||||
},
|
||||
io.io,
|
||||
{ renderStoredMemoryFlow },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(renderStoredMemoryFlow).not.toHaveBeenCalled();
|
||||
expect(io.stdout()).toContain('Run: run-raw-missing-stored-viz-run');
|
||||
expect(io.stdout()).toContain('Job: raw-missing-stored-viz-run');
|
||||
expect(io.stdout()).not.toContain('KTX memory flow');
|
||||
expect(io.stderr()).toContain(
|
||||
'Visualization requested but stdin raw mode is unavailable; printing plain output.',
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps stored --viz snapshot-only when input is disabled', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('no-input-viz-run'));
|
||||
|
||||
const io = makeIo({ isTTY: true, columns: 120 });
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'replay',
|
||||
projectDir,
|
||||
runId: 'no-input-viz-run',
|
||||
outputMode: 'viz',
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('KTX memory flow warehouse/fake done');
|
||||
expect(io.stdout()).not.toContain('\u001b[2J\u001b[H');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('keeps disabled-input stored --viz snapshot output even when stdin raw mode is unavailable', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('disabled-raw-missing-viz-run'));
|
||||
|
||||
const io = makeIo({ isTTY: true, stdinIsTTY: true, rawMode: false, columns: 120 });
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'replay',
|
||||
projectDir,
|
||||
runId: 'disabled-raw-missing-viz-run',
|
||||
outputMode: 'viz',
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('KTX memory flow warehouse/fake done');
|
||||
expect(io.stdout()).not.toContain('\u001b[2J\u001b[H');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('degrades stored --viz snapshots to plain status when stdout is redirected even when input is disabled', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('redirected-no-input-viz-run'));
|
||||
|
||||
const io = makeIo({ isTTY: false });
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'replay',
|
||||
projectDir,
|
||||
runId: 'redirected-no-input-viz-run',
|
||||
outputMode: 'viz',
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Run: run-redirected-no-input-viz-run');
|
||||
expect(io.stdout()).toContain('Job: redirected-no-input-viz-run');
|
||||
expect(io.stdout()).not.toContain('KTX memory flow');
|
||||
expect(io.stderr()).toContain(
|
||||
'Visualization requested but stdout is not an interactive terminal; printing plain output.',
|
||||
);
|
||||
});
|
||||
|
||||
it('degrades ingest replay --viz to plain status when TERM is dumb', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('dumb-terminal-viz-run'));
|
||||
|
||||
const io = makeIo({ isTTY: true });
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{ command: 'replay', projectDir, runId: 'dumb-terminal-viz-run', outputMode: 'viz' },
|
||||
io.io,
|
||||
{ env: { ...process.env, TERM: 'dumb' } },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Run: run-dumb-terminal-viz-run');
|
||||
expect(io.stdout()).toContain('Job: dumb-terminal-viz-run');
|
||||
expect(io.stdout()).not.toContain('KTX memory flow');
|
||||
expect(io.stderr()).toContain(
|
||||
'Visualization requested but TERM=dumb does not support the visual renderer; printing plain output.',
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to plain status for --viz when stdout is not interactive', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('viz-run-2'));
|
||||
|
||||
const io = makeIo({ isTTY: false });
|
||||
await expect(
|
||||
runKtxIngest({ command: 'replay', projectDir, runId: 'viz-run-2', outputMode: 'viz' }, io.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Run: run-viz-run-2');
|
||||
expect(io.stdout()).toContain('Job: viz-run-2');
|
||||
expect(io.stdout()).not.toContain('KTX memory flow');
|
||||
expect(io.stderr()).toContain(
|
||||
'Visualization requested but stdout is not an interactive terminal; printing plain output.',
|
||||
);
|
||||
});
|
||||
|
||||
it('prints JSON for status --json', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
await persistLocalBundleReport(projectDir, localFakeBundleReport('json-run-1'));
|
||||
|
||||
const io = makeIo();
|
||||
await expect(
|
||||
runKtxIngest({ command: 'status', projectDir, runId: 'json-run-1', outputMode: 'json' }, io.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(JSON.parse(io.stdout())).toMatchObject({
|
||||
runId: 'run-json-run-1',
|
||||
jobId: 'json-run-1',
|
||||
sourceKey: 'fake',
|
||||
connectionId: 'warehouse',
|
||||
});
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
});
|
||||
746
packages/cli/src/ingest.test-utils.ts
Normal file
746
packages/cli/src/ingest.test-utils.ts
Normal file
|
|
@ -0,0 +1,746 @@
|
|||
import { EventEmitter } from 'node:events';
|
||||
import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { AgentRunnerService, type RunLoopParams } from '@ktx/context/agent';
|
||||
import {
|
||||
LocalLookerRuntimeStore,
|
||||
LocalMetabaseSourceStateReader,
|
||||
MetabaseSourceAdapter,
|
||||
getLocalIngestStatus,
|
||||
type ChunkResult,
|
||||
type FetchContext,
|
||||
type IngestReportSnapshot,
|
||||
type LocalIngestResult,
|
||||
type LocalMetabaseFanoutProgress,
|
||||
type LookerMappingClient,
|
||||
type LookerRuntimeClient,
|
||||
type LookerTableIdentifierParser,
|
||||
type MemoryFlowEventSink,
|
||||
type MemoryFlowReplayInput,
|
||||
type MetabaseCard,
|
||||
type MetabaseCardSummary,
|
||||
type MetabaseClientFactory,
|
||||
type MetabaseRuntimeClient,
|
||||
type RunLocalIngestOptions,
|
||||
type SourceAdapter,
|
||||
type SqliteBundleIngestStore,
|
||||
} from '@ktx/context/ingest';
|
||||
import { ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project';
|
||||
import { expect, vi } from 'vitest';
|
||||
import { type KtxIngestArgs, runKtxIngest } from './ingest.js';
|
||||
|
||||
export function makeIo(
|
||||
options: {
|
||||
isTTY?: boolean;
|
||||
stdinIsTTY?: boolean;
|
||||
columns?: number;
|
||||
rawMode?: boolean;
|
||||
keypresses?: { name?: string; ctrl?: boolean }[];
|
||||
} = {},
|
||||
) {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
type TestKey = { name?: string; ctrl?: boolean };
|
||||
|
||||
class TestStdin extends EventEmitter {
|
||||
isTTY = options.stdinIsTTY ?? false;
|
||||
isRaw = false;
|
||||
|
||||
setRawMode =
|
||||
options.rawMode === false
|
||||
? undefined
|
||||
: (value: boolean): void => {
|
||||
this.isRaw = value;
|
||||
};
|
||||
|
||||
resume(): void {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
pause(): void {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
override on(eventName: string | symbol, listener: (chunk: string, key: TestKey) => void): this {
|
||||
const result = super.on(eventName, listener);
|
||||
if (eventName === 'keypress') {
|
||||
for (const key of options.keypresses ?? []) {
|
||||
queueMicrotask(() => listener('', key));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
override off(eventName: string | symbol, listener: (chunk: string, key: TestKey) => void): this {
|
||||
return super.off(eventName, listener);
|
||||
}
|
||||
|
||||
override removeListener(eventName: string | symbol, listener: (chunk: string, key: TestKey) => void): this {
|
||||
return super.removeListener(eventName, listener);
|
||||
}
|
||||
}
|
||||
|
||||
const stdin = new TestStdin();
|
||||
|
||||
return {
|
||||
io: {
|
||||
stdin,
|
||||
stdout: {
|
||||
isTTY: options.isTTY,
|
||||
columns: options.columns,
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
export async function writeWarehouseConfig(projectDir: string): Promise<void> {
|
||||
await mkdir(projectDir, { recursive: true });
|
||||
await writeFile(
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
' prod-metabase:',
|
||||
' driver: metabase',
|
||||
' warehouse_a:',
|
||||
' driver: postgres',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - fake',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
|
||||
export async function writeMetabaseConfig(projectDir: string): Promise<void> {
|
||||
await mkdir(projectDir, { recursive: true });
|
||||
await writeFile(
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: postgres',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - metabase',
|
||||
' embeddings:',
|
||||
' backend: deterministic',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
|
||||
export function bundleReportSnapshot(): IngestReportSnapshot {
|
||||
return {
|
||||
id: 'report-1',
|
||||
runId: 'run-1',
|
||||
jobId: 'job-1',
|
||||
connectionId: 'warehouse',
|
||||
sourceKey: 'metabase',
|
||||
createdAt: '2026-04-30T12:00:00.000Z',
|
||||
body: {
|
||||
syncId: 'sync-1',
|
||||
diffSummary: { added: 2, modified: 0, deleted: 0, unchanged: 0 },
|
||||
commitSha: 'abc12345',
|
||||
workUnits: [
|
||||
{
|
||||
unitKey: 'cards',
|
||||
rawFiles: ['cards/1.json', 'cards/2.json'],
|
||||
status: 'success',
|
||||
actions: [
|
||||
{ target: 'wiki', type: 'created', key: 'knowledge/global/revenue.md', detail: 'Revenue overview' },
|
||||
{ target: 'sl', type: 'updated', key: 'warehouse.orders', detail: 'Added order amount measure' },
|
||||
],
|
||||
touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'warehouse.orders' }],
|
||||
},
|
||||
],
|
||||
failedWorkUnits: [],
|
||||
reconciliationSkipped: false,
|
||||
conflictsResolved: [],
|
||||
evictionsApplied: [],
|
||||
unmappedFallbacks: [],
|
||||
evictionInputs: [],
|
||||
unresolvedCards: [],
|
||||
supersededBy: null,
|
||||
overrideOf: null,
|
||||
provenanceRows: [
|
||||
{
|
||||
rawPath: 'cards/1.json',
|
||||
artifactKind: 'wiki',
|
||||
artifactKey: 'knowledge/global/revenue.md',
|
||||
actionType: 'wiki_written',
|
||||
},
|
||||
{
|
||||
rawPath: 'cards/2.json',
|
||||
artifactKind: 'sl',
|
||||
artifactKey: 'warehouse.orders',
|
||||
actionType: 'measure_added',
|
||||
},
|
||||
],
|
||||
toolTranscripts: [
|
||||
{
|
||||
unitKey: 'cards',
|
||||
path: 'tool-transcripts/cards.jsonl',
|
||||
toolCallCount: 4,
|
||||
errorCount: 0,
|
||||
toolNames: ['ingest_triage', 'knowledge_capture', 'sl_capture'],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function completedLocalBundleRun(input: RunLocalIngestOptions, jobId: string): LocalIngestResult {
|
||||
const nextReport = localFakeBundleReport(jobId, {
|
||||
id: 'report-live-1',
|
||||
runId: 'run-live-1',
|
||||
connectionId: input.connectionId,
|
||||
sourceKey: input.adapter,
|
||||
});
|
||||
return {
|
||||
result: {
|
||||
jobId,
|
||||
runId: nextReport.runId,
|
||||
syncId: nextReport.body.syncId,
|
||||
diffSummary: nextReport.body.diffSummary,
|
||||
workUnitCount: nextReport.body.workUnits.length,
|
||||
failedWorkUnits: nextReport.body.failedWorkUnits,
|
||||
artifactsWritten: nextReport.body.provenanceRows.length,
|
||||
commitSha: nextReport.body.commitSha,
|
||||
},
|
||||
report: nextReport,
|
||||
};
|
||||
}
|
||||
|
||||
export function failedLocalBundleRun(input: RunLocalIngestOptions, jobId: string): LocalIngestResult {
|
||||
const failedWorkUnit = {
|
||||
...bundleReportSnapshot().body.workUnits[0],
|
||||
status: 'failed' as const,
|
||||
reason: 'writer tool failed',
|
||||
actions: [],
|
||||
touchedSlSources: [],
|
||||
};
|
||||
const nextReport = localFakeBundleReport(jobId, {
|
||||
id: 'report-failed-1',
|
||||
runId: 'run-failed-1',
|
||||
connectionId: input.connectionId,
|
||||
sourceKey: input.adapter,
|
||||
body: {
|
||||
workUnits: [failedWorkUnit],
|
||||
failedWorkUnits: [failedWorkUnit.unitKey],
|
||||
},
|
||||
});
|
||||
return {
|
||||
result: {
|
||||
jobId,
|
||||
runId: nextReport.runId,
|
||||
syncId: nextReport.body.syncId,
|
||||
diffSummary: nextReport.body.diffSummary,
|
||||
workUnitCount: nextReport.body.workUnits.length,
|
||||
failedWorkUnits: nextReport.body.failedWorkUnits,
|
||||
artifactsWritten: nextReport.body.provenanceRows.length,
|
||||
commitSha: nextReport.body.commitSha,
|
||||
},
|
||||
report: nextReport,
|
||||
};
|
||||
}
|
||||
|
||||
export class CliLookerSlWritingAgentRunner extends AgentRunnerService {
|
||||
override runLoop = vi.fn(async (params: RunLoopParams) => {
|
||||
if (
|
||||
params.telemetryTags?.operationName === 'ingest-bundle-wu' &&
|
||||
params.telemetryTags?.unitKey === 'looker-explore-ecommerce-orders'
|
||||
) {
|
||||
const slWrite = params.toolSet.sl_write_source;
|
||||
if (!slWrite?.execute) {
|
||||
throw new Error('sl_write_source tool was not available to the Looker WorkUnit');
|
||||
}
|
||||
const result = await slWrite.execute(
|
||||
{
|
||||
connectionId: 'prod-warehouse',
|
||||
sourceName: 'looker__ecommerce__orders',
|
||||
source: {
|
||||
name: 'looker__ecommerce__orders',
|
||||
table: 'public.orders',
|
||||
grain: ['id'],
|
||||
columns: [
|
||||
{ name: 'id', type: 'number' },
|
||||
{ name: 'revenue', type: 'number' },
|
||||
],
|
||||
measures: [{ name: 'total_revenue', expr: 'sum(revenue)' }],
|
||||
},
|
||||
},
|
||||
{ toolCallId: 'cli-looker-sl-write', messages: [] },
|
||||
);
|
||||
if (!result.structured.success) {
|
||||
throw new Error(result.markdown);
|
||||
}
|
||||
}
|
||||
return { stopReason: 'natural' as const };
|
||||
});
|
||||
|
||||
constructor() {
|
||||
super({ llmProvider: { getModel: () => ({}) as never } as never });
|
||||
}
|
||||
}
|
||||
|
||||
export class CliMetabaseAgentRunner extends AgentRunnerService {
|
||||
override runLoop = vi.fn(async () => ({ stopReason: 'natural' as const }));
|
||||
|
||||
constructor() {
|
||||
super({ llmProvider: { getModel: () => ({}) as never } as never });
|
||||
}
|
||||
}
|
||||
|
||||
export class CliMetabaseSourceAdapter implements SourceAdapter {
|
||||
readonly source = 'metabase';
|
||||
readonly skillNames: string[] = [];
|
||||
readonly fetchCalls: Array<{ metabaseConnectionId: string; metabaseDatabaseId: number; connectionId: string }> = [];
|
||||
private readonly databaseByStagedDir = new Map<string, number>();
|
||||
|
||||
detect(): Promise<boolean> {
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
|
||||
async fetch(pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise<void> {
|
||||
const config = pullConfig as { metabaseConnectionId: string; metabaseDatabaseId: number };
|
||||
this.fetchCalls.push({
|
||||
metabaseConnectionId: config.metabaseConnectionId,
|
||||
metabaseDatabaseId: config.metabaseDatabaseId,
|
||||
connectionId: ctx.connectionId,
|
||||
});
|
||||
this.databaseByStagedDir.set(stagedDir, config.metabaseDatabaseId);
|
||||
await mkdir(join(stagedDir, 'cards'), { recursive: true });
|
||||
await mkdir(join(stagedDir, 'databases'), { recursive: true });
|
||||
await writeFile(
|
||||
join(stagedDir, 'cards', `${config.metabaseDatabaseId}.json`),
|
||||
JSON.stringify({ connectionId: ctx.connectionId, databaseId: config.metabaseDatabaseId }),
|
||||
'utf-8',
|
||||
);
|
||||
await writeFile(
|
||||
join(stagedDir, 'databases', `${config.metabaseDatabaseId}.json`),
|
||||
JSON.stringify({ metabaseConnectionId: config.metabaseConnectionId }),
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
|
||||
async chunk(stagedDir: string): Promise<ChunkResult> {
|
||||
const databaseId = this.databaseByStagedDir.get(stagedDir);
|
||||
if (!databaseId) {
|
||||
throw new Error(`Missing Metabase database id for staged dir ${stagedDir}`);
|
||||
}
|
||||
return {
|
||||
workUnits: [
|
||||
{
|
||||
unitKey: `metabase-db-${databaseId}`,
|
||||
rawFiles: [`cards/${databaseId}.json`],
|
||||
peerFileIndex: [],
|
||||
dependencyPaths: [`databases/${databaseId}.json`],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const SYNC_MODE_METABASE_CARDS: MetabaseCard[] = [
|
||||
{
|
||||
id: 101,
|
||||
name: 'Collection 12 Revenue',
|
||||
description: null,
|
||||
type: 'question',
|
||||
query_type: 'native',
|
||||
database_id: 1,
|
||||
collection_id: 12,
|
||||
archived: false,
|
||||
result_metadata: [],
|
||||
dataset_query: { type: 'native', database: 1, native: { query: 'select 101 as id' } },
|
||||
parameters: [],
|
||||
dashboard_count: 0,
|
||||
},
|
||||
{
|
||||
id: 102,
|
||||
name: 'Collection 12 Margin',
|
||||
description: null,
|
||||
type: 'question',
|
||||
query_type: 'native',
|
||||
database_id: 1,
|
||||
collection_id: 12,
|
||||
archived: false,
|
||||
result_metadata: [],
|
||||
dataset_query: { type: 'native', database: 1, native: { query: 'select 102 as id' } },
|
||||
parameters: [],
|
||||
dashboard_count: 0,
|
||||
},
|
||||
{
|
||||
id: 103,
|
||||
name: 'Collection 13 Pipeline',
|
||||
description: null,
|
||||
type: 'question',
|
||||
query_type: 'native',
|
||||
database_id: 1,
|
||||
collection_id: 13,
|
||||
archived: false,
|
||||
result_metadata: [],
|
||||
dataset_query: { type: 'native', database: 1, native: { query: 'select 103 as id' } },
|
||||
parameters: [],
|
||||
dashboard_count: 0,
|
||||
},
|
||||
];
|
||||
|
||||
function metabaseCardSummary(card: MetabaseCard): MetabaseCardSummary {
|
||||
return {
|
||||
id: card.id,
|
||||
name: card.name,
|
||||
archived: card.archived,
|
||||
database_id: card.database_id,
|
||||
collection_id: card.collection_id,
|
||||
};
|
||||
}
|
||||
|
||||
function createSyncModeMetabaseClient(): MetabaseRuntimeClient {
|
||||
const cardsById = new Map(SYNC_MODE_METABASE_CARDS.map((card) => [card.id, card]));
|
||||
return {
|
||||
testConnection: async () => ({ success: true }),
|
||||
getCurrentUser: async () => ({ id: 1, email: 'local@example.test' }),
|
||||
getDatabases: async () => [{ id: 1, name: 'Warehouse A', engine: 'postgres' }],
|
||||
getDatabase: async (id) => ({ id, name: 'Warehouse A', engine: 'postgres' }),
|
||||
getCollectionTree: async () => [
|
||||
{ id: 12, name: 'Selected Collection', parent_id: 'root', children: [] },
|
||||
{ id: 13, name: 'Other Collection', parent_id: 'root', children: [] },
|
||||
],
|
||||
getCollection: async (id) => ({
|
||||
id,
|
||||
name: id === 12 ? 'Selected Collection' : 'Other Collection',
|
||||
parent_id: 'root',
|
||||
children: [],
|
||||
}),
|
||||
getCollectionItems: async (collectionId) =>
|
||||
SYNC_MODE_METABASE_CARDS.filter((card) => card.collection_id === collectionId).map((card) => ({
|
||||
id: card.id,
|
||||
model: 'card',
|
||||
name: card.name,
|
||||
collection_id: card.collection_id,
|
||||
database_id: card.database_id,
|
||||
})),
|
||||
getCard: async (id) => {
|
||||
const card = cardsById.get(id);
|
||||
if (!card) {
|
||||
throw new Error(`unexpected card ${id}`);
|
||||
}
|
||||
return card;
|
||||
},
|
||||
getAllCards: async () => SYNC_MODE_METABASE_CARDS.map(metabaseCardSummary),
|
||||
convertMbqlToNative: async () => ({ query: 'select 1' }),
|
||||
getNativeSql: (card) => card.dataset_query?.native?.query ?? null,
|
||||
getTemplateTags: () => ({}),
|
||||
getCardSql: async (card) => card.dataset_query?.native?.query ?? null,
|
||||
getResolvedSql: async (card) => ({
|
||||
resolvedSql: card.dataset_query?.native?.query ?? `select ${card.id} as id`,
|
||||
templateTags: [],
|
||||
resolutionStatus: 'resolved',
|
||||
}),
|
||||
cleanup: async () => undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export class StaticMetabaseClientFactory implements MetabaseClientFactory {
|
||||
constructor(private readonly client: MetabaseRuntimeClient) {}
|
||||
|
||||
createClient(): MetabaseRuntimeClient {
|
||||
return this.client;
|
||||
}
|
||||
}
|
||||
|
||||
type SyncModeCase = {
|
||||
name: string;
|
||||
syncMode: 'ALL' | 'ONLY' | 'EXCEPT';
|
||||
selections: Array<{ selectionType: 'collection' | 'item'; metabaseObjectId: number }>;
|
||||
expectedRawFiles: string[];
|
||||
expectedWorkUnitKeys: string[];
|
||||
};
|
||||
|
||||
export async function runPublicMetabaseSyncModeCase(tempDir: string, input: SyncModeCase): Promise<void> {
|
||||
const projectDir = join(tempDir, `metabase-sync-mode-${input.name}`);
|
||||
await mkdir(projectDir, { recursive: true });
|
||||
await writeFile(
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
`project: metabase-sync-mode-${input.name}`,
|
||||
'connections:',
|
||||
' prod-metabase:',
|
||||
' driver: metabase',
|
||||
' api_url: https://metabase.example.test',
|
||||
' api_key: literal-test-key',
|
||||
' warehouse_a:',
|
||||
' driver: postgres',
|
||||
' url: postgresql://readonly@db.example.test/warehouse_a',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - metabase',
|
||||
' embeddings:',
|
||||
' backend: deterministic',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const project = await loadKtxProject({ projectDir });
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
|
||||
await store.replaceSourceState({
|
||||
connectionId: 'prod-metabase',
|
||||
syncMode: input.syncMode,
|
||||
defaultTagNames: ['sync-mode-smoke'],
|
||||
selections: input.selections,
|
||||
mappings: [
|
||||
{
|
||||
metabaseDatabaseId: 1,
|
||||
metabaseDatabaseName: 'Warehouse A',
|
||||
metabaseEngine: 'postgres',
|
||||
metabaseHost: 'db.example.test',
|
||||
metabaseDbName: 'warehouse_a',
|
||||
targetConnectionId: 'warehouse_a',
|
||||
syncEnabled: true,
|
||||
source: 'refresh',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const adapter = new MetabaseSourceAdapter({
|
||||
clientFactory: new StaticMetabaseClientFactory(createSyncModeMetabaseClient()),
|
||||
sourceStateReader: store,
|
||||
});
|
||||
const jobId = `metabase-sync-mode-${input.name}-child`;
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'prod-metabase',
|
||||
adapter: 'metabase',
|
||||
outputMode: 'plain',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
createAdapters: vi.fn(() => [adapter]),
|
||||
jobIdFactory: () => jobId,
|
||||
localIngestOptions: {
|
||||
agentRunner: new CliMetabaseAgentRunner(),
|
||||
},
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stderr()).toBe('');
|
||||
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
|
||||
expect(io.stdout()).toContain(`target=warehouse_a database=1 status=done job=${jobId}`);
|
||||
|
||||
const report = await getLocalIngestStatus(project, jobId);
|
||||
expect(report).not.toBeNull();
|
||||
expect(report?.body.workUnits.map((wu) => wu.unitKey).sort()).toEqual(input.expectedWorkUnitKeys);
|
||||
expect(report?.body.workUnits.flatMap((wu) => wu.rawFiles).sort()).toEqual(input.expectedRawFiles);
|
||||
}
|
||||
|
||||
type CliLookerRuntimeClient = LookerRuntimeClient &
|
||||
Pick<LookerMappingClient, 'listLookerConnections'> & {
|
||||
cleanup: ReturnType<typeof vi.fn<NonNullable<LookerRuntimeClient['cleanup']>>>;
|
||||
};
|
||||
|
||||
export function makeCliLookerRuntimeClient(): CliLookerRuntimeClient {
|
||||
const lookerModels = {
|
||||
source: 'looker',
|
||||
fetchedAt: '2026-05-05T00:00:00.000Z',
|
||||
models: [{ name: 'ecommerce', label: 'Ecommerce', explores: [{ name: 'orders', label: 'Orders' }] }],
|
||||
};
|
||||
const lookerExplore = {
|
||||
source: 'looker',
|
||||
modelName: 'ecommerce',
|
||||
exploreName: 'orders',
|
||||
label: 'Orders',
|
||||
description: null,
|
||||
connectionName: 'analytics',
|
||||
viewName: 'orders',
|
||||
rawSqlTableName: 'public.orders',
|
||||
fields: {
|
||||
dimensions: [{ name: 'orders.id', label: null, type: null, sql: null, description: null }],
|
||||
measures: [{ name: 'orders.revenue', label: null, type: null, sql: null, description: null }],
|
||||
},
|
||||
joins: [
|
||||
{
|
||||
name: 'users',
|
||||
type: 'left_outer',
|
||||
relationship: 'many_to_one',
|
||||
rawSqlTableName: 'public.users',
|
||||
sqlOn: '${orders.user_id} = ${users.id}',
|
||||
from: null,
|
||||
targetTable: null,
|
||||
},
|
||||
],
|
||||
targetWarehouseConnectionId: null,
|
||||
targetTable: null,
|
||||
};
|
||||
|
||||
return {
|
||||
listLookerConnections: vi.fn().mockResolvedValue([
|
||||
{
|
||||
name: 'analytics',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
schema: null,
|
||||
dialect: 'postgres',
|
||||
},
|
||||
]),
|
||||
listDashboards: vi.fn().mockResolvedValue([{ id: '10', updatedAt: '2026-05-05T08:00:00.000Z' }]),
|
||||
getDashboard: vi.fn().mockResolvedValue({
|
||||
lookerId: '10',
|
||||
title: 'Revenue Overview',
|
||||
description: 'Revenue dashboard',
|
||||
folderId: '7',
|
||||
ownerId: '3',
|
||||
updatedAt: '2026-05-05T08:00:00.000Z',
|
||||
tiles: [{ id: '100', title: 'Revenue', lookId: null, query: { model: 'ecommerce', view: 'orders' } }],
|
||||
}),
|
||||
listLooks: vi.fn().mockResolvedValue([{ id: '20', updatedAt: '2026-05-05T08:10:00.000Z' }]),
|
||||
getLook: vi.fn().mockResolvedValue({
|
||||
lookerId: '20',
|
||||
title: 'Revenue Look',
|
||||
description: null,
|
||||
folderId: '7',
|
||||
ownerId: '3',
|
||||
updatedAt: '2026-05-05T08:10:00.000Z',
|
||||
query: { model: 'ecommerce', view: 'orders', fields: ['orders.revenue'] },
|
||||
}),
|
||||
listFolders: vi.fn().mockResolvedValue({ folders: [{ id: '7', name: 'Shared', parentId: null, path: ['Shared'] }] }),
|
||||
listUsers: vi.fn().mockResolvedValue([{ id: '3', displayName: 'Ada Lovelace', email: 'ada@example.test' }]),
|
||||
listGroups: vi.fn().mockResolvedValue([{ id: '4', name: 'Analysts' }]),
|
||||
listLookmlModels: vi.fn().mockResolvedValue(lookerModels),
|
||||
getExplore: vi.fn().mockResolvedValue(lookerExplore),
|
||||
getSignals: vi.fn().mockResolvedValue({
|
||||
dashboardUsage: [{ contentId: '10', queryCount30d: 12, uniqueUsers30d: 3, lastRunAt: null, topUsers: ['3'] }],
|
||||
lookUsage: [{ contentId: '20', queryCount30d: 4, uniqueUsers30d: 2, lastRunAt: null, topUsers: ['3'] }],
|
||||
scheduledPlans: [
|
||||
{ contentId: '10', contentType: 'dashboard', isScheduled: true, scheduleCount: 1, recipientCount: 4 },
|
||||
],
|
||||
favorites: [{ contentId: '10', contentType: 'dashboard', favoriteCount: 2 }],
|
||||
}),
|
||||
cleanup: vi.fn<NonNullable<LookerRuntimeClient['cleanup']>>().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
interface TestLookerTableIdentifierParser extends LookerTableIdentifierParser {
|
||||
parse: ReturnType<typeof vi.fn<LookerTableIdentifierParser['parse']>>;
|
||||
}
|
||||
|
||||
export function makeCliLookerParser(): TestLookerTableIdentifierParser {
|
||||
return {
|
||||
parse: vi.fn<LookerTableIdentifierParser['parse']>().mockResolvedValue({
|
||||
'ecommerce.orders': {
|
||||
ok: true,
|
||||
catalog: null,
|
||||
schema: 'public',
|
||||
name: 'orders',
|
||||
canonical_table: 'public.orders',
|
||||
},
|
||||
'ecommerce.orders.users': {
|
||||
ok: true,
|
||||
catalog: null,
|
||||
schema: 'public',
|
||||
name: 'users',
|
||||
canonical_table: 'public.users',
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function localFakeBundleReport(
|
||||
jobId: string,
|
||||
overrides: Partial<Omit<IngestReportSnapshot, 'body'>> & { body?: Partial<IngestReportSnapshot['body']> } = {},
|
||||
): IngestReportSnapshot {
|
||||
const report = bundleReportSnapshot();
|
||||
return {
|
||||
...report,
|
||||
id: `report-${jobId}`,
|
||||
runId: `run-${jobId}`,
|
||||
jobId,
|
||||
connectionId: 'warehouse',
|
||||
sourceKey: 'fake',
|
||||
...overrides,
|
||||
body: {
|
||||
...report.body,
|
||||
syncId: 'sync-live-1',
|
||||
...(overrides.body ?? {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function localBundleStore(projectDir: string, ids: [string, string]): Promise<SqliteBundleIngestStore> {
|
||||
const { SqliteBundleIngestStore } = await import('@ktx/context/ingest');
|
||||
const project = await loadKtxProject({ projectDir });
|
||||
return new SqliteBundleIngestStore({
|
||||
dbPath: ktxLocalStateDbPath(project),
|
||||
idFactory: (() => {
|
||||
let index = 0;
|
||||
return () => ids[index++] ?? `generated-${index}`;
|
||||
})(),
|
||||
});
|
||||
}
|
||||
|
||||
export async function persistLocalBundleReport(projectDir: string, report = bundleReportSnapshot()): Promise<void> {
|
||||
const store = await localBundleStore(projectDir, [report.runId, report.id]);
|
||||
const run = await store.create({
|
||||
jobId: report.jobId,
|
||||
connectionId: report.connectionId,
|
||||
sourceKey: report.sourceKey,
|
||||
syncId: report.body.syncId,
|
||||
trigger: 'manual_resync',
|
||||
});
|
||||
await store.markCompleted(run.id, report.body.diffSummary);
|
||||
await store.create({
|
||||
runId: run.id,
|
||||
jobId: report.jobId,
|
||||
connectionId: report.connectionId,
|
||||
sourceKey: report.sourceKey,
|
||||
body: report.body,
|
||||
});
|
||||
}
|
||||
|
||||
export async function writeBundleReportFile(tempDir: string, report = bundleReportSnapshot()): Promise<string> {
|
||||
const reportFile = join(tempDir, 'bundle-report.json');
|
||||
await writeFile(reportFile, `${JSON.stringify(report, null, 2)}\n`, 'utf-8');
|
||||
return reportFile;
|
||||
}
|
||||
|
||||
export function emitLiveLocalMemoryFlow(memoryFlow: MemoryFlowEventSink | undefined): void {
|
||||
memoryFlow?.emit({ type: 'source_acquired', adapter: 'fake', trigger: 'manual_resync', fileCount: 1 });
|
||||
memoryFlow?.update({ syncId: 'sync-live-1' });
|
||||
memoryFlow?.emit({ type: 'raw_snapshot_written', syncId: 'sync-live-1', rawFileCount: 1 });
|
||||
memoryFlow?.emit({ type: 'diff_computed', added: 1, modified: 0, deleted: 0, unchanged: 0 });
|
||||
memoryFlow?.update({
|
||||
plannedWorkUnits: [
|
||||
{
|
||||
unitKey: 'fake-orders',
|
||||
rawFiles: ['orders/orders.json'],
|
||||
peerFileCount: 0,
|
||||
dependencyCount: 0,
|
||||
},
|
||||
],
|
||||
});
|
||||
memoryFlow?.emit({ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 });
|
||||
memoryFlow?.emit({ type: 'report_created', runId: 'live-viz-run' });
|
||||
memoryFlow?.finish('done');
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -114,6 +114,16 @@ function writeReportStatus(report: IngestReportSnapshot, io: KtxIngestIo): void
|
|||
}
|
||||
|
||||
function writeMetabaseFanoutStatus(result: LocalMetabaseFanoutResult, io: KtxIngestIo): void {
|
||||
const counts = result.children.reduce(
|
||||
(acc, child) => {
|
||||
const childCounts = reportActionCounts(child.report);
|
||||
return {
|
||||
wikiCount: acc.wikiCount + childCounts.wikiCount,
|
||||
slCount: acc.slCount + childCounts.slCount,
|
||||
};
|
||||
},
|
||||
{ wikiCount: 0, slCount: 0 },
|
||||
);
|
||||
io.stdout.write(`Metabase fan-out: ${result.status}\n`);
|
||||
io.stdout.write(`Source: ${result.metabaseConnectionId}\n`);
|
||||
io.stdout.write(`Children: ${result.children.length}\n`);
|
||||
|
|
@ -121,10 +131,11 @@ function writeMetabaseFanoutStatus(result: LocalMetabaseFanoutResult, io: KtxIng
|
|||
io.stdout.write(`Work units: ${result.totals.workUnits}\n`);
|
||||
io.stdout.write(`Failed work units: ${result.totals.failedWorkUnits}\n`);
|
||||
}
|
||||
io.stdout.write(`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} SL\n`);
|
||||
for (const child of result.children) {
|
||||
const status = reportStatus(child.report);
|
||||
io.stdout.write(
|
||||
`- target=${child.targetConnectionId} database=${child.metabaseDatabaseId} status=${status} job=${child.jobId}\n`,
|
||||
`- target=${child.targetConnectionId} database=${child.metabaseDatabaseId} status=${status} job=${child.jobId} report=${child.report.id}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -345,7 +356,7 @@ export async function runKtxIngest(
|
|||
} else {
|
||||
writeMetabaseFanoutStatus(result, io);
|
||||
}
|
||||
return 0;
|
||||
return result.status === 'all_succeeded' ? 0 : 1;
|
||||
}
|
||||
|
||||
const jobId = deps.jobIdFactory?.();
|
||||
|
|
@ -397,14 +408,14 @@ export async function runKtxIngest(
|
|||
liveTui?.close();
|
||||
liveTui = null;
|
||||
io.stdout.write(formatMemoryFlowFinalSummary(latestMemoryFlowSnapshot));
|
||||
return 0;
|
||||
return reportStatus(result.report) === 'done' ? 0 : 1;
|
||||
}
|
||||
await writeReportRecord(result.report, runOutputMode, io, {
|
||||
interactive: (args.inputMode ?? 'auto') === 'auto',
|
||||
renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
|
||||
env,
|
||||
});
|
||||
return 0;
|
||||
return reportStatus(result.report) === 'done' ? 0 : 1;
|
||||
} finally {
|
||||
liveTui?.close();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,29 +95,6 @@ describe('createKtxCliScanConnector', () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it('does not create a standalone PostHog scan connector', async () => {
|
||||
await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
' product:',
|
||||
' driver: posthog',
|
||||
' api_key: phx_test',
|
||||
' project_id: "157881"',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const project = await loadKtxProject({ projectDir: tempDir });
|
||||
|
||||
await expect(createKtxCliScanConnector(project, 'product')).rejects.toThrow(
|
||||
'Connection "product" uses driver "posthog", which has no native standalone KTX scan connector',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws for structural daemon-only fallback configs', async () => {
|
||||
await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
|
||||
await writeFile(
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ function commandLines(commands: ReadonlyArray<{ command: string; description: st
|
|||
export function formatNextStepLines(indent = ' '): string[] {
|
||||
return [
|
||||
`${indent}KTX context is ready for agents.`,
|
||||
`${indent}Preferred route: CLI + Skills; installed rules call \`ktx agent ...\` directly, so no MCP server is required.`,
|
||||
`${indent}Preferred route: CLI + Skills; installed rules call the pinned local CLI directly, so no MCP server is required.`,
|
||||
`${indent}Direct CLI checks:`,
|
||||
...commandLines(KTX_NEXT_STEP_DIRECT_COMMANDS, indent),
|
||||
`${indent}Optional MCP:`,
|
||||
|
|
|
|||
|
|
@ -80,13 +80,6 @@ describe('buildPublicIngestPlan', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('does not plan PostHog connections as CLI ingest targets', () => {
|
||||
const project = projectWithConnections({ product: { driver: 'posthog' } });
|
||||
|
||||
expect(() =>
|
||||
buildPublicIngestPlan(project, { projectDir: '/tmp/project', targetConnectionId: 'product', all: false }),
|
||||
).toThrow('Connection "product" uses unsupported public ingest driver "posthog"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runKtxPublicIngest', () => {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { tmpdir } from 'node:os';
|
|||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
formatInstallSummary,
|
||||
plannedKtxAgentFiles,
|
||||
readKtxAgentInstallManifest,
|
||||
removeKtxAgentInstall,
|
||||
|
|
@ -37,11 +38,13 @@ describe('setup agents', () => {
|
|||
|
||||
it('plans project-scoped CLI and MCP files for every target', () => {
|
||||
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'claude-code', scope: 'project', mode: 'both' })).toEqual([
|
||||
{ kind: 'file', path: join(tempDir, '.claude/skills/ktx/SKILL.md') },
|
||||
{ kind: 'file', path: join(tempDir, '.claude/skills/ktx/SKILL.md'), role: 'skill' },
|
||||
{ kind: 'file', path: join(tempDir, '.claude/rules/ktx.md'), role: 'rule' },
|
||||
{ kind: 'json-key', path: join(tempDir, '.mcp.json'), jsonPath: ['mcpServers', 'ktx'] },
|
||||
]);
|
||||
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'codex', scope: 'project', mode: 'cli' })).toEqual([
|
||||
{ kind: 'file', path: join(tempDir, '.agents/skills/ktx/SKILL.md') },
|
||||
{ kind: 'file', path: join(tempDir, '.agents/skills/ktx/SKILL.md'), role: 'skill' },
|
||||
{ kind: 'file', path: join(tempDir, '.codex/instructions/ktx.md'), role: 'rule' },
|
||||
]);
|
||||
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'cursor', scope: 'project', mode: 'mcp' })).toEqual([
|
||||
{ kind: 'json-key', path: join(tempDir, '.cursor/mcp.json'), jsonPath: ['mcpServers', 'ktx'] },
|
||||
|
|
@ -83,7 +86,7 @@ describe('setup agents', () => {
|
|||
const skill = await readFile(join(tempDir, '.agents/skills/ktx/SKILL.md'), 'utf-8');
|
||||
expect(skill).toContain(`--project-dir ${tempDir}`);
|
||||
expect(skill).toContain('must not print secrets');
|
||||
expect(skill).toContain('ktx agent sql execute');
|
||||
expect(skill).toContain('agent sql execute');
|
||||
expect(await readKtxAgentInstallManifest(tempDir)).toMatchObject({
|
||||
version: 1,
|
||||
projectDir: tempDir,
|
||||
|
|
@ -93,6 +96,47 @@ describe('setup agents', () => {
|
|||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('writes PATH-independent launcher commands for skills and MCP configs', async () => {
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxSetupAgentsStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'disabled',
|
||||
yes: true,
|
||||
agents: true,
|
||||
target: 'universal',
|
||||
scope: 'project',
|
||||
mode: 'both',
|
||||
skipAgents: false,
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toMatchObject({ status: 'ready' });
|
||||
|
||||
const skill = await readFile(join(tempDir, '.agents/skills/ktx/SKILL.md'), 'utf-8');
|
||||
expect(skill).not.toContain('`ktx agent');
|
||||
expect(skill).toContain('agent context --json');
|
||||
expect(skill).toContain('agent sql execute');
|
||||
|
||||
const mcp = JSON.parse(await readFile(join(tempDir, '.agents/mcp/ktx.json'), 'utf-8')) as {
|
||||
mcpServers?: { ktx?: { command?: string; args?: string[] } };
|
||||
};
|
||||
expect(mcp.mcpServers?.ktx?.command).toBe(process.execPath);
|
||||
expect(mcp.mcpServers?.ktx?.args?.[0]).toMatch(/packages\/cli\/(src|dist)\/bin\.(ts|js)$/);
|
||||
expect(mcp.mcpServers?.ktx?.args).toEqual([
|
||||
expect.stringMatching(/packages\/cli\/(src|dist)\/bin\.(ts|js)$/),
|
||||
'--project-dir',
|
||||
tempDir,
|
||||
'serve',
|
||||
'--mcp',
|
||||
'stdio',
|
||||
'--semantic-compute',
|
||||
'--execute-queries',
|
||||
]);
|
||||
});
|
||||
|
||||
it('removes only manifest-listed files and JSON keys', async () => {
|
||||
const io = makeIo();
|
||||
await runKtxSetupAgentsStep(
|
||||
|
|
@ -113,6 +157,7 @@ describe('setup agents', () => {
|
|||
await expect(removeKtxAgentInstall(tempDir, io.io)).resolves.toBe(0);
|
||||
|
||||
await expect(stat(join(tempDir, '.claude/skills/ktx/SKILL.md'))).rejects.toThrow();
|
||||
await expect(stat(join(tempDir, '.claude/rules/ktx.md'))).rejects.toThrow();
|
||||
await expect(stat(join(tempDir, '.claude/skills/ktx/keep.txt'))).resolves.toBeDefined();
|
||||
await expect(readKtxAgentInstallManifest(tempDir)).resolves.toEqual(null);
|
||||
});
|
||||
|
|
@ -173,4 +218,71 @@ describe('setup agents', () => {
|
|||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('prints per-agent install summary after successful installation', async () => {
|
||||
const io = makeIo();
|
||||
|
||||
await runKtxSetupAgentsStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'disabled',
|
||||
yes: true,
|
||||
agents: true,
|
||||
target: 'claude-code',
|
||||
scope: 'project',
|
||||
mode: 'both',
|
||||
skipAgents: false,
|
||||
},
|
||||
io.io,
|
||||
);
|
||||
|
||||
const output = io.stdout();
|
||||
expect(output).toContain('Agent integration complete');
|
||||
expect(output).toContain('Claude Code');
|
||||
expect(output).toContain('+ Skill installed');
|
||||
expect(output).toContain('.claude/skills/ktx/SKILL.md');
|
||||
expect(output).toContain('+ Rule installed');
|
||||
expect(output).toContain('.claude/rules/ktx.md');
|
||||
expect(output).toContain('+ MCP config added');
|
||||
expect(output).toContain('.mcp.json');
|
||||
});
|
||||
|
||||
it('formats summary with relative paths for project scope', () => {
|
||||
const summary = formatInstallSummary(
|
||||
[{ target: 'cursor', scope: 'project', mode: 'both' }],
|
||||
[
|
||||
{ kind: 'file', path: join(tempDir, '.cursor/rules/ktx.mdc') },
|
||||
{ kind: 'json-key', path: join(tempDir, '.cursor/mcp.json'), jsonPath: ['mcpServers', 'ktx'] },
|
||||
],
|
||||
tempDir,
|
||||
);
|
||||
|
||||
expect(summary).toContain('Cursor');
|
||||
expect(summary).toContain('+ Rule installed');
|
||||
expect(summary).toContain('.cursor/rules/ktx.mdc');
|
||||
expect(summary).toContain('+ MCP config added');
|
||||
expect(summary).toContain('.cursor/mcp.json');
|
||||
expect(summary).not.toContain(tempDir);
|
||||
});
|
||||
|
||||
it('formats summary with multiple agent targets', () => {
|
||||
const summary = formatInstallSummary(
|
||||
[
|
||||
{ target: 'claude-code', scope: 'project', mode: 'cli' },
|
||||
{ target: 'codex', scope: 'project', mode: 'mcp' },
|
||||
],
|
||||
[
|
||||
{ kind: 'file', path: join(tempDir, '.claude/skills/ktx/SKILL.md'), role: 'skill' },
|
||||
{ kind: 'file', path: join(tempDir, '.claude/rules/ktx.md'), role: 'rule' },
|
||||
{ kind: 'json-key', path: join(tempDir, '.agents/mcp/ktx.json'), jsonPath: ['mcpServers', 'ktx'] },
|
||||
],
|
||||
tempDir,
|
||||
);
|
||||
|
||||
expect(summary).toContain('Claude Code');
|
||||
expect(summary).toContain('+ Skill installed');
|
||||
expect(summary).toContain('+ Rule installed');
|
||||
expect(summary).toContain('Codex');
|
||||
expect(summary).toContain('+ MCP config added');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { dirname, join, relative, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { cancel, isCancel, multiselect, select } from '@clack/prompts';
|
||||
import { loadKtxProject, markKtxSetupStepComplete, serializeKtxProjectConfig } from '@ktx/context/project';
|
||||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
|
|
@ -37,11 +38,19 @@ export interface KtxAgentInstallManifest {
|
|||
projectDir: string;
|
||||
installedAt: string;
|
||||
installs: Array<{ target: KtxAgentTarget; scope: KtxAgentScope; mode: KtxAgentInstallMode }>;
|
||||
entries: Array<{ kind: 'file'; path: string } | { kind: 'json-key'; path: string; jsonPath: string[] }>;
|
||||
entries: Array<
|
||||
| { kind: 'file'; path: string; role?: 'skill' | 'rule' }
|
||||
| { kind: 'json-key'; path: string; jsonPath: string[] }
|
||||
>;
|
||||
}
|
||||
|
||||
type InstallEntry = KtxAgentInstallManifest['entries'][number];
|
||||
|
||||
interface KtxCliLauncher {
|
||||
command: string;
|
||||
args: string[];
|
||||
}
|
||||
|
||||
export function agentInstallManifestPath(projectDir: string): string {
|
||||
return join(resolve(projectDir), '.ktx/agents/install-manifest.json');
|
||||
}
|
||||
|
|
@ -54,11 +63,17 @@ export function plannedKtxAgentFiles(input: {
|
|||
}): InstallEntry[] {
|
||||
if (input.scope === 'global') {
|
||||
if (input.target === 'claude-code') {
|
||||
return [{ kind: 'file', path: join(process.env.HOME ?? '', '.claude/skills/ktx/SKILL.md') }];
|
||||
const home = process.env.HOME ?? '';
|
||||
return [
|
||||
{ kind: 'file', path: join(home, '.claude/skills/ktx/SKILL.md'), role: 'skill' as const },
|
||||
{ kind: 'file', path: join(home, '.claude/rules/ktx.md'), role: 'rule' as const },
|
||||
];
|
||||
}
|
||||
if (input.target === 'codex') {
|
||||
const codexHome = process.env.CODEX_HOME ?? join(process.env.HOME ?? '', '.codex');
|
||||
return [
|
||||
{ kind: 'file', path: join(process.env.CODEX_HOME ?? join(process.env.HOME ?? '', '.codex'), 'skills/ktx/SKILL.md') },
|
||||
{ kind: 'file', path: join(codexHome, 'skills/ktx/SKILL.md'), role: 'skill' as const },
|
||||
{ kind: 'file', path: join(codexHome, 'instructions/ktx.md'), role: 'rule' as const },
|
||||
];
|
||||
}
|
||||
throw new Error(`Global ${input.target} installation is not supported; use --project.`);
|
||||
|
|
@ -66,12 +81,16 @@ export function plannedKtxAgentFiles(input: {
|
|||
|
||||
const root = resolve(input.projectDir);
|
||||
const cliEntries: Partial<Record<KtxAgentTarget, InstallEntry>> = {
|
||||
'claude-code': { kind: 'file', path: join(root, '.claude/skills/ktx/SKILL.md') },
|
||||
codex: { kind: 'file', path: join(root, '.agents/skills/ktx/SKILL.md') },
|
||||
'claude-code': { kind: 'file', path: join(root, '.claude/skills/ktx/SKILL.md'), role: 'skill' },
|
||||
codex: { kind: 'file', path: join(root, '.agents/skills/ktx/SKILL.md'), role: 'skill' },
|
||||
cursor: { kind: 'file', path: join(root, '.cursor/rules/ktx.mdc') },
|
||||
opencode: { kind: 'file', path: join(root, '.opencode/commands/ktx.md') },
|
||||
universal: { kind: 'file', path: join(root, '.agents/skills/ktx/SKILL.md') },
|
||||
};
|
||||
const ruleEntries: Partial<Record<KtxAgentTarget, InstallEntry>> = {
|
||||
'claude-code': { kind: 'file', path: join(root, '.claude/rules/ktx.md'), role: 'rule' },
|
||||
codex: { kind: 'file', path: join(root, '.codex/instructions/ktx.md'), role: 'rule' },
|
||||
};
|
||||
const mcpEntries: Record<KtxAgentTarget, InstallEntry> = {
|
||||
'claude-code': { kind: 'json-key', path: join(root, '.mcp.json'), jsonPath: ['mcpServers', 'ktx'] },
|
||||
codex: { kind: 'json-key', path: join(root, '.agents/mcp/ktx.json'), jsonPath: ['mcpServers', 'ktx'] },
|
||||
|
|
@ -80,12 +99,31 @@ export function plannedKtxAgentFiles(input: {
|
|||
universal: { kind: 'json-key', path: join(root, '.agents/mcp/ktx.json'), jsonPath: ['mcpServers', 'ktx'] },
|
||||
};
|
||||
return [
|
||||
...(input.mode === 'cli' || input.mode === 'both' ? [cliEntries[input.target]] : []),
|
||||
...(input.mode === 'cli' || input.mode === 'both' ? [cliEntries[input.target], ruleEntries[input.target]] : []),
|
||||
...(input.mode === 'mcp' || input.mode === 'both' ? [mcpEntries[input.target]] : []),
|
||||
].filter((entry): entry is InstallEntry => entry !== undefined);
|
||||
}
|
||||
|
||||
function cliInstructionContent(input: { projectDir: string; target: KtxAgentTarget }): string {
|
||||
function ktxCliLauncher(): KtxCliLauncher {
|
||||
return {
|
||||
command: process.execPath,
|
||||
args: [fileURLToPath(new URL('./bin.js', import.meta.url))],
|
||||
};
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
if (/^[A-Za-z0-9_/:=.,@%+-]+$/.test(value)) {
|
||||
return value;
|
||||
}
|
||||
return `'${value.replaceAll("'", "'\\''")}'`;
|
||||
}
|
||||
|
||||
function ktxCommandLine(launcher: KtxCliLauncher, args: string[]): string {
|
||||
return [launcher.command, ...launcher.args, ...args].map(shellQuote).join(' ');
|
||||
}
|
||||
|
||||
function cliInstructionContent(input: { projectDir: string; launcher: KtxCliLauncher }): string {
|
||||
const projectDirArgs = ['--json', '--project-dir', input.projectDir];
|
||||
return [
|
||||
'---',
|
||||
'name: ktx',
|
||||
|
|
@ -95,28 +133,64 @@ function cliInstructionContent(input: { projectDir: string; target: KtxAgentTarg
|
|||
'# KTX Local Context',
|
||||
'',
|
||||
`Use this project with \`--project-dir ${input.projectDir}\`.`,
|
||||
'Commands are pinned to the local KTX CLI path that created this file, so agents do not need `ktx` in PATH.',
|
||||
'If the CLI path no longer exists after moving this checkout or reinstalling KTX, rerun `ktx setup --agents`.',
|
||||
'',
|
||||
'Agents must not print secrets, credential references, environment variable values, or file contents from `.ktx/secrets`.',
|
||||
'',
|
||||
'Available commands:',
|
||||
'',
|
||||
`- \`ktx agent context --json --project-dir ${input.projectDir}\``,
|
||||
`- \`ktx agent sl list --json --project-dir ${input.projectDir}\``,
|
||||
`- \`ktx agent sl read <sourceName> --json --project-dir ${input.projectDir}\``,
|
||||
`- \`ktx agent sl query --json --project-dir ${input.projectDir} --connection-id <id> --query-file <path> --execute --max-rows 100\``,
|
||||
`- \`ktx agent wiki search <query> --json --project-dir ${input.projectDir}\``,
|
||||
`- \`ktx agent wiki read <pageId> --json --project-dir ${input.projectDir}\``,
|
||||
`- \`ktx agent sql execute --json --project-dir ${input.projectDir} --connection-id <id> --sql-file <path> --max-rows 100\``,
|
||||
`- \`${ktxCommandLine(input.launcher, ['agent', 'context', ...projectDirArgs])}\``,
|
||||
`- \`${ktxCommandLine(input.launcher, ['agent', 'sl', 'list', ...projectDirArgs])}\``,
|
||||
`- \`${ktxCommandLine(input.launcher, ['agent', 'sl', 'read', '<sourceName>', ...projectDirArgs])}\``,
|
||||
`- \`${ktxCommandLine(input.launcher, [
|
||||
'agent',
|
||||
'sl',
|
||||
'query',
|
||||
...projectDirArgs,
|
||||
'--connection-id',
|
||||
'<id>',
|
||||
'--query-file',
|
||||
'<path>',
|
||||
'--execute',
|
||||
'--max-rows',
|
||||
'100',
|
||||
])}\``,
|
||||
`- \`${ktxCommandLine(input.launcher, ['agent', 'wiki', 'search', '<query>', ...projectDirArgs])}\``,
|
||||
`- \`${ktxCommandLine(input.launcher, ['agent', 'wiki', 'read', '<pageId>', ...projectDirArgs])}\``,
|
||||
`- \`${ktxCommandLine(input.launcher, [
|
||||
'agent',
|
||||
'sql',
|
||||
'execute',
|
||||
...projectDirArgs,
|
||||
'--connection-id',
|
||||
'<id>',
|
||||
'--sql-file',
|
||||
'<path>',
|
||||
'--max-rows',
|
||||
'100',
|
||||
])}\``,
|
||||
'',
|
||||
'SQL execution is read-only, requires an explicit row limit, and should use the smallest useful limit.',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function mcpConfig(projectDir: string): Record<string, unknown> {
|
||||
function ruleInstructionContent(input: { projectDir: string }): string {
|
||||
return [
|
||||
`Use the \`ktx\` CLI to query local semantic context, wiki knowledge, and execute safe SQL for this project (\`--project-dir ${input.projectDir}\`).`,
|
||||
'',
|
||||
'Use when the user asks about data schemas, metrics, dimensions, database structure, or wants to run SQL queries.',
|
||||
'',
|
||||
'Do not use for general programming, code review, or tasks unrelated to data and analytics.',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function mcpConfig(projectDir: string, launcher: KtxCliLauncher): Record<string, unknown> {
|
||||
return {
|
||||
command: 'ktx',
|
||||
args: ['--project-dir', projectDir, 'serve', '--mcp', 'stdio', '--semantic-compute', '--execute-queries'],
|
||||
command: launcher.command,
|
||||
args: [...launcher.args, '--project-dir', projectDir, 'serve', '--mcp', 'stdio', '--semantic-compute', '--execute-queries'],
|
||||
env: {},
|
||||
};
|
||||
}
|
||||
|
|
@ -245,6 +319,55 @@ function createPromptAdapter(): KtxSetupAgentsPromptAdapter {
|
|||
};
|
||||
}
|
||||
|
||||
const targetDisplayNames: Record<KtxAgentTarget, string> = {
|
||||
'claude-code': 'Claude Code',
|
||||
codex: 'Codex',
|
||||
cursor: 'Cursor',
|
||||
opencode: 'OpenCode',
|
||||
universal: 'Universal .agents',
|
||||
};
|
||||
|
||||
const fileEntryLabels: Record<KtxAgentTarget, string> = {
|
||||
'claude-code': 'Skill installed',
|
||||
codex: 'Skill installed',
|
||||
cursor: 'Rule installed',
|
||||
opencode: 'Command installed',
|
||||
universal: 'Skill installed',
|
||||
};
|
||||
|
||||
export function formatInstallSummary(
|
||||
installs: Array<{ target: KtxAgentTarget; scope: KtxAgentScope; mode: KtxAgentInstallMode }>,
|
||||
entries: InstallEntry[],
|
||||
projectDir: string,
|
||||
): string {
|
||||
const entriesByTarget = new Map<KtxAgentTarget, InstallEntry[]>();
|
||||
let idx = 0;
|
||||
for (const install of installs) {
|
||||
const planned = plannedKtxAgentFiles({ projectDir, ...install });
|
||||
entriesByTarget.set(install.target, entries.slice(idx, idx + planned.length));
|
||||
idx += planned.length;
|
||||
}
|
||||
|
||||
const lines: string[] = [];
|
||||
for (const install of installs) {
|
||||
const targetEntries = entriesByTarget.get(install.target) ?? [];
|
||||
lines.push(` ${targetDisplayNames[install.target]}`);
|
||||
for (const entry of targetEntries) {
|
||||
const displayPath =
|
||||
install.scope === 'global' ? entry.path : relative(projectDir, entry.path);
|
||||
if (entry.kind === 'file') {
|
||||
const label = entry.role === 'rule' ? 'Rule installed' : fileEntryLabels[install.target];
|
||||
lines.push(` + ${label}`);
|
||||
lines.push(` ${displayPath}`);
|
||||
} else {
|
||||
lines.push(` + MCP config added`);
|
||||
lines.push(` ${displayPath}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async function installTarget(input: {
|
||||
projectDir: string;
|
||||
target: KtxAgentTarget;
|
||||
|
|
@ -252,12 +375,17 @@ async function installTarget(input: {
|
|||
mode: KtxAgentInstallMode;
|
||||
}): Promise<InstallEntry[]> {
|
||||
const entries = plannedKtxAgentFiles(input);
|
||||
const launcher = ktxCliLauncher();
|
||||
for (const entry of entries) {
|
||||
if (entry.kind === 'file') {
|
||||
const content =
|
||||
entry.role === 'rule'
|
||||
? ruleInstructionContent({ projectDir: input.projectDir })
|
||||
: cliInstructionContent({ projectDir: input.projectDir, launcher });
|
||||
await mkdir(dirname(entry.path), { recursive: true });
|
||||
await writeFile(entry.path, cliInstructionContent({ projectDir: input.projectDir, target: input.target }), 'utf-8');
|
||||
await writeFile(entry.path, content, 'utf-8');
|
||||
} else {
|
||||
await writeJsonKey(entry.path, entry.jsonPath, mcpConfig(input.projectDir));
|
||||
await writeJsonKey(entry.path, entry.jsonPath, mcpConfig(input.projectDir, launcher));
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
|
|
@ -311,7 +439,6 @@ export async function runKtxSetupAgentsStep(
|
|||
{ value: 'cursor', label: 'Cursor' },
|
||||
{ value: 'opencode', label: 'OpenCode' },
|
||||
{ value: 'universal', label: 'Universal .agents' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
required: true,
|
||||
})) as KtxAgentTarget[]);
|
||||
|
|
@ -327,7 +454,7 @@ export async function runKtxSetupAgentsStep(
|
|||
for (const install of installs) entries.push(...(await installTarget({ projectDir: args.projectDir, ...install })));
|
||||
await writeManifest(args.projectDir, mergeManifest(args.projectDir, await readKtxAgentInstallManifest(args.projectDir), installs, entries));
|
||||
await markAgentsComplete(args.projectDir);
|
||||
io.stdout.write(`Agent integration installed for ${installs.map((install) => install.target).join(', ')}.\n`);
|
||||
io.stdout.write(`\nAgent integration complete\n\n${formatInstallSummary(installs, entries, args.projectDir)}\n`);
|
||||
return { status: 'ready', projectDir: args.projectDir, installs };
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
|
|
|
|||
|
|
@ -166,7 +166,12 @@ describe('setup context build state', () => {
|
|||
it('runs setup context build, verifies readiness, and marks context complete', async () => {
|
||||
await writeReadyProject(tempDir);
|
||||
const io = makeIo();
|
||||
const runContextBuildMock = vi.fn(async () => ({ exitCode: 0, detached: false }));
|
||||
const runContextBuildMock = vi.fn(async () => ({
|
||||
exitCode: 0,
|
||||
detached: false,
|
||||
reportIds: ['report-docs-1'],
|
||||
artifactPaths: ['raw-sources/warehouse/live-database/sync-1/scan-report.json'],
|
||||
}));
|
||||
const verifyContextReady = vi.fn(async () => ({
|
||||
ready: true,
|
||||
agentContextReady: true,
|
||||
|
|
@ -204,6 +209,8 @@ describe('setup context build state', () => {
|
|||
runId: 'setup-context-local-abc123',
|
||||
status: 'completed',
|
||||
completedAt: '2026-05-09T10:00:00.000Z',
|
||||
reportIds: ['report-docs-1'],
|
||||
artifactPaths: ['raw-sources/warehouse/live-database/sync-1/scan-report.json'],
|
||||
});
|
||||
expect(io.stdout()).toContain('KTX context is ready for agents.');
|
||||
});
|
||||
|
|
@ -340,6 +347,207 @@ describe('setup context build state', () => {
|
|||
expect(io.stderr()).toContain('No primary or context sources are configured for a KTX context build.');
|
||||
});
|
||||
|
||||
it('watches an already-running setup context build from the resume prompt', async () => {
|
||||
await writeReadyProject(tempDir);
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-resume-watch',
|
||||
status: 'detached',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:00:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: ['docs'],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-resume-watch'),
|
||||
});
|
||||
const io = makeIo();
|
||||
const completeRun = async () => {
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-resume-watch',
|
||||
status: 'completed',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:02:00.000Z',
|
||||
completedAt: '2026-05-09T10:02:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: ['docs'],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-resume-watch'),
|
||||
});
|
||||
};
|
||||
const select = vi.fn(async (options: { options: Array<{ value: string; label: string }> }) => {
|
||||
expect(options.options.map((option) => option.label)).toContain('Watch progress');
|
||||
return 'watch';
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupContextStep(
|
||||
{ projectDir: tempDir, inputMode: 'auto' },
|
||||
io.io,
|
||||
{
|
||||
prompts: { select, cancel: vi.fn() },
|
||||
sleep: completeRun,
|
||||
watchIntervalMs: 1,
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir: tempDir, runId: 'setup-context-local-resume-watch' });
|
||||
expect(io.stdout()).toContain('KTX context built: detached');
|
||||
expect(io.stdout()).toContain('KTX context built: yes');
|
||||
});
|
||||
|
||||
it('auto-watches a running build without prompting when autoWatch is true', async () => {
|
||||
await writeReadyProject(tempDir);
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-auto-watch',
|
||||
status: 'detached',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:00:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: [],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-auto-watch'),
|
||||
});
|
||||
const io = makeIo();
|
||||
const completeRun = async () => {
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-auto-watch',
|
||||
status: 'completed',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:02:00.000Z',
|
||||
completedAt: '2026-05-09T10:02:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: [],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-auto-watch'),
|
||||
});
|
||||
};
|
||||
const select = vi.fn(async () => {
|
||||
throw new Error('should not prompt when autoWatch is true');
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupContextStep(
|
||||
{ projectDir: tempDir, inputMode: 'auto', autoWatch: true },
|
||||
io.io,
|
||||
{
|
||||
prompts: { select, cancel: vi.fn() },
|
||||
sleep: completeRun,
|
||||
watchIntervalMs: 1,
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir: tempDir, runId: 'setup-context-local-auto-watch' });
|
||||
expect(select).not.toHaveBeenCalled();
|
||||
expect(io.stdout()).toContain('KTX context built: yes');
|
||||
});
|
||||
|
||||
it('renders the progress view when watching a build with sourceProgress', async () => {
|
||||
await writeReadyProject(tempDir);
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-progress',
|
||||
status: 'detached',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:00:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: ['docs'],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-progress'),
|
||||
sourceProgress: [
|
||||
{ connectionId: 'warehouse', operation: 'scan' as const, status: 'done' as const, elapsedMs: 30000 },
|
||||
{ connectionId: 'docs', operation: 'source-ingest' as const, status: 'running' as const, startedAtMs: Date.now() - 5000 },
|
||||
],
|
||||
});
|
||||
const io = makeIo();
|
||||
const completeRun = async () => {
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-progress',
|
||||
status: 'completed',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:02:00.000Z',
|
||||
completedAt: '2026-05-09T10:02:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: ['docs'],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-progress'),
|
||||
sourceProgress: [
|
||||
{ connectionId: 'warehouse', operation: 'scan' as const, status: 'done' as const, elapsedMs: 30000 },
|
||||
{ connectionId: 'docs', operation: 'source-ingest' as const, status: 'done' as const, elapsedMs: 60000 },
|
||||
],
|
||||
});
|
||||
};
|
||||
const select = vi.fn(async () => 'watch');
|
||||
|
||||
await expect(
|
||||
runKtxSetupContextStep(
|
||||
{ projectDir: tempDir, inputMode: 'auto' },
|
||||
io.io,
|
||||
{
|
||||
prompts: { select, cancel: vi.fn() },
|
||||
sleep: completeRun,
|
||||
watchIntervalMs: 1,
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir: tempDir, runId: 'setup-context-local-progress' });
|
||||
|
||||
const output = io.stdout();
|
||||
expect(output).toContain('Building KTX context');
|
||||
expect(output).toContain('Primary sources:');
|
||||
expect(output).toContain('warehouse');
|
||||
expect(output).toContain('Context sources:');
|
||||
expect(output).toContain('docs');
|
||||
expect(output).not.toContain('KTX context built: detached');
|
||||
});
|
||||
|
||||
it('supports d to detach from the progress watch view', async () => {
|
||||
await writeReadyProject(tempDir);
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-detach',
|
||||
status: 'running',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:00:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: [],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-detach'),
|
||||
sourceProgress: [
|
||||
{ connectionId: 'warehouse', operation: 'scan' as const, status: 'running' as const, startedAtMs: Date.now() },
|
||||
],
|
||||
});
|
||||
const io = makeIo();
|
||||
let triggerDetach: (() => void) | null = null;
|
||||
|
||||
await expect(
|
||||
runKtxSetupContextStep(
|
||||
{ projectDir: tempDir, inputMode: 'auto', autoWatch: true },
|
||||
io.io,
|
||||
{
|
||||
sleep: async () => { triggerDetach?.(); },
|
||||
watchIntervalMs: 1,
|
||||
setupKeystroke: (onDetach) => {
|
||||
triggerDetach = onDetach;
|
||||
return () => {};
|
||||
},
|
||||
},
|
||||
),
|
||||
).resolves.toMatchObject({ status: 'detached' });
|
||||
|
||||
const output = io.stdout();
|
||||
expect(output).toContain('Building KTX context');
|
||||
expect(output).toContain('Context build continuing in the background.');
|
||||
expect(output).toContain('Resume: ktx setup --project-dir');
|
||||
});
|
||||
|
||||
it('prints JSON setup context command status with watch and resume commands', async () => {
|
||||
await mkdir(join(tempDir, '.ktx', 'setup'), { recursive: true });
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
|
|
@ -372,6 +580,48 @@ describe('setup context build state', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('watches setup context command status until the run reaches a terminal state', async () => {
|
||||
await mkdir(join(tempDir, '.ktx', 'setup'), { recursive: true });
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-watch',
|
||||
status: 'running',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:00:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: ['docs'],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-watch'),
|
||||
});
|
||||
const io = makeIo();
|
||||
const completeRun = async () => {
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-watch',
|
||||
status: 'completed',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:02:00.000Z',
|
||||
completedAt: '2026-05-09T10:02:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: ['docs'],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-watch'),
|
||||
});
|
||||
};
|
||||
|
||||
await expect(
|
||||
runKtxSetupContextCommand(
|
||||
{ command: 'watch', projectDir: tempDir, runId: 'setup-context-local-watch', inputMode: 'disabled' },
|
||||
io.io,
|
||||
{ sleep: completeRun, watchIntervalMs: 1 },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
expect(io.stdout()).toContain('KTX context built: running');
|
||||
expect(io.stdout()).toContain('KTX context built: yes');
|
||||
});
|
||||
|
||||
it('runs direct build commands without asking for setup confirmation first', async () => {
|
||||
await writeReadyProject(tempDir);
|
||||
const io = makeIo();
|
||||
|
|
|
|||
|
|
@ -10,7 +10,14 @@ import {
|
|||
} from '@ktx/context/project';
|
||||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import { buildPublicIngestPlan } from './public-ingest.js';
|
||||
import { runContextBuild } from './context-build-view.js';
|
||||
import {
|
||||
type ContextBuildSourceProgressUpdate,
|
||||
createRepainter,
|
||||
defaultSetupKeystroke,
|
||||
renderContextBuildView,
|
||||
runContextBuild,
|
||||
viewStateFromSourceProgress,
|
||||
} from './context-build-view.js';
|
||||
import { withMenuOptionsSpacing } from './prompt-navigation.js';
|
||||
import { withSetupInterruptConfirmation } from './setup-interrupt.js';
|
||||
|
||||
|
|
@ -45,6 +52,7 @@ export interface KtxSetupContextState {
|
|||
retryableFailedTargets: string[];
|
||||
commands: KtxSetupContextCommands;
|
||||
failureReason?: string;
|
||||
sourceProgress?: ContextBuildSourceProgressUpdate[];
|
||||
}
|
||||
|
||||
export interface KtxSetupContextStatusSummary {
|
||||
|
|
@ -80,6 +88,7 @@ export interface KtxSetupContextStepArgs {
|
|||
forcePrompt?: boolean;
|
||||
allowEmpty?: boolean;
|
||||
prompt?: boolean;
|
||||
autoWatch?: boolean;
|
||||
}
|
||||
|
||||
export type KtxSetupContextCommandArgs =
|
||||
|
|
@ -99,6 +108,9 @@ export interface KtxSetupContextDeps {
|
|||
now?: () => Date;
|
||||
runContextBuild?: typeof runContextBuild;
|
||||
verifyContextReady?: (projectDir: string) => Promise<KtxSetupContextReadiness>;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
watchIntervalMs?: number;
|
||||
setupKeystroke?: (onDetach: () => void, onCtrlC: () => void) => (() => void) | null;
|
||||
}
|
||||
|
||||
interface KtxSetupContextTargets {
|
||||
|
|
@ -109,6 +121,7 @@ interface KtxSetupContextTargets {
|
|||
const SETUP_CONTEXT_STATE_PATH = ['.ktx', 'setup', 'context-build.json'] as const;
|
||||
const LIVE_DATABASE_ADAPTER = 'live-database';
|
||||
const SCAN_REPORT_FILE = 'scan-report.json';
|
||||
const DEFAULT_WATCH_INTERVAL_MS = 2_000;
|
||||
|
||||
function createPromptAdapter(): KtxSetupContextPromptAdapter {
|
||||
return {
|
||||
|
|
@ -193,9 +206,34 @@ function normalizeState(projectDir: string, value: unknown): KtxSetupContextStat
|
|||
: [],
|
||||
commands: contextBuildCommands(projectDir, runId),
|
||||
...(typeof record.failureReason === 'string' ? { failureReason: record.failureReason } : {}),
|
||||
...(normalizeSourceProgress(record.sourceProgress) ? { sourceProgress: normalizeSourceProgress(record.sourceProgress) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
const VALID_SOURCE_OPERATIONS = new Set(['scan', 'source-ingest']);
|
||||
const VALID_SOURCE_STATUSES = new Set(['queued', 'running', 'done', 'failed']);
|
||||
|
||||
function normalizeSourceProgress(value: unknown): ContextBuildSourceProgressUpdate[] | undefined {
|
||||
if (!Array.isArray(value)) return undefined;
|
||||
const entries: ContextBuildSourceProgressUpdate[] = [];
|
||||
for (const item of value) {
|
||||
if (typeof item !== 'object' || item === null || Array.isArray(item)) continue;
|
||||
const rec = item as Record<string, unknown>;
|
||||
if (typeof rec.connectionId !== 'string') continue;
|
||||
if (!VALID_SOURCE_OPERATIONS.has(String(rec.operation))) continue;
|
||||
if (!VALID_SOURCE_STATUSES.has(String(rec.status))) continue;
|
||||
entries.push({
|
||||
connectionId: rec.connectionId,
|
||||
operation: rec.operation as 'scan' | 'source-ingest',
|
||||
status: rec.status as 'queued' | 'running' | 'done' | 'failed',
|
||||
...(typeof rec.startedAtMs === 'number' ? { startedAtMs: rec.startedAtMs } : {}),
|
||||
...(typeof rec.elapsedMs === 'number' ? { elapsedMs: rec.elapsedMs } : {}),
|
||||
...(typeof rec.summaryText === 'string' ? { summaryText: rec.summaryText } : {}),
|
||||
});
|
||||
}
|
||||
return entries.length > 0 ? entries : undefined;
|
||||
}
|
||||
|
||||
export async function readKtxSetupContextState(projectDir: string): Promise<KtxSetupContextState> {
|
||||
const filePath = statePath(projectDir);
|
||||
if (!(await pathExists(filePath))) {
|
||||
|
|
@ -514,6 +552,7 @@ async function runBuild(
|
|||
};
|
||||
await writeKtxSetupContextState(args.projectDir, runningState);
|
||||
|
||||
let lastSourceProgress: ContextBuildSourceProgressUpdate[] | undefined;
|
||||
const contextBuild = deps.runContextBuild ?? runContextBuild;
|
||||
const buildResult = await contextBuild(
|
||||
project,
|
||||
|
|
@ -532,14 +571,39 @@ async function runBuild(
|
|||
...runningState,
|
||||
status: 'detached',
|
||||
updatedAt: new Date().toISOString(),
|
||||
...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}),
|
||||
});
|
||||
writeFileSync(statePath(resolvedDir), `${JSON.stringify(detachedState, null, 2)}\n`);
|
||||
},
|
||||
onSourceProgress: (sources) => {
|
||||
lastSourceProgress = sources;
|
||||
try {
|
||||
const resolvedDir = resolve(args.projectDir);
|
||||
mkdirSync(join(resolvedDir, '.ktx', 'setup'), { recursive: true });
|
||||
const progressState = normalizeState(resolvedDir, {
|
||||
...runningState,
|
||||
sourceProgress: sources,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
writeFileSync(statePath(resolvedDir), `${JSON.stringify(progressState, null, 2)}\n`);
|
||||
} catch {
|
||||
// Progress reporting is supplementary — don't crash the build
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
const completedReportIds = buildResult.reportIds ?? [];
|
||||
const completedArtifactPaths = buildResult.artifactPaths ?? [];
|
||||
if (buildResult.detached) {
|
||||
const updatedAt = now().toISOString();
|
||||
await writeKtxSetupContextState(args.projectDir, { ...runningState, status: 'detached', updatedAt });
|
||||
await writeKtxSetupContextState(args.projectDir, {
|
||||
...runningState,
|
||||
status: 'detached',
|
||||
updatedAt,
|
||||
reportIds: completedReportIds,
|
||||
artifactPaths: completedArtifactPaths,
|
||||
...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}),
|
||||
});
|
||||
return { status: 'detached', projectDir: args.projectDir, runId };
|
||||
}
|
||||
if (buildResult.exitCode !== 0) {
|
||||
|
|
@ -548,8 +612,11 @@ async function runBuild(
|
|||
...runningState,
|
||||
status: 'failed',
|
||||
updatedAt,
|
||||
reportIds: completedReportIds,
|
||||
artifactPaths: completedArtifactPaths,
|
||||
retryableFailedTargets: [...targets.primarySourceConnectionIds, ...targets.contextSourceConnectionIds],
|
||||
failureReason: 'Context build failed.',
|
||||
...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}),
|
||||
});
|
||||
return { status: 'failed', projectDir: args.projectDir };
|
||||
}
|
||||
|
|
@ -561,8 +628,11 @@ async function runBuild(
|
|||
...runningState,
|
||||
status: 'failed',
|
||||
updatedAt,
|
||||
reportIds: completedReportIds,
|
||||
artifactPaths: completedArtifactPaths,
|
||||
retryableFailedTargets: readiness.failedTargets ?? [],
|
||||
failureReason: readiness.details.join(' '),
|
||||
...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}),
|
||||
});
|
||||
io.stderr.write('KTX context build did not pass agent-readiness verification.\n');
|
||||
for (const detail of readiness.details) {
|
||||
|
|
@ -578,7 +648,10 @@ async function runBuild(
|
|||
status: 'completed',
|
||||
updatedAt: completedAt,
|
||||
completedAt,
|
||||
reportIds: completedReportIds,
|
||||
artifactPaths: completedArtifactPaths,
|
||||
retryableFailedTargets: [],
|
||||
...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}),
|
||||
});
|
||||
writeSuccess(readiness, targets, io);
|
||||
return { status: 'ready', projectDir: args.projectDir, runId };
|
||||
|
|
@ -632,17 +705,46 @@ export async function runKtxSetupContextStep(
|
|||
(existingState.status === 'running' || existingState.status === 'detached') &&
|
||||
args.inputMode !== 'disabled'
|
||||
) {
|
||||
if (args.autoWatch) {
|
||||
const watched = await watchContextStatus(
|
||||
{
|
||||
command: 'watch',
|
||||
projectDir: args.projectDir,
|
||||
...(existingState.runId ? { runId: existingState.runId } : {}),
|
||||
inputMode: args.inputMode,
|
||||
},
|
||||
existingState,
|
||||
io,
|
||||
deps,
|
||||
);
|
||||
return setupResultFromWatchedState(args.projectDir, watched.state);
|
||||
}
|
||||
const prompts = deps.prompts ?? createPromptAdapter();
|
||||
const choice = await prompts.select({
|
||||
message:
|
||||
'A context build is running in the background.\n\n' +
|
||||
'You can wait for it to finish, check its status, or start a fresh build.',
|
||||
'You can watch it until it finishes, check its status once, or start a fresh build.',
|
||||
options: [
|
||||
{ value: 'watch', label: 'Watch progress' },
|
||||
{ value: 'status', label: 'Check status' },
|
||||
{ value: 'rebuild', label: 'Start a fresh context build' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
if (choice === 'watch') {
|
||||
const watched = await watchContextStatus(
|
||||
{
|
||||
command: 'watch',
|
||||
projectDir: args.projectDir,
|
||||
...(existingState.runId ? { runId: existingState.runId } : {}),
|
||||
inputMode: args.inputMode,
|
||||
},
|
||||
existingState,
|
||||
io,
|
||||
deps,
|
||||
);
|
||||
return setupResultFromWatchedState(args.projectDir, watched.state);
|
||||
}
|
||||
if (choice === 'status') {
|
||||
const commands = contextBuildCommands(args.projectDir, existingState.runId);
|
||||
io.stdout.write(`\nRun: ${commands.status}\n`);
|
||||
|
|
@ -698,6 +800,18 @@ function stateMatchesRunId(state: KtxSetupContextState, runId: string | undefine
|
|||
return !runId || state.runId === runId;
|
||||
}
|
||||
|
||||
function isActiveStatus(status: KtxSetupContextBuildStatus): boolean {
|
||||
return status === 'running' || status === 'detached';
|
||||
}
|
||||
|
||||
function watchExitCode(status: KtxSetupContextBuildStatus): number {
|
||||
return status === 'failed' || status === 'interrupted' || status === 'stale' ? 1 : 0;
|
||||
}
|
||||
|
||||
function defaultSleep(ms: number): Promise<void> {
|
||||
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
||||
}
|
||||
|
||||
function statusPayload(state: KtxSetupContextState): KtxSetupContextStatusSummary {
|
||||
return setupContextStatusFromState(state, { completedStep: state.status === 'completed' });
|
||||
}
|
||||
|
|
@ -714,6 +828,149 @@ function writeContextStatus(state: KtxSetupContextState, io: KtxCliIo): void {
|
|||
}
|
||||
}
|
||||
|
||||
async function watchContextStatus(
|
||||
args: Extract<KtxSetupContextCommandArgs, { command: 'watch' }>,
|
||||
initialState: KtxSetupContextState,
|
||||
io: KtxCliIo,
|
||||
deps: KtxSetupContextDeps,
|
||||
): Promise<{ exitCode: number; state: KtxSetupContextState }> {
|
||||
if (initialState.sourceProgress && initialState.sourceProgress.length > 0) {
|
||||
return watchContextStatusWithProgressView(args, initialState, io, deps);
|
||||
}
|
||||
return watchContextStatusText(args, initialState, io, deps);
|
||||
}
|
||||
|
||||
async function watchContextStatusText(
|
||||
args: Extract<KtxSetupContextCommandArgs, { command: 'watch' }>,
|
||||
initialState: KtxSetupContextState,
|
||||
io: KtxCliIo,
|
||||
deps: KtxSetupContextDeps,
|
||||
): Promise<{ exitCode: number; state: KtxSetupContextState }> {
|
||||
const sleep = deps.sleep ?? defaultSleep;
|
||||
const intervalMs = deps.watchIntervalMs ?? DEFAULT_WATCH_INTERVAL_MS;
|
||||
let state = initialState;
|
||||
let lastRenderedStatus = '';
|
||||
|
||||
io.stdout.write('KTX context build\n');
|
||||
while (true) {
|
||||
const renderedStatus = `${state.status}:${state.updatedAt ?? ''}:${state.completedAt ?? ''}:${state.failureReason ?? ''}`;
|
||||
if (renderedStatus !== lastRenderedStatus) {
|
||||
writeContextStatus(state, io);
|
||||
lastRenderedStatus = renderedStatus;
|
||||
}
|
||||
|
||||
if (!isActiveStatus(state.status)) {
|
||||
return { exitCode: watchExitCode(state.status), state };
|
||||
}
|
||||
|
||||
await sleep(intervalMs);
|
||||
state = await readKtxSetupContextState(args.projectDir);
|
||||
if (!stateMatchesRunId(state, args.runId)) {
|
||||
io.stderr.write(`KTX setup context run "${args.runId}" was not found.\n`);
|
||||
return { exitCode: 1, state };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function watchContextStatusWithProgressView(
|
||||
args: Extract<KtxSetupContextCommandArgs, { command: 'watch' }>,
|
||||
initialState: KtxSetupContextState,
|
||||
io: KtxCliIo,
|
||||
deps: KtxSetupContextDeps,
|
||||
): Promise<{ exitCode: number; state: KtxSetupContextState }> {
|
||||
const sleep = deps.sleep ?? defaultSleep;
|
||||
const intervalMs = deps.watchIntervalMs ?? DEFAULT_WATCH_INTERVAL_MS;
|
||||
const isTTY = io.stdout.isTTY === true;
|
||||
const repainter = isTTY ? createRepainter(io) : null;
|
||||
const projectDir = resolve(args.projectDir);
|
||||
const viewOpts = { styled: isTTY, showHint: true, projectDir };
|
||||
let state = initialState;
|
||||
let lastProgressKey = '';
|
||||
let detached = false;
|
||||
|
||||
let viewState = viewStateFromSourceProgress(state.sourceProgress ?? [], Date.now(),
|
||||
state.startedAt ? new Date(state.startedAt).getTime() : undefined);
|
||||
|
||||
const cleanupKeystroke = (isTTY || deps.setupKeystroke)
|
||||
? (deps.setupKeystroke ?? defaultSetupKeystroke)(
|
||||
() => { detached = true; },
|
||||
() => { detached = true; },
|
||||
)
|
||||
: null;
|
||||
|
||||
let spinnerInterval: ReturnType<typeof setInterval> | null = null;
|
||||
if (repainter) {
|
||||
repainter.paint(renderContextBuildView(viewState, viewOpts));
|
||||
spinnerInterval = setInterval(() => {
|
||||
viewState.frame++;
|
||||
const now = Date.now();
|
||||
viewState.totalElapsedMs = viewState.startedAt !== null ? now - viewState.startedAt : 0;
|
||||
for (const t of [...viewState.primarySources, ...viewState.contextSources]) {
|
||||
if (t.status === 'running' && t.startedAt !== null) {
|
||||
t.elapsedMs = now - t.startedAt;
|
||||
}
|
||||
}
|
||||
repainter.paint(renderContextBuildView(viewState, viewOpts));
|
||||
}, 140);
|
||||
}
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
if (!repainter) {
|
||||
const currentKey = JSON.stringify(state.sourceProgress?.map((s) => s.status));
|
||||
if (currentKey !== lastProgressKey || !isActiveStatus(state.status)) {
|
||||
io.stdout.write(renderContextBuildView(viewState, viewOpts));
|
||||
lastProgressKey = currentKey;
|
||||
}
|
||||
}
|
||||
|
||||
if (!isActiveStatus(state.status)) {
|
||||
return { exitCode: watchExitCode(state.status), state };
|
||||
}
|
||||
if (detached) break;
|
||||
|
||||
await sleep(intervalMs);
|
||||
if (detached) break;
|
||||
|
||||
try {
|
||||
state = await readKtxSetupContextState(args.projectDir);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!stateMatchesRunId(state, args.runId)) {
|
||||
io.stderr.write(`KTX setup context run "${args.runId}" was not found.\n`);
|
||||
return { exitCode: 1, state };
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const startedAtMs = state.startedAt ? new Date(state.startedAt).getTime() : undefined;
|
||||
viewState = viewStateFromSourceProgress(state.sourceProgress ?? [], now, startedAtMs);
|
||||
}
|
||||
} finally {
|
||||
if (spinnerInterval) clearInterval(spinnerInterval);
|
||||
cleanupKeystroke?.();
|
||||
}
|
||||
|
||||
io.stdout.write('\n\nContext build continuing in the background.\n');
|
||||
io.stdout.write(`Resume: ktx setup --project-dir ${projectDir}\n`);
|
||||
io.stdout.write(`Status: ktx setup context status --project-dir ${projectDir}\n`);
|
||||
return { exitCode: 0, state };
|
||||
}
|
||||
|
||||
function setupResultFromWatchedState(projectDir: string, state: KtxSetupContextState): KtxSetupContextResult {
|
||||
if (state.status === 'completed') {
|
||||
return { status: 'ready', projectDir, runId: state.runId ?? 'setup-context-completed' };
|
||||
}
|
||||
if (state.status === 'paused') {
|
||||
return { status: 'paused', projectDir, runId: state.runId ?? '' };
|
||||
}
|
||||
if (state.status === 'running' || state.status === 'detached') {
|
||||
return { status: 'detached', projectDir, runId: state.runId ?? '' };
|
||||
}
|
||||
return { status: 'failed', projectDir };
|
||||
}
|
||||
|
||||
export async function runKtxSetupContextCommand(
|
||||
args: KtxSetupContextCommandArgs,
|
||||
io: KtxCliIo,
|
||||
|
|
@ -744,9 +1001,7 @@ export async function runKtxSetupContextCommand(
|
|||
}
|
||||
|
||||
if (args.command === 'watch') {
|
||||
io.stdout.write('KTX context build\n');
|
||||
writeContextStatus(state, io);
|
||||
return 0;
|
||||
return (await watchContextStatus(args, state, io, deps)).exitCode;
|
||||
}
|
||||
|
||||
const updatedAt = new Date().toISOString();
|
||||
|
|
|
|||
|
|
@ -962,10 +962,95 @@ describe('setup databases step', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('prompts for discovered Postgres schemas before the first scan', async () => {
|
||||
const io = makeIo();
|
||||
const prompts = makePromptAdapter({
|
||||
selectValues: ['url'],
|
||||
textValues: ['', 'env:DATABASE_URL'],
|
||||
multiselectValues: [['orbit_analytics', 'orbit_raw']],
|
||||
});
|
||||
const testConnection = vi.fn(async () => 0);
|
||||
const scanConnection = vi.fn(async asyncScanProjectDir => {
|
||||
const config = parseKtxProjectConfig(await readFile(join(asyncScanProjectDir, 'ktx.yaml'), 'utf-8'));
|
||||
expect(config.connections['postgres-warehouse']).toMatchObject({
|
||||
schemas: ['orbit_analytics', 'orbit_raw'],
|
||||
});
|
||||
return 0;
|
||||
});
|
||||
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
|
||||
|
||||
const result = await runKtxSetupDatabasesStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'auto',
|
||||
databaseDrivers: ['postgres'],
|
||||
databaseSchemas: [],
|
||||
skipDatabases: false,
|
||||
},
|
||||
io.io,
|
||||
{ prompts, testConnection, scanConnection, listSchemas },
|
||||
);
|
||||
|
||||
expect(result.status).toBe('ready');
|
||||
expect(listSchemas).toHaveBeenCalledWith(tempDir, 'postgres-warehouse');
|
||||
expect(prompts.multiselect).toHaveBeenCalledWith({
|
||||
message: expect.stringContaining('PostgreSQL schemas to scan'),
|
||||
options: [
|
||||
{ value: 'orbit_analytics', label: 'orbit_analytics' },
|
||||
{ value: 'orbit_raw', label: 'orbit_raw' },
|
||||
{ value: 'public', label: 'public' },
|
||||
],
|
||||
initialValues: ['orbit_analytics', 'orbit_raw'],
|
||||
required: true,
|
||||
});
|
||||
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
|
||||
expect(config.connections['postgres-warehouse']).toMatchObject({
|
||||
schemas: ['orbit_analytics', 'orbit_raw'],
|
||||
});
|
||||
expect(io.stdout()).toContain('Schemas: orbit_analytics, orbit_raw');
|
||||
});
|
||||
|
||||
it('auto-selects all discovered Postgres schemas in non-interactive setup', async () => {
|
||||
const io = makeIo();
|
||||
const prompts = makePromptAdapter({});
|
||||
const testConnection = vi.fn(async () => 0);
|
||||
const scanConnection = vi.fn(async asyncScanProjectDir => {
|
||||
const config = parseKtxProjectConfig(await readFile(join(asyncScanProjectDir, 'ktx.yaml'), 'utf-8'));
|
||||
expect(config.connections.warehouse).toMatchObject({
|
||||
schemas: ['orbit_analytics', 'orbit_raw', 'public'],
|
||||
});
|
||||
return 0;
|
||||
});
|
||||
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
|
||||
|
||||
const result = await runKtxSetupDatabasesStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'disabled',
|
||||
databaseDrivers: ['postgres'],
|
||||
databaseConnectionId: 'warehouse',
|
||||
databaseUrl: 'env:DATABASE_URL',
|
||||
databaseSchemas: [],
|
||||
skipDatabases: false,
|
||||
},
|
||||
io.io,
|
||||
{ prompts, testConnection, scanConnection, listSchemas },
|
||||
);
|
||||
|
||||
expect(result.status).toBe('ready');
|
||||
expect(prompts.multiselect).not.toHaveBeenCalled();
|
||||
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
|
||||
expect(config.connections.warehouse).toMatchObject({
|
||||
schemas: ['orbit_analytics', 'orbit_raw', 'public'],
|
||||
});
|
||||
expect(io.stdout()).toContain('Schemas: orbit_analytics, orbit_raw, public');
|
||||
});
|
||||
|
||||
it('adds one non-interactive Postgres URL connection, tests it, scans it, and marks databases complete', async () => {
|
||||
const io = makeIo();
|
||||
const testConnection = vi.fn(async () => 0);
|
||||
const scanConnection = vi.fn(async () => 0);
|
||||
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
|
||||
|
||||
const result = await runKtxSetupDatabasesStep(
|
||||
{
|
||||
|
|
@ -978,10 +1063,11 @@ describe('setup databases step', () => {
|
|||
skipDatabases: false,
|
||||
},
|
||||
io.io,
|
||||
{ testConnection, scanConnection },
|
||||
{ testConnection, scanConnection, listSchemas },
|
||||
);
|
||||
|
||||
expect(result.status).toBe('ready');
|
||||
expect(listSchemas).not.toHaveBeenCalled();
|
||||
expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything());
|
||||
expect(scanConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything());
|
||||
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ export interface KtxSetupDatabasesPromptAdapter {
|
|||
message: string;
|
||||
options: Array<{ value: string; label: string }>;
|
||||
required?: boolean;
|
||||
initialValues?: string[];
|
||||
}): Promise<string[]>;
|
||||
select(options: { message: string; options: Array<{ value: string; label: string }> }): Promise<string>;
|
||||
text(options: { message: string; placeholder?: string; initialValue?: string }): Promise<string | undefined>;
|
||||
|
|
@ -76,6 +77,7 @@ export interface KtxSetupDatabasesDeps {
|
|||
prompts?: KtxSetupDatabasesPromptAdapter;
|
||||
testConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise<number>;
|
||||
scanConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise<number>;
|
||||
listSchemas?: (projectDir: string, connectionId: string) => Promise<string[]>;
|
||||
historicSqlProbe?: KtxSetupHistoricSqlProbe;
|
||||
}
|
||||
|
||||
|
|
@ -255,6 +257,21 @@ async function defaultHistoricSqlProbe(input: KtxSetupHistoricSqlProbeInput): Pr
|
|||
}
|
||||
}
|
||||
|
||||
async function defaultListSchemas(projectDir: string, connectionId: string): Promise<string[]> {
|
||||
const project = await loadKtxProject({ projectDir });
|
||||
const connection = project.config.connections[connectionId];
|
||||
const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres');
|
||||
if (!isKtxPostgresConnectionConfig(connection)) {
|
||||
return [];
|
||||
}
|
||||
const connector = new KtxPostgresScanConnector({ connectionId, connection });
|
||||
try {
|
||||
return await connector.listSchemas();
|
||||
} finally {
|
||||
await connector.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
function existingConnectionIdsByDriver(
|
||||
connections: Record<string, KtxProjectConnectionConfig>,
|
||||
driver: KtxSetupDatabaseDriver,
|
||||
|
|
@ -814,6 +831,113 @@ async function writeConnectionConfig(input: {
|
|||
}
|
||||
}
|
||||
|
||||
function configuredSchemas(connection: KtxProjectConnectionConfig | undefined): string[] {
|
||||
if (!connection) return [];
|
||||
if (Array.isArray(connection.schemas)) {
|
||||
return connection.schemas
|
||||
.filter((schema): schema is string => typeof schema === 'string' && schema.trim().length > 0)
|
||||
.map((schema) => schema.trim());
|
||||
}
|
||||
return typeof connection.schema === 'string' && connection.schema.trim().length > 0 ? [connection.schema.trim()] : [];
|
||||
}
|
||||
|
||||
function defaultSchemaSelection(schemas: string[]): string[] {
|
||||
const nonPublic = schemas.filter((schema) => schema !== 'public');
|
||||
return nonPublic.length > 0 ? nonPublic : schemas;
|
||||
}
|
||||
|
||||
async function writeConnectionSchemas(input: {
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
schemas: string[];
|
||||
}): Promise<void> {
|
||||
const project = await loadKtxProject({ projectDir: input.projectDir });
|
||||
const connection = project.config.connections[input.connectionId];
|
||||
if (!connection) return;
|
||||
const { schema: _schema, ...connectionWithoutLegacySchema } = connection;
|
||||
await writeConnectionConfig({
|
||||
projectDir: input.projectDir,
|
||||
connectionId: input.connectionId,
|
||||
connection: {
|
||||
...connectionWithoutLegacySchema,
|
||||
schemas: unique(input.schemas),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function maybeConfigurePostgresSchemas(input: {
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
args: KtxSetupDatabasesArgs;
|
||||
prompts: KtxSetupDatabasesPromptAdapter;
|
||||
deps: KtxSetupDatabasesDeps;
|
||||
io: KtxCliIo;
|
||||
}): Promise<boolean> {
|
||||
const project = await loadKtxProject({ projectDir: input.projectDir });
|
||||
const connection = project.config.connections[input.connectionId];
|
||||
if (normalizeDriver(connection?.driver) !== 'postgres') {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (configuredSchemas(connection).length > 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (input.args.databaseSchemas.length > 0) {
|
||||
await writeConnectionSchemas({
|
||||
projectDir: input.projectDir,
|
||||
connectionId: input.connectionId,
|
||||
schemas: input.args.databaseSchemas,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
let discoveredSchemas: string[];
|
||||
try {
|
||||
discoveredSchemas = unique(
|
||||
await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId),
|
||||
);
|
||||
} catch (error) {
|
||||
input.io.stderr.write(
|
||||
`Could not discover PostgreSQL schemas for ${input.connectionId}; continuing with existing schema scope. ` +
|
||||
`Pass --database-schema to set it explicitly. ${error instanceof Error ? error.message : String(error)}\n`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
if (discoveredSchemas.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let selectedSchemas: string[];
|
||||
if (input.args.inputMode === 'disabled' || discoveredSchemas.length === 1) {
|
||||
selectedSchemas = discoveredSchemas;
|
||||
} else {
|
||||
const initialValues = defaultSchemaSelection(discoveredSchemas);
|
||||
const choices = await input.prompts.multiselect({
|
||||
message: withMultiselectNavigation(
|
||||
'PostgreSQL schemas to scan\nKTX found multiple non-system schemas. Select every schema agents should use.',
|
||||
),
|
||||
options: discoveredSchemas.map((schema) => ({ value: schema, label: schema })),
|
||||
initialValues,
|
||||
required: true,
|
||||
});
|
||||
if (choices.includes('back')) {
|
||||
return false;
|
||||
}
|
||||
selectedSchemas = choices.length > 0 ? choices : initialValues;
|
||||
}
|
||||
|
||||
await writeConnectionSchemas({
|
||||
projectDir: input.projectDir,
|
||||
connectionId: input.connectionId,
|
||||
schemas: selectedSchemas,
|
||||
});
|
||||
writeSetupSection(input.io, `Selecting schemas for ${input.connectionId}`, [
|
||||
`Schemas: ${selectedSchemas.join(', ')}`,
|
||||
]);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function ensureHistoricSqlAdapterEnabled(projectDir: string): Promise<void> {
|
||||
const project = await loadKtxProject({ projectDir });
|
||||
if (project.config.ingest.adapters.includes('historic-sql')) {
|
||||
|
|
@ -902,6 +1026,8 @@ async function validateAndScanConnection(input: {
|
|||
connectionId: string;
|
||||
io: KtxCliIo;
|
||||
deps: KtxSetupDatabasesDeps;
|
||||
args: KtxSetupDatabasesArgs;
|
||||
prompts: KtxSetupDatabasesPromptAdapter;
|
||||
}): Promise<boolean> {
|
||||
const testConnection = input.deps.testConnection ?? defaultTestConnection;
|
||||
const scanConnection = input.deps.scanConnection ?? defaultScanConnection;
|
||||
|
|
@ -923,6 +1049,10 @@ async function validateAndScanConnection(input: {
|
|||
testLines.push(`Driver: ${driverDisplay}${Number.isFinite(tableCount) ? ` · Tables: ${tableCount}` : ''}`);
|
||||
writeSetupSection(input.io, `Testing ${input.connectionId}`, testLines);
|
||||
|
||||
if (!(await maybeConfigurePostgresSchemas(input))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await maybeRunHistoricSqlSetupProbe({
|
||||
projectDir: input.projectDir,
|
||||
connectionId: input.connectionId,
|
||||
|
|
@ -1069,7 +1199,7 @@ export async function runKtxSetupDatabasesStep(
|
|||
prompts,
|
||||
});
|
||||
if (historicSqlResult === 'back') return { status: 'back', projectDir: args.projectDir };
|
||||
if (!(await validateAndScanConnection({ projectDir: args.projectDir, connectionId, io, deps }))) {
|
||||
if (!(await validateAndScanConnection({ projectDir: args.projectDir, connectionId, io, deps, args, prompts }))) {
|
||||
return { status: 'failed', projectDir: args.projectDir };
|
||||
}
|
||||
selectedConnectionIds.push(connectionId);
|
||||
|
|
@ -1209,6 +1339,8 @@ export async function runKtxSetupDatabasesStep(
|
|||
connectionId: connectionChoice.connectionId,
|
||||
io,
|
||||
deps,
|
||||
args,
|
||||
prompts,
|
||||
}))
|
||||
) {
|
||||
if (args.inputMode === 'disabled') return { status: 'failed', projectDir: args.projectDir };
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { isKtxSetupReady, runKtxSetupReadyChangeMenu } from './setup-ready-menu.js';
|
||||
import { isKtxPreAgentSetupReady, isKtxSetupReady, runKtxSetupReadyChangeMenu } from './setup-ready-menu.js';
|
||||
import type { KtxSetupStatus } from './setup.js';
|
||||
|
||||
const readyStatus: KtxSetupStatus = {
|
||||
|
|
@ -20,6 +20,13 @@ describe('setup ready menu', () => {
|
|||
expect(isKtxSetupReady({ ...readyStatus, agents: [] })).toBe(false);
|
||||
});
|
||||
|
||||
it('recognizes pre-agent readiness without requiring agents', () => {
|
||||
expect(isKtxPreAgentSetupReady(readyStatus)).toBe(true);
|
||||
expect(isKtxPreAgentSetupReady({ ...readyStatus, agents: [] })).toBe(true);
|
||||
expect(isKtxPreAgentSetupReady({ ...readyStatus, embeddings: { ready: false } })).toBe(false);
|
||||
expect(isKtxPreAgentSetupReady({ ...readyStatus, context: { ready: false, status: 'not_started' } })).toBe(false);
|
||||
});
|
||||
|
||||
it('maps ready-project menu choices to setup sections', async () => {
|
||||
const prompts = { select: vi.fn(async () => 'agents'), cancel: vi.fn() };
|
||||
|
||||
|
|
|
|||
|
|
@ -14,18 +14,21 @@ export interface KtxSetupReadyMenuDeps {
|
|||
prompts?: KtxSetupReadyMenuPromptAdapter;
|
||||
}
|
||||
|
||||
export function isKtxSetupReady(status: KtxSetupStatus): boolean {
|
||||
export function isKtxPreAgentSetupReady(status: KtxSetupStatus): boolean {
|
||||
return (
|
||||
status.project.ready &&
|
||||
status.llm.ready &&
|
||||
status.embeddings.ready &&
|
||||
status.databases.every((database) => database.ready) &&
|
||||
status.sources.every((source) => source.ready) &&
|
||||
status.context.ready &&
|
||||
status.agents.some((agent) => agent.ready)
|
||||
status.context.ready
|
||||
);
|
||||
}
|
||||
|
||||
export function isKtxSetupReady(status: KtxSetupStatus): boolean {
|
||||
return isKtxPreAgentSetupReady(status) && status.agents.some((agent) => agent.ready);
|
||||
}
|
||||
|
||||
function createPromptAdapter(): KtxSetupReadyMenuPromptAdapter {
|
||||
return {
|
||||
async select(options) {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
|
|
@ -8,6 +8,7 @@ import {
|
|||
serializeKtxProjectConfig,
|
||||
} from '@ktx/context/project';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import {
|
||||
runKtxSetupSourcesStep,
|
||||
type KtxSetupSourcesDeps,
|
||||
|
|
@ -41,14 +42,17 @@ function prompts(values: {
|
|||
multiselect?: string[][];
|
||||
select?: string[];
|
||||
text?: Array<string | undefined>;
|
||||
password?: Array<string | undefined>;
|
||||
}): KtxSetupSourcesPromptAdapter {
|
||||
const multiselectValues = [...(values.multiselect ?? [])];
|
||||
const selectValues = [...(values.select ?? [])];
|
||||
const textValues = [...(values.text ?? [])];
|
||||
const passwordValues = [...(values.password ?? [])];
|
||||
return {
|
||||
multiselect: vi.fn(async () => multiselectValues.shift() ?? []),
|
||||
select: vi.fn(async () => selectValues.shift() ?? 'skip'),
|
||||
text: vi.fn(async () => (textValues.length > 0 ? textValues.shift() : '')),
|
||||
password: vi.fn(async () => (passwordValues.length > 0 ? passwordValues.shift() : undefined)),
|
||||
cancel: vi.fn(),
|
||||
log: vi.fn(),
|
||||
};
|
||||
|
|
@ -201,12 +205,199 @@ describe('setup sources step', () => {
|
|||
mappings: {
|
||||
databaseMappings: { '1': 'warehouse' },
|
||||
syncEnabled: { '1': true },
|
||||
syncMode: 'ONLY',
|
||||
syncMode: 'ALL',
|
||||
},
|
||||
});
|
||||
expect(runMapping).toHaveBeenCalledWith(projectDir, 'prod_metabase', io.io);
|
||||
});
|
||||
|
||||
it('defaults interactive Metabase and Looker source setup to the only warehouse connection', async () => {
|
||||
await addPrimarySource();
|
||||
const cases: Array<{
|
||||
source: 'metabase' | 'looker';
|
||||
text: string[];
|
||||
deps: KtxSetupSourcesDeps;
|
||||
expectedConnection: Record<string, unknown>;
|
||||
}> = [
|
||||
{
|
||||
source: 'metabase',
|
||||
text: ['metabase-main', 'https://metabase.example.com'],
|
||||
deps: {
|
||||
discoverMetabaseDatabases: vi.fn(async () => [
|
||||
{ id: 1, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' },
|
||||
]),
|
||||
validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })),
|
||||
runMapping: vi.fn(async () => 0),
|
||||
},
|
||||
expectedConnection: {
|
||||
driver: 'metabase',
|
||||
mappings: { databaseMappings: { '1': 'warehouse' } },
|
||||
},
|
||||
},
|
||||
{
|
||||
source: 'looker',
|
||||
text: ['looker-main', 'https://looker.example.com', 'client-id', ''],
|
||||
deps: {
|
||||
validateLooker: vi.fn(async () => ({ ok: true as const, detail: 'mapping refreshed' })),
|
||||
runMapping: vi.fn(async () => 0),
|
||||
},
|
||||
expectedConnection: {
|
||||
driver: 'looker',
|
||||
mappings: { connectionMappings: { warehouse: 'warehouse' } },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
const testPrompts = prompts({
|
||||
multiselect: [[testCase.source]],
|
||||
select: ['env', 'done'],
|
||||
text: testCase.text,
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
makeIo().io,
|
||||
{
|
||||
prompts: testPrompts,
|
||||
...testCase.deps,
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: [`${testCase.source}-main`] });
|
||||
|
||||
expect(
|
||||
vi.mocked(testPrompts.text).mock.calls.some(([options]) => options.message.includes('Mapped warehouse')),
|
||||
).toBe(false);
|
||||
if (testCase.source === 'metabase') {
|
||||
expect(
|
||||
vi.mocked(testPrompts.text).mock.calls.some(([options]) => options.message.includes('Metabase database id')),
|
||||
).toBe(false);
|
||||
}
|
||||
expect((await readConfig()).connections[`${testCase.source}-main`]).toMatchObject(testCase.expectedConnection);
|
||||
}
|
||||
});
|
||||
|
||||
it('prompts for the mapped warehouse when interactive Metabase and Looker source setup has multiple choices', async () => {
|
||||
await addPrimarySource();
|
||||
await addConnection('analytics_warehouse', {
|
||||
driver: 'snowflake',
|
||||
account: 'acme',
|
||||
database: 'analytics',
|
||||
readonly: true,
|
||||
});
|
||||
|
||||
const cases: Array<{
|
||||
source: 'metabase' | 'looker';
|
||||
text: string[];
|
||||
deps: KtxSetupSourcesDeps;
|
||||
expectedConnection: Record<string, unknown>;
|
||||
}> = [
|
||||
{
|
||||
source: 'metabase',
|
||||
text: ['metabase-main', 'https://metabase.example.com'],
|
||||
deps: {
|
||||
discoverMetabaseDatabases: vi.fn(async () => [
|
||||
{ id: 1, name: 'Finance', engine: 'postgres', host: 'db.example.com', dbName: 'finance' },
|
||||
{ id: 2, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' },
|
||||
]),
|
||||
validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })),
|
||||
runMapping: vi.fn(async () => 0),
|
||||
},
|
||||
expectedConnection: {
|
||||
driver: 'metabase',
|
||||
mappings: { databaseMappings: { '2': 'analytics_warehouse' } },
|
||||
},
|
||||
},
|
||||
{
|
||||
source: 'looker',
|
||||
text: ['looker-main', 'https://looker.example.com', 'client-id', 'analytics'],
|
||||
deps: {
|
||||
validateLooker: vi.fn(async () => ({ ok: true as const, detail: 'mapping refreshed' })),
|
||||
runMapping: vi.fn(async () => 0),
|
||||
},
|
||||
expectedConnection: {
|
||||
driver: 'looker',
|
||||
mappings: { connectionMappings: { analytics: 'analytics_warehouse' } },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
const testPrompts = prompts({
|
||||
multiselect: [[testCase.source]],
|
||||
select: testCase.source === 'metabase' ? ['env', 'analytics_warehouse', '2', 'done'] : ['env', 'analytics_warehouse', 'done'],
|
||||
text: testCase.text,
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
makeIo().io,
|
||||
{
|
||||
prompts: testPrompts,
|
||||
...testCase.deps,
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: [`${testCase.source}-main`] });
|
||||
|
||||
expect(testPrompts.select).toHaveBeenCalledWith({
|
||||
message: 'Mapped warehouse connection',
|
||||
options: [
|
||||
{ value: 'analytics_warehouse', label: 'analytics_warehouse (SNOWFLAKE)' },
|
||||
{ value: 'warehouse', label: 'warehouse (POSTGRESQL)' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
if (testCase.source === 'metabase') {
|
||||
expect(testPrompts.select).toHaveBeenCalledWith({
|
||||
message: 'Metabase database',
|
||||
options: [
|
||||
{ value: '1', label: '1: Finance (postgres)' },
|
||||
{ value: '2', label: '2: Analytics (postgres)' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
expect(
|
||||
vi.mocked(testPrompts.text).mock.calls.some(([options]) => options.message.includes('Metabase database id')),
|
||||
).toBe(false);
|
||||
}
|
||||
expect((await readConfig()).connections[`${testCase.source}-main`]).toMatchObject(testCase.expectedConnection);
|
||||
}
|
||||
});
|
||||
|
||||
it('lets visible Metabase mapping surface refresh and validation failures', async () => {
|
||||
await addPrimarySource();
|
||||
const runMapping = vi.fn(async (_projectDir: string, _connectionId: string, io: KtxCliIo) => {
|
||||
io.stderr.write('1: Metabase database does not match KTX connection database\n');
|
||||
return 1;
|
||||
});
|
||||
const io = makeIo();
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['metabase']],
|
||||
select: ['env'],
|
||||
text: ['metabase-main', 'https://metabase.example.com'],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
io.io,
|
||||
{
|
||||
prompts: testPrompts,
|
||||
discoverMetabaseDatabases: vi.fn(async () => [
|
||||
{ id: 1, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' },
|
||||
]),
|
||||
runMapping,
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ status: 'failed', projectDir });
|
||||
|
||||
expect(runMapping).toHaveBeenCalledWith(projectDir, 'metabase-main', io.io);
|
||||
expect(io.stderr()).toContain('1: Metabase database does not match KTX connection database');
|
||||
expect(io.stderr()).not.toContain('Metabase mapping validation failed');
|
||||
});
|
||||
|
||||
it('does not mark sources complete when validation fails', async () => {
|
||||
await addPrimarySource();
|
||||
const io = makeIo();
|
||||
|
|
@ -253,7 +444,6 @@ describe('setup sources step', () => {
|
|||
);
|
||||
const options = vi.mocked(testPrompts.multiselect).mock.calls[0]?.[0].options ?? [];
|
||||
expect(options).toContainEqual({ value: 'notion', label: 'Notion' });
|
||||
expect(options).not.toContainEqual({ value: 'posthog', label: 'PostHog' });
|
||||
});
|
||||
|
||||
it('uses a source-specific editable connection name for new interactive connections', async () => {
|
||||
|
|
@ -333,8 +523,8 @@ describe('setup sources step', () => {
|
|||
const io = makeIo();
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['dbt']],
|
||||
select: ['git'],
|
||||
text: ['dbt-main', 'https://github.com/acme-org/private-repo', 'main', '', 'env:GITHUB_TOKEN'],
|
||||
select: ['git', 'env'],
|
||||
text: ['dbt-main', 'https://github.com/acme-org/private-repo', 'main', ''],
|
||||
});
|
||||
|
||||
await expect(
|
||||
|
|
@ -350,19 +540,16 @@ describe('setup sources step', () => {
|
|||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
|
||||
|
||||
expect(testGitRepo).toHaveBeenCalledWith({ repoUrl: 'https://github.com/acme-org/private-repo' });
|
||||
expect(testPrompts.text).toHaveBeenNthCalledWith(5, {
|
||||
message: textInputPrompt(
|
||||
[
|
||||
'This repo requires authentication.',
|
||||
'Generate a token at: https://github.com/settings/tokens/new',
|
||||
'Store it in an env var, then enter env:VARIABLE_NAME here (e.g. env:GITHUB_TOKEN).',
|
||||
'Or use file:/absolute/path if the token is stored in a file.',
|
||||
'Press Enter to skip and try without authentication anyway.',
|
||||
].join('\n'),
|
||||
),
|
||||
placeholder: 'env:GITHUB_TOKEN',
|
||||
expect(testPrompts.select).toHaveBeenCalledWith({
|
||||
message: 'This repo requires authentication.',
|
||||
options: [
|
||||
{ value: 'env', label: 'Use GITHUB_TOKEN from the environment' },
|
||||
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
|
||||
{ value: 'skip', label: 'Skip — try without authentication' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
expect(testPrompts.text).toHaveBeenCalledTimes(5);
|
||||
expect(testPrompts.text).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it('enables the dbt adapter when adding a dbt source connection', async () => {
|
||||
|
|
@ -520,7 +707,7 @@ describe('setup sources step', () => {
|
|||
mappings: {
|
||||
databaseMappings: { '1': 'warehouse' },
|
||||
syncEnabled: { '1': true },
|
||||
syncMode: 'ONLY',
|
||||
syncMode: 'ALL',
|
||||
},
|
||||
},
|
||||
deps: {
|
||||
|
|
@ -692,13 +879,11 @@ describe('setup sources step', () => {
|
|||
},
|
||||
{
|
||||
source: 'metabase',
|
||||
select: ['back', 'env'],
|
||||
text: [
|
||||
'metabase-main',
|
||||
'https://old-metabase.example.com',
|
||||
undefined,
|
||||
'https://metabase.example.com',
|
||||
'env:METABASE_API_KEY',
|
||||
'warehouse',
|
||||
'1',
|
||||
],
|
||||
deps: {
|
||||
|
|
@ -709,14 +894,13 @@ describe('setup sources step', () => {
|
|||
},
|
||||
{
|
||||
source: 'looker',
|
||||
select: ['env'],
|
||||
text: [
|
||||
'looker-main',
|
||||
'https://old-looker.example.com',
|
||||
undefined,
|
||||
'https://looker.example.com',
|
||||
'client-id',
|
||||
'env:LOOKER_CLIENT_SECRET',
|
||||
'warehouse',
|
||||
'',
|
||||
],
|
||||
deps: {
|
||||
|
|
@ -727,10 +911,10 @@ describe('setup sources step', () => {
|
|||
},
|
||||
{
|
||||
source: 'notion',
|
||||
select: ['back', 'all_accessible'],
|
||||
text: ['notion-main', 'env:NOTION_TOKEN', 'env:NOTION_TOKEN'],
|
||||
select: ['env', 'back', 'env', 'all_accessible'],
|
||||
text: ['notion-main'],
|
||||
deps: { validateNotion: vi.fn(async () => ({ ok: true as const, detail: 'roots=0' })) },
|
||||
repeatedTextMessage: textInputPrompt('Notion token ref'),
|
||||
repeatedSelectMessage: 'How should KTX find your Notion integration token?',
|
||||
},
|
||||
];
|
||||
|
||||
|
|
@ -787,4 +971,102 @@ describe('setup sources step', () => {
|
|||
expect(io.stdout()).toContain('Connect a primary source before adding context sources.');
|
||||
expect((await readConfig()).setup?.completed_steps ?? []).not.toContain('sources');
|
||||
});
|
||||
|
||||
it('auto-detects dbt_project.yml at the root of a local path', async () => {
|
||||
await addPrimarySource();
|
||||
const dbtDir = join(tempDir, 'dbt-repo');
|
||||
await mkdir(dbtDir, { recursive: true });
|
||||
await writeFile(join(dbtDir, 'dbt_project.yml'), 'name: analytics\n');
|
||||
|
||||
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
|
||||
const io = makeIo();
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['dbt']],
|
||||
select: ['path'],
|
||||
text: ['dbt-main', dbtDir],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
io.io,
|
||||
{ prompts: testPrompts, validateDbt },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
|
||||
|
||||
expect(testPrompts.text).toHaveBeenCalledTimes(2);
|
||||
const config = await readConfig();
|
||||
expect(config.connections['dbt-main']).toMatchObject({ driver: 'dbt', source_dir: dbtDir });
|
||||
expect(config.connections['dbt-main']).not.toHaveProperty('path');
|
||||
});
|
||||
|
||||
it('auto-detects dbt_project.yml in a subdirectory of a local path', async () => {
|
||||
await addPrimarySource();
|
||||
const dbtDir = join(tempDir, 'monorepo');
|
||||
await mkdir(join(dbtDir, 'analytics', 'dbt'), { recursive: true });
|
||||
await writeFile(join(dbtDir, 'analytics', 'dbt', 'dbt_project.yml'), 'name: analytics\n');
|
||||
|
||||
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
|
||||
const io = makeIo();
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['dbt']],
|
||||
select: ['path'],
|
||||
text: ['dbt-main', dbtDir],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
io.io,
|
||||
{ prompts: testPrompts, validateDbt },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
|
||||
|
||||
expect(testPrompts.text).toHaveBeenCalledTimes(2);
|
||||
expect(testPrompts.log).toHaveBeenCalledWith('Found dbt_project.yml in analytics/dbt/');
|
||||
const config = await readConfig();
|
||||
expect(config.connections['dbt-main']).toMatchObject({
|
||||
driver: 'dbt',
|
||||
source_dir: dbtDir,
|
||||
path: 'analytics/dbt',
|
||||
});
|
||||
});
|
||||
|
||||
it('shows a picker when multiple dbt projects are found in a local path', async () => {
|
||||
await addPrimarySource();
|
||||
const dbtDir = join(tempDir, 'multi-dbt');
|
||||
await mkdir(join(dbtDir, 'analytics'), { recursive: true });
|
||||
await mkdir(join(dbtDir, 'staging'), { recursive: true });
|
||||
await writeFile(join(dbtDir, 'analytics', 'dbt_project.yml'), 'name: analytics\n');
|
||||
await writeFile(join(dbtDir, 'staging', 'dbt_project.yml'), 'name: staging\n');
|
||||
|
||||
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
|
||||
const io = makeIo();
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['dbt']],
|
||||
select: ['path', 'staging'],
|
||||
text: ['dbt-main', dbtDir],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
io.io,
|
||||
{ prompts: testPrompts, validateDbt },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
|
||||
|
||||
expect(testPrompts.select).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: 'Multiple dbt projects found — which one should KTX use?',
|
||||
}),
|
||||
);
|
||||
expect(testPrompts.text).toHaveBeenCalledTimes(2);
|
||||
const config = await readConfig();
|
||||
expect(config.connections['dbt-main']).toMatchObject({
|
||||
driver: 'dbt',
|
||||
source_dir: dbtDir,
|
||||
path: 'staging',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
import { mkdtemp, readdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { join, relative, resolve } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { cancel, isCancel, log, multiselect, select, text } from '@clack/prompts';
|
||||
import { resolveNotionAuthToken } from '@ktx/context/connections';
|
||||
import { cancel, isCancel, log, multiselect, password, select, text } from '@clack/prompts';
|
||||
import { localConnectionTypeForConfig, resolveNotionAuthToken } from '@ktx/context/connections';
|
||||
import { resolveKtxConfigReference } from '@ktx/context/core';
|
||||
import {
|
||||
cloneOrPull,
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
discoverMetabaseDatabases,
|
||||
type DiscoveredMetabaseDatabase,
|
||||
loadDbtSchemaFiles,
|
||||
loadProjectInfo,
|
||||
MetabaseClient,
|
||||
type NotionApi,
|
||||
NotionClient,
|
||||
parseLookmlStagedDir,
|
||||
|
|
@ -28,6 +32,7 @@ import { runKtxConnection } from './connection.js';
|
|||
import { withMenuOptionsSpacing, withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
|
||||
import { runKtxPublicIngest } from './public-ingest.js';
|
||||
import { withSetupInterruptConfirmation } from './setup-interrupt.js';
|
||||
import { writeProjectLocalSecretReference } from './setup-secrets.js';
|
||||
|
||||
export type KtxSetupSourceType = 'dbt' | 'metricflow' | 'metabase' | 'looker' | 'lookml' | 'notion';
|
||||
|
||||
|
|
@ -71,6 +76,7 @@ export interface KtxSetupSourcesPromptAdapter {
|
|||
}): Promise<string[]>;
|
||||
select(options: { message: string; options: Array<{ value: string; label: string }> }): Promise<string>;
|
||||
text(options: { message: string; placeholder?: string; initialValue?: string }): Promise<string | undefined>;
|
||||
password(options: { message: string }): Promise<string | undefined>;
|
||||
cancel(message: string): void;
|
||||
log?(message: string): void;
|
||||
}
|
||||
|
|
@ -86,6 +92,11 @@ export interface KtxSetupSourcesDeps {
|
|||
validateLooker?: (projectDir: string, connectionId: string) => Promise<SourceValidationResult>;
|
||||
validateLookml?: (connection: KtxProjectConnectionConfig) => Promise<SourceValidationResult>;
|
||||
validateNotion?: (connection: KtxProjectConnectionConfig) => Promise<SourceValidationResult>;
|
||||
discoverMetabaseDatabases?: (args: {
|
||||
sourceUrl: string;
|
||||
sourceApiKeyRef: string;
|
||||
sourceConnectionId: string;
|
||||
}) => Promise<DiscoveredMetabaseDatabase[]>;
|
||||
runMapping?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise<number>;
|
||||
runInitialIngest?: (
|
||||
projectDir: string,
|
||||
|
|
@ -143,6 +154,12 @@ function createPromptAdapter(): KtxSetupSourcesPromptAdapter {
|
|||
);
|
||||
return isCancel(value) ? undefined : String(value);
|
||||
},
|
||||
async password(options) {
|
||||
const value = await withSetupInterruptConfirmation(() =>
|
||||
password({ ...options, message: withTextInputNavigation(options.message) }),
|
||||
);
|
||||
return isCancel(value) ? undefined : String(value);
|
||||
},
|
||||
cancel(message) {
|
||||
cancel(message);
|
||||
},
|
||||
|
|
@ -172,17 +189,6 @@ function connectionNamePrompt(label: string): string {
|
|||
return `Name this ${label} connection\nKTX will use this short name in commands and config. You can rename it now.`;
|
||||
}
|
||||
|
||||
function gitAuthAfterFailurePrompt(source: KtxSetupSourceType): string {
|
||||
const label = source === 'dbt' ? 'This' : `This ${sourceLabel(source)}`;
|
||||
return [
|
||||
`${label} repo requires authentication.`,
|
||||
'Generate a token at: https://github.com/settings/tokens/new',
|
||||
'Store it in an env var, then enter env:VARIABLE_NAME here (e.g. env:GITHUB_TOKEN).',
|
||||
'Or use file:/absolute/path if the token is stored in a file.',
|
||||
'Press Enter to skip and try without authentication anyway.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function sourceSubpathPrompt(source: KtxSetupSourceType): string {
|
||||
if (source === 'dbt') {
|
||||
return [
|
||||
|
|
@ -198,6 +204,21 @@ function sourceSubpathPrompt(source: KtxSetupSourceType): string {
|
|||
].join('\n');
|
||||
}
|
||||
|
||||
const SCAN_SKIP_DIRS = new Set(['.git', 'node_modules', '.venv', 'target', 'dbt_packages', 'dbt_modules', '__pycache__']);
|
||||
|
||||
async function findDbtProjectSubpaths(rootDir: string): Promise<string[]> {
|
||||
const entries = await readdir(rootDir, { withFileTypes: true, recursive: true });
|
||||
const subpaths: string[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
if (entry.name !== 'dbt_project.yml' && entry.name !== 'dbt_project.yaml') continue;
|
||||
const relDir = relative(rootDir, entry.parentPath);
|
||||
if (relDir.split('/').some((part) => SCAN_SKIP_DIRS.has(part))) continue;
|
||||
subpaths.push(relDir);
|
||||
}
|
||||
return subpaths;
|
||||
}
|
||||
|
||||
async function promptText(
|
||||
prompts: KtxSetupSourcesPromptAdapter,
|
||||
options: { message: string; placeholder?: string; initialValue?: string },
|
||||
|
|
@ -222,6 +243,75 @@ function credentialRef(value: string | undefined, label: string): string {
|
|||
return ref;
|
||||
}
|
||||
|
||||
async function chooseSourceCredentialRef(input: {
|
||||
prompts: KtxSetupSourcesPromptAdapter;
|
||||
projectDir: string;
|
||||
label: string;
|
||||
envName: string;
|
||||
secretFileName: string;
|
||||
}): Promise<string | 'back'> {
|
||||
while (true) {
|
||||
const choice = await input.prompts.select({
|
||||
message: `How should KTX find your ${input.label}?`,
|
||||
options: [
|
||||
{ value: 'env', label: `Use ${input.envName} from the environment` },
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
if (choice === 'back') return 'back';
|
||||
if (choice === 'paste') {
|
||||
const value = await input.prompts.password({ message: input.label });
|
||||
if (value === undefined) continue;
|
||||
if (!value.trim()) continue;
|
||||
const ref = await writeProjectLocalSecretReference({
|
||||
projectDir: input.projectDir,
|
||||
fileName: input.secretFileName,
|
||||
value,
|
||||
});
|
||||
input.prompts.log?.(`Saved to .ktx/secrets/${input.secretFileName}`);
|
||||
return ref;
|
||||
}
|
||||
return `env:${input.envName}`;
|
||||
}
|
||||
}
|
||||
|
||||
async function chooseGitAuthCredentialRef(input: {
|
||||
prompts: KtxSetupSourcesPromptAdapter;
|
||||
projectDir: string;
|
||||
source: KtxSetupSourceType;
|
||||
connectionId: string;
|
||||
}): Promise<string | undefined | 'back'> {
|
||||
const label = input.source === 'dbt' ? 'This' : `This ${sourceLabel(input.source)}`;
|
||||
while (true) {
|
||||
const choice = await input.prompts.select({
|
||||
message: `${label} repo requires authentication.`,
|
||||
options: [
|
||||
{ value: 'env', label: 'Use GITHUB_TOKEN from the environment' },
|
||||
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
|
||||
{ value: 'skip', label: 'Skip — try without authentication' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
if (choice === 'back') return 'back';
|
||||
if (choice === 'skip') return undefined;
|
||||
if (choice === 'paste') {
|
||||
const value = await input.prompts.password({ message: 'Git access token' });
|
||||
if (value === undefined) continue;
|
||||
if (!value.trim()) continue;
|
||||
const fileName = `${input.connectionId}-auth-token`;
|
||||
const ref = await writeProjectLocalSecretReference({
|
||||
projectDir: input.projectDir,
|
||||
fileName,
|
||||
value,
|
||||
});
|
||||
input.prompts.log?.(`Saved to .ktx/secrets/${fileName}`);
|
||||
return ref;
|
||||
}
|
||||
return 'env:GITHUB_TOKEN';
|
||||
}
|
||||
}
|
||||
|
||||
function repoOrLocalSource(args: KtxSetupSourcesArgs): { sourceDir?: string; repoUrl?: string } {
|
||||
if (args.sourcePath && args.sourceGitUrl) {
|
||||
throw new Error('Choose only one source location: --source-path or --source-git-url.');
|
||||
|
|
@ -373,7 +463,7 @@ function buildMetabaseConnection(args: KtxSetupSourcesArgs): KtxProjectConnectio
|
|||
mappings: {
|
||||
databaseMappings: { [String(args.metabaseDatabaseId)]: args.sourceWarehouseConnectionId },
|
||||
syncEnabled: { [String(args.metabaseDatabaseId)]: true },
|
||||
syncMode: 'ONLY',
|
||||
syncMode: 'ALL',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -512,16 +602,6 @@ async function defaultValidateMetricflow(connection: KtxProjectConnectionConfig)
|
|||
};
|
||||
}
|
||||
|
||||
async function defaultValidateMetabase(projectDir: string, connectionId: string): Promise<SourceValidationResult> {
|
||||
const code = await runKtxConnection(
|
||||
{ command: 'map', projectDir, sourceConnectionId: connectionId, json: true },
|
||||
{ stdout: { write() {} }, stderr: { write() {} } },
|
||||
);
|
||||
return code === 0
|
||||
? { ok: true, detail: 'mapping validated' }
|
||||
: { ok: false, message: 'Metabase mapping validation failed' };
|
||||
}
|
||||
|
||||
async function defaultValidateLooker(projectDir: string, connectionId: string): Promise<SourceValidationResult> {
|
||||
const code = await runKtxConnectionMapping(
|
||||
{ command: 'refresh', projectDir, connectionId, autoAccept: true },
|
||||
|
|
@ -559,8 +639,37 @@ async function defaultValidateNotion(connection: KtxProjectConnectionConfig): Pr
|
|||
return { ok: true, detail: `roots=${roots.length}` };
|
||||
}
|
||||
|
||||
interface MappingJsonOutput {
|
||||
connectionId: string;
|
||||
refresh: { ok: boolean; output: string[] };
|
||||
validation: { ok: boolean; output: string[] };
|
||||
mappings: unknown[];
|
||||
}
|
||||
|
||||
function summarizeMappingResult(parsed: MappingJsonOutput): string {
|
||||
const mappingCount = parsed.mappings.length;
|
||||
const mappingNoun = mappingCount === 1 ? 'mapping' : 'mappings';
|
||||
return `Mapping validated — ${mappingCount} ${mappingNoun} configured`;
|
||||
}
|
||||
|
||||
async function defaultRunMapping(projectDir: string, connectionId: string, io: KtxCliIo): Promise<number> {
|
||||
return await runKtxConnection({ command: 'map', projectDir, sourceConnectionId: connectionId, json: false }, io);
|
||||
let captured = '';
|
||||
const captureIo: KtxCliIo = {
|
||||
stdout: { write(chunk: string) { captured += chunk; } },
|
||||
stderr: io.stderr,
|
||||
};
|
||||
const code = await runKtxConnection(
|
||||
{ command: 'map', projectDir, sourceConnectionId: connectionId, json: true },
|
||||
captureIo,
|
||||
);
|
||||
if (code !== 0) return code;
|
||||
try {
|
||||
const parsed = JSON.parse(captured.trim()) as MappingJsonOutput;
|
||||
io.stdout.write(`${summarizeMappingResult(parsed)}\n`);
|
||||
} catch {
|
||||
io.stdout.write(captured);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function defaultRunInitialIngest(
|
||||
|
|
@ -634,6 +743,11 @@ type SourcePromptState = KtxSetupSourcesArgs & {
|
|||
|
||||
type SourcePromptStep = (state: SourcePromptState) => Promise<'next' | 'back'>;
|
||||
|
||||
interface WarehouseConnectionChoice {
|
||||
id: string;
|
||||
connectionType: string;
|
||||
}
|
||||
|
||||
type InteractiveSourceConnectionChoice =
|
||||
| { kind: 'existing'; connectionId: string; connection: KtxProjectConnectionConfig }
|
||||
| { kind: 'new'; args: KtxSetupSourcesArgs }
|
||||
|
|
@ -672,6 +786,107 @@ function resetRepoLocationFields(state: SourcePromptState): void {
|
|||
delete state.sourceProjectName;
|
||||
}
|
||||
|
||||
function warehouseConnectionChoices(config: KtxProjectConfig): WarehouseConnectionChoice[] {
|
||||
return Object.entries(config.connections)
|
||||
.filter(([, connection]) => PRIMARY_SOURCE_DRIVERS.has(String(connection.driver ?? '').toLowerCase()))
|
||||
.map(([id, connection]) => ({ id, connectionType: localConnectionTypeForConfig(id, connection) }))
|
||||
.sort((left, right) => left.id.localeCompare(right.id));
|
||||
}
|
||||
|
||||
async function chooseMappedWarehouseConnectionId(input: {
|
||||
projectDir: string;
|
||||
prompts: KtxSetupSourcesPromptAdapter;
|
||||
}): Promise<string | 'back'> {
|
||||
const project = await loadKtxProject({ projectDir: input.projectDir });
|
||||
const choices = warehouseConnectionChoices(project.config);
|
||||
if (choices.length === 1) {
|
||||
return choices[0].id;
|
||||
}
|
||||
if (choices.length === 0) {
|
||||
const entered = await promptText(input.prompts, { message: 'Mapped warehouse connection id' });
|
||||
return entered === undefined ? 'back' : entered;
|
||||
}
|
||||
|
||||
const selected = await input.prompts.select({
|
||||
message: 'Mapped warehouse connection',
|
||||
options: [
|
||||
...choices.map((choice) => ({
|
||||
value: choice.id,
|
||||
label: `${choice.id} (${choice.connectionType})`,
|
||||
})),
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
return selected === 'back' ? 'back' : selected;
|
||||
}
|
||||
|
||||
async function defaultDiscoverMetabaseDatabases(input: {
|
||||
sourceUrl: string;
|
||||
sourceApiKeyRef: string;
|
||||
}): Promise<DiscoveredMetabaseDatabase[]> {
|
||||
const apiKey = resolveKtxConfigReference(input.sourceApiKeyRef, process.env);
|
||||
if (!apiKey) {
|
||||
throw new Error('Metabase API key ref could not be resolved');
|
||||
}
|
||||
const client = new MetabaseClient(
|
||||
{ apiUrl: input.sourceUrl, apiKey },
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
);
|
||||
try {
|
||||
return await discoverMetabaseDatabases(client);
|
||||
} finally {
|
||||
await client.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
function metabaseDatabaseLabel(database: DiscoveredMetabaseDatabase): string {
|
||||
const detail = [database.engine].filter(Boolean).join(', ');
|
||||
return detail ? `${database.id}: ${database.name} (${detail})` : `${database.id}: ${database.name}`;
|
||||
}
|
||||
|
||||
async function chooseMetabaseDatabaseId(input: {
|
||||
state: SourcePromptState;
|
||||
prompts: KtxSetupSourcesPromptAdapter;
|
||||
deps: KtxSetupSourcesDeps;
|
||||
}): Promise<number | 'back'> {
|
||||
const sourceUrl = input.state.sourceUrl;
|
||||
const sourceApiKeyRef = input.state.sourceApiKeyRef;
|
||||
if (sourceUrl && sourceApiKeyRef) {
|
||||
try {
|
||||
const discovered = await (input.deps.discoverMetabaseDatabases ?? defaultDiscoverMetabaseDatabases)({
|
||||
sourceUrl,
|
||||
sourceApiKeyRef,
|
||||
sourceConnectionId: input.state.sourceConnectionId ?? 'metabase-main',
|
||||
});
|
||||
if (discovered.length === 1) {
|
||||
return discovered[0].id;
|
||||
}
|
||||
if (discovered.length > 1) {
|
||||
const selected = await input.prompts.select({
|
||||
message: 'Metabase database',
|
||||
options: [
|
||||
...discovered
|
||||
.slice()
|
||||
.sort((left, right) => left.id - right.id)
|
||||
.map((database) => ({
|
||||
value: String(database.id),
|
||||
label: metabaseDatabaseLabel(database),
|
||||
})),
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
return selected === 'back' ? 'back' : Number.parseInt(selected, 10);
|
||||
}
|
||||
} catch {
|
||||
// Discovery is a convenience. Fall back to the raw id prompt when credentials
|
||||
// are unavailable locally or the Metabase API cannot be reached yet.
|
||||
}
|
||||
}
|
||||
|
||||
const databaseId = await promptText(input.prompts, { message: 'Metabase database id' });
|
||||
return databaseId === undefined ? 'back' : Number.parseInt(databaseId, 10);
|
||||
}
|
||||
|
||||
function connectionIdPromptSteps(
|
||||
args: KtxSetupSourcesArgs,
|
||||
source: KtxSetupSourceType,
|
||||
|
|
@ -703,6 +918,7 @@ async function promptForInteractiveSource(
|
|||
prompts: KtxSetupSourcesPromptAdapter,
|
||||
defaultConnectionId = `${source}-main`,
|
||||
testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection,
|
||||
discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'],
|
||||
): Promise<KtxSetupSourcesArgs | 'back'> {
|
||||
const initialState: SourcePromptState = { ...args, source };
|
||||
if (args.sourceConnectionId) {
|
||||
|
|
@ -757,23 +973,6 @@ async function promptForInteractiveSource(
|
|||
},
|
||||
]
|
||||
: []),
|
||||
...(state.sourceLocation
|
||||
? [
|
||||
async (currentState: SourcePromptState) => {
|
||||
const subpath = await promptText(prompts, {
|
||||
message: sourceSubpathPrompt(source),
|
||||
placeholder: 'optional',
|
||||
});
|
||||
if (subpath === undefined) return 'back';
|
||||
if (subpath) {
|
||||
currentState.sourceSubpath = subpath;
|
||||
} else {
|
||||
delete currentState.sourceSubpath;
|
||||
}
|
||||
return 'next';
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(state.sourceLocation === 'git'
|
||||
? [
|
||||
async (currentState: SourcePromptState) => {
|
||||
|
|
@ -783,11 +982,13 @@ async function promptForInteractiveSource(
|
|||
prompts.log?.('Repository connected.');
|
||||
return 'next';
|
||||
}
|
||||
const authRef = await promptText(prompts, {
|
||||
message: gitAuthAfterFailurePrompt(source),
|
||||
placeholder: 'env:GITHUB_TOKEN',
|
||||
const authRef = await chooseGitAuthCredentialRef({
|
||||
prompts,
|
||||
projectDir: args.projectDir,
|
||||
source,
|
||||
connectionId: currentState.sourceConnectionId ?? `${source}-main`,
|
||||
});
|
||||
if (authRef === undefined) return 'back';
|
||||
if (authRef === 'back') return 'back';
|
||||
if (authRef) {
|
||||
currentState.sourceAuthTokenRef = authRef;
|
||||
} else {
|
||||
|
|
@ -797,6 +998,79 @@ async function promptForInteractiveSource(
|
|||
},
|
||||
]
|
||||
: []),
|
||||
...(state.sourceLocation
|
||||
? [
|
||||
async (currentState: SourcePromptState) => {
|
||||
if (source === 'dbt') {
|
||||
let scanDir: string | undefined;
|
||||
if (currentState.sourceLocation === 'path' && currentState.sourcePath) {
|
||||
scanDir = currentState.sourcePath;
|
||||
} else if (currentState.sourceLocation === 'git' && currentState.sourceGitUrl) {
|
||||
try {
|
||||
const cacheDir = await mkdtemp(join(tmpdir(), 'ktx-setup-dbt-scan-'));
|
||||
const authToken = currentState.sourceAuthTokenRef
|
||||
? resolveKtxConfigReference(currentState.sourceAuthTokenRef, process.env)
|
||||
: null;
|
||||
await cloneOrPull({
|
||||
repoUrl: currentState.sourceGitUrl,
|
||||
authToken,
|
||||
cacheDir,
|
||||
branch: currentState.sourceBranch ?? 'main',
|
||||
});
|
||||
scanDir = cacheDir;
|
||||
} catch {
|
||||
// Clone failed — fall through to manual prompt
|
||||
}
|
||||
}
|
||||
if (scanDir) {
|
||||
try {
|
||||
const subpaths = await findDbtProjectSubpaths(scanDir);
|
||||
if (subpaths.length === 1) {
|
||||
const found = subpaths[0]!;
|
||||
if (found) {
|
||||
currentState.sourceSubpath = found;
|
||||
prompts.log?.(`Found dbt_project.yml in ${found}/`);
|
||||
} else {
|
||||
delete currentState.sourceSubpath;
|
||||
}
|
||||
return 'next';
|
||||
}
|
||||
if (subpaths.length > 1) {
|
||||
const selected = await prompts.select({
|
||||
message: 'Multiple dbt projects found — which one should KTX use?',
|
||||
options: [
|
||||
...subpaths.map((p) => ({ value: p || '.', label: p || '(project root)' })),
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
if (selected === 'back') return 'back';
|
||||
const subpath = selected === '.' ? '' : selected;
|
||||
if (subpath) {
|
||||
currentState.sourceSubpath = subpath;
|
||||
} else {
|
||||
delete currentState.sourceSubpath;
|
||||
}
|
||||
return 'next';
|
||||
}
|
||||
} catch {
|
||||
// Directory unreadable — fall through to manual prompt
|
||||
}
|
||||
}
|
||||
}
|
||||
const subpath = await promptText(prompts, {
|
||||
message: sourceSubpathPrompt(source),
|
||||
placeholder: 'optional',
|
||||
});
|
||||
if (subpath === undefined) return 'back';
|
||||
if (subpath) {
|
||||
currentState.sourceSubpath = subpath;
|
||||
} else {
|
||||
delete currentState.sourceSubpath;
|
||||
}
|
||||
return 'next';
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]);
|
||||
}
|
||||
|
||||
|
|
@ -810,24 +1084,34 @@ async function promptForInteractiveSource(
|
|||
return 'next';
|
||||
},
|
||||
async (state) => {
|
||||
const sourceApiKeyRef = await promptText(prompts, {
|
||||
message: 'Metabase API key ref',
|
||||
placeholder: 'env:METABASE_API_KEY',
|
||||
const ref = await chooseSourceCredentialRef({
|
||||
prompts,
|
||||
projectDir: args.projectDir,
|
||||
label: 'Metabase API key',
|
||||
envName: 'METABASE_API_KEY',
|
||||
secretFileName: `${state.sourceConnectionId ?? 'metabase-main'}-api-key`,
|
||||
});
|
||||
if (sourceApiKeyRef === undefined) return 'back';
|
||||
state.sourceApiKeyRef = sourceApiKeyRef;
|
||||
if (ref === 'back') return 'back';
|
||||
state.sourceApiKeyRef = ref;
|
||||
return 'next';
|
||||
},
|
||||
async (state) => {
|
||||
const sourceWarehouseConnectionId = await promptText(prompts, { message: 'Mapped warehouse connection id' });
|
||||
if (sourceWarehouseConnectionId === undefined) return 'back';
|
||||
const sourceWarehouseConnectionId = await chooseMappedWarehouseConnectionId({
|
||||
projectDir: args.projectDir,
|
||||
prompts,
|
||||
});
|
||||
if (sourceWarehouseConnectionId === 'back') return 'back';
|
||||
state.sourceWarehouseConnectionId = sourceWarehouseConnectionId;
|
||||
return 'next';
|
||||
},
|
||||
async (state) => {
|
||||
const databaseId = await promptText(prompts, { message: 'Metabase database id' });
|
||||
if (databaseId === undefined) return 'back';
|
||||
state.metabaseDatabaseId = Number.parseInt(databaseId, 10);
|
||||
const databaseId = await chooseMetabaseDatabaseId({
|
||||
state,
|
||||
prompts,
|
||||
deps: { discoverMetabaseDatabases: discoverMetabaseDatabaseList },
|
||||
});
|
||||
if (databaseId === 'back') return 'back';
|
||||
state.metabaseDatabaseId = databaseId;
|
||||
return 'next';
|
||||
},
|
||||
]);
|
||||
|
|
@ -849,17 +1133,23 @@ async function promptForInteractiveSource(
|
|||
return 'next';
|
||||
},
|
||||
async (state) => {
|
||||
const sourceClientSecretRef = await promptText(prompts, {
|
||||
message: 'Looker client secret ref',
|
||||
placeholder: 'env:LOOKER_CLIENT_SECRET',
|
||||
const ref = await chooseSourceCredentialRef({
|
||||
prompts,
|
||||
projectDir: args.projectDir,
|
||||
label: 'Looker client secret',
|
||||
envName: 'LOOKER_CLIENT_SECRET',
|
||||
secretFileName: `${state.sourceConnectionId ?? 'looker-main'}-client-secret`,
|
||||
});
|
||||
if (sourceClientSecretRef === undefined) return 'back';
|
||||
state.sourceClientSecretRef = sourceClientSecretRef;
|
||||
if (ref === 'back') return 'back';
|
||||
state.sourceClientSecretRef = ref;
|
||||
return 'next';
|
||||
},
|
||||
async (state) => {
|
||||
const sourceWarehouseConnectionId = await promptText(prompts, { message: 'Mapped warehouse connection id' });
|
||||
if (sourceWarehouseConnectionId === undefined) return 'back';
|
||||
const sourceWarehouseConnectionId = await chooseMappedWarehouseConnectionId({
|
||||
projectDir: args.projectDir,
|
||||
prompts,
|
||||
});
|
||||
if (sourceWarehouseConnectionId === 'back') return 'back';
|
||||
state.sourceWarehouseConnectionId = sourceWarehouseConnectionId;
|
||||
return 'next';
|
||||
},
|
||||
|
|
@ -882,12 +1172,15 @@ async function promptForInteractiveSource(
|
|||
return await runSourcePromptSteps(initialState, (state) => [
|
||||
...connectionSteps,
|
||||
async (currentState) => {
|
||||
const sourceApiKeyRef = await promptText(prompts, {
|
||||
message: 'Notion token ref',
|
||||
placeholder: 'env:NOTION_TOKEN',
|
||||
const ref = await chooseSourceCredentialRef({
|
||||
prompts,
|
||||
projectDir: args.projectDir,
|
||||
label: 'Notion integration token',
|
||||
envName: 'NOTION_TOKEN',
|
||||
secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`,
|
||||
});
|
||||
if (sourceApiKeyRef === undefined) return 'back';
|
||||
currentState.sourceApiKeyRef = sourceApiKeyRef;
|
||||
if (ref === 'back') return 'back';
|
||||
currentState.sourceApiKeyRef = ref;
|
||||
return 'next';
|
||||
},
|
||||
async (currentState) => {
|
||||
|
|
@ -956,13 +1249,21 @@ async function chooseInteractiveSourceConnection(input: {
|
|||
connections: Record<string, KtxProjectConnectionConfig>;
|
||||
prompts: KtxSetupSourcesPromptAdapter;
|
||||
testGitRepo?: KtxSetupSourcesDeps['testGitRepo'];
|
||||
discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases'];
|
||||
}): Promise<InteractiveSourceConnectionChoice> {
|
||||
const existingIds = existingConnectionIdsBySource(input.connections, input.source);
|
||||
const defaultConnectionId = defaultConnectionIdForSource(input.connections, input.source);
|
||||
const label = sourceLabel(input.source);
|
||||
|
||||
if (existingIds.length === 0) {
|
||||
const sourceArgs = await promptForInteractiveSource(input.args, input.source, input.prompts, defaultConnectionId, input.testGitRepo);
|
||||
const sourceArgs = await promptForInteractiveSource(
|
||||
input.args,
|
||||
input.source,
|
||||
input.prompts,
|
||||
defaultConnectionId,
|
||||
input.testGitRepo,
|
||||
input.discoverMetabaseDatabases,
|
||||
);
|
||||
return sourceArgs === 'back' ? 'back' : { kind: 'new', args: sourceArgs };
|
||||
}
|
||||
|
||||
|
|
@ -987,7 +1288,14 @@ async function chooseInteractiveSourceConnection(input: {
|
|||
}
|
||||
continue;
|
||||
}
|
||||
const sourceArgs = await promptForInteractiveSource(input.args, input.source, input.prompts, defaultConnectionId, input.testGitRepo);
|
||||
const sourceArgs = await promptForInteractiveSource(
|
||||
input.args,
|
||||
input.source,
|
||||
input.prompts,
|
||||
defaultConnectionId,
|
||||
input.testGitRepo,
|
||||
input.discoverMetabaseDatabases,
|
||||
);
|
||||
if (sourceArgs === 'back') {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -1026,7 +1334,9 @@ async function validateSource(
|
|||
return await (deps.validateMetricflow ?? defaultValidateMetricflow)(args.connection);
|
||||
}
|
||||
if (source === 'metabase') {
|
||||
return await (deps.validateMetabase ?? defaultValidateMetabase)(args.projectDir, args.connectionId);
|
||||
return deps.validateMetabase
|
||||
? await deps.validateMetabase(args.projectDir, args.connectionId)
|
||||
: { ok: true, detail: 'mapping validation runs after the connection is saved' };
|
||||
}
|
||||
if (source === 'looker') {
|
||||
return await (deps.validateLooker ?? defaultValidateLooker)(args.projectDir, args.connectionId);
|
||||
|
|
@ -1097,6 +1407,7 @@ export async function runKtxSetupSourcesStep(
|
|||
connections: (await loadKtxProject({ projectDir: args.projectDir })).config.connections,
|
||||
prompts,
|
||||
testGitRepo: deps.testGitRepo,
|
||||
discoverMetabaseDatabases: deps.discoverMetabaseDatabases,
|
||||
});
|
||||
if (sourceChoice === 'back') {
|
||||
if (args.source) {
|
||||
|
|
@ -1126,6 +1437,7 @@ export async function runKtxSetupSourcesStep(
|
|||
return { status: 'failed', projectDir: args.projectDir };
|
||||
}
|
||||
if (source === 'metabase' || source === 'looker') {
|
||||
prompts.log?.(`Validating ${sourceLabel(source)} mapping…`);
|
||||
const mappingCode = await (deps.runMapping ?? defaultRunMapping)(args.projectDir, connectionId, io);
|
||||
if (mappingCode !== 0) {
|
||||
await rollback?.();
|
||||
|
|
|
|||
|
|
@ -1367,6 +1367,142 @@ describe('setup status', () => {
|
|||
expect(calls).toEqual(['context']);
|
||||
});
|
||||
|
||||
it('resumes an active context build before prompting for earlier setup steps', async () => {
|
||||
const io = makeIo();
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: revenue',
|
||||
'setup:',
|
||||
' database_connection_ids:',
|
||||
' - warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-active',
|
||||
status: 'running',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:00:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: [],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-active'),
|
||||
});
|
||||
const context = vi.fn(async () => ({
|
||||
status: 'detached' as const,
|
||||
projectDir: tempDir,
|
||||
runId: 'setup-context-local-active',
|
||||
}));
|
||||
const databases = vi.fn(async () => {
|
||||
throw new Error('database setup should not run while context build is active');
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetup(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: tempDir,
|
||||
mode: 'existing',
|
||||
agents: false,
|
||||
inputMode: 'auto',
|
||||
yes: false,
|
||||
cliVersion: '0.2.0',
|
||||
skipLlm: false,
|
||||
skipEmbeddings: false,
|
||||
skipDatabases: false,
|
||||
skipSources: false,
|
||||
skipAgents: false,
|
||||
databaseSchemas: [],
|
||||
},
|
||||
io.io,
|
||||
{ context, databases },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(context).toHaveBeenCalledWith(
|
||||
{ projectDir: tempDir, inputMode: 'auto', allowEmpty: true },
|
||||
io.io,
|
||||
);
|
||||
expect(databases).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips entry menu and auto-watches when context build is active and showEntryMenu is true', async () => {
|
||||
const io = makeIo();
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: revenue',
|
||||
'setup:',
|
||||
' database_connection_ids:',
|
||||
' - warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-active',
|
||||
status: 'detached',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:00:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: [],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-active'),
|
||||
});
|
||||
const context = vi.fn(async () => ({
|
||||
status: 'detached' as const,
|
||||
projectDir: tempDir,
|
||||
runId: 'setup-context-local-active',
|
||||
}));
|
||||
const entryMenuSelect = vi.fn(async () => 'exit');
|
||||
|
||||
await expect(
|
||||
runKtxSetup(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: tempDir,
|
||||
mode: 'existing',
|
||||
agents: false,
|
||||
inputMode: 'auto',
|
||||
yes: false,
|
||||
cliVersion: '0.2.0',
|
||||
skipLlm: false,
|
||||
skipEmbeddings: false,
|
||||
skipDatabases: false,
|
||||
skipSources: false,
|
||||
skipAgents: false,
|
||||
databaseSchemas: [],
|
||||
showEntryMenu: true,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
context,
|
||||
entryMenuDeps: { prompts: { select: entryMenuSelect, cancel: vi.fn() } },
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(entryMenuSelect).not.toHaveBeenCalled();
|
||||
expect(context).toHaveBeenCalledWith(
|
||||
{ projectDir: tempDir, inputMode: 'auto', allowEmpty: true, autoWatch: true },
|
||||
io.io,
|
||||
);
|
||||
});
|
||||
|
||||
it('routes a ready project menu selection to agent setup', async () => {
|
||||
const calls: string[] = [];
|
||||
const io = makeIo();
|
||||
|
|
@ -1479,6 +1615,103 @@ describe('setup status', () => {
|
|||
expect(calls).toEqual(['agents']);
|
||||
});
|
||||
|
||||
it('skips to agent setup when context is ready but agents are not configured', async () => {
|
||||
const calls: string[] = [];
|
||||
const io = makeIo();
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: revenue',
|
||||
'setup:',
|
||||
' completed_steps:',
|
||||
' - project',
|
||||
' - llm',
|
||||
' - embeddings',
|
||||
' - sources',
|
||||
' - context',
|
||||
' database_connection_ids: []',
|
||||
'connections: {}',
|
||||
'llm:',
|
||||
' provider:',
|
||||
' backend: anthropic',
|
||||
' models:',
|
||||
' default: claude-sonnet-4-6',
|
||||
'ingest:',
|
||||
' embeddings:',
|
||||
' backend: openai',
|
||||
' model: text-embedding-3-small',
|
||||
' dimensions: 1536',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-ready',
|
||||
status: 'completed',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:02:00.000Z',
|
||||
completedAt: '2026-05-09T10:02:00.000Z',
|
||||
primarySourceConnectionIds: [],
|
||||
contextSourceConnectionIds: [],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-ready'),
|
||||
});
|
||||
|
||||
const readyMenuSelect = vi.fn();
|
||||
await expect(
|
||||
runKtxSetup(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: tempDir,
|
||||
mode: 'existing',
|
||||
agents: false,
|
||||
inputMode: 'auto',
|
||||
yes: false,
|
||||
cliVersion: '0.2.0',
|
||||
skipLlm: false,
|
||||
skipEmbeddings: false,
|
||||
skipDatabases: false,
|
||||
skipSources: false,
|
||||
skipAgents: false,
|
||||
databaseSchemas: [],
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
readyMenuDeps: { prompts: { select: readyMenuSelect, cancel: vi.fn() } },
|
||||
model: async (args) => {
|
||||
expect(args.skipLlm).toBe(true);
|
||||
return { status: 'skipped', projectDir: tempDir };
|
||||
},
|
||||
embeddings: async (args) => {
|
||||
expect(args.skipEmbeddings).toBe(true);
|
||||
return { status: 'skipped', projectDir: tempDir };
|
||||
},
|
||||
databases: async (args) => {
|
||||
expect(args.skipDatabases).toBe(true);
|
||||
return { status: 'skipped', projectDir: tempDir };
|
||||
},
|
||||
sources: async (args) => {
|
||||
expect(args.skipSources).toBe(true);
|
||||
return { status: 'skipped', projectDir: tempDir };
|
||||
},
|
||||
agents: async () => {
|
||||
calls.push('agents');
|
||||
return {
|
||||
status: 'ready',
|
||||
projectDir: tempDir,
|
||||
installs: [{ target: 'codex', scope: 'project', mode: 'cli' }],
|
||||
};
|
||||
},
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(readyMenuSelect).not.toHaveBeenCalled();
|
||||
expect(calls).toEqual(['agents']);
|
||||
});
|
||||
|
||||
it('runs only project resolution, context gate, and agent setup in --agents mode', async () => {
|
||||
const io = makeIo();
|
||||
const context = vi.fn(async () => ({ status: 'ready' as const, projectDir: tempDir, runId: 'setup-context-local-test' }));
|
||||
|
|
|
|||
|
|
@ -24,7 +24,12 @@ import {
|
|||
import { type KtxSetupEmbeddingsDeps, runKtxSetupEmbeddingsStep } from './setup-embeddings.js';
|
||||
import { type KtxSetupModelDeps, runKtxSetupAnthropicModelStep } from './setup-models.js';
|
||||
import { type KtxSetupProjectDeps, runKtxSetupProjectStep } from './setup-project.js';
|
||||
import { isKtxSetupReady, type KtxSetupReadyMenuDeps, runKtxSetupReadyChangeMenu } from './setup-ready-menu.js';
|
||||
import {
|
||||
isKtxPreAgentSetupReady,
|
||||
isKtxSetupReady,
|
||||
type KtxSetupReadyMenuDeps,
|
||||
runKtxSetupReadyChangeMenu,
|
||||
} from './setup-ready-menu.js';
|
||||
import { type KtxSetupSourcesDeps, type KtxSetupSourceType, runKtxSetupSourcesStep } from './setup-sources.js';
|
||||
import { withMenuOptionsSpacing } from './prompt-navigation.js';
|
||||
import {
|
||||
|
|
@ -392,6 +397,10 @@ function setupContextReady(status: KtxSetupStatus): boolean {
|
|||
return status.context.ready;
|
||||
}
|
||||
|
||||
function setupContextActive(status: KtxSetupStatus): boolean {
|
||||
return status.context.status === 'running' || status.context.status === 'detached';
|
||||
}
|
||||
|
||||
function writeContextNotReadyForAgents(projectDir: string, io: KtxCliIo): void {
|
||||
io.stderr.write('KTX context is not ready for agents.\n\n');
|
||||
io.stderr.write(`Build context first:\n ktx setup context build --project-dir ${resolve(projectDir)}\n\n`);
|
||||
|
|
@ -462,22 +471,27 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
|
|||
args.inputMode !== 'disabled' &&
|
||||
!args.agents &&
|
||||
(io.stdout.isTTY === true || deps.entryMenuDeps?.prompts !== undefined);
|
||||
let autoWatchActiveBuild = false;
|
||||
|
||||
setupLoop: while (true) {
|
||||
entryAction = undefined;
|
||||
if (canShowEntryMenu) {
|
||||
const status = await readKtxSetupStatus(args.projectDir);
|
||||
entryAction = (await runKtxSetupEntryMenu(status, deps.entryMenuDeps)).action;
|
||||
if (entryAction === 'exit') {
|
||||
(deps.entryMenuDeps?.prompts ?? createEntryMenuPromptAdapter()).cancel('Setup cancelled.');
|
||||
return 0;
|
||||
}
|
||||
if (entryAction === 'status') {
|
||||
io.stdout.write(formatKtxSetupStatus(status));
|
||||
return 0;
|
||||
}
|
||||
if (entryAction === 'demo') {
|
||||
return await runKtxSetupDemoFromEntryMenu(args, io, deps);
|
||||
if (setupContextActive(status)) {
|
||||
autoWatchActiveBuild = true;
|
||||
} else {
|
||||
entryAction = (await runKtxSetupEntryMenu(status, deps.entryMenuDeps)).action;
|
||||
if (entryAction === 'exit') {
|
||||
(deps.entryMenuDeps?.prompts ?? createEntryMenuPromptAdapter()).cancel('Setup cancelled.');
|
||||
return 0;
|
||||
}
|
||||
if (entryAction === 'status') {
|
||||
io.stdout.write(formatKtxSetupStatus(status));
|
||||
return 0;
|
||||
}
|
||||
if (entryAction === 'demo') {
|
||||
return await runKtxSetupDemoFromEntryMenu(args, io, deps);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -505,9 +519,38 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
|
|||
const agentsRequested = args.agents || entryAction === 'agents';
|
||||
const currentStatus = await readKtxSetupStatus(projectResult.projectDir);
|
||||
let readyAction: string | undefined;
|
||||
if (args.inputMode !== 'disabled' && !agentsRequested && isKtxSetupReady(currentStatus)) {
|
||||
readyAction = (await runKtxSetupReadyChangeMenu(currentStatus, deps.readyMenuDeps)).action;
|
||||
if (readyAction === 'exit') return 0;
|
||||
|
||||
if (args.inputMode !== 'disabled' && !agentsRequested && setupContextActive(currentStatus)) {
|
||||
const contextRunner =
|
||||
deps.context ?? ((contextArgs, contextIo) => runKtxSetupContextStep(contextArgs, contextIo, deps.contextDeps));
|
||||
const contextResult = await contextRunner(
|
||||
{
|
||||
projectDir: projectResult.projectDir,
|
||||
inputMode: args.inputMode,
|
||||
allowEmpty: true,
|
||||
...(autoWatchActiveBuild ? { autoWatch: true } : {}),
|
||||
},
|
||||
io,
|
||||
);
|
||||
autoWatchActiveBuild = false;
|
||||
if (contextResult.status === 'back') {
|
||||
continue;
|
||||
}
|
||||
if (contextResult.status === 'failed' || contextResult.status === 'missing-input') {
|
||||
return 1;
|
||||
}
|
||||
if (contextResult.status !== 'ready') {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (args.inputMode !== 'disabled' && !agentsRequested) {
|
||||
if (isKtxSetupReady(currentStatus)) {
|
||||
readyAction = (await runKtxSetupReadyChangeMenu(currentStatus, deps.readyMenuDeps)).action;
|
||||
if (readyAction === 'exit') return 0;
|
||||
} else if (isKtxPreAgentSetupReady(currentStatus)) {
|
||||
readyAction = 'agents';
|
||||
}
|
||||
}
|
||||
|
||||
const runOnly = readyAction;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue