mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-25 08:48:08 +02:00
Merge origin/main into fix-metabase-readiness
This commit is contained in:
Andrey Avtomonov
commit
60b29bb1e6
173 changed files with 9803 additions and 1140 deletions
|
|
@ -1,7 +1,8 @@
|
|||
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { initKtxProject, parseKtxProjectConfig } from '@ktx/context/project';
|
||||
import type { MetabaseRuntimeClient } from '@ktx/context/ingest';
|
||||
import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from '@ktx/context/project';
|
||||
import type { KtxConnectionDriver, KtxScanConnector, KtxSchemaSnapshot } from '@ktx/context/scan';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKtxConnection } from './connection.js';
|
||||
|
|
@ -598,6 +599,61 @@ describe('runKtxConnection', () => {
|
|||
expect(io.stdout()).toContain('Tables: 2');
|
||||
});
|
||||
|
||||
it('tests a configured Metabase connection through the Metabase runtime client', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
const projectConfig = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
|
||||
await writeFile(
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
serializeKtxProjectConfig({
|
||||
...projectConfig,
|
||||
connections: {
|
||||
...projectConfig.connections,
|
||||
prod_metabase: {
|
||||
driver: 'metabase',
|
||||
api_url: 'http://metabase.example.test',
|
||||
api_key: 'mb_test',
|
||||
},
|
||||
},
|
||||
}),
|
||||
'utf-8',
|
||||
);
|
||||
const testConnection = vi.fn(async () => ({ success: true as const }));
|
||||
const getDatabases = vi.fn(async () => [
|
||||
{ id: 1, name: 'Analytics', engine: 'postgres', details: {}, is_sample: false },
|
||||
{ id: 2, name: 'Sample Database', engine: 'h2', details: {}, is_sample: true },
|
||||
]);
|
||||
const cleanup = vi.fn(async () => undefined);
|
||||
const createMetabaseClient = vi.fn(
|
||||
async (): Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>> => ({
|
||||
testConnection,
|
||||
getDatabases,
|
||||
cleanup,
|
||||
}),
|
||||
);
|
||||
const createScanConnector = vi.fn(async () => {
|
||||
throw new Error('native scanner should not be used for Metabase');
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxConnection({ command: 'test', projectDir, connectionId: 'prod_metabase' }, io.io, {
|
||||
createScanConnector,
|
||||
createMetabaseClient,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(createScanConnector).not.toHaveBeenCalled();
|
||||
expect(createMetabaseClient).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), 'prod_metabase');
|
||||
expect(testConnection).toHaveBeenCalledTimes(1);
|
||||
expect(getDatabases).toHaveBeenCalledTimes(1);
|
||||
expect(cleanup).toHaveBeenCalledTimes(1);
|
||||
expect(io.stdout()).toContain('Connection test passed: prod_metabase');
|
||||
expect(io.stdout()).toContain('Driver: metabase');
|
||||
expect(io.stdout()).toContain('Databases: 1');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('cleans up the native scan connector when connection testing fails', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
|
|
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
import { cancel, confirm, isCancel } from '@clack/prompts';
|
||||
import {
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
DefaultMetabaseConnectionClientFactory,
|
||||
type MetabaseRuntimeClient,
|
||||
metabaseRuntimeConfigFromLocalConnection,
|
||||
} from '@ktx/context/ingest';
|
||||
import { type KtxLocalProject, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
|
||||
import type { KtxScanConnector } from '@ktx/context/scan';
|
||||
import type { KtxConnectionMappingArgs } from './commands/connection-mapping.js';
|
||||
|
|
@ -61,6 +67,7 @@ interface KtxConnectionIo extends KtxCliIo {
|
|||
|
||||
interface KtxConnectionDeps {
|
||||
createScanConnector?: typeof createKtxCliScanConnector;
|
||||
createMetabaseClient?: typeof createDefaultMetabaseClient;
|
||||
runMapping?: (argv: string[], io: KtxCliIo) => Promise<number>;
|
||||
prompts?: KtxConnectionPromptAdapter;
|
||||
}
|
||||
|
|
@ -104,6 +111,12 @@ async function cleanupConnector(connector: KtxScanConnector | null): Promise<voi
|
|||
}
|
||||
}
|
||||
|
||||
function normalizedConnectionDriver(project: KtxLocalProject, connectionId: string): string {
|
||||
return String(project.config.connections[connectionId]?.driver ?? '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
async function testNativeConnection(
|
||||
project: KtxLocalProject,
|
||||
connectionId: string,
|
||||
|
|
@ -131,6 +144,48 @@ async function testNativeConnection(
|
|||
}
|
||||
}
|
||||
|
||||
async function createDefaultMetabaseClient(
|
||||
project: KtxLocalProject,
|
||||
connectionId: string,
|
||||
): Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>> {
|
||||
const factory = new DefaultMetabaseConnectionClientFactory(
|
||||
(metabaseConnectionId) =>
|
||||
metabaseRuntimeConfigFromLocalConnection(
|
||||
metabaseConnectionId,
|
||||
project.config.connections[metabaseConnectionId],
|
||||
),
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
);
|
||||
return factory.createClient(connectionId);
|
||||
}
|
||||
|
||||
async function testMetabaseConnection(
|
||||
project: KtxLocalProject,
|
||||
connectionId: string,
|
||||
createMetabaseClient: typeof createDefaultMetabaseClient,
|
||||
): Promise<{ driver: 'metabase'; databaseCount: number }> {
|
||||
let client: Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'> | null = null;
|
||||
try {
|
||||
client = await createMetabaseClient(project, connectionId);
|
||||
const testResult = await client.testConnection();
|
||||
if (!testResult.success) {
|
||||
throw new Error(
|
||||
`Metabase connection test failed: ${testResult.error ?? testResult.message ?? 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
|
||||
const databases = await client.getDatabases();
|
||||
const databaseCount = databases.filter((database) => database.is_sample !== true).length;
|
||||
if (databaseCount === 0) {
|
||||
throw new Error('Metabase auth worked but no usable databases were returned');
|
||||
}
|
||||
|
||||
return { driver: 'metabase', databaseCount };
|
||||
} finally {
|
||||
await client?.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
interface BufferedIo extends KtxCliIo {
|
||||
stdoutText(): string;
|
||||
stderrText(): string;
|
||||
|
|
@ -399,6 +454,18 @@ export async function runKtxConnection(
|
|||
return 0;
|
||||
}
|
||||
|
||||
if (normalizedConnectionDriver(project, args.connectionId) === 'metabase') {
|
||||
const result = await testMetabaseConnection(
|
||||
project,
|
||||
args.connectionId,
|
||||
deps.createMetabaseClient ?? createDefaultMetabaseClient,
|
||||
);
|
||||
io.stdout.write(`Connection test passed: ${args.connectionId}\n`);
|
||||
io.stdout.write(`Driver: ${result.driver}\n`);
|
||||
io.stdout.write(`Databases: ${result.databaseCount}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const result = await testNativeConnection(
|
||||
project,
|
||||
args.connectionId,
|
||||
|
|
|
|||
|
|
@ -91,22 +91,17 @@ describe('demo assets', () => {
|
|||
expect(manifest.sources.bi.explores).toBeGreaterThanOrEqual(2);
|
||||
expect(manifest.sources.bi.dashboards).toBeGreaterThanOrEqual(2);
|
||||
expect(manifest.sources.notion.pages).toBeGreaterThanOrEqual(5);
|
||||
expect(manifest.generated.semanticLayer.sourceCount).toBeGreaterThanOrEqual(5);
|
||||
expect(manifest.generated.knowledge.pageCount).toBeGreaterThanOrEqual(10);
|
||||
expect(manifest.generated.semanticLayer.sourceCount).toBeGreaterThanOrEqual(40);
|
||||
expect(manifest.generated.knowledge.pageCount).toBeGreaterThanOrEqual(20);
|
||||
expect(manifest.generated.links.linkCount).toBeGreaterThanOrEqual(10);
|
||||
|
||||
const dbStat = await stat(packagedDemoAssetPath('demo.db'));
|
||||
expect(dbStat.size).toBeGreaterThan(0);
|
||||
expect(dbStat.size).toBeLessThan(10 * 1024 * 1024);
|
||||
|
||||
await expect(access(packagedDemoAssetPath('raw-sources/warehouse/accounts.csv'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('raw-sources/dbt/schema.yml'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('raw-sources/bi/revenue_exec.dashboard.lookml'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('raw-sources/notion/revenue-reporting-policy.md'))).resolves.toBeUndefined();
|
||||
expect(manifest.generated.semanticLayer.path).toBe('semantic-layer/orbit_demo');
|
||||
|
||||
await expect(access(packagedDemoAssetPath('semantic-layer/orbit_demo/accounts.yaml'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('knowledge/global/arr-contract-first.md'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('semantic-layer/dbt-main/mart_arr_daily.yaml'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('semantic-layer/postgres-warehouse/mart_account_activity.yaml'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('knowledge/global/orbit-company-overview.md'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('links/provenance.json'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('reports/seeded-demo-report.json'))).resolves.toBeUndefined();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -45,14 +45,9 @@ const REQUIRED_SEEDED_ASSET_PATHS = [
|
|||
'demo.db',
|
||||
'manifest.json',
|
||||
DEMO_REPLAY_FILE,
|
||||
join('raw-sources', 'warehouse', 'accounts.csv'),
|
||||
join('raw-sources', 'dbt', 'schema.yml'),
|
||||
join('raw-sources', 'bi', 'revenue_exec.dashboard.lookml'),
|
||||
join('raw-sources', 'notion', 'revenue-reporting-policy.md'),
|
||||
join('semantic-layer', 'orbit_demo', 'accounts.yaml'),
|
||||
join('knowledge', 'global', 'arr-contract-first.md'),
|
||||
join('links', 'provenance.json'),
|
||||
join('reports', 'seeded-demo-report.json'),
|
||||
join('semantic-layer', 'dbt-main', 'mart_arr_daily.yaml'),
|
||||
join('semantic-layer', 'postgres-warehouse', 'mart_account_activity.yaml'),
|
||||
join('knowledge', 'global', 'orbit-company-overview.md'),
|
||||
] as const;
|
||||
|
||||
function assetDir(): string {
|
||||
|
|
|
|||
|
|
@ -53,8 +53,8 @@ describe('seeded demo inspect contract', () => {
|
|||
notion: { label: 'Notion', path: 'raw-sources/notion', pageCount: 8 },
|
||||
},
|
||||
generatedOutputs: {
|
||||
semanticLayer: { path: 'semantic-layer/orbit_demo', manifestSourceCount: 6, fileCount: 6 },
|
||||
knowledge: { path: 'knowledge/global', manifestPageCount: 10, fileCount: 10 },
|
||||
semanticLayer: { path: 'semantic-layer', manifestSourceCount: 46, fileCount: 46 },
|
||||
knowledge: { path: 'knowledge/global', manifestPageCount: 28, fileCount: 28 },
|
||||
links: { path: 'links/provenance.json', manifestLinkCount: 23, linkCount: 23 },
|
||||
reports: { primaryPath: 'reports/seeded-demo-report.json', fileCount: 1 },
|
||||
replays: { primaryPath: 'replays/replay.memory-flow.v1.json', latestPath: 'replays/latest.memory-flow.v1.json' },
|
||||
|
|
@ -83,8 +83,8 @@ describe('seeded demo inspect contract', () => {
|
|||
expect(output).toContain('dbt: 3 models, 8 source tables');
|
||||
expect(output).toContain('BI: 5 explores, 2 dashboards');
|
||||
expect(output).toContain('Notion: 8 pages');
|
||||
expect(output).toContain('Semantic-layer sources: 6 manifest, 6 files');
|
||||
expect(output).toContain('Knowledge pages: 10 manifest, 10 files');
|
||||
expect(output).toContain('Semantic-layer sources: 46 manifest, 46 files');
|
||||
expect(output).toContain('Knowledge pages: 28 manifest, 28 files');
|
||||
expect(output).toContain('Evidence links: 23 manifest, 23 links');
|
||||
expect(output).toContain('Report: reports/seeded-demo-report.json');
|
||||
expect(output).toContain('Replay: replays/replay.memory-flow.v1.json');
|
||||
|
|
|
|||
|
|
@ -71,12 +71,9 @@ const REQUIRED_SEEDED_PROJECT_PATHS = [
|
|||
'state.sqlite',
|
||||
'manifest.json',
|
||||
join('replays', 'replay.memory-flow.v1.json'),
|
||||
join('raw-sources', 'warehouse', 'accounts.csv'),
|
||||
join('raw-sources', 'dbt', 'schema.yml'),
|
||||
join('raw-sources', 'bi', 'revenue_exec.dashboard.lookml'),
|
||||
join('raw-sources', 'notion', 'revenue-reporting-policy.md'),
|
||||
join('semantic-layer', 'orbit_demo', 'accounts.yaml'),
|
||||
join('knowledge', 'global', 'arr-contract-first.md'),
|
||||
join('semantic-layer', 'dbt-main', 'mart_arr_daily.yaml'),
|
||||
join('semantic-layer', 'postgres-warehouse', 'mart_account_activity.yaml'),
|
||||
join('knowledge', 'global', 'orbit-company-overview.md'),
|
||||
join('links', 'provenance.json'),
|
||||
join('reports', 'seeded-demo-report.json'),
|
||||
] as const;
|
||||
|
|
|
|||
|
|
@ -19,11 +19,9 @@ describe('demo seeded mode', () => {
|
|||
await expect(access(join(projectDir, 'demo.db'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'ktx.yaml'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'manifest.json'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'semantic-layer/orbit_demo/accounts.yaml'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'knowledge/global/arr-contract-first.md'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'raw-sources/dbt/schema.yml'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'raw-sources/bi/revenue_exec.dashboard.lookml'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'raw-sources/notion/revenue-reporting-policy.md'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'semantic-layer/dbt-main/mart_arr_daily.yaml'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'semantic-layer/postgres-warehouse/mart_account_activity.yaml'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'knowledge/global/orbit-company-overview.md'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'links/provenance.json'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'reports/seeded-demo-report.json'))).resolves.toBeUndefined();
|
||||
});
|
||||
|
|
@ -88,8 +86,8 @@ describe('demo seeded mode', () => {
|
|||
|
||||
it('SL YAML validates correctly', async () => {
|
||||
await ensureSeededDemoProject({ projectDir, force: false });
|
||||
const slYaml = await readFile(join(projectDir, 'semantic-layer/orbit_demo/accounts.yaml'), 'utf-8');
|
||||
expect(slYaml).toContain('name: accounts');
|
||||
const slYaml = await readFile(join(projectDir, 'semantic-layer/dbt-main/mart_arr_daily.yaml'), 'utf-8');
|
||||
expect(slYaml).toContain('name: mart_arr_daily');
|
||||
expect(slYaml).toContain('grain:');
|
||||
expect(slYaml).toContain('columns:');
|
||||
expect(slYaml).toContain('measures:');
|
||||
|
|
@ -98,11 +96,11 @@ describe('demo seeded mode', () => {
|
|||
|
||||
it('wiki pages have valid frontmatter', async () => {
|
||||
await ensureSeededDemoProject({ projectDir, force: false });
|
||||
const wiki = await readFile(join(projectDir, 'knowledge/global/arr-contract-first.md'), 'utf-8');
|
||||
const wiki = await readFile(join(projectDir, 'knowledge/global/orbit-company-overview.md'), 'utf-8');
|
||||
expect(wiki).toContain('---');
|
||||
expect(wiki).toContain('summary:');
|
||||
expect(wiki).toContain('tags:');
|
||||
expect(wiki).toContain('sl_refs:');
|
||||
expect(wiki).toContain('refs:');
|
||||
expect(wiki).toContain('usage_mode: auto');
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import {
|
|||
writeWarehouseConfig,
|
||||
} from './ingest.test-utils.js';
|
||||
import { resetVizFallbackWarningsForTest } from './viz-fallback.js';
|
||||
import { runKtxSetup } from './setup.js';
|
||||
|
||||
describe('runKtxIngest', () => {
|
||||
let tempDir: string;
|
||||
|
|
@ -105,6 +106,75 @@ describe('runKtxIngest', () => {
|
|||
expect(statusIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('prints provider setup guidance when a skip-llm setup project runs dev ingest', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const setupIo = makeIo();
|
||||
await expect(
|
||||
runKtxSetup(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
mode: 'new',
|
||||
agents: false,
|
||||
agentScope: 'project',
|
||||
agentInstallMode: 'cli',
|
||||
skipAgents: true,
|
||||
inputMode: 'disabled',
|
||||
yes: true,
|
||||
cliVersion: '0.0.0-test',
|
||||
skipLlm: true,
|
||||
skipEmbeddings: true,
|
||||
databaseDrivers: ['postgres'],
|
||||
databaseConnectionId: 'warehouse',
|
||||
databaseUrl: 'env:WAREHOUSE_URL',
|
||||
databaseSchemas: [],
|
||||
enableHistoricSql: true,
|
||||
skipDatabases: false,
|
||||
skipSources: true,
|
||||
},
|
||||
setupIo.io,
|
||||
{
|
||||
databasesDeps: {
|
||||
testConnection: async (_projectDir, _connectionId, io) => {
|
||||
io.stdout.write('Driver: postgres\nTables: 1\n');
|
||||
return 0;
|
||||
},
|
||||
scanConnection: async () => 0,
|
||||
historicSqlProbe: async () => ({ ok: true, lines: ['PASS Historic SQL probe skipped in test'] }),
|
||||
},
|
||||
context: async () => ({ status: 'skipped', projectDir }),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
const runIo = makeIo();
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'historic-sql',
|
||||
sourceDir,
|
||||
outputMode: 'plain',
|
||||
},
|
||||
runIo.io,
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(runIo.stdout()).toBe('');
|
||||
expect(runIo.stderr()).toContain(
|
||||
'ktx dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner.',
|
||||
);
|
||||
expect(runIo.stderr()).toContain(
|
||||
`ktx setup --project-dir ${projectDir} --anthropic-api-key-env ANTHROPIC_API_KEY --anthropic-model claude-sonnet-4-6 --no-input`,
|
||||
);
|
||||
});
|
||||
|
||||
it('routes metabase scheduled pulls to the fan-out runner and prints child summaries', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeMetabaseConfig(projectDir);
|
||||
|
|
@ -918,6 +988,97 @@ describe('runKtxIngest', () => {
|
|||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('prints plain WorkUnit step progress during long-running local ingest', async () => {
|
||||
const projectDir = join(tempDir, 'historic-sql-step-progress-project');
|
||||
await mkdir(projectDir, { recursive: true });
|
||||
await writeFile(
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: historic-sql-step-progress-project',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:WAREHOUSE_DATABASE_URL',
|
||||
' historicSql:',
|
||||
' enabled: true',
|
||||
' dialect: postgres',
|
||||
' minExecutions: 2',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - historic-sql',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const createdAdapters: SourceAdapter[] = [
|
||||
{ source: 'historic-sql', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) },
|
||||
];
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions) => {
|
||||
input.memoryFlow?.update({
|
||||
plannedWorkUnits: [
|
||||
{
|
||||
unitKey: 'historic-sql-table-public-orders',
|
||||
rawFiles: ['tables/public/orders.json'],
|
||||
peerFileCount: 0,
|
||||
dependencyCount: 0,
|
||||
},
|
||||
{
|
||||
unitKey: 'historic-sql-table-public-customers',
|
||||
rawFiles: ['tables/public/customers.json'],
|
||||
peerFileCount: 0,
|
||||
dependencyCount: 0,
|
||||
},
|
||||
],
|
||||
});
|
||||
input.memoryFlow?.emit({ type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 });
|
||||
input.memoryFlow?.emit({
|
||||
type: 'work_unit_started',
|
||||
unitKey: 'historic-sql-table-public-orders',
|
||||
skills: ['historic_sql_table_digest'],
|
||||
stepBudget: 40,
|
||||
});
|
||||
input.memoryFlow?.emit({
|
||||
type: 'work_unit_step',
|
||||
unitKey: 'historic-sql-table-public-orders',
|
||||
stepIndex: 7,
|
||||
stepBudget: 40,
|
||||
});
|
||||
input.memoryFlow?.emit({
|
||||
type: 'work_unit_finished',
|
||||
unitKey: 'historic-sql-table-public-orders',
|
||||
status: 'success',
|
||||
});
|
||||
input.memoryFlow?.finish('done');
|
||||
return completedLocalBundleRun(input, input.jobId ?? 'historic-step-progress-job');
|
||||
});
|
||||
const io = makeIo({ isTTY: true });
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'historic-sql',
|
||||
outputMode: 'plain',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
env: interactiveEnv(),
|
||||
createAdapters: vi.fn(() => createdAdapters as never),
|
||||
runLocalIngest: runLocal,
|
||||
jobIdFactory: () => 'historic-step-progress-job',
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const stdout = io.stdout();
|
||||
expect(stdout).toContain('[45%] Planned 2 work units');
|
||||
expect(stdout).toContain('[55%] Processing 1/2 work units: historic-sql-table-public-orders');
|
||||
expect(stdout).toContain('[58%] Processing 1/2 work units: historic-sql-table-public-orders step 7/40');
|
||||
expect(stdout).toContain('[68%] Processed 1/2 work units');
|
||||
});
|
||||
|
||||
it('passes local Looker pull-config options and agent runner into scheduled ingest for Looker scheduled ingest', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
|
|
|
|||
|
|
@ -168,13 +168,37 @@ function formatDiffProgress(event: Extract<MemoryFlowEvent, { type: 'diff_comput
|
|||
return `+${event.added}/~${event.modified}/-${event.deleted}/=${event.unchanged}`;
|
||||
}
|
||||
|
||||
function completedWorkUnitCount(snapshot: MemoryFlowReplayInput): number {
|
||||
return snapshot.events.filter((event) => event.type === 'work_unit_finished').length;
|
||||
function workUnitEventsThrough(snapshot: MemoryFlowReplayInput, eventIndex: number): MemoryFlowEvent[] {
|
||||
return snapshot.events.slice(0, eventIndex + 1);
|
||||
}
|
||||
|
||||
function completedWorkUnitCountThrough(snapshot: MemoryFlowReplayInput, eventIndex: number): number {
|
||||
return workUnitEventsThrough(snapshot, eventIndex).filter((event) => event.type === 'work_unit_finished').length;
|
||||
}
|
||||
|
||||
function plannedWorkUnitCountThrough(snapshot: MemoryFlowReplayInput, eventIndex: number): number {
|
||||
if (snapshot.plannedWorkUnits.length > 0) {
|
||||
return snapshot.plannedWorkUnits.length;
|
||||
}
|
||||
const planEvent = workUnitEventsThrough(snapshot, eventIndex)
|
||||
.filter((event) => event.type === 'chunks_planned')
|
||||
.at(-1);
|
||||
return planEvent?.workUnitCount ?? completedWorkUnitCountThrough(snapshot, eventIndex);
|
||||
}
|
||||
|
||||
function workUnitOrdinalThrough(snapshot: MemoryFlowReplayInput, eventIndex: number, unitKey: string): number {
|
||||
const events = workUnitEventsThrough(snapshot, eventIndex);
|
||||
const startedIndex = events.findIndex((event) => event.type === 'work_unit_started' && event.unitKey === unitKey);
|
||||
if (startedIndex === -1) {
|
||||
return completedWorkUnitCountThrough(snapshot, eventIndex) + 1;
|
||||
}
|
||||
return events.slice(0, startedIndex + 1).filter((event) => event.type === 'work_unit_started').length;
|
||||
}
|
||||
|
||||
function plainIngestEventProgress(
|
||||
event: MemoryFlowEvent,
|
||||
snapshot: MemoryFlowReplayInput,
|
||||
eventIndex: number,
|
||||
): { percent: number; message: string } | null {
|
||||
switch (event.type) {
|
||||
case 'source_acquired':
|
||||
|
|
@ -196,11 +220,27 @@ function plainIngestEventProgress(
|
|||
};
|
||||
case 'stage_skipped':
|
||||
return { percent: 45, message: `Skipped ${event.stage}: ${event.reason}` };
|
||||
case 'work_unit_started':
|
||||
return { percent: 55, message: `Processing ${event.unitKey}` };
|
||||
case 'work_unit_started': {
|
||||
const total = plannedWorkUnitCountThrough(snapshot, eventIndex);
|
||||
const ordinal = workUnitOrdinalThrough(snapshot, eventIndex, event.unitKey);
|
||||
const progress = total > 0 ? `${ordinal}/${total} work units: ` : '';
|
||||
return { percent: 55, message: `Processing ${progress}${event.unitKey}` };
|
||||
}
|
||||
case 'work_unit_step': {
|
||||
const total = plannedWorkUnitCountThrough(snapshot, eventIndex);
|
||||
const completed = completedWorkUnitCountThrough(snapshot, eventIndex);
|
||||
const ordinal = workUnitOrdinalThrough(snapshot, eventIndex, event.unitKey);
|
||||
const stepFraction = event.stepBudget > 0 ? Math.min(1, event.stepIndex / event.stepBudget) : 0;
|
||||
const percent = total > 0 ? 55 + Math.ceil(((completed + stepFraction) / total) * 25) : 55;
|
||||
const progress = total > 0 ? `${ordinal}/${total} work units: ` : '';
|
||||
return {
|
||||
percent,
|
||||
message: `Processing ${progress}${event.unitKey} step ${event.stepIndex}/${event.stepBudget}`,
|
||||
};
|
||||
}
|
||||
case 'work_unit_finished': {
|
||||
const total = snapshot.plannedWorkUnits.length || completedWorkUnitCount(snapshot);
|
||||
const completed = completedWorkUnitCount(snapshot);
|
||||
const total = plannedWorkUnitCountThrough(snapshot, eventIndex);
|
||||
const completed = completedWorkUnitCountThrough(snapshot, eventIndex);
|
||||
const percent = total > 0 ? 55 + Math.round((completed / total) * 25) : 80;
|
||||
return {
|
||||
percent,
|
||||
|
|
@ -225,7 +265,6 @@ function plainIngestEventProgress(
|
|||
case 'report_created':
|
||||
return { percent: 98, message: `Created ingest report ${event.reportPath ?? event.runId}` };
|
||||
case 'scope_detected':
|
||||
case 'work_unit_step':
|
||||
case 'candidate_action':
|
||||
return null;
|
||||
}
|
||||
|
|
@ -259,11 +298,12 @@ function createPlainIngestProgressRenderer(
|
|||
},
|
||||
update(snapshot) {
|
||||
while (printedEvents < snapshot.events.length) {
|
||||
const eventIndex = printedEvents;
|
||||
const event = snapshot.events[printedEvents++];
|
||||
if (!event) {
|
||||
continue;
|
||||
}
|
||||
const progress = plainIngestEventProgress(event, snapshot);
|
||||
const progress = plainIngestEventProgress(event, snapshot, eventIndex);
|
||||
if (progress) {
|
||||
write(progress.percent, progress.message);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -170,6 +170,41 @@ describe('managed Python daemon lifecycle', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('makes a final health probe before reporting startup failure', async () => {
|
||||
const spawnDaemon = makeSpawn(5556);
|
||||
const installRuntime = vi.fn(async () => installResult(tempDir));
|
||||
const fetch = vi
|
||||
.fn<ManagedPythonDaemonFetch>()
|
||||
.mockRejectedValueOnce(new Error('fetch failed'))
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({ status: 'healthy', version: '0.2.0' }),
|
||||
text: async () => '',
|
||||
});
|
||||
|
||||
const result = await startManagedPythonDaemon({
|
||||
cliVersion: '0.2.0',
|
||||
runtimeRoot: join(tempDir, 'runtime'),
|
||||
features: ['core'],
|
||||
installRuntime,
|
||||
spawnDaemon,
|
||||
fetch,
|
||||
allocatePort: vi.fn(async () => 61234),
|
||||
now: () => new Date('2026-05-11T00:00:00.000Z'),
|
||||
startupTimeoutMs: 5,
|
||||
pollIntervalMs: 20,
|
||||
});
|
||||
|
||||
expect(result.status).toBe('started');
|
||||
expect(fetch).toHaveBeenCalledTimes(2);
|
||||
expect(JSON.parse(await readFile(layout(tempDir).daemonStatePath, 'utf8'))).toMatchObject({
|
||||
pid: 5556,
|
||||
port: 61234,
|
||||
version: '0.2.0',
|
||||
});
|
||||
});
|
||||
|
||||
it('reuses a healthy daemon with the requested feature set', async () => {
|
||||
await mkdir(layout(tempDir).versionDir, { recursive: true });
|
||||
await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`);
|
||||
|
|
|
|||
|
|
@ -273,6 +273,15 @@ async function waitForHealth(input: {
|
|||
lastDetail = health.detail;
|
||||
await delay(input.pollIntervalMs);
|
||||
}
|
||||
const finalHealth = await healthOk({
|
||||
state: input.state,
|
||||
cliVersion: input.cliVersion,
|
||||
fetch: input.fetch,
|
||||
});
|
||||
if (finalHealth.ok) {
|
||||
return;
|
||||
}
|
||||
lastDetail = finalHealth.detail;
|
||||
throw new Error(`KTX Python daemon failed to start: ${lastDetail}. stderr: ${input.state.stderrLog}`);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -161,6 +161,14 @@ describe('verifyRuntimeAsset', () => {
|
|||
|
||||
await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow(/Unsafe runtime wheel filename/);
|
||||
});
|
||||
|
||||
it('reports the source-checkout artifact command when the bundled manifest is missing', async () => {
|
||||
const assetDir = join(tempDir, 'packages', 'cli', 'assets', 'python');
|
||||
|
||||
await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow(
|
||||
/Missing bundled Python runtime manifest.*pnpm run artifacts:build/s,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('installManagedPythonRuntime', () => {
|
||||
|
|
@ -210,6 +218,30 @@ describe('installManagedPythonRuntime', () => {
|
|||
expect(manifest.python.daemonExecutable).toBe(result.layout.daemonPath);
|
||||
});
|
||||
|
||||
it('disables repo uv config for managed runtime uv commands', async () => {
|
||||
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
|
||||
const commands: Array<{ command: string; args: string[]; env?: NodeJS.ProcessEnv }> = [];
|
||||
const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args, options) => {
|
||||
commands.push({ command, args, env: options?.env });
|
||||
return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.11.13\n' : '', stderr: '' };
|
||||
});
|
||||
|
||||
await installManagedPythonRuntime({
|
||||
cliVersion: '0.2.0',
|
||||
runtimeRoot: join(tempDir, 'runtime'),
|
||||
assetDir,
|
||||
env: { PATH: '/opt/homebrew/bin', UV_NO_CONFIG: '0' },
|
||||
features: ['core'],
|
||||
exec,
|
||||
});
|
||||
|
||||
expect(commands.map((call) => [call.command, call.args[0], call.env?.UV_NO_CONFIG, call.env?.PATH])).toEqual([
|
||||
['uv', '--version', '1', '/opt/homebrew/bin'],
|
||||
['uv', 'venv', '1', '/opt/homebrew/bin'],
|
||||
['uv', 'pip', '1', '/opt/homebrew/bin'],
|
||||
]);
|
||||
});
|
||||
|
||||
it('installs the local-embeddings extra when requested', async () => {
|
||||
const { assetDir } = await writeAsset(tempDir, 'embedding-wheel');
|
||||
const commands: Array<{ command: string; args: string[] }> = [];
|
||||
|
|
|
|||
|
|
@ -186,9 +186,28 @@ async function readJsonFile(path: string): Promise<unknown> {
|
|||
return JSON.parse(await readFile(path, 'utf8')) as unknown;
|
||||
}
|
||||
|
||||
function isErrnoException(error: unknown, code: string): boolean {
|
||||
return typeof error === 'object' && error !== null && 'code' in error && error.code === code;
|
||||
}
|
||||
|
||||
export async function verifyRuntimeAsset(input: { assetDir: string }): Promise<ManagedRuntimeAsset> {
|
||||
const manifestPath = join(input.assetDir, 'manifest.json');
|
||||
const manifest = runtimeAssetManifestSchema.parse(await readJsonFile(manifestPath));
|
||||
let manifestData: unknown;
|
||||
try {
|
||||
manifestData = await readJsonFile(manifestPath);
|
||||
} catch (error) {
|
||||
if (isErrnoException(error, 'ENOENT')) {
|
||||
throw new Error(
|
||||
[
|
||||
`Missing bundled Python runtime manifest: ${manifestPath}`,
|
||||
'In a source checkout, build the local runtime assets with: pnpm run artifacts:build',
|
||||
'Then retry the runtime-backed KTX command.',
|
||||
].join('\n'),
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const manifest = runtimeAssetManifestSchema.parse(manifestData);
|
||||
assertSafeWheelFilename(manifest.wheel.file);
|
||||
const wheelPath = join(input.assetDir, manifest.wheel.file);
|
||||
const wheel = await readFile(wheelPath);
|
||||
|
|
@ -243,10 +262,11 @@ async function runLogged(input: {
|
|||
command: string;
|
||||
args: string[];
|
||||
cwd?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): Promise<{ stdout: string; stderr: string }> {
|
||||
await appendFile(input.logPath, `$ ${input.command} ${input.args.join(' ')}\n`);
|
||||
try {
|
||||
const result = await input.exec(input.command, input.args, { cwd: input.cwd });
|
||||
const result = await input.exec(input.command, input.args, { cwd: input.cwd, env: input.env });
|
||||
if (result.stdout) {
|
||||
await appendFile(input.logPath, result.stdout.endsWith('\n') ? result.stdout : `${result.stdout}\n`);
|
||||
}
|
||||
|
|
@ -266,9 +286,13 @@ async function runLogged(input: {
|
|||
}
|
||||
}
|
||||
|
||||
async function ensureUv(exec: ManagedPythonRuntimeExec): Promise<string> {
|
||||
function managedRuntimeUvEnv(baseEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
||||
return { ...baseEnv, UV_NO_CONFIG: '1' };
|
||||
}
|
||||
|
||||
async function ensureUv(exec: ManagedPythonRuntimeExec, env?: NodeJS.ProcessEnv): Promise<string> {
|
||||
try {
|
||||
const result = await exec('uv', ['--version']);
|
||||
const result = await exec('uv', ['--version'], { env });
|
||||
return result.stdout.trim() || 'uv available';
|
||||
} catch {
|
||||
throw new Error(MISSING_UV_RUNTIME_INSTALL_MESSAGE);
|
||||
|
|
@ -282,6 +306,7 @@ export async function installManagedPythonRuntime(
|
|||
const exec = options.exec ?? defaultExec;
|
||||
const features = normalizeFeatures(options.features);
|
||||
const asset = await verifyRuntimeAsset({ assetDir: layout.assetDir });
|
||||
const uvEnv = managedRuntimeUvEnv(options.env ?? process.env);
|
||||
const existing = await readInstalledManifest(layout.manifestPath);
|
||||
if (
|
||||
options.force !== true &&
|
||||
|
|
@ -298,14 +323,21 @@ export async function installManagedPythonRuntime(
|
|||
await rm(layout.versionDir, { recursive: true, force: true });
|
||||
await mkdir(layout.versionDir, { recursive: true });
|
||||
await writeFile(layout.installLogPath, '');
|
||||
await ensureUv(exec);
|
||||
await runLogged({ exec, logPath: layout.installLogPath, command: 'uv', args: ['venv', layout.venvDir] });
|
||||
await ensureUv(exec, uvEnv);
|
||||
await runLogged({
|
||||
exec,
|
||||
logPath: layout.installLogPath,
|
||||
command: 'uv',
|
||||
args: ['venv', layout.venvDir],
|
||||
env: uvEnv,
|
||||
});
|
||||
const wheelSpec = features.includes('local-embeddings') ? `${asset.wheelPath}[local-embeddings]` : asset.wheelPath;
|
||||
await runLogged({
|
||||
exec,
|
||||
logPath: layout.installLogPath,
|
||||
command: 'uv',
|
||||
args: ['pip', 'install', '--python', layout.pythonPath, wheelSpec],
|
||||
env: uvEnv,
|
||||
});
|
||||
|
||||
const manifest: InstalledKtxRuntimeManifest = {
|
||||
|
|
@ -371,7 +403,7 @@ export async function doctorManagedPythonRuntime(
|
|||
const exec = options.exec ?? defaultExec;
|
||||
const checks: ManagedPythonRuntimeDoctorCheck[] = [];
|
||||
try {
|
||||
const version = await ensureUv(exec);
|
||||
const version = await ensureUv(exec, managedRuntimeUvEnv(options.env ?? process.env));
|
||||
checks.push(check('pass', { id: 'uv', label: 'uv', detail: version }));
|
||||
} catch (error) {
|
||||
checks.push(
|
||||
|
|
|
|||
|
|
@ -1295,6 +1295,7 @@ describe('setup databases step', () => {
|
|||
expect(config.connections.warehouse.historicSql).not.toHaveProperty('redactionPatterns');
|
||||
expect(config.connections.warehouse.historicSql).not.toHaveProperty(legacyHistoricSqlServiceAccountPatternsKey);
|
||||
expect(config.ingest.adapters).toContain('historic-sql');
|
||||
expect(config.ingest.workUnits.maxConcurrency).toBe(6);
|
||||
expect(io.stdout()).toContain('Historic SQL probe...');
|
||||
expect(io.stdout()).toContain('pg_stat_statements ready');
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ import { runKtxScan } from './scan.js';
|
|||
import { withSetupInterruptConfirmation } from './setup-interrupt.js';
|
||||
import { writeProjectLocalSecretReference } from './setup-secrets.js';
|
||||
|
||||
const HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY = 6;
|
||||
|
||||
export type KtxSetupDatabaseDriver =
|
||||
| 'sqlite'
|
||||
| 'postgres'
|
||||
|
|
@ -843,7 +845,7 @@ async function writeConnectionConfig(input: {
|
|||
? (input.connection.historicSql as Record<string, unknown>)
|
||||
: null;
|
||||
if (historicSql?.enabled === true) {
|
||||
await ensureHistoricSqlAdapterEnabled(input.projectDir);
|
||||
await ensureHistoricSqlIngestDefaults(input.projectDir);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -954,9 +956,19 @@ async function maybeConfigurePostgresSchemas(input: {
|
|||
return true;
|
||||
}
|
||||
|
||||
async function ensureHistoricSqlAdapterEnabled(projectDir: string): Promise<void> {
|
||||
async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise<void> {
|
||||
const project = await loadKtxProject({ projectDir });
|
||||
if (project.config.ingest.adapters.includes('historic-sql')) {
|
||||
const adapters = project.config.ingest.adapters.includes('historic-sql')
|
||||
? project.config.ingest.adapters
|
||||
: [...project.config.ingest.adapters, 'historic-sql'];
|
||||
const maxConcurrency = Math.max(
|
||||
project.config.ingest.workUnits.maxConcurrency,
|
||||
HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY,
|
||||
);
|
||||
if (
|
||||
adapters === project.config.ingest.adapters &&
|
||||
maxConcurrency === project.config.ingest.workUnits.maxConcurrency
|
||||
) {
|
||||
return;
|
||||
}
|
||||
await writeFile(
|
||||
|
|
@ -965,7 +977,11 @@ async function ensureHistoricSqlAdapterEnabled(projectDir: string): Promise<void
|
|||
...project.config,
|
||||
ingest: {
|
||||
...project.config.ingest,
|
||||
adapters: [...project.config.ingest.adapters, 'historic-sql'],
|
||||
adapters,
|
||||
workUnits: {
|
||||
...project.config.ingest.workUnits,
|
||||
maxConcurrency,
|
||||
},
|
||||
},
|
||||
}),
|
||||
'utf-8',
|
||||
|
|
|
|||
270
packages/cli/src/setup-demo-tour.test.ts
Normal file
270
packages/cli/src/setup-demo-tour.test.ts
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { KtxSetupAgentsResult } from './setup-agents.js';
|
||||
import {
|
||||
buildDemoReplayTimeline,
|
||||
DEMO_REPLAY_TARGETS,
|
||||
renderDemoAgentTransition,
|
||||
renderDemoBanner,
|
||||
renderDemoCardContent,
|
||||
renderDemoCompletionSummary,
|
||||
runDemoTour,
|
||||
} from './setup-demo-tour.js';
|
||||
|
||||
/** Strip ANSI escape sequences for plain-text assertions. */
|
||||
function stripAnsi(text: string): string {
|
||||
return text.replace(/\x1b\[[0-9;]*m/g, '');
|
||||
}
|
||||
|
||||
describe('renderDemoBanner', () => {
|
||||
it('contains "Demo mode"', () => {
|
||||
const plain = stripAnsi(renderDemoBanner());
|
||||
expect(plain).toContain('Demo mode');
|
||||
});
|
||||
|
||||
it('mentions pre-processed data', () => {
|
||||
const plain = stripAnsi(renderDemoBanner());
|
||||
expect(plain).toContain('pre-processed');
|
||||
});
|
||||
|
||||
it('mentions read-only', () => {
|
||||
const plain = stripAnsi(renderDemoBanner());
|
||||
expect(plain).toContain('read-only');
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderDemoCardContent', () => {
|
||||
it('contains the title', () => {
|
||||
const plain = stripAnsi(renderDemoCardContent('Database connection', ['Postgres']));
|
||||
expect(plain).toContain('Database connection');
|
||||
});
|
||||
|
||||
it('contains each selection', () => {
|
||||
const plain = stripAnsi(renderDemoCardContent('Sources', ['dbt', 'metabase']));
|
||||
expect(plain).toContain('dbt');
|
||||
expect(plain).toContain('metabase');
|
||||
});
|
||||
|
||||
it('contains navigation hints', () => {
|
||||
const plain = stripAnsi(renderDemoCardContent('Title', ['a']));
|
||||
expect(plain).toContain('Press Enter to continue');
|
||||
expect(plain).toContain('Escape to go back');
|
||||
});
|
||||
|
||||
it('works with multiple selections', () => {
|
||||
const result = renderDemoCardContent('Pick', ['one', 'two', 'three']);
|
||||
const plain = stripAnsi(result);
|
||||
expect(plain).toContain('one');
|
||||
expect(plain).toContain('two');
|
||||
expect(plain).toContain('three');
|
||||
// Each selection gets a ▸ bullet
|
||||
const bullets = (plain.match(/▸/g) ?? []).length;
|
||||
expect(bullets).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderDemoAgentTransition', () => {
|
||||
it('contains "Demo project is ready"', () => {
|
||||
const plain = stripAnsi(renderDemoAgentTransition());
|
||||
expect(plain).toContain('Demo project is ready');
|
||||
});
|
||||
|
||||
it('mentions connecting an agent', () => {
|
||||
const plain = stripAnsi(renderDemoAgentTransition());
|
||||
expect(plain).toContain('connect your agent');
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderDemoCompletionSummary', () => {
|
||||
const projectDir = '/tmp/ktx-demo-123';
|
||||
|
||||
it('includes the project path', () => {
|
||||
const plain = stripAnsi(renderDemoCompletionSummary(projectDir, true));
|
||||
expect(plain).toContain(projectDir);
|
||||
});
|
||||
|
||||
it('includes a temp directory warning', () => {
|
||||
const plain = stripAnsi(renderDemoCompletionSummary(projectDir, true));
|
||||
expect(plain).toContain('temporary directory');
|
||||
});
|
||||
|
||||
it('points to ktx setup for real data', () => {
|
||||
const plain = stripAnsi(renderDemoCompletionSummary(projectDir, true));
|
||||
expect(plain).toContain('ktx setup');
|
||||
});
|
||||
|
||||
it('shows agent-connected message when installed', () => {
|
||||
const plain = stripAnsi(renderDemoCompletionSummary(projectDir, true));
|
||||
expect(plain).toContain('agent is connected');
|
||||
});
|
||||
|
||||
it('includes star headline', () => {
|
||||
const plain = stripAnsi(renderDemoCompletionSummary(projectDir, true));
|
||||
expect(plain).toContain('★ KTX demo is ready');
|
||||
});
|
||||
|
||||
it('shows manual instructions when agent not installed', () => {
|
||||
const plain = stripAnsi(renderDemoCompletionSummary(projectDir, false));
|
||||
expect(plain).toContain('--agents');
|
||||
expect(plain).toContain(`--project-dir ${projectDir}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildDemoReplayTimeline', () => {
|
||||
const timeline = buildDemoReplayTimeline();
|
||||
const connectionIds = new Set(timeline.map((e) => e.connectionId));
|
||||
|
||||
it('produces events for all 4 targets', () => {
|
||||
expect(connectionIds.size).toBe(4);
|
||||
expect(connectionIds).toContain('postgres-warehouse');
|
||||
expect(connectionIds).toContain('dbt-main');
|
||||
expect(connectionIds).toContain('metabase-main');
|
||||
expect(connectionIds).toContain('notion-main');
|
||||
});
|
||||
|
||||
it('all targets end as done', () => {
|
||||
for (const id of connectionIds) {
|
||||
const events = timeline.filter((e) => e.connectionId === id);
|
||||
const last = events[events.length - 1];
|
||||
expect(last.status).toBe('done');
|
||||
}
|
||||
});
|
||||
|
||||
it('events are sorted by delayMs', () => {
|
||||
for (let i = 1; i < timeline.length; i++) {
|
||||
expect(timeline[i].delayMs).toBeGreaterThanOrEqual(timeline[i - 1].delayMs);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('DEMO_REPLAY_TARGETS', () => {
|
||||
it('has 1 primary source', () => {
|
||||
expect(DEMO_REPLAY_TARGETS.primarySources).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('has 3 context sources', () => {
|
||||
expect(DEMO_REPLAY_TARGETS.contextSources).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('primary source is a scan operation', () => {
|
||||
expect(DEMO_REPLAY_TARGETS.primarySources[0].operation).toBe('scan');
|
||||
});
|
||||
|
||||
it('context sources are source-ingest operations', () => {
|
||||
for (const source of DEMO_REPLAY_TARGETS.contextSources) {
|
||||
expect(source.operation).toBe('source-ingest');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('runDemoTour', () => {
|
||||
function createMockIo() {
|
||||
const chunks: string[] = [];
|
||||
return {
|
||||
io: {
|
||||
stdout: { isTTY: true, columns: 80, write: (chunk: string) => { chunks.push(chunk); } },
|
||||
stderr: { write: () => {} },
|
||||
},
|
||||
chunks,
|
||||
};
|
||||
}
|
||||
|
||||
it('returns 0 on successful tour with agent installed', async () => {
|
||||
const { io, chunks } = createMockIo();
|
||||
const mockAgents = vi.fn().mockResolvedValue({
|
||||
status: 'ready',
|
||||
projectDir: '/tmp/test',
|
||||
installs: [{ target: 'claude-code', scope: 'project', mode: 'both' }],
|
||||
} satisfies KtxSetupAgentsResult);
|
||||
|
||||
const navigation = vi.fn().mockResolvedValue('forward');
|
||||
|
||||
const result = await runDemoTour(
|
||||
{ inputMode: 'auto' },
|
||||
io,
|
||||
{
|
||||
agents: mockAgents,
|
||||
waitForNavigation: navigation,
|
||||
skipReplayAnimation: true,
|
||||
ensureProject: vi.fn().mockResolvedValue({ projectDir: '/tmp/test' }),
|
||||
},
|
||||
);
|
||||
expect(result).toBe(0);
|
||||
expect(mockAgents).toHaveBeenCalled();
|
||||
// Should have rendered completion summary
|
||||
const allOutput = chunks.join('');
|
||||
expect(allOutput).toContain('agent is connected');
|
||||
});
|
||||
|
||||
it('handles back navigation from first step by exiting', async () => {
|
||||
const { io } = createMockIo();
|
||||
const navigation = vi.fn().mockResolvedValue('back');
|
||||
|
||||
const result = await runDemoTour(
|
||||
{ inputMode: 'auto' },
|
||||
io,
|
||||
{
|
||||
waitForNavigation: navigation,
|
||||
skipReplayAnimation: true,
|
||||
ensureProject: vi.fn().mockResolvedValue({ projectDir: '/tmp/test' }),
|
||||
},
|
||||
);
|
||||
expect(result).toBe(0);
|
||||
// Navigation called once for databases step, then exits
|
||||
expect(navigation).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('goes back from sources to databases', async () => {
|
||||
const { io } = createMockIo();
|
||||
let callCount = 0;
|
||||
const navigation = vi.fn().mockImplementation(() => {
|
||||
callCount++;
|
||||
// First call (databases): forward
|
||||
// Second call (sources): back
|
||||
// Third call (databases again): back (exit)
|
||||
if (callCount === 1) return Promise.resolve('forward');
|
||||
return Promise.resolve('back');
|
||||
});
|
||||
|
||||
const result = await runDemoTour(
|
||||
{ inputMode: 'auto' },
|
||||
io,
|
||||
{
|
||||
waitForNavigation: navigation,
|
||||
skipReplayAnimation: true,
|
||||
ensureProject: vi.fn().mockResolvedValue({ projectDir: '/tmp/test' }),
|
||||
},
|
||||
);
|
||||
expect(result).toBe(0);
|
||||
expect(navigation).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('handles agent step returning back', async () => {
|
||||
const { io } = createMockIo();
|
||||
let navCount = 0;
|
||||
const navigation = vi.fn().mockImplementation(() => {
|
||||
navCount++;
|
||||
// Forward through databases, sources, context
|
||||
// Then back from context (after agents returns back)
|
||||
// Then back from sources, then back from databases (exit)
|
||||
if (navCount <= 3) return Promise.resolve('forward');
|
||||
return Promise.resolve('back');
|
||||
});
|
||||
|
||||
const mockAgents = vi.fn().mockResolvedValue({
|
||||
status: 'back',
|
||||
projectDir: '/tmp/test',
|
||||
} satisfies KtxSetupAgentsResult);
|
||||
|
||||
const result = await runDemoTour(
|
||||
{ inputMode: 'auto' },
|
||||
io,
|
||||
{
|
||||
agents: mockAgents,
|
||||
waitForNavigation: navigation,
|
||||
skipReplayAnimation: true,
|
||||
ensureProject: vi.fn().mockResolvedValue({ projectDir: '/tmp/test' }),
|
||||
},
|
||||
);
|
||||
expect(result).toBe(0);
|
||||
});
|
||||
});
|
||||
390
packages/cli/src/setup-demo-tour.ts
Normal file
390
packages/cli/src/setup-demo-tour.ts
Normal file
|
|
@ -0,0 +1,390 @@
|
|||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import type {
|
||||
ContextBuildTargetState,
|
||||
ContextBuildViewState,
|
||||
} from './context-build-view.js';
|
||||
import { createRepainter, renderContextBuildView } from './context-build-view.js';
|
||||
import { defaultDemoProjectDir, ensureSeededDemoProject } from './demo-assets.js';
|
||||
import type { KtxPublicIngestPlanTarget } from './public-ingest.js';
|
||||
import type { KtxSetupAgentsResult } from './setup-agents.js';
|
||||
import { runKtxSetupAgentsStep } from './setup-agents.js';
|
||||
import { KtxSetupExitError } from './setup-interrupt.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ANSI helpers (internal)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ESC = String.fromCharCode(0x1b);
|
||||
|
||||
function cyan(text: string): string {
|
||||
return `${ESC}[36m${text}${ESC}[39m`;
|
||||
}
|
||||
|
||||
function dim(text: string): string {
|
||||
return `${ESC}[2m${text}${ESC}[22m`;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Demo target helpers (internal)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function createDemoTarget(
|
||||
connectionId: string,
|
||||
operation: 'scan' | 'source-ingest',
|
||||
driver: string,
|
||||
): KtxPublicIngestPlanTarget {
|
||||
const adapter = operation === 'source-ingest' ? driver : undefined;
|
||||
return {
|
||||
connectionId,
|
||||
driver,
|
||||
operation,
|
||||
...(adapter ? { adapter } : {}),
|
||||
debugCommand: `ktx setup context build --target ${connectionId}`,
|
||||
steps: operation === 'scan'
|
||||
? ['scan', 'enrich', 'memory-update']
|
||||
: ['source-ingest', 'enrich', 'memory-update'],
|
||||
};
|
||||
}
|
||||
|
||||
function createTargetState(target: KtxPublicIngestPlanTarget): ContextBuildTargetState {
|
||||
return {
|
||||
target,
|
||||
status: 'queued',
|
||||
detailLine: null,
|
||||
summaryText: null,
|
||||
startedAt: null,
|
||||
elapsedMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pure rendering functions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function renderDemoBanner(): string {
|
||||
const lines = [
|
||||
'',
|
||||
`┌ ${cyan('Demo mode')} — data has been pre-processed and KTX context is already built.`,
|
||||
'│ This walkthrough illustrates the setup steps. Selections are pre-filled and read-only.',
|
||||
];
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export function renderDemoCardContent(title: string, selections: string[]): string {
|
||||
const lines = [
|
||||
`┌ ${title}`,
|
||||
'│',
|
||||
...selections.map((s) => `│ ${cyan('▸')} ${s}`),
|
||||
'│',
|
||||
`│ ${dim('Press Enter to continue, Escape to go back')}`,
|
||||
'└',
|
||||
];
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export function renderDemoAgentTransition(): string {
|
||||
const lines = [
|
||||
'┌ Demo project is ready — let\'s connect your agent',
|
||||
'│',
|
||||
'│ Your KTX context has been built with demo data.',
|
||||
'│ Select an agent to start using it.',
|
||||
'└',
|
||||
];
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export function renderDemoCompletionSummary(projectDir: string, agentInstalled: boolean): string {
|
||||
const lines: string[] = [
|
||||
'',
|
||||
`${cyan('★')} KTX demo is ready`,
|
||||
'',
|
||||
];
|
||||
|
||||
if (agentInstalled) {
|
||||
lines.push(' Your agent is connected to a demo KTX project.');
|
||||
} else {
|
||||
lines.push(' Demo project created. Connect an agent to start using it:');
|
||||
lines.push(` $ ${cyan(`ktx setup --agents --project-dir ${projectDir}`)}`);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
'',
|
||||
` ${dim('⚠')} This project is in a temporary directory and will be`,
|
||||
' cleaned up by your system. To set up KTX with your own',
|
||||
' data, run: ktx setup',
|
||||
'',
|
||||
` Project: ${projectDir}`,
|
||||
);
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Keypress navigation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function waitForDemoNavigation(
|
||||
stdin?: NodeJS.ReadStream,
|
||||
): Promise<'forward' | 'back'> {
|
||||
const input = stdin ?? process.stdin;
|
||||
const hadRawMode = input.isRaw ?? false;
|
||||
|
||||
return new Promise<'forward' | 'back'>((resolve, reject) => {
|
||||
if (typeof input.setRawMode === 'function') {
|
||||
input.setRawMode(true);
|
||||
}
|
||||
input.resume();
|
||||
|
||||
const cleanup = () => {
|
||||
input.off('data', onData);
|
||||
if (typeof input.setRawMode === 'function') {
|
||||
input.setRawMode(hadRawMode);
|
||||
}
|
||||
};
|
||||
|
||||
const onData = (data: Buffer) => {
|
||||
const char = data.toString();
|
||||
if (char === '\r' || char === '\n') {
|
||||
cleanup();
|
||||
resolve('forward');
|
||||
} else if (char === '\x1b') {
|
||||
cleanup();
|
||||
resolve('back');
|
||||
} else if (char === '\x03') {
|
||||
cleanup();
|
||||
reject(new KtxSetupExitError());
|
||||
}
|
||||
};
|
||||
|
||||
input.on('data', onData);
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Interactive card
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function renderDemoCard(
|
||||
title: string,
|
||||
selections: string[],
|
||||
io: KtxCliIo,
|
||||
stdin?: NodeJS.ReadStream,
|
||||
waitNav: (stdin?: NodeJS.ReadStream) => Promise<'forward' | 'back'> = waitForDemoNavigation,
|
||||
): Promise<'forward' | 'back'> {
|
||||
io.stdout.write(renderDemoBanner() + '\n\n');
|
||||
io.stdout.write(renderDemoCardContent(title, selections) + '\n');
|
||||
return waitNav(stdin);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Context build replay
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface DemoReplayEvent {
|
||||
delayMs: number;
|
||||
connectionId: string;
|
||||
status: 'running' | 'done';
|
||||
detailLine: string | null;
|
||||
summaryText: string | null;
|
||||
}
|
||||
|
||||
export const DEMO_REPLAY_TARGETS = {
|
||||
primarySources: [
|
||||
createDemoTarget('postgres-warehouse', 'scan', 'postgres'),
|
||||
],
|
||||
contextSources: [
|
||||
createDemoTarget('dbt-main', 'source-ingest', 'dbt'),
|
||||
createDemoTarget('metabase-main', 'source-ingest', 'metabase'),
|
||||
createDemoTarget('notion-main', 'source-ingest', 'notion'),
|
||||
],
|
||||
} as const;
|
||||
|
||||
export function buildDemoReplayTimeline(): DemoReplayEvent[] {
|
||||
return [
|
||||
// postgres-warehouse: scan
|
||||
{ delayMs: 0, connectionId: 'postgres-warehouse', status: 'running', detailLine: null, summaryText: null },
|
||||
{ delayMs: 1200, connectionId: 'postgres-warehouse', status: 'running', detailLine: '[50%] scanning tables...', summaryText: null },
|
||||
{ delayMs: 2400, connectionId: 'postgres-warehouse', status: 'done', detailLine: null, summaryText: '56 tables scanned' },
|
||||
// dbt-main
|
||||
{ delayMs: 2400, connectionId: 'dbt-main', status: 'running', detailLine: null, summaryText: null },
|
||||
{ delayMs: 3600, connectionId: 'dbt-main', status: 'running', detailLine: '[60%] ingesting models...', summaryText: null },
|
||||
{ delayMs: 4400, connectionId: 'dbt-main', status: 'done', detailLine: null, summaryText: '34 models ingested' },
|
||||
// metabase-main
|
||||
{ delayMs: 4400, connectionId: 'metabase-main', status: 'running', detailLine: null, summaryText: null },
|
||||
{ delayMs: 5600, connectionId: 'metabase-main', status: 'done', detailLine: null, summaryText: '80 cards ingested' },
|
||||
// notion-main
|
||||
{ delayMs: 5600, connectionId: 'notion-main', status: 'running', detailLine: null, summaryText: null },
|
||||
{ delayMs: 6800, connectionId: 'notion-main', status: 'done', detailLine: null, summaryText: '9 pages ingested' },
|
||||
];
|
||||
}
|
||||
|
||||
function renderDemoContextCompletionSummary(): string {
|
||||
const lines = [
|
||||
'',
|
||||
`${cyan('★')} KTX finished building context`,
|
||||
'',
|
||||
' KTX created:',
|
||||
` ${cyan('📊')} 46 semantic layer definitions`,
|
||||
` ${cyan('📝')} 28 knowledge pages`,
|
||||
'',
|
||||
` ${dim('Press Enter to continue, Escape to go back')}`,
|
||||
'',
|
||||
];
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export async function runDemoContextReplay(
|
||||
io: KtxCliIo,
|
||||
stdin?: NodeJS.ReadStream,
|
||||
): Promise<'forward' | 'back'> {
|
||||
const allPrimary = DEMO_REPLAY_TARGETS.primarySources.map(createTargetState);
|
||||
const allContext = DEMO_REPLAY_TARGETS.contextSources.map(createTargetState);
|
||||
|
||||
const state: ContextBuildViewState = {
|
||||
primarySources: allPrimary,
|
||||
contextSources: allContext,
|
||||
frame: 0,
|
||||
startedAt: Date.now(),
|
||||
totalElapsedMs: 0,
|
||||
};
|
||||
|
||||
const allTargets = [...allPrimary, ...allContext];
|
||||
const timeline = buildDemoReplayTimeline();
|
||||
|
||||
const repainter = createRepainter(io);
|
||||
const paint = () => repainter.paint(renderContextBuildView(state, { styled: true }));
|
||||
|
||||
paint();
|
||||
|
||||
let eventIndex = 0;
|
||||
const startTime = Date.now();
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
const frameInterval = setInterval(() => {
|
||||
const elapsed = Date.now() - startTime;
|
||||
state.frame++;
|
||||
state.totalElapsedMs = elapsed;
|
||||
|
||||
// Apply all events up to the current elapsed time
|
||||
while (eventIndex < timeline.length && timeline[eventIndex].delayMs <= elapsed) {
|
||||
const event = timeline[eventIndex];
|
||||
const target = allTargets.find((t) => t.target.connectionId === event.connectionId);
|
||||
if (target) {
|
||||
target.status = event.status;
|
||||
target.detailLine = event.detailLine;
|
||||
if (event.summaryText !== null) {
|
||||
target.summaryText = event.summaryText;
|
||||
}
|
||||
if (event.status === 'running' && target.startedAt === null) {
|
||||
target.startedAt = Date.now();
|
||||
}
|
||||
if (event.status === 'done') {
|
||||
target.elapsedMs = target.startedAt !== null ? Date.now() - target.startedAt : 0;
|
||||
}
|
||||
}
|
||||
eventIndex++;
|
||||
}
|
||||
|
||||
// Update running target elapsed times
|
||||
for (const t of allTargets) {
|
||||
if (t.status === 'running' && t.startedAt !== null) {
|
||||
t.elapsedMs = Date.now() - t.startedAt;
|
||||
}
|
||||
}
|
||||
|
||||
paint();
|
||||
|
||||
// Check if all events have been applied
|
||||
if (eventIndex >= timeline.length) {
|
||||
clearInterval(frameInterval);
|
||||
resolve();
|
||||
}
|
||||
}, 120);
|
||||
});
|
||||
|
||||
// Final paint with all done
|
||||
paint();
|
||||
|
||||
// Show completion summary and wait for navigation
|
||||
io.stdout.write(renderDemoContextCompletionSummary() + '\n');
|
||||
return waitForDemoNavigation(stdin);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Demo tour orchestrator
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type DemoStep = 'databases' | 'sources' | 'context' | 'agents';
|
||||
|
||||
const DEMO_STEPS: DemoStep[] = ['databases', 'sources', 'context', 'agents'];
|
||||
|
||||
export interface DemoTourDeps {
|
||||
agents?: (args: Parameters<typeof runKtxSetupAgentsStep>[0], io: KtxCliIo) => Promise<KtxSetupAgentsResult>;
|
||||
waitForNavigation?: (stdin?: NodeJS.ReadStream) => Promise<'forward' | 'back'>;
|
||||
ensureProject?: typeof ensureSeededDemoProject;
|
||||
skipReplayAnimation?: boolean;
|
||||
}
|
||||
|
||||
export async function runDemoTour(
|
||||
args: { inputMode: 'auto' | 'disabled' },
|
||||
io: KtxCliIo,
|
||||
deps: DemoTourDeps = {},
|
||||
): Promise<number> {
|
||||
const waitNav = deps.waitForNavigation ?? waitForDemoNavigation;
|
||||
const ensureProject = deps.ensureProject ?? ensureSeededDemoProject;
|
||||
|
||||
const projectDir = defaultDemoProjectDir();
|
||||
await ensureProject({ projectDir, force: false });
|
||||
|
||||
let stepIndex = 0;
|
||||
|
||||
while (stepIndex < DEMO_STEPS.length) {
|
||||
const step = DEMO_STEPS[stepIndex]!;
|
||||
let direction: 'forward' | 'back';
|
||||
|
||||
if (step === 'databases') {
|
||||
direction = await renderDemoCard('Database connection', ['PostgreSQL — Orbit Analytics (56 tables, 2 schemas)'], io, undefined, waitNav);
|
||||
} else if (step === 'sources') {
|
||||
direction = await renderDemoCard('Context sources', ['dbt — 34 transformation models', 'Metabase — 80 dashboard cards', 'Notion — 9 knowledge pages'], io, undefined, waitNav);
|
||||
} else if (step === 'context') {
|
||||
io.stdout.write(renderDemoBanner() + '\n\n');
|
||||
if (deps.skipReplayAnimation) {
|
||||
direction = await waitNav();
|
||||
} else {
|
||||
direction = await runDemoContextReplay(io);
|
||||
}
|
||||
} else {
|
||||
// agents step — real interactive
|
||||
io.stdout.write(renderDemoAgentTransition() + '\n');
|
||||
const agentsRunner = deps.agents ?? runKtxSetupAgentsStep;
|
||||
const agentsResult = await agentsRunner(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: args.inputMode,
|
||||
yes: false,
|
||||
agents: true,
|
||||
scope: 'project',
|
||||
mode: 'both',
|
||||
skipAgents: false,
|
||||
},
|
||||
io,
|
||||
);
|
||||
const agentInstalled = agentsResult.status === 'ready';
|
||||
if (agentsResult.status === 'back') {
|
||||
direction = 'back';
|
||||
} else {
|
||||
io.stdout.write(renderDemoCompletionSummary(projectDir, agentInstalled) + '\n');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (direction === 'back') {
|
||||
if (stepIndex === 0) return 0;
|
||||
stepIndex -= 1;
|
||||
} else {
|
||||
stepIndex += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -676,4 +676,53 @@ describe('setup Anthropic model step', () => {
|
|||
).resolves.toMatchObject({ status: 'ready' });
|
||||
expect(healthCheck).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
backend: 'vertex',
|
||||
providerLines: [' backend: vertex', ' vertex:', ' project: kaelio-dev', ' location: us-east5'],
|
||||
model: 'claude-sonnet-4-6',
|
||||
},
|
||||
{
|
||||
backend: 'gateway',
|
||||
providerLines: [' backend: gateway', ' gateway:', ' api_key: env:AI_GATEWAY_API_KEY'],
|
||||
model: 'anthropic/claude-sonnet-4-6',
|
||||
},
|
||||
])('preserves already configured $backend llm setup without asking for Anthropic credentials', async (fixture) => {
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'setup:',
|
||||
' database_connection_ids: []',
|
||||
' completed_steps:',
|
||||
' - project',
|
||||
' - llm',
|
||||
'connections: {}',
|
||||
'llm:',
|
||||
' provider:',
|
||||
...fixture.providerLines,
|
||||
' models:',
|
||||
` default: ${fixture.model}`,
|
||||
'ingest:',
|
||||
' embeddings:',
|
||||
' backend: deterministic',
|
||||
' model: deterministic',
|
||||
' dimensions: 8',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const healthCheck = vi.fn(async () => ({ ok: true as const }));
|
||||
const io = makeIo();
|
||||
await expect(
|
||||
runKtxSetupAnthropicModelStep({ projectDir: tempDir, inputMode: 'disabled', skipLlm: false }, io.io, {
|
||||
healthCheck,
|
||||
}),
|
||||
).resolves.toMatchObject({ status: 'ready' });
|
||||
|
||||
expect(healthCheck).not.toHaveBeenCalled();
|
||||
expect(io.stdout()).toContain(`LLM ready: yes (${fixture.model})`);
|
||||
expect(io.stderr()).not.toContain('Anthropic');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { writeFile } from 'node:fs/promises';
|
||||
import { cancel, isCancel, password, select, text } from '@clack/prompts';
|
||||
import { resolveLocalKtxLlmConfig } from '@ktx/context';
|
||||
import { resolveKtxConfigReference } from '@ktx/context/core';
|
||||
import {
|
||||
type KtxProjectConfig,
|
||||
|
|
@ -170,13 +171,26 @@ export async function fetchAnthropicModels(
|
|||
return models.map((item, index) => ({ ...item, recommended: index === Math.max(recommendedIndex, 0) }));
|
||||
}
|
||||
|
||||
function hasCompletedLlm(config: KtxProjectConfig): boolean {
|
||||
return (
|
||||
config.setup?.completed_steps.includes('llm') === true &&
|
||||
config.llm.provider.backend === 'anthropic' &&
|
||||
typeof config.llm.models.default === 'string' &&
|
||||
config.llm.models.default.length > 0
|
||||
);
|
||||
export function isKtxSetupLlmConfigReady(config: KtxProjectLlmConfig): boolean {
|
||||
let resolved: KtxLlmConfig | null;
|
||||
try {
|
||||
resolved = resolveLocalKtxLlmConfig(config, process.env);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (!resolved) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (resolved.backend === 'vertex') {
|
||||
return typeof resolved.vertex?.location === 'string' && resolved.vertex.location.trim().length > 0;
|
||||
}
|
||||
|
||||
return resolved.backend === 'anthropic' || resolved.backend === 'gateway';
|
||||
}
|
||||
|
||||
function hasUsableConfiguredLlm(config: KtxProjectConfig): boolean {
|
||||
return isKtxSetupLlmConfigReady(config.llm);
|
||||
}
|
||||
|
||||
function buildProjectLlmConfig(
|
||||
|
|
@ -386,7 +400,7 @@ export async function runKtxSetupAnthropicModelStep(
|
|||
const project = await loadKtxProject({ projectDir: args.projectDir });
|
||||
if (
|
||||
args.forcePrompt !== true &&
|
||||
hasCompletedLlm(project.config) &&
|
||||
hasUsableConfiguredLlm(project.config) &&
|
||||
!args.anthropicApiKeyEnv &&
|
||||
!args.anthropicApiKeyFile &&
|
||||
!args.anthropicModel
|
||||
|
|
|
|||
|
|
@ -5,8 +5,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|||
|
||||
import { localFakeBundleReport, persistLocalBundleReport } from './ingest.test-utils.js';
|
||||
import { contextBuildCommands, writeKtxSetupContextState } from './setup-context.js';
|
||||
import { runDemoTour } from './setup-demo-tour.js';
|
||||
import { readKtxSetupStatus, runKtxSetup } from './setup.js';
|
||||
|
||||
vi.mock('./setup-demo-tour.js', () => ({
|
||||
runDemoTour: vi.fn(async () => 0),
|
||||
}));
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
|
@ -83,6 +88,38 @@ describe('setup status', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
backend: 'vertex',
|
||||
providerLines: [' backend: vertex', ' vertex:', ' project: kaelio-dev', ' location: us-east5'],
|
||||
model: 'claude-sonnet-4-6',
|
||||
},
|
||||
{
|
||||
backend: 'gateway',
|
||||
providerLines: [' backend: gateway', ' gateway:', ' api_key: env:AI_GATEWAY_API_KEY'],
|
||||
model: 'anthropic/claude-sonnet-4-6',
|
||||
},
|
||||
])('reports configured $backend llm backends as setup-ready', async (fixture) => {
|
||||
await mkdir(tempDir, { recursive: true });
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: revenue',
|
||||
'llm:',
|
||||
' provider:',
|
||||
...fixture.providerLines,
|
||||
' models:',
|
||||
` default: ${fixture.model}`,
|
||||
'connections: {}',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({
|
||||
llm: { backend: fixture.backend, ready: true, model: fixture.model },
|
||||
});
|
||||
});
|
||||
|
||||
it('uses setup database connection ids when present', async () => {
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
|
|
@ -404,10 +441,10 @@ describe('setup status', () => {
|
|||
expect(labels).toEqual([
|
||||
'Set up KTX for my data',
|
||||
'Check setup status',
|
||||
'Try KTX with packaged demo data',
|
||||
'Explore a pre-built KTX project',
|
||||
'Exit',
|
||||
]);
|
||||
expect(labels.indexOf('Try KTX with packaged demo data')).toBe(labels.length - 2);
|
||||
expect(labels.indexOf('Explore a pre-built KTX project')).toBe(labels.length - 2);
|
||||
return 'exit';
|
||||
});
|
||||
const cancel = vi.fn();
|
||||
|
|
@ -453,7 +490,7 @@ describe('setup status', () => {
|
|||
'Create a new KTX project',
|
||||
'Connect a coding agent to KTX',
|
||||
'Check setup status',
|
||||
'Try KTX with packaged demo data',
|
||||
'Explore a pre-built KTX project',
|
||||
'Exit',
|
||||
]);
|
||||
return 'exit';
|
||||
|
|
@ -748,9 +785,8 @@ describe('setup status', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('runs the seeded demo when the first setup intent menu chooses packaged demo data', async () => {
|
||||
it('runs the demo tour when the first setup intent menu chooses demo', async () => {
|
||||
const testIo = makeIo();
|
||||
const demo = vi.fn(async (_args: { projectDir: string }, _io: unknown) => 0);
|
||||
|
||||
await expect(
|
||||
runKtxSetup(
|
||||
|
|
@ -771,19 +807,15 @@ describe('setup status', () => {
|
|||
showEntryMenu: true,
|
||||
},
|
||||
testIo.io,
|
||||
{ entryMenuDeps: { prompts: { select: vi.fn(async () => 'demo'), cancel: vi.fn() } }, demo },
|
||||
{ entryMenuDeps: { prompts: { select: vi.fn(async () => 'demo'), cancel: vi.fn() } } },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(demo).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
command: 'seeded',
|
||||
outputMode: 'viz',
|
||||
inputMode: 'auto',
|
||||
}),
|
||||
expect(runDemoTour).toHaveBeenCalledWith(
|
||||
{ inputMode: 'auto' },
|
||||
testIo.io,
|
||||
expect.objectContaining({}),
|
||||
);
|
||||
expect(demo.mock.calls[0]?.[0].projectDir).toMatch(/ktx-demo-/);
|
||||
});
|
||||
|
||||
it('creates a project through run mode when --new is selected', async () => {
|
||||
|
|
@ -1231,6 +1263,77 @@ describe('setup status', () => {
|
|||
expect(calls).toEqual(['model', 'embeddings', 'databases', 'sources']);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
backend: 'vertex',
|
||||
providerLines: [' backend: vertex', ' vertex:', ' project: kaelio-dev', ' location: us-east5'],
|
||||
model: 'claude-sonnet-4-6',
|
||||
},
|
||||
{
|
||||
backend: 'gateway',
|
||||
providerLines: [' backend: gateway', ' gateway:', ' api_key: env:AI_GATEWAY_API_KEY'],
|
||||
model: 'anthropic/claude-sonnet-4-6',
|
||||
},
|
||||
])('adds a dbt source in non-interactive setup with existing $backend llm config', async (fixture) => {
|
||||
const io = makeIo();
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: revenue',
|
||||
'setup:',
|
||||
' database_connection_ids:',
|
||||
' - warehouse',
|
||||
' completed_steps:',
|
||||
' - project',
|
||||
' - databases',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:WAREHOUSE_URL',
|
||||
'llm:',
|
||||
' provider:',
|
||||
...fixture.providerLines,
|
||||
' models:',
|
||||
` default: ${fixture.model}`,
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
await expect(
|
||||
runKtxSetup(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: tempDir,
|
||||
mode: 'existing',
|
||||
agents: false,
|
||||
skipAgents: true,
|
||||
inputMode: 'disabled',
|
||||
yes: true,
|
||||
cliVersion: '0.2.0',
|
||||
skipLlm: false,
|
||||
skipEmbeddings: true,
|
||||
skipDatabases: true,
|
||||
source: 'dbt',
|
||||
sourceConnectionId: 'dbt-main',
|
||||
sourceGitUrl: 'https://github.com/Kaelio/klo-dbt-demo',
|
||||
sourceBranch: 'main',
|
||||
sourceProjectName: 'orbit_analytics',
|
||||
sourceWarehouseConnectionId: 'warehouse',
|
||||
skipSources: false,
|
||||
databaseSchemas: [],
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
sourcesDeps: { validateDbt: vi.fn(async () => ({ ok: true as const, detail: 'dbt project valid' })) },
|
||||
context: vi.fn(async () => ({ status: 'ready' as const, projectDir: tempDir, runId: 'setup-context-test' })),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stderr()).not.toContain('Anthropic');
|
||||
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).toContain('dbt-main:');
|
||||
});
|
||||
|
||||
it('does not fail context build when prerequisites were explicitly skipped and agents are skipped', async () => {
|
||||
const calls: string[] = [];
|
||||
const io = makeIo();
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@ import { cancel, isCancel, select } from '@clack/prompts';
|
|||
import { getLatestLocalIngestStatus, savedMemoryCountsForReport } from '@ktx/context/ingest';
|
||||
import { ktxLocalStateDbPath, loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
|
||||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import type { KtxDemoArgs } from './demo.js';
|
||||
import { defaultDemoProjectDir } from './demo-assets.js';
|
||||
import { formatSetupNextStepLines } from './next-steps.js';
|
||||
import { isKtxSetupExitError, withSetupInterruptConfirmation } from './setup-interrupt.js';
|
||||
import {
|
||||
|
|
@ -23,7 +21,7 @@ import {
|
|||
runKtxSetupDatabasesStep,
|
||||
} from './setup-databases.js';
|
||||
import { type KtxSetupEmbeddingsDeps, runKtxSetupEmbeddingsStep } from './setup-embeddings.js';
|
||||
import { type KtxSetupModelDeps, runKtxSetupAnthropicModelStep } from './setup-models.js';
|
||||
import { type KtxSetupModelDeps, isKtxSetupLlmConfigReady, runKtxSetupAnthropicModelStep } from './setup-models.js';
|
||||
import { type KtxSetupProjectDeps, runKtxSetupProjectStep } from './setup-project.js';
|
||||
import {
|
||||
isKtxPreAgentSetupReady,
|
||||
|
|
@ -149,11 +147,9 @@ export interface KtxSetupDeps {
|
|||
removeAgents?: typeof removeKtxAgentInstall;
|
||||
readyMenuDeps?: KtxSetupReadyMenuDeps;
|
||||
entryMenuDeps?: KtxSetupEntryMenuDeps;
|
||||
demo?: (args: KtxDemoArgs, io: KtxCliIo) => Promise<number>;
|
||||
}
|
||||
|
||||
const SOURCE_DRIVERS = new Set(['dbt', 'metricflow', 'metabase', 'looker', 'lookml', 'notion']);
|
||||
const READY_LLM_BACKENDS = new Set(['anthropic', 'vertex', 'gateway']);
|
||||
|
||||
type KtxSetupEntryAction = 'setup' | 'new-project' | 'agents' | 'status' | 'demo' | 'exit';
|
||||
type KtxSetupFlowStep = 'models' | 'embeddings' | 'databases' | 'sources' | 'context' | 'agents';
|
||||
|
|
@ -202,13 +198,13 @@ async function runKtxSetupEntryMenu(
|
|||
{ value: 'new-project', label: 'Create a new KTX project' },
|
||||
{ value: 'agents', label: 'Connect a coding agent to KTX' },
|
||||
{ value: 'status', label: 'Check setup status' },
|
||||
{ value: 'demo', label: 'Try KTX with packaged demo data' },
|
||||
{ value: 'demo', label: 'Explore a pre-built KTX project' },
|
||||
{ value: 'exit', label: 'Exit' },
|
||||
]
|
||||
: [
|
||||
{ value: 'setup', label: 'Set up KTX for my data' },
|
||||
{ value: 'status', label: 'Check setup status' },
|
||||
{ value: 'demo', label: 'Try KTX with packaged demo data' },
|
||||
{ value: 'demo', label: 'Explore a pre-built KTX project' },
|
||||
{ value: 'exit', label: 'Exit' },
|
||||
];
|
||||
const action = (await prompts.select({
|
||||
|
|
@ -223,24 +219,11 @@ async function runKtxSetupDemoFromEntryMenu(
|
|||
io: KtxCliIo,
|
||||
deps: KtxSetupDeps,
|
||||
): Promise<number> {
|
||||
const runner = deps.demo ?? (await import('./demo.js')).runKtxDemo;
|
||||
return await runner(
|
||||
{
|
||||
command: 'seeded',
|
||||
projectDir: defaultDemoProjectDir(),
|
||||
outputMode: 'viz',
|
||||
inputMode: args.inputMode,
|
||||
},
|
||||
const { runDemoTour } = await import('./setup-demo-tour.js');
|
||||
return await runDemoTour(
|
||||
{ inputMode: args.inputMode },
|
||||
io,
|
||||
);
|
||||
}
|
||||
|
||||
function llmReady(status: KtxSetupStatus['llm']): boolean {
|
||||
return (
|
||||
status.backend !== undefined &&
|
||||
READY_LLM_BACKENDS.has(status.backend) &&
|
||||
typeof status.model === 'string' &&
|
||||
status.model.length > 0
|
||||
{ agents: deps.agents },
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -308,10 +291,9 @@ export async function readKtxSetupStatus(projectDir: string): Promise<KtxSetupSt
|
|||
const project = await loadKtxProject({ projectDir: resolvedProjectDir });
|
||||
const llm = {
|
||||
backend: project.config.llm.provider.backend,
|
||||
ready: false,
|
||||
ready: isKtxSetupLlmConfigReady(project.config.llm),
|
||||
model: project.config.llm.models.default,
|
||||
};
|
||||
llm.ready = llmReady(llm);
|
||||
|
||||
const embeddings = {
|
||||
backend: project.config.ingest.embeddings.backend,
|
||||
|
|
@ -419,7 +401,7 @@ function setupStatusReady(status: KtxSetupStatus): boolean {
|
|||
return true;
|
||||
}
|
||||
return (
|
||||
llmReady(status.llm) &&
|
||||
status.llm.ready &&
|
||||
embeddingsReady(status.embeddings) &&
|
||||
status.databases.every((database) => database.ready) &&
|
||||
status.sources.every((source) => source.ready)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue