mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-04 10:52:13 +02:00
Merge branch 'main' into review-workspace-changes
This commit is contained in:
Andrey Avtomonov
GitHub
commit
b30c320408
50 changed files with 745 additions and 922 deletions
|
|
@ -228,11 +228,11 @@ describe('renderContextBuildView', () => {
|
|||
|
||||
const rendered = renderContextBuildView(state, {
|
||||
styled: false,
|
||||
warnings: ['--deep affects database ingest only; ignoring it for docs.'],
|
||||
warnings: ['--query-history affects database ingest only; ignoring it for docs.'],
|
||||
});
|
||||
|
||||
expect(rendered).toContain('Warnings:');
|
||||
expect(rendered).toContain('--deep affects database ingest only; ignoring it for docs.');
|
||||
expect(rendered).toContain('--query-history affects database ingest only; ignoring it for docs.');
|
||||
});
|
||||
|
||||
it('renders public notices in the foreground view before warnings', () => {
|
||||
|
|
@ -243,7 +243,6 @@ describe('renderContextBuildView', () => {
|
|||
operation: 'database-ingest',
|
||||
debugCommand: 'ktx ingest warehouse --debug',
|
||||
steps: ['database-schema', 'query-history'],
|
||||
databaseDepth: 'deep',
|
||||
detectRelationships: true,
|
||||
queryHistory: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
|
|
@ -252,12 +251,12 @@ describe('renderContextBuildView', () => {
|
|||
const rendered = renderContextBuildView(state, {
|
||||
styled: false,
|
||||
notices: ['Schema ingest runs before query history for warehouse.'],
|
||||
warnings: ['--query-history requires deep ingest; running warehouse with --deep.'],
|
||||
warnings: ['--query-history is not supported for sqlite; running schema ingest for local.'],
|
||||
});
|
||||
|
||||
expect(rendered.indexOf('Notices:')).toBeLessThan(rendered.indexOf('Warnings:'));
|
||||
expect(rendered).toContain('Schema ingest runs before query history for warehouse.');
|
||||
expect(rendered).toContain('--query-history requires deep ingest; running warehouse with --deep.');
|
||||
expect(rendered).toContain('--query-history is not supported for sqlite; running schema ingest for local.');
|
||||
});
|
||||
|
||||
it('renders dynamic separator matching header width', () => {
|
||||
|
|
@ -653,7 +652,6 @@ describe('runContextBuild', () => {
|
|||
inputMode: 'disabled',
|
||||
targetConnectionId: 'warehouse',
|
||||
all: false,
|
||||
depth: 'fast',
|
||||
queryHistory: 'default',
|
||||
},
|
||||
io.io,
|
||||
|
|
@ -665,7 +663,6 @@ describe('runContextBuild', () => {
|
|||
expect(executeTarget.mock.calls[0]?.[0]).toMatchObject({
|
||||
connectionId: 'warehouse',
|
||||
operation: 'database-ingest',
|
||||
databaseDepth: 'fast',
|
||||
});
|
||||
expect(io.stdout()).toContain('Databases:');
|
||||
expect(io.stdout()).toContain('warehouse');
|
||||
|
|
@ -716,7 +713,7 @@ describe('runContextBuild', () => {
|
|||
it('renders localhost SQL analysis refusal as a runtime failure during query history', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep', queryHistory: { enabled: true } } },
|
||||
warehouse: { driver: 'postgres', context: { queryHistory: { enabled: true } } },
|
||||
});
|
||||
const executeTarget = vi.fn(async (target, _args, targetIo) => {
|
||||
targetIo.stderr.write('connect ECONNREFUSED 127.0.0.1:8765\n');
|
||||
|
|
@ -751,7 +748,7 @@ describe('runContextBuild', () => {
|
|||
it('uses captured query-history stderr instead of generic failed-at detail', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep', queryHistory: { enabled: true } } },
|
||||
warehouse: { driver: 'postgres', context: { queryHistory: { enabled: true } } },
|
||||
});
|
||||
const executeTarget = vi.fn(async (target, _args, targetIo) => {
|
||||
targetIo.stdout.write('KTX scan completed\n');
|
||||
|
|
@ -768,7 +765,7 @@ describe('runContextBuild', () => {
|
|||
operation: 'query-history',
|
||||
status: 'failed',
|
||||
detail:
|
||||
'warehouse failed at query-history. Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history',
|
||||
'warehouse failed at query-history. Retry: ktx ingest warehouse --project-dir /tmp/project --query-history',
|
||||
},
|
||||
{ operation: 'source-ingest', status: 'skipped' },
|
||||
{ operation: 'memory-update', status: 'skipped' },
|
||||
|
|
@ -785,7 +782,7 @@ describe('runContextBuild', () => {
|
|||
|
||||
expect(result).toEqual({ exitCode: 1 });
|
||||
expect(io.stdout()).toContain('Missing bundled Python runtime manifest: /tmp/assets/python/manifest.json.');
|
||||
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history');
|
||||
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --query-history');
|
||||
expect(io.stdout()).not.toContain('Then retry the runtime-backed KTX command');
|
||||
expect(io.stdout()).not.toContain('warehouse failed at query-history');
|
||||
expect(io.stdout().match(/Retry: /g)).toHaveLength(1);
|
||||
|
|
@ -899,12 +896,12 @@ describe('runContextBuild', () => {
|
|||
const io = makeIo();
|
||||
const project: KtxPublicIngestProject = {
|
||||
...projectWithConnections({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres' },
|
||||
}),
|
||||
config: {
|
||||
...projectWithConnections({ warehouse: { driver: 'postgres' } }).config,
|
||||
connections: {
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres' },
|
||||
},
|
||||
llm: {
|
||||
provider: { backend: 'gateway', gateway: { api_key: 'env:KTX_GATEWAY_API_KEY' } }, // pragma: allowlist secret
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|||
import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js';
|
||||
import { LocalMetabaseDiscoveryCache } from '../../../src/context/ingest/adapters/metabase/local-source-state-store.js';
|
||||
import { getLocalIngestStatus, runLocalMetabaseIngest } from '../../../src/context/ingest/local-ingest.js';
|
||||
import { ingestReportOutcome } from '../../../src/context/ingest/reports.js';
|
||||
import type { ChunkResult, FetchContext, SourceAdapter } from '../../../src/context/ingest/types.js';
|
||||
|
||||
class TestAgentRunner implements AgentRunnerPort {
|
||||
|
|
@ -202,6 +203,24 @@ describe('runLocalMetabaseIngest', () => {
|
|||
expect(result.children[1]?.report.body.failedWorkUnits).toEqual(['metabase-db-2']);
|
||||
});
|
||||
|
||||
it('keeps a child that saved memory out of all_failed when another child fails', async () => {
|
||||
await seedMetabaseState();
|
||||
const agentRunner = new TestAgentRunner();
|
||||
const ids = ['metabase-child-1', 'metabase-child-2'];
|
||||
|
||||
const result = await runLocalMetabaseIngest({
|
||||
project,
|
||||
adapters: [new FakeMetabaseSourceAdapter()],
|
||||
metabaseConnectionId: 'prod-metabase',
|
||||
agentRunner,
|
||||
jobIdFactory: () => ids.shift() ?? 'metabase-child-extra',
|
||||
});
|
||||
|
||||
expect(result.status).toBe('partial_failure');
|
||||
expect(ingestReportOutcome(result.children[0].report)).toBe('done');
|
||||
expect(ingestReportOutcome(result.children[1].report)).toBe('error');
|
||||
});
|
||||
|
||||
it('captures fetch-time child failures and continues later mappings', async () => {
|
||||
await seedMetabaseState();
|
||||
project.config.connections.warehouse_c = { driver: 'postgres', url: 'postgres://localhost/c' };
|
||||
|
|
|
|||
|
|
@ -166,7 +166,7 @@ describe('memory-flow event mapping', () => {
|
|||
runId: 'run-1',
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'lookml',
|
||||
status: 'error',
|
||||
status: 'done',
|
||||
sourceDir: null,
|
||||
syncId: 'sync-2',
|
||||
reportId: 'report-1',
|
||||
|
|
@ -308,7 +308,7 @@ describe('memory-flow event mapping', () => {
|
|||
sourceReportPath: 'report-1',
|
||||
fallbackReason: null,
|
||||
});
|
||||
expect(replay.status).toBe('error');
|
||||
expect(replay.status).toBe('done');
|
||||
expect(replay.reportId).toBe('report-1');
|
||||
expect(replay.reportPath).toBe('report-1');
|
||||
expect(replay.events[0]).toMatchObject({ type: 'source_acquired', emittedAt: '2026-05-01T10:00:00.000Z' });
|
||||
|
|
|
|||
71
packages/cli/test/context/ingest/reports.test.ts
Normal file
71
packages/cli/test/context/ingest/reports.test.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { ingestReportOutcome } from '../../../src/context/ingest/reports.js';
|
||||
import type { IngestReportSnapshot } from '../../../src/context/ingest/reports.js';
|
||||
|
||||
function report(body: Partial<IngestReportSnapshot['body']>): IngestReportSnapshot {
|
||||
return {
|
||||
id: 'r',
|
||||
runId: 'run',
|
||||
jobId: 'job',
|
||||
connectionId: 'warehouse',
|
||||
sourceKey: 'metabase',
|
||||
createdAt: '2026-05-29T00:00:00.000Z',
|
||||
body: {
|
||||
syncId: 'sync',
|
||||
diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 0 },
|
||||
commitSha: null,
|
||||
workUnits: [],
|
||||
failedWorkUnits: [],
|
||||
reconciliationSkipped: false,
|
||||
conflictsResolved: [],
|
||||
evictionsApplied: [],
|
||||
unmappedFallbacks: [],
|
||||
evictionInputs: [],
|
||||
unresolvedCards: [],
|
||||
supersededBy: null,
|
||||
overrideOf: null,
|
||||
provenanceRows: [],
|
||||
toolTranscripts: [],
|
||||
...body,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const savingWorkUnit = {
|
||||
unitKey: 'ok',
|
||||
rawFiles: ['cards/1.json'],
|
||||
status: 'success' as const,
|
||||
actions: [{ target: 'sl' as const, type: 'updated' as const, key: 'warehouse.orders', detail: 'measure' }],
|
||||
touchedSlSources: [],
|
||||
};
|
||||
|
||||
const failedWorkUnit = {
|
||||
unitKey: 'bad',
|
||||
rawFiles: ['cards/2.json'],
|
||||
status: 'failed' as const,
|
||||
reason: 'tool write failed',
|
||||
actions: [],
|
||||
touchedSlSources: [],
|
||||
};
|
||||
|
||||
describe('ingestReportOutcome', () => {
|
||||
it('returns done when there are no failed work units', () => {
|
||||
expect(ingestReportOutcome(report({ workUnits: [savingWorkUnit] }))).toBe('done');
|
||||
});
|
||||
|
||||
it('returns partial when failed work units coexist with saved memory', () => {
|
||||
expect(
|
||||
ingestReportOutcome(report({ workUnits: [savingWorkUnit, failedWorkUnit], failedWorkUnits: ['bad'] })),
|
||||
).toBe('partial');
|
||||
});
|
||||
|
||||
it('returns error when failed work units produced no saved memory', () => {
|
||||
expect(ingestReportOutcome(report({ workUnits: [failedWorkUnit], failedWorkUnits: ['bad'] }))).toBe('error');
|
||||
});
|
||||
|
||||
it('returns error for a stage-level failure even if artifacts were recorded', () => {
|
||||
expect(ingestReportOutcome(report({ status: 'failed', workUnits: [savingWorkUnit], failedWorkUnits: [] }))).toBe(
|
||||
'error',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -702,7 +702,7 @@ describe('runKtxCli', () => {
|
|||
const publicIngest = vi.fn().mockResolvedValue(0);
|
||||
|
||||
await expect(
|
||||
runKtxCli(['--project-dir', tempDir, 'ingest', 'warehouse', '--fast', '--no-input'], testIo.io, {
|
||||
runKtxCli(['--project-dir', tempDir, 'ingest', 'warehouse', '--no-input'], testIo.io, {
|
||||
publicIngest,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
|
@ -715,7 +715,6 @@ describe('runKtxCli', () => {
|
|||
all: false,
|
||||
json: false,
|
||||
inputMode: 'disabled',
|
||||
depth: 'fast',
|
||||
queryHistory: 'default',
|
||||
cliVersion,
|
||||
runtimeInstallPolicy: 'never',
|
||||
|
|
@ -725,12 +724,12 @@ describe('runKtxCli', () => {
|
|||
expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`);
|
||||
});
|
||||
|
||||
it('routes public ingest --all --deep with JSON output', async () => {
|
||||
it('routes public ingest --all with JSON output', async () => {
|
||||
const testIo = makeIo();
|
||||
const publicIngest = vi.fn().mockResolvedValue(0);
|
||||
|
||||
await expect(
|
||||
runKtxCli(['--project-dir', tempDir, 'ingest', '--all', '--deep', '--json'], testIo.io, {
|
||||
runKtxCli(['--project-dir', tempDir, 'ingest', '--all', '--json'], testIo.io, {
|
||||
publicIngest,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
|
@ -742,7 +741,6 @@ describe('runKtxCli', () => {
|
|||
all: true,
|
||||
json: true,
|
||||
inputMode: 'auto',
|
||||
depth: 'deep',
|
||||
queryHistory: 'default',
|
||||
cliVersion,
|
||||
runtimeInstallPolicy: 'prompt',
|
||||
|
|
@ -786,20 +784,6 @@ describe('runKtxCli', () => {
|
|||
expect(testIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
|
||||
});
|
||||
|
||||
it('rejects mutually exclusive public ingest depth flags before dispatch', async () => {
|
||||
const testIo = makeIo();
|
||||
const publicIngest = vi.fn().mockResolvedValue(0);
|
||||
|
||||
await expect(
|
||||
runKtxCli(['--project-dir', '/tmp/project', 'ingest', 'warehouse', '--fast', '--deep'], testIo.io, {
|
||||
publicIngest,
|
||||
}),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(publicIngest).not.toHaveBeenCalled();
|
||||
expect(testIo.stderr()).toMatch(/option '--(deep|fast)' cannot be used with option '--(fast|deep)'/);
|
||||
});
|
||||
|
||||
it.each(['run', 'status', 'watch', 'replay'])(
|
||||
'routes former ingest subcommand name "%s" as a connection id',
|
||||
async (connectionId) => {
|
||||
|
|
@ -890,8 +874,6 @@ describe('runKtxCli', () => {
|
|||
expect(testIo.stdout()).toContain('Usage: ktx ingest');
|
||||
expect(testIo.stdout()).toContain('Build or inspect KTX context');
|
||||
expect(testIo.stdout()).toContain('--all');
|
||||
expect(testIo.stdout()).toContain('--fast');
|
||||
expect(testIo.stdout()).toContain('--deep');
|
||||
expect(testIo.stdout()).toContain('--query-history');
|
||||
expect(testIo.stdout()).toContain('--no-query-history');
|
||||
expect(testIo.stdout()).toContain('--query-history-window-days <days>');
|
||||
|
|
|
|||
|
|
@ -403,7 +403,7 @@ describe('runKtxIngest', () => {
|
|||
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
|
||||
});
|
||||
|
||||
it('returns a non-zero code when Metabase fanout has failed children', async () => {
|
||||
it('returns a non-zero code when a Metabase fanout child fully fails', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeMetabaseConfig(projectDir);
|
||||
const io = makeIo();
|
||||
|
|
@ -441,7 +441,7 @@ describe('runKtxIngest', () => {
|
|||
{
|
||||
runLocalMetabaseIngest: async () => ({
|
||||
metabaseConnectionId: 'prod-metabase',
|
||||
status: 'partial_failure',
|
||||
status: 'all_failed',
|
||||
totals: { workUnits: 1, failedWorkUnits: 1 },
|
||||
children: [
|
||||
{
|
||||
|
|
@ -467,9 +467,83 @@ describe('runKtxIngest', () => {
|
|||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stdout()).toContain('Metabase fanout: partial_failure');
|
||||
expect(io.stdout()).toContain('Failed tasks: 1');
|
||||
expect(io.stdout()).toContain('Metabase fanout: all_failed');
|
||||
expect(io.stdout()).toContain('status=error');
|
||||
});
|
||||
|
||||
it('exits 0 and reports status=partial when a Metabase child saved memory despite a failure', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeMetabaseConfig(projectDir);
|
||||
const io = makeIo();
|
||||
const report = localFakeBundleReport('metabase-child-1', {
|
||||
id: 'report-metabase-child-1',
|
||||
runId: 'run-a',
|
||||
jobId: 'metabase-child-1',
|
||||
connectionId: 'warehouse_a',
|
||||
sourceKey: 'metabase',
|
||||
body: {
|
||||
failedWorkUnits: ['metabase-db-2'],
|
||||
workUnits: [
|
||||
{
|
||||
unitKey: 'metabase-db-1',
|
||||
rawFiles: ['cards/1.json'],
|
||||
status: 'success',
|
||||
actions: [{ target: 'sl', type: 'updated', key: 'warehouse.orders', detail: 'measure' }],
|
||||
touchedSlSources: [],
|
||||
},
|
||||
{
|
||||
unitKey: 'metabase-db-2',
|
||||
rawFiles: ['cards/2.json'],
|
||||
status: 'failed',
|
||||
reason: 'bad SQL',
|
||||
actions: [],
|
||||
touchedSlSources: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'prod-metabase',
|
||||
adapter: 'metabase',
|
||||
outputMode: 'plain',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
runLocalMetabaseIngest: async () => ({
|
||||
metabaseConnectionId: 'prod-metabase',
|
||||
status: 'partial_failure',
|
||||
totals: { workUnits: 2, failedWorkUnits: 1 },
|
||||
children: [
|
||||
{
|
||||
jobId: 'metabase-child-1',
|
||||
metabaseConnectionId: 'prod-metabase',
|
||||
metabaseDatabaseId: 1,
|
||||
targetConnectionId: 'warehouse_a',
|
||||
result: {
|
||||
jobId: 'metabase-child-1',
|
||||
runId: 'run-a',
|
||||
syncId: 'sync-a',
|
||||
diffSummary: { added: 1, modified: 0, deleted: 0, unchanged: 0 },
|
||||
workUnitCount: 2,
|
||||
failedWorkUnits: ['metabase-db-2'],
|
||||
artifactsWritten: 1,
|
||||
commitSha: 'abc',
|
||||
},
|
||||
report,
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Metabase fanout: partial_failure');
|
||||
expect(io.stdout()).toContain('status=partial');
|
||||
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
|
||||
});
|
||||
|
||||
|
|
@ -1140,6 +1214,63 @@ describe('runKtxIngest', () => {
|
|||
expect(io.stdout()).toContain('Status: error\n');
|
||||
});
|
||||
|
||||
it('exits 0 and reports Status: partial when a single-source ingest saved memory despite a failure', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
||||
const partialReport = localFakeBundleReport('local-job-partial', {
|
||||
connectionId: 'warehouse',
|
||||
sourceKey: 'fake',
|
||||
body: {
|
||||
failedWorkUnits: ['orders-bad'],
|
||||
workUnits: [
|
||||
{
|
||||
unitKey: 'orders-ok',
|
||||
rawFiles: ['orders/orders.json'],
|
||||
status: 'success',
|
||||
actions: [{ target: 'wiki', type: 'created', key: 'wiki/orders.md', detail: 'orders' }],
|
||||
touchedSlSources: [],
|
||||
},
|
||||
{
|
||||
unitKey: 'orders-bad',
|
||||
rawFiles: ['orders/bad.json'],
|
||||
status: 'failed',
|
||||
reason: 'writer tool failed',
|
||||
actions: [],
|
||||
touchedSlSources: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const runLocal = vi.fn(async (_input: RunLocalIngestOptions) => ({
|
||||
result: {
|
||||
jobId: 'local-job-partial',
|
||||
runId: partialReport.runId,
|
||||
syncId: partialReport.body.syncId,
|
||||
diffSummary: partialReport.body.diffSummary,
|
||||
workUnitCount: partialReport.body.workUnits.length,
|
||||
failedWorkUnits: partialReport.body.failedWorkUnits,
|
||||
artifactsWritten: 1,
|
||||
commitSha: partialReport.body.commitSha,
|
||||
},
|
||||
report: partialReport,
|
||||
}));
|
||||
|
||||
const io = makeIo();
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{ command: 'run', projectDir, connectionId: 'warehouse', adapter: 'fake', sourceDir, outputMode: 'plain' },
|
||||
io.io,
|
||||
{ runLocalIngest: runLocal, jobIdFactory: () => 'local-job-partial' },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Status: partial\n');
|
||||
});
|
||||
|
||||
it('prints trace path and error status for stored failed ingest reports', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ function deepReadyProject(
|
|||
|
||||
describe('buildPublicIngestPlan', () => {
|
||||
it('plans warehouse connections as scan targets and source connections as source ingest targets', () => {
|
||||
const project = projectWithConnections({
|
||||
const project = deepReadyProject({
|
||||
warehouse: { driver: 'postgres' },
|
||||
prod_metabase: { driver: 'metabase', api_url: 'https://metabase.example.com' },
|
||||
docs: { driver: 'notion' },
|
||||
|
|
@ -103,8 +103,7 @@ describe('buildPublicIngestPlan', () => {
|
|||
operation: 'database-ingest',
|
||||
debugCommand: 'ktx ingest warehouse --debug',
|
||||
steps: ['database-schema'],
|
||||
databaseDepth: 'fast',
|
||||
detectRelationships: false,
|
||||
detectRelationships: true,
|
||||
queryHistory: { enabled: false },
|
||||
},
|
||||
{
|
||||
|
|
@ -139,61 +138,6 @@ describe('buildPublicIngestPlan', () => {
|
|||
expect(plan.targets.map((target) => target.connectionId).sort()).toEqual(['docs', 'warehouse']);
|
||||
});
|
||||
|
||||
it('resolves database depth from flags, stored context, and defaults', () => {
|
||||
const project = projectWithConnections({
|
||||
fast_default: { driver: 'postgres' },
|
||||
deep_default: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
docs: { driver: 'notion' },
|
||||
});
|
||||
|
||||
expect(
|
||||
buildPublicIngestPlan(project, {
|
||||
projectDir: '/tmp/project',
|
||||
targetConnectionId: 'fast_default',
|
||||
all: false,
|
||||
queryHistory: 'default',
|
||||
}).targets[0],
|
||||
).toMatchObject({ connectionId: 'fast_default', databaseDepth: 'fast', queryHistory: { enabled: false } });
|
||||
|
||||
expect(
|
||||
buildPublicIngestPlan(project, {
|
||||
projectDir: '/tmp/project',
|
||||
targetConnectionId: 'deep_default',
|
||||
all: false,
|
||||
queryHistory: 'default',
|
||||
}).targets[0],
|
||||
).toMatchObject({ connectionId: 'deep_default', databaseDepth: 'deep' });
|
||||
|
||||
expect(
|
||||
buildPublicIngestPlan(project, {
|
||||
projectDir: '/tmp/project',
|
||||
targetConnectionId: 'docs',
|
||||
all: false,
|
||||
depth: 'deep',
|
||||
queryHistory: 'default',
|
||||
}).warnings,
|
||||
).toEqual(['--deep affects database ingest only; ignoring it for docs.']);
|
||||
});
|
||||
|
||||
it('does not infer deep ingest from legacy scanMode values', () => {
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres' },
|
||||
});
|
||||
|
||||
const plan = buildPublicIngestPlan(project, {
|
||||
projectDir: '/tmp/project',
|
||||
targetConnectionId: 'warehouse',
|
||||
all: false,
|
||||
scanMode: 'enriched',
|
||||
});
|
||||
|
||||
expect(plan.targets[0]).toMatchObject({
|
||||
connectionId: 'warehouse',
|
||||
databaseDepth: 'fast',
|
||||
steps: ['database-schema'],
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects stale local Looker source driver aliases', () => {
|
||||
const project = projectWithConnections({
|
||||
local_looker: { driver: 'local_looker' } as never,
|
||||
|
|
@ -204,8 +148,8 @@ describe('buildPublicIngestPlan', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('upgrades effective depth when query history is explicitly enabled', () => {
|
||||
const project = projectWithConnections({
|
||||
it('enables query history when explicitly requested even if stored config disables it', () => {
|
||||
const project = deepReadyProject({
|
||||
warehouse: { driver: 'postgres', context: { queryHistory: { enabled: false } } },
|
||||
});
|
||||
|
||||
|
|
@ -213,17 +157,16 @@ describe('buildPublicIngestPlan', () => {
|
|||
projectDir: '/tmp/project',
|
||||
targetConnectionId: 'warehouse',
|
||||
all: false,
|
||||
depth: 'fast',
|
||||
queryHistory: 'enabled',
|
||||
queryHistoryWindowDays: 30,
|
||||
});
|
||||
|
||||
expect(plan.targets[0]).toMatchObject({
|
||||
connectionId: 'warehouse',
|
||||
databaseDepth: 'deep',
|
||||
queryHistory: { enabled: true, windowDays: 30, dialect: 'postgres' },
|
||||
steps: ['database-schema', 'query-history'],
|
||||
});
|
||||
expect(plan.warnings).toEqual(['--query-history requires deep ingest; running warehouse with --deep.']);
|
||||
expect(plan.warnings).toEqual([]);
|
||||
});
|
||||
|
||||
it('warns and skips query history for unsupported database drivers', () => {
|
||||
|
|
@ -238,7 +181,6 @@ describe('buildPublicIngestPlan', () => {
|
|||
|
||||
expect(plan.targets[0]).toMatchObject({
|
||||
connectionId: 'local',
|
||||
databaseDepth: 'fast',
|
||||
queryHistory: { enabled: false, unsupported: true },
|
||||
});
|
||||
expect(plan.warnings).toEqual(['--query-history is not supported for sqlite; running schema ingest for local.']);
|
||||
|
|
@ -249,12 +191,11 @@ describe('buildPublicIngestPlan', () => {
|
|||
deepReadyProject({
|
||||
local: { driver: 'sqlite' },
|
||||
mysql_warehouse: { driver: 'mysql' },
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres' },
|
||||
}),
|
||||
{
|
||||
projectDir: '/tmp/project',
|
||||
all: true,
|
||||
depth: 'deep',
|
||||
queryHistory: 'enabled',
|
||||
},
|
||||
);
|
||||
|
|
@ -326,7 +267,6 @@ describe('buildPublicIngestPlan', () => {
|
|||
|
||||
expect(plan.targets[0]).toMatchObject({
|
||||
connectionId: 'warehouse',
|
||||
databaseDepth: 'deep',
|
||||
queryHistory: { enabled: true, dialect: 'postgres', windowDays: 30 },
|
||||
steps: ['database-schema', 'query-history'],
|
||||
});
|
||||
|
|
@ -334,7 +274,7 @@ describe('buildPublicIngestPlan', () => {
|
|||
|
||||
it('adds a schema-first notice when query history is explicitly enabled', () => {
|
||||
const project = deepReadyProject({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres' },
|
||||
});
|
||||
|
||||
expect(
|
||||
|
|
@ -363,34 +303,15 @@ describe('buildPublicIngestPlan', () => {
|
|||
|
||||
expect(plan.targets[0]).toMatchObject({
|
||||
connectionId: 'local',
|
||||
databaseDepth: 'fast',
|
||||
queryHistory: { enabled: false, windowDays: 30, unsupported: true },
|
||||
steps: ['database-schema'],
|
||||
});
|
||||
expect(plan.warnings).toEqual(['--query-history is not supported for sqlite; running schema ingest for local.']);
|
||||
});
|
||||
|
||||
it('aggregates ignored database-depth warnings for all source targets', () => {
|
||||
const plan = buildPublicIngestPlan(
|
||||
projectWithConnections({
|
||||
warehouse: { driver: 'postgres' },
|
||||
docs: { driver: 'notion' },
|
||||
dbt: { driver: 'dbt' },
|
||||
}),
|
||||
{
|
||||
projectDir: '/tmp/project',
|
||||
all: true,
|
||||
depth: 'deep',
|
||||
queryHistory: 'default',
|
||||
},
|
||||
);
|
||||
|
||||
expect(plan.warnings).toEqual(['--deep ignored for 2 non-database sources.']);
|
||||
});
|
||||
|
||||
it('records a preflight failure for deep database ingest when readiness config is missing', () => {
|
||||
it('records a preflight failure for database ingest when enrichment readiness config is missing', () => {
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres' },
|
||||
});
|
||||
|
||||
const plan = buildPublicIngestPlan(project, {
|
||||
|
|
@ -402,15 +323,14 @@ describe('buildPublicIngestPlan', () => {
|
|||
|
||||
expect(plan.targets[0]).toMatchObject({
|
||||
connectionId: 'warehouse',
|
||||
databaseDepth: 'deep',
|
||||
preflightFailure:
|
||||
'warehouse requires deep ingest readiness: model configuration, scan enrichment mode, scan embeddings. Run ktx setup or rerun with --fast.',
|
||||
'warehouse cannot be ingested: enrichment is not configured (model configuration, scan enrichment mode, scan embeddings). Run ktx setup to configure a model and embeddings.',
|
||||
});
|
||||
});
|
||||
|
||||
it('honors scan.relationships.enabled when planning deep database ingest', () => {
|
||||
it('honors scan.relationships.enabled when planning database ingest', () => {
|
||||
const plan = buildPublicIngestPlan(
|
||||
deepReadyProject({ warehouse: { driver: 'postgres', context: { depth: 'deep' } } }, false),
|
||||
deepReadyProject({ warehouse: { driver: 'postgres' } }, false),
|
||||
{
|
||||
projectDir: '/tmp/project',
|
||||
targetConnectionId: 'warehouse',
|
||||
|
|
@ -421,7 +341,6 @@ describe('buildPublicIngestPlan', () => {
|
|||
|
||||
expect(plan.targets[0]).toMatchObject({
|
||||
connectionId: 'warehouse',
|
||||
databaseDepth: 'deep',
|
||||
detectRelationships: false,
|
||||
});
|
||||
});
|
||||
|
|
@ -432,11 +351,11 @@ describe('runKtxPublicIngest', () => {
|
|||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it('maps fast and deep database targets to scan internals', async () => {
|
||||
it('maps database targets to enriched scan internals', async () => {
|
||||
const io = makeIo();
|
||||
const project = deepReadyProject({
|
||||
fast: { driver: 'postgres' },
|
||||
deep: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
first: { driver: 'postgres' },
|
||||
second: { driver: 'postgres' },
|
||||
});
|
||||
const runScan = vi.fn(async () => 0);
|
||||
|
||||
|
|
@ -450,12 +369,12 @@ describe('runKtxPublicIngest', () => {
|
|||
|
||||
expect(runScan).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ connectionId: 'deep', mode: 'enriched', detectRelationships: true }),
|
||||
expect.objectContaining({ connectionId: 'first', mode: 'enriched', detectRelationships: true }),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(runScan).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({ connectionId: 'fast', mode: 'structural', detectRelationships: false }),
|
||||
expect.objectContaining({ connectionId: 'second', mode: 'enriched', detectRelationships: true }),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
|
@ -467,7 +386,7 @@ describe('runKtxPublicIngest', () => {
|
|||
try {
|
||||
await initKtxProject({ projectDir });
|
||||
const io = makeIo({ isTTY: true });
|
||||
const project = projectWithConnections({
|
||||
const project = deepReadyProject({
|
||||
warehouse: { driver: 'sqlite', path: join(projectDir, 'warehouse.sqlite') },
|
||||
});
|
||||
|
||||
|
|
@ -614,7 +533,7 @@ describe('runKtxPublicIngest', () => {
|
|||
it('prints the schema-first notice for explicit query-history runs', async () => {
|
||||
const io = makeIo();
|
||||
const project = deepReadyProject({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres' },
|
||||
});
|
||||
const runScan = vi.fn(async () => 0);
|
||||
const runIngest = vi.fn(async () => 0);
|
||||
|
|
@ -640,7 +559,7 @@ describe('runKtxPublicIngest', () => {
|
|||
|
||||
it('suppresses internal scan output for public database ingest summaries', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
|
||||
const project = deepReadyProject({ warehouse: { driver: 'postgres' } });
|
||||
const runScan = vi.fn(async (_args, scanIo) => {
|
||||
scanIo.stdout.write('KTX scan completed\n');
|
||||
scanIo.stdout.write('Mode: structural\n');
|
||||
|
|
@ -674,7 +593,7 @@ describe('runKtxPublicIngest', () => {
|
|||
|
||||
it('sanitizes captured database scan failure details in direct public output', async () => {
|
||||
const io = makeIo();
|
||||
const project = deepReadyProject({ warehouse: { driver: 'postgres', context: { depth: 'deep' } } });
|
||||
const project = deepReadyProject({ warehouse: { driver: 'postgres' } });
|
||||
const runScan = vi.fn(async (_args, scanIo) => {
|
||||
scanIo.stdout.write('KTX scan enrichment failed after structural scan completed: embedding service timed out\n');
|
||||
return 1;
|
||||
|
|
@ -689,7 +608,6 @@ describe('runKtxPublicIngest', () => {
|
|||
all: false,
|
||||
json: false,
|
||||
inputMode: 'disabled',
|
||||
depth: 'deep',
|
||||
},
|
||||
io.io,
|
||||
{ loadProject: vi.fn(async () => project), runScan },
|
||||
|
|
@ -699,7 +617,7 @@ describe('runKtxPublicIngest', () => {
|
|||
expect(io.stdout()).toContain(
|
||||
'warehouse failed: Database enrichment failed after schema context completed: embedding service timed out.',
|
||||
);
|
||||
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep');
|
||||
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project');
|
||||
expect(io.stdout()).not.toContain('KTX scan enrichment failed');
|
||||
expect(io.stdout()).not.toContain('structural scan');
|
||||
});
|
||||
|
|
@ -743,7 +661,7 @@ describe('runKtxPublicIngest', () => {
|
|||
it('suppresses historic-sql report output during direct public query-history ingest', async () => {
|
||||
const io = makeIo();
|
||||
const project = deepReadyProject({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres' },
|
||||
});
|
||||
const runScan = vi.fn(async () => 0);
|
||||
const runIngest = vi.fn(async (_args, ingestIo) => {
|
||||
|
|
@ -794,7 +712,6 @@ describe('runKtxPublicIngest', () => {
|
|||
all: false,
|
||||
json: false,
|
||||
inputMode: 'auto',
|
||||
depth: 'fast',
|
||||
queryHistory: 'default',
|
||||
},
|
||||
io.io,
|
||||
|
|
@ -809,7 +726,6 @@ describe('runKtxPublicIngest', () => {
|
|||
targetConnectionId: 'warehouse',
|
||||
all: false,
|
||||
entrypoint: 'ingest',
|
||||
depth: 'fast',
|
||||
queryHistory: 'default',
|
||||
}),
|
||||
io.io,
|
||||
|
|
@ -821,7 +737,7 @@ describe('runKtxPublicIngest', () => {
|
|||
const io = makeIo({ isTTY: true, interactive: true });
|
||||
const calls: string[] = [];
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres' },
|
||||
});
|
||||
const ensureRuntime = vi.fn(async (): Promise<ManagedPythonCommandRuntime> => {
|
||||
calls.push('runtime');
|
||||
|
|
@ -923,10 +839,13 @@ describe('runKtxPublicIngest', () => {
|
|||
|
||||
it('runs all independent targets and reports partial failures', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres' },
|
||||
prod_metabase: { driver: 'metabase', api_url: 'https://metabase.example.com' },
|
||||
});
|
||||
const project = deepReadyProject(
|
||||
{
|
||||
warehouse: { driver: 'postgres' },
|
||||
prod_metabase: { driver: 'metabase', api_url: 'https://metabase.example.com' },
|
||||
},
|
||||
false,
|
||||
);
|
||||
const runScan = vi.fn(async () => 1);
|
||||
const runIngest = vi.fn(async () => 0);
|
||||
|
||||
|
|
@ -959,7 +878,7 @@ describe('runKtxPublicIngest', () => {
|
|||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: 'warehouse',
|
||||
mode: 'structural',
|
||||
mode: 'enriched',
|
||||
detectRelationships: false,
|
||||
dryRun: false,
|
||||
},
|
||||
|
|
@ -967,14 +886,14 @@ describe('runKtxPublicIngest', () => {
|
|||
);
|
||||
expect(io.stdout()).toContain('Ingest finished with partial failures');
|
||||
expect(io.stdout()).toContain('warehouse failed at database-schema.');
|
||||
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --fast');
|
||||
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project');
|
||||
expect(io.stdout()).not.toContain('Debug:');
|
||||
});
|
||||
|
||||
it('skips the query-history facet but keeps the target green when query-history fails', async () => {
|
||||
const io = makeIo();
|
||||
const project = deepReadyProject({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres' },
|
||||
});
|
||||
const runScan = vi.fn(async () => 0);
|
||||
const runIngest = vi.fn(async (_args, ingestIo) => {
|
||||
|
|
@ -1007,14 +926,14 @@ describe('runKtxPublicIngest', () => {
|
|||
'Query history failed for 60 tasks. First failure: Google Cloud authentication failed while analyzing query history',
|
||||
);
|
||||
expect(io.stdout()).not.toContain('warehouse failed: Error:');
|
||||
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history');
|
||||
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --query-history');
|
||||
expect(io.stdout()).not.toContain('historic-sql');
|
||||
});
|
||||
|
||||
it('prints the runtime artifact build hint for missing query-history runtime assets', async () => {
|
||||
const io = makeIo();
|
||||
const project = deepReadyProject({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres' },
|
||||
});
|
||||
const runScan = vi.fn(async () => 0);
|
||||
const runIngest = vi.fn(async (_args, ingestIo) => {
|
||||
|
|
@ -1045,14 +964,14 @@ describe('runKtxPublicIngest', () => {
|
|||
expect(io.stdout()).toContain(
|
||||
'In a source checkout, build the local runtime assets with: pnpm run artifacts:build',
|
||||
);
|
||||
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history');
|
||||
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --query-history');
|
||||
expect(io.stdout()).not.toContain('Then retry the runtime-backed KTX command');
|
||||
});
|
||||
|
||||
it('fails deep-readiness targets before work starts while continuing independent --all targets', async () => {
|
||||
it('fails enrichment-readiness targets before work starts while continuing independent --all targets', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres' },
|
||||
docs: { driver: 'notion' },
|
||||
});
|
||||
const runScan = vi.fn(async () => 0);
|
||||
|
|
@ -1071,12 +990,12 @@ describe('runKtxPublicIngest', () => {
|
|||
expect.objectContaining({ command: 'run', connectionId: 'docs', adapter: 'notion' }),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(io.stdout()).toContain('warehouse requires deep ingest readiness');
|
||||
expect(io.stdout()).toContain('warehouse cannot be ingested: enrichment is not configured');
|
||||
});
|
||||
|
||||
it('does not infer enriched relationship scans from legacy scanMode values', async () => {
|
||||
it('drives scan relationship detection from project config, not from legacy args', async () => {
|
||||
const io = makeIo();
|
||||
const project = deepReadyProject({ warehouse: { driver: 'postgres' } });
|
||||
const project = deepReadyProject({ warehouse: { driver: 'postgres' } }, false);
|
||||
const runScan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
|
|
@ -1103,7 +1022,7 @@ describe('runKtxPublicIngest', () => {
|
|||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: 'warehouse',
|
||||
mode: 'structural',
|
||||
mode: 'enriched',
|
||||
detectRelationships: false,
|
||||
dryRun: false,
|
||||
},
|
||||
|
|
@ -1113,7 +1032,7 @@ describe('runKtxPublicIngest', () => {
|
|||
|
||||
it('prints stable JSON results', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
|
||||
const project = deepReadyProject({ warehouse: { driver: 'postgres' } });
|
||||
|
||||
await expect(
|
||||
runKtxPublicIngest(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { buildDefaultKtxProjectConfig, parseKtxProjectConfig, serializeKtxProjectConfig, type KtxProjectConfig } from '../src/context/project/config.js';
|
||||
import { buildDefaultKtxProjectConfig, serializeKtxProjectConfig, type KtxProjectConfig } from '../src/context/project/config.js';
|
||||
import { readKtxSetupState, writeKtxSetupState } from '../src/context/project/setup-config.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
|
|
@ -49,7 +49,7 @@ async function writeReadyProject(projectDir: string, overrides: ReadyProjectOver
|
|||
...defaults,
|
||||
setup: { database_connection_ids: ['warehouse'] },
|
||||
connections: {
|
||||
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' },
|
||||
docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' },
|
||||
},
|
||||
llm: {
|
||||
|
|
@ -407,130 +407,10 @@ describe('setup context build state', () => {
|
|||
expect(io.stdout()).not.toContain('Existing context artifacts were found from setup ingest.');
|
||||
});
|
||||
|
||||
it('treats fast database context as ready from schema manifest shards without AI artifacts', async () => {
|
||||
it('requires completed relationships for database context when relationship discovery is enabled', async () => {
|
||||
await writeReadyProject(tempDir, {
|
||||
connections: {
|
||||
warehouse: { driver: 'postgres', readonly: true, context: { depth: 'fast' } },
|
||||
},
|
||||
llm: { provider: { backend: 'none' }, models: {} },
|
||||
scan: { enrichment: { mode: 'none' } },
|
||||
});
|
||||
await mkdir(join(tempDir, 'semantic-layer', 'warehouse', '_schema'), { recursive: true });
|
||||
await writeFile(join(tempDir, 'semantic-layer', 'warehouse', '_schema', 'public.yaml'), 'tables: {}\n');
|
||||
await writeScanReport(tempDir, '2026-05-09T10:00:00.000Z', {
|
||||
mode: 'structural',
|
||||
tableDescriptions: 'skipped',
|
||||
columnDescriptions: 'skipped',
|
||||
embeddings: 'skipped',
|
||||
manifestShards: ['semantic-layer/warehouse/_schema/public.yaml'],
|
||||
});
|
||||
const io = makeIo();
|
||||
const runContextBuildMock = vi.fn<NonNullable<KtxSetupContextDeps['runContextBuild']>>(async () => ({
|
||||
exitCode: 0,
|
||||
}));
|
||||
|
||||
await expect(
|
||||
runKtxSetupContextStep(
|
||||
{ projectDir: tempDir, inputMode: 'disabled' },
|
||||
io.io,
|
||||
{
|
||||
runContextBuild: runContextBuildMock,
|
||||
},
|
||||
),
|
||||
).resolves.toMatchObject({ status: 'ready' });
|
||||
|
||||
expect(runContextBuildMock).not.toHaveBeenCalled();
|
||||
expect(io.stdout()).toContain('Existing context artifacts were found from setup ingest.');
|
||||
});
|
||||
|
||||
it('stores fast context depth non-interactively when deep readiness is missing', async () => {
|
||||
await writeReadyProject(tempDir, {
|
||||
connections: { warehouse: { driver: 'postgres', readonly: true } },
|
||||
llm: { provider: { backend: 'none' }, models: {} },
|
||||
scan: { enrichment: { mode: 'none' } },
|
||||
});
|
||||
const io = makeIo();
|
||||
const runContextBuildMock = vi.fn<NonNullable<KtxSetupContextDeps['runContextBuild']>>(async () => ({
|
||||
exitCode: 0,
|
||||
}));
|
||||
const verifyContextReady = vi.fn(async () => ({
|
||||
ready: true,
|
||||
agentContextReady: true,
|
||||
semanticSearchReady: true,
|
||||
details: ['ready'],
|
||||
}));
|
||||
|
||||
await expect(
|
||||
runKtxSetupContextStep(
|
||||
{ projectDir: tempDir, inputMode: 'disabled' },
|
||||
io.io,
|
||||
{ runContextBuild: runContextBuildMock, verifyContextReady },
|
||||
),
|
||||
).resolves.toMatchObject({ status: 'ready' });
|
||||
|
||||
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
|
||||
expect(config.connections.warehouse.context).toMatchObject({ depth: 'fast' });
|
||||
expect(runContextBuildMock).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({ projectDir: tempDir, inputMode: 'disabled' }),
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(runContextBuildMock.mock.calls[0]?.[1]).not.toMatchObject({
|
||||
scanMode: 'enriched',
|
||||
detectRelationships: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('prompts for database context depth after final readiness is known', async () => {
|
||||
await writeReadyProject(tempDir, {
|
||||
connections: { warehouse: { driver: 'postgres', readonly: true } },
|
||||
llm: {
|
||||
provider: { backend: 'gateway', gateway: { api_key: 'env:KTX_GATEWAY_API_KEY' } }, // pragma: allowlist secret
|
||||
models: { default: 'gpt-test' },
|
||||
},
|
||||
scan: {
|
||||
enrichment: {
|
||||
mode: 'llm',
|
||||
embeddings: { backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536 },
|
||||
},
|
||||
},
|
||||
});
|
||||
const io = makeIo();
|
||||
const select = vi.fn(async () => 'deep');
|
||||
const runContextBuildMock = vi.fn(async () => ({ exitCode: 0 }));
|
||||
const verifyContextReady = vi.fn(async () => ({
|
||||
ready: true,
|
||||
agentContextReady: true,
|
||||
semanticSearchReady: true,
|
||||
details: ['ready'],
|
||||
}));
|
||||
|
||||
await expect(
|
||||
runKtxSetupContextStep(
|
||||
{ projectDir: tempDir, inputMode: 'auto' },
|
||||
io.io,
|
||||
{
|
||||
prompts: { select, cancel: vi.fn() },
|
||||
runContextBuild: runContextBuildMock,
|
||||
verifyContextReady,
|
||||
},
|
||||
),
|
||||
).resolves.toMatchObject({ status: 'ready' });
|
||||
|
||||
expect(select).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: expect.stringContaining('How much database context should KTX build?'),
|
||||
}),
|
||||
);
|
||||
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
|
||||
expect(config.connections.warehouse.context).toMatchObject({ depth: 'deep' });
|
||||
});
|
||||
|
||||
it('requires completed relationships for deep context when relationship discovery is enabled', async () => {
|
||||
await writeReadyProject(tempDir, {
|
||||
connections: {
|
||||
warehouse: { driver: 'postgres', readonly: true, context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres', readonly: true },
|
||||
},
|
||||
scan: { relationships: { enabled: true } },
|
||||
});
|
||||
|
|
@ -560,10 +440,10 @@ describe('setup context build state', () => {
|
|||
expect(runContextBuildMock).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('does not require relationships for deep context when relationship discovery is disabled', async () => {
|
||||
it('does not require relationships for database context when relationship discovery is disabled', async () => {
|
||||
await writeReadyProject(tempDir, {
|
||||
connections: {
|
||||
warehouse: { driver: 'postgres', readonly: true, context: { depth: 'deep' } },
|
||||
warehouse: { driver: 'postgres', readonly: true },
|
||||
},
|
||||
scan: { relationships: { enabled: false } },
|
||||
});
|
||||
|
|
@ -620,7 +500,7 @@ describe('setup context build state', () => {
|
|||
|
||||
it('starts a fresh foreground build when stale state is found', async () => {
|
||||
await writeReadyProject(tempDir, {
|
||||
connections: { warehouse: { driver: 'postgres', readonly: true, context: { depth: 'fast' } } },
|
||||
connections: { warehouse: { driver: 'postgres', readonly: true } },
|
||||
});
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-stale',
|
||||
|
|
|
|||
|
|
@ -262,48 +262,6 @@ describe('setup databases step', () => {
|
|||
expect(prompts.select).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('preserves context.depth when editing an existing database connection', async () => {
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
[
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
' path: ./warehouse.sqlite',
|
||||
' context:',
|
||||
' depth: deep',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const prompts = makePromptAdapter({
|
||||
selectValues: ['edit', 'warehouse', 'continue'],
|
||||
textValues: ['./warehouse.sqlite'],
|
||||
});
|
||||
const testConnection = vi.fn(async () => 0);
|
||||
const scanConnection = vi.fn(async () => 0);
|
||||
const io = makeIo();
|
||||
const result = await runKtxSetupDatabasesStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'auto',
|
||||
skipDatabases: false,
|
||||
databaseSchemas: [],
|
||||
disableQueryHistory: true,
|
||||
},
|
||||
io.io,
|
||||
{ prompts, testConnection, scanConnection },
|
||||
);
|
||||
|
||||
expect(result.status, io.stderr()).toBe('ready');
|
||||
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
|
||||
expect(config.connections.warehouse).toMatchObject({
|
||||
driver: 'sqlite',
|
||||
path: './warehouse.sqlite',
|
||||
context: { depth: 'deep' },
|
||||
});
|
||||
});
|
||||
|
||||
it('labels existing database connections with the database type', async () => {
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
|
|
@ -376,7 +334,6 @@ describe('setup databases step', () => {
|
|||
expect(config.connections['postgres-warehouse']).toEqual({
|
||||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
context: { depth: 'fast' },
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1558,7 +1515,7 @@ describe('setup databases step', () => {
|
|||
);
|
||||
expect(io.stdout()).not.toContain('Tables: 2');
|
||||
expect(io.stdout()).toContain('◇ Building schema context for postgres-warehouse');
|
||||
expect(io.stdout()).toContain('│ Running fast database ingest…');
|
||||
expect(io.stdout()).toContain('│ Running database scan…');
|
||||
expect(io.stdout()).toContain('◇ Schema context complete for postgres-warehouse');
|
||||
expect(io.stdout()).toContain('│ Changes: 2 new tables');
|
||||
expect(io.stdout()).toContain('◇ Database ready');
|
||||
|
|
@ -1907,7 +1864,7 @@ describe('setup databases step', () => {
|
|||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
schemas: ['public'],
|
||||
context: { queryHistory: { enabled: false }, depth: 'fast' },
|
||||
context: { queryHistory: { enabled: false } },
|
||||
});
|
||||
expect(config.setup).toEqual({
|
||||
database_connection_ids: ['warehouse'],
|
||||
|
|
@ -1946,7 +1903,6 @@ describe('setup databases step', () => {
|
|||
expect(config.connections.warehouse).toEqual({
|
||||
driver: 'sqlite',
|
||||
path: './warehouse.sqlite',
|
||||
context: { depth: 'fast' },
|
||||
});
|
||||
expect(config.setup).toEqual({
|
||||
database_connection_ids: ['warehouse'],
|
||||
|
|
@ -2023,11 +1979,11 @@ describe('setup databases step', () => {
|
|||
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
|
||||
expect(config.connections.warehouse).toMatchObject({ driver: 'postgres', url: 'env:DATABASE_URL' });
|
||||
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
|
||||
expect(io.stderr()).toContain('Fast database ingest failed for warehouse.');
|
||||
expect(io.stderr()).toContain('│ Fast database ingest failed for warehouse.');
|
||||
expect(io.stderr()).toContain(`Debug command: ktx ingest warehouse --project-dir ${tempDir} --fast --debug`);
|
||||
expect(io.stderr()).toContain('Database scan failed for warehouse.');
|
||||
expect(io.stderr()).toContain('│ Database scan failed for warehouse.');
|
||||
expect(io.stderr()).toContain(`Debug command: ktx ingest warehouse --project-dir ${tempDir} --debug`);
|
||||
expect(io.stderr()).not.toContain('Structural scan failed for warehouse.');
|
||||
expect(io.stderr()).not.toMatch(/^Fast database ingest failed for warehouse\./m);
|
||||
expect(io.stderr()).not.toMatch(/^Database scan failed for warehouse\./m);
|
||||
});
|
||||
|
||||
it('prints the native SQLite rebuild command when scanning hits a Node ABI mismatch', async () => {
|
||||
|
|
@ -2066,7 +2022,7 @@ describe('setup databases step', () => {
|
|||
expect(io.stderr()).toContain('Native SQLite is built for a different Node.js ABI.');
|
||||
expect(io.stderr()).toContain('│ Native SQLite is built for a different Node.js ABI.');
|
||||
expect(io.stderr()).toContain('Fix: pnpm run native:rebuild');
|
||||
expect(io.stderr()).toContain(`Retry: ktx ingest warehouse --project-dir ${tempDir} --fast`);
|
||||
expect(io.stderr()).toContain(`Retry: ktx ingest warehouse --project-dir ${tempDir}`);
|
||||
expect(io.stderr()).not.toContain('ktx scan');
|
||||
expect(io.stderr()).not.toContain('npm rebuild');
|
||||
expect(io.stderr()).not.toMatch(/^Native SQLite is built for a different Node.js ABI\./m);
|
||||
|
|
@ -2364,7 +2320,7 @@ describe('setup databases step', () => {
|
|||
'utf-8',
|
||||
);
|
||||
const io = makeIo();
|
||||
const prompts = makePromptAdapter({ selectValues: ['yes', 'deep'] });
|
||||
const prompts = makePromptAdapter({ selectValues: ['yes'] });
|
||||
const runner = fakeHistoricSqlRunner('postgres', 'pg_stat_statements');
|
||||
const historicSqlReadinessProbe = vi.fn(async () => ({
|
||||
ok: true as const,
|
||||
|
|
@ -2399,12 +2355,6 @@ describe('setup databases step', () => {
|
|||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
expect(prompts.select).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
message: expect.stringContaining('How much database context should KTX build?'),
|
||||
}),
|
||||
);
|
||||
expect(historicSqlReadinessProbe).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
projectDir: tempDir,
|
||||
|
|
@ -2420,7 +2370,6 @@ describe('setup databases step', () => {
|
|||
minExecutions: 5,
|
||||
filters: { dropTrivialProbes: true },
|
||||
},
|
||||
depth: 'deep',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ describe('setup sources step', () => {
|
|||
inputMode: 'disabled',
|
||||
source: 'notion',
|
||||
sourceConnectionId: 'notion-main',
|
||||
sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
|
||||
sourceAuthTokenRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
|
||||
notionCrawlMode: 'selected_roots',
|
||||
notionRootPageIds: ['page-1'],
|
||||
runInitialSourceIngest: false,
|
||||
|
|
@ -281,6 +281,81 @@ describe('setup sources step', () => {
|
|||
expect((await readConfig()).connections['notion-main']?.last_successful_cursor).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects --source-api-key-ref for Notion and points at --source-auth-token-ref', async () => {
|
||||
await addPrimarySource();
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'disabled',
|
||||
source: 'notion',
|
||||
sourceConnectionId: 'notion-main',
|
||||
sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
|
||||
notionCrawlMode: 'selected_roots',
|
||||
notionRootPageIds: ['page-1'],
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
io.io,
|
||||
{},
|
||||
),
|
||||
).resolves.toEqual({ status: 'failed', projectDir });
|
||||
|
||||
expect(io.stderr()).toContain('--source-api-key-ref does not apply to --source notion; use --source-auth-token-ref.');
|
||||
expect((await readConfig()).connections['notion-main']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects --source-auth-token-ref for Metabase and points at --source-api-key-ref', async () => {
|
||||
await addPrimarySource();
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'disabled',
|
||||
source: 'metabase',
|
||||
sourceConnectionId: 'prod_metabase',
|
||||
sourceUrl: 'https://metabase.example.com',
|
||||
sourceAuthTokenRef: 'env:METABASE_API_KEY', // pragma: allowlist secret
|
||||
sourceWarehouseConnectionId: 'warehouse',
|
||||
metabaseDatabaseId: 1,
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
io.io,
|
||||
{},
|
||||
),
|
||||
).resolves.toEqual({ status: 'failed', projectDir });
|
||||
|
||||
expect(io.stderr()).toContain('--source-auth-token-ref does not apply to --source metabase; use --source-api-key-ref.');
|
||||
});
|
||||
|
||||
it('rejects --source-client-secret-ref for dbt and points at --source-auth-token-ref', async () => {
|
||||
await addPrimarySource();
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'disabled',
|
||||
source: 'dbt',
|
||||
sourceConnectionId: 'dbt-main',
|
||||
sourceClientSecretRef: 'env:DBT_SECRET', // pragma: allowlist secret
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
io.io,
|
||||
{},
|
||||
),
|
||||
).resolves.toEqual({ status: 'failed', projectDir });
|
||||
|
||||
expect(io.stderr()).toContain('--source-client-secret-ref does not apply to --source dbt; use --source-auth-token-ref.');
|
||||
});
|
||||
|
||||
it('accepts former ingest subcommand names as interactive source connection ids', async () => {
|
||||
await addPrimarySource();
|
||||
const io = makeIo();
|
||||
|
|
@ -323,7 +398,7 @@ describe('setup sources step', () => {
|
|||
inputMode: 'disabled',
|
||||
source: 'notion',
|
||||
sourceConnectionId: 'notion-main',
|
||||
sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
|
||||
sourceAuthTokenRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
|
||||
notionCrawlMode: 'all_accessible',
|
||||
notionRootPageIds: ['page-1'],
|
||||
runInitialSourceIngest: false,
|
||||
|
|
|
|||
|
|
@ -398,6 +398,59 @@ describe('setup status', () => {
|
|||
expect(rendered).toContain('KTX context built: yes');
|
||||
});
|
||||
|
||||
it('reports context ready after a partial ingest report saved memory', async () => {
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
[
|
||||
'setup:',
|
||||
' database_connection_ids:',
|
||||
' - warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
'ingest:',
|
||||
' embeddings:',
|
||||
' backend: none',
|
||||
' dimensions: 8',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
await writeKtxSetupState(tempDir, { completed_steps: ['project', 'databases'] });
|
||||
await persistLocalBundleReport(
|
||||
tempDir,
|
||||
localFakeBundleReport('warehouse-job-partial', {
|
||||
connectionId: 'warehouse',
|
||||
sourceKey: 'fake',
|
||||
body: {
|
||||
failedWorkUnits: ['orders-bad'],
|
||||
workUnits: [
|
||||
{
|
||||
unitKey: 'orders-ok',
|
||||
rawFiles: ['orders/orders.json'],
|
||||
status: 'success',
|
||||
actions: [{ target: 'wiki', type: 'created', key: 'wiki/orders.md', detail: 'orders' }],
|
||||
touchedSlSources: [],
|
||||
},
|
||||
{
|
||||
unitKey: 'orders-bad',
|
||||
rawFiles: ['orders/bad.json'],
|
||||
status: 'failed',
|
||||
reason: 'writer tool failed',
|
||||
actions: [],
|
||||
touchedSlSources: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const status = await readKtxSetupStatus(tempDir);
|
||||
|
||||
expect(status.context).toMatchObject({ ready: true, status: 'completed' });
|
||||
});
|
||||
|
||||
it('formats plain and JSON setup status payloads', async () => {
|
||||
const status = await readKtxSetupStatus(tempDir);
|
||||
const rendered = formatKtxSetupStatus(status);
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ describe('standalone built ktx CLI smoke', () => {
|
|||
expect([0, 1]).toContain(result.code);
|
||||
});
|
||||
|
||||
it('runs fast public database ingest through the built binary with manifest artifacts', async () => {
|
||||
it('blocks public database ingest through the built binary when enrichment is not configured', async () => {
|
||||
const projectDir = join(tempDir, 'database-ingest-project');
|
||||
const init = await runSetupNewProject(projectDir);
|
||||
expectSetupStderr(init);
|
||||
|
|
@ -200,19 +200,10 @@ describe('standalone built ktx CLI smoke', () => {
|
|||
expect(connectionTest.stdout).toContain('Driver: sqlite');
|
||||
expect(connectionTest.stdout).toContain('Status: ok');
|
||||
|
||||
const ingest = await runBuiltCli(['ingest', 'warehouse', '--project-dir', projectDir, '--fast', '--no-input']);
|
||||
expectProjectStderr(ingest, projectDir);
|
||||
expect(ingest.stdout).toContain('Ingest finished');
|
||||
expect(ingest.stdout).toContain('warehouse');
|
||||
expect(ingest.stdout).toContain('Database schema');
|
||||
expect(ingest.stdout).toContain('warehouse done');
|
||||
const ingest = await runBuiltCli(['ingest', 'warehouse', '--project-dir', projectDir, '--no-input']);
|
||||
expect(ingest.code).toBe(1);
|
||||
expect(ingest.stdout).toContain('warehouse cannot be ingested: enrichment is not configured');
|
||||
expect(ingest.stdout).not.toContain('KTX scan completed');
|
||||
|
||||
const manifest = await readFile(join(projectDir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8');
|
||||
expect(manifest).toContain('customers:');
|
||||
expect(manifest).toContain('orders:');
|
||||
expect(manifest).toContain('source: formal');
|
||||
expect(manifest).not.toContain('ai:');
|
||||
}, 30_000);
|
||||
|
||||
it('parses gateway LLM config and OpenAI enrichment embeddings used by standalone scans without network calls', async () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue