apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-13 08:15:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

test: align ingest surface expectations

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-13 18:14:01 +02:00
parent 220fb5f8ea
commit 43bf5bfdc5
6 changed files with 22 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/cli/src/commands/ingest-commands.ts
View file

@ -47,7 +47,7 @@ function inputMode(options: OutputModeOptions): Pick<KtxIngestArgs, 'inputMode'>
return options.input === false ? { inputMode: 'disabled' } : {};
}
function resolvedOptions<T extends object>(command: Command, fallback: T): T {
function resolvedOptions<T extends object>(command: { optsWithGlobals?: () => object }, fallback: T): T {
return (command.optsWithGlobals ? command.optsWithGlobals() : fallback) as T;
}

1
packages/cli/src/ingest.test.ts
View file

@ -238,6 +238,7 @@ describe('runKtxIngest', () => {
connectionId: 'warehouse',
adapter: 'historic-sql',
sourceDir,
allowImplicitAdapter: true,
outputMode: 'plain',
},
runIo.io,

20
packages/cli/src/setup-context.test.ts
View file

@ -145,7 +145,7 @@ describe('setup context build state', () => {
sourceProgress: [
{
connectionId: 'warehouse',
operation: 'scan',
operation: 'database-ingest',
status: 'running',
percent: 42,
message: 'Generating descriptions 4/10 tables',
@ -168,7 +168,7 @@ describe('setup context build state', () => {
sourceProgress: [
{
connectionId: 'warehouse',
operation: 'scan',
operation: 'database-ingest',
status: 'running',
percent: 42,
message: 'Generating descriptions 4/10 tables',
@ -238,7 +238,7 @@ describe('setup context build state', () => {
const io = makeIo();
const runContextBuildMock = vi.fn(async (_project, _args, _io, hooks) => {
hooks.onSourceProgress?.([
{ connectionId: 'warehouse', operation: 'scan', status: 'done', elapsedMs: 1000 },
{ connectionId: 'warehouse', operation: 'database-ingest', status: 'done', elapsedMs: 1000 },
{ connectionId: 'docs', operation: 'source-ingest', status: 'failed', elapsedMs: 2000 },
]);
return {
@ -268,7 +268,7 @@ describe('setup context build state', () => {
artifactPaths: ['raw-sources/docs/notion/sync-1/ingest-report.json'],
retryableFailedTargets: ['docs'],
sourceProgress: [
{ connectionId: 'warehouse', operation: 'scan', status: 'done', elapsedMs: 1000 },
{ connectionId: 'warehouse', operation: 'database-ingest', status: 'done', elapsedMs: 1000 },
{ connectionId: 'docs', operation: 'source-ingest', status: 'failed', elapsedMs: 2000 },
],
});
@ -520,7 +520,7 @@ describe('setup context build state', () => {
retryableFailedTargets: [],
commands: contextBuildCommands(tempDir, 'setup-context-local-progress'),
sourceProgress: [
{ connectionId: 'warehouse', operation: 'scan' as const, status: 'done' as const, elapsedMs: 30000 },
{ connectionId: 'warehouse', operation: 'database-ingest' as const, status: 'done' as const, elapsedMs: 30000 },
{ connectionId: 'docs', operation: 'source-ingest' as const, status: 'running' as const, startedAtMs: Date.now() - 5000 },
],
});
@ -539,7 +539,7 @@ describe('setup context build state', () => {
retryableFailedTargets: [],
commands: contextBuildCommands(tempDir, 'setup-context-local-progress'),
sourceProgress: [
{ connectionId: 'warehouse', operation: 'scan' as const, status: 'done' as const, elapsedMs: 30000 },
{ connectionId: 'warehouse', operation: 'database-ingest' as const, status: 'done' as const, elapsedMs: 30000 },
{ connectionId: 'docs', operation: 'source-ingest' as const, status: 'done' as const, elapsedMs: 60000 },
],
});
@ -560,7 +560,7 @@ describe('setup context build state', () => {
const output = io.stdout();
expect(output).toContain('Building KTX context');
expect(output).toContain('Primary sources:');
expect(output).toContain('Databases:');
expect(output).toContain('warehouse');
expect(output).toContain('Context sources:');
expect(output).toContain('docs');
@ -583,7 +583,7 @@ describe('setup context build state', () => {
sourceProgress: [
{
connectionId: 'warehouse',
operation: 'scan' as const,
operation: 'database-ingest' as const,
status: 'running' as const,
startedAtMs: Date.now() - 5000,
percent: 35,
@ -611,7 +611,7 @@ describe('setup context build state', () => {
sourceProgress: [
{
connectionId: 'warehouse',
operation: 'scan' as const,
operation: 'database-ingest' as const,
status: polls === 1 ? ('running' as const) : ('done' as const),
startedAtMs: Date.now() - 5000,
elapsedMs: polls === 1 ? undefined : 6000,
@ -654,7 +654,7 @@ describe('setup context build state', () => {
retryableFailedTargets: [],
commands: contextBuildCommands(tempDir, 'setup-context-local-detach'),
sourceProgress: [
{ connectionId: 'warehouse', operation: 'scan' as const, status: 'running' as const, startedAtMs: Date.now() },
{ connectionId: 'warehouse', operation: 'database-ingest' as const, status: 'running' as const, startedAtMs: Date.now() },
],
});
const io = makeIo();

6
packages/cli/src/setup-context.ts
View file

@ -199,7 +199,7 @@ function normalizeState(projectDir: string, value: unknown): KtxSetupContextStat
};
}
const VALID_SOURCE_OPERATIONS = new Set(['scan', 'source-ingest']);
const VALID_SOURCE_OPERATIONS = new Set(['database-ingest', 'source-ingest']);
const VALID_SOURCE_STATUSES = new Set(['queued', 'running', 'done', 'failed']);
function normalizeSourceProgress(value: unknown): ContextBuildSourceProgressUpdate[] | undefined {
@ -213,7 +213,7 @@ function normalizeSourceProgress(value: unknown): ContextBuildSourceProgressUpda
if (!VALID_SOURCE_STATUSES.has(String(rec.status))) continue;
entries.push({
connectionId: rec.connectionId,
operation: rec.operation as 'scan' | 'source-ingest',
operation: rec.operation as 'database-ingest' | 'source-ingest',
status: rec.status as 'queued' | 'running' | 'done' | 'failed',
...(typeof rec.startedAtMs === 'number' ? { startedAtMs: rec.startedAtMs } : {}),
...(typeof rec.elapsedMs === 'number' ? { elapsedMs: rec.elapsedMs } : {}),
@ -289,7 +289,7 @@ function listContextTargets(project: KtxLocalProject): KtxSetupContextTargets {
const plan = buildPublicIngestPlan(project, { projectDir: project.projectDir, all: true });
return {
primarySourceConnectionIds: plan.targets
.filter((target) => target.operation === 'scan')
.filter((target) => target.operation === 'database-ingest')
.map((target) => target.connectionId),
contextSourceConnectionIds: plan.targets
.filter((target) => target.operation === 'source-ingest')

4
packages/cli/src/setup-demo-tour.test.ts
View file

@ -145,8 +145,8 @@ describe('DEMO_REPLAY_TARGETS', () => {
expect(DEMO_REPLAY_TARGETS.contextSources).toHaveLength(3);
});
it('primary source is a scan operation', () => {
expect(DEMO_REPLAY_TARGETS.primarySources[0].operation).toBe('scan');
it('primary source is a database-ingest operation', () => {
expect(DEMO_REPLAY_TARGETS.primarySources[0].operation).toBe('database-ingest');
});
it('context sources are source-ingest operations', () => {

10
packages/cli/src/setup-demo-tour.ts
View file

@ -30,7 +30,7 @@ function dim(text: string): string {
function createDemoTarget(
connectionId: string,
operation: 'scan' | 'source-ingest',
operation: 'database-ingest' | 'source-ingest',
driver: string,
): KtxPublicIngestPlanTarget {
const adapter = operation === 'source-ingest' ? driver : undefined;
@ -40,9 +40,9 @@ function createDemoTarget(
operation,
...(adapter ? { adapter } : {}),
debugCommand: `ktx setup --project-dir <project-dir>`,
steps: operation === 'scan'
? ['scan', 'enrich', 'memory-update']
: ['source-ingest', 'enrich', 'memory-update'],
steps: operation === 'database-ingest'
? ['database-schema']
: ['source-ingest', 'memory-update'],
};
}
@ -195,7 +195,7 @@ export interface DemoReplayEvent {
export const DEMO_REPLAY_TARGETS = {
primarySources: [
createDemoTarget('postgres-warehouse', 'scan', 'postgres'),
createDemoTarget('postgres-warehouse', 'database-ingest', 'postgres'),
],
contextSources: [
createDemoTarget('dbt-main', 'source-ingest', 'dbt'),