apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-07-01 08:59:39 +02:00
Code Issues Projects Releases Packages Wiki Activity

Improve ingest progress updates

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-18 02:21:59 +02:00
parent 93f992ed5c
commit a00285fd42
9 changed files with 361 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

111
packages/cli/src/ingest.test.ts
View file

@ -635,6 +635,117 @@ describe('runKtxIngest', () => {
expect(io.stderr()).not.toContain('Metabase ingest: prod-metabase');
});
it('emits structured child ingest progress during Metabase fan-out', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo();
const progressEvents: Array<{ percent: number; message: string; transient?: boolean }> = [];
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'prod-metabase',
adapter: 'metabase',
outputMode: 'json',
},
io.io,
{
progress: (event) => progressEvents.push(event),
runLocalMetabaseIngest: async (input) => {
input.progress?.onMetabaseFanoutPlanned?.({
metabaseConnectionId: 'prod-metabase',
children: [{ metabaseDatabaseId: 1, targetConnectionId: 'warehouse_a' }],
});
input.progress?.onMetabaseChildStarted?.({
metabaseConnectionId: 'prod-metabase',
metabaseDatabaseId: 1,
targetConnectionId: 'warehouse_a',
jobId: 'metabase-child-1',
});
input.memoryFlow?.update({
plannedWorkUnits: [
{
unitKey: 'metabase-col-6',
rawFiles: ['cards/40.json'],
peerFileCount: 0,
dependencyCount: 0,
},
],
});
input.memoryFlow?.emit({ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 });
input.memoryFlow?.emit({
type: 'work_unit_started',
unitKey: 'metabase-col-6',
skills: ['sl_capture'],
stepBudget: 40,
});
input.memoryFlow?.emit({
type: 'work_unit_step',
unitKey: 'metabase-col-6',
stepIndex: 7,
stepBudget: 40,
});
input.memoryFlow?.emit({
type: 'stage_progress',
stage: 'integration',
percent: 81,
message: 'Resolving text conflict for metabase-col-6',
});
input.memoryFlow?.emit({ type: 'work_unit_finished', unitKey: 'metabase-col-6', status: 'success' });
input.memoryFlow?.update({
plannedWorkUnits: [
{
unitKey: 'metabase-col-7',
rawFiles: ['cards/48.json'],
peerFileCount: 0,
dependencyCount: 0,
},
],
});
input.memoryFlow?.emit({ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 });
input.memoryFlow?.emit({
type: 'work_unit_started',
unitKey: 'metabase-col-7',
skills: ['sl_capture'],
stepBudget: 40,
});
input.progress?.onMetabaseChildCompleted?.({
metabaseConnectionId: 'prod-metabase',
metabaseDatabaseId: 1,
targetConnectionId: 'warehouse_a',
jobId: 'metabase-child-1',
status: 'done',
});
return {
metabaseConnectionId: 'prod-metabase',
status: 'all_succeeded',
totals: { workUnits: 1, failedWorkUnits: 0 },
children: [],
};
},
},
),
).resolves.toBe(0);
expect(progressEvents).toEqual(
expect.arrayContaining([
{ percent: 45, message: 'Planned 1 task' },
{ percent: 55, message: 'Processing 1/1 tasks: metabase-col-6' },
{
percent: 60,
message: 'Processing tasks: 0/1 complete, 1 active; latest metabase-col-6 step 7/40',
transient: true,
},
{ percent: 81, message: 'Resolving text conflict for metabase-col-6' },
{ percent: 81, message: 'Processing 1/1 tasks: metabase-col-7' },
]),
);
expect(io.stdout()).toContain('"status": "all_succeeded"');
expect(io.stderr()).not.toContain('Metabase ingest: prod-metabase');
});
it('runs Metabase scheduled ingest through the public CLI command path with real fan-out', async () => {
const projectDir = join(tempDir, 'metabase-cli-project');
await writeWarehouseConfig(projectDir);