apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-10 08:05:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix(cli): keep ingest progress off stdout

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-12 01:36:41 +02:00
parent 5cdae74825
commit cd2b50d7d2
4 changed files with 30 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

6
packages/cli/src/ingest-viz.test.ts
View file

@ -331,8 +331,9 @@ describe('runKtxIngest viz and replay', () => {
).resolves.toBe(0);
expect(runLocal).toHaveBeenCalledWith(expect.objectContaining({ memoryFlow: expect.anything() }));
expect(io.stdout()).toContain('[5%] Fetching source files for warehouse/fake');
expect(io.stderr()).toContain('[5%] Fetching source files for warehouse/fake');
expect(io.stdout()).toContain('Job: plain-run');
expect(io.stdout()).not.toContain('[5%]');
expect(io.stdout()).not.toContain('KTX memory flow');
});
@ -407,8 +408,9 @@ describe('runKtxIngest viz and replay', () => {
expect(startLiveMemoryFlow).not.toHaveBeenCalled();
expect(runLocal).toHaveBeenCalledWith(expect.objectContaining({ memoryFlow: expect.anything() }));
expect(io.stdout()).toContain('[5%] Fetching source files for warehouse/fake');
expect(io.stderr()).toContain('[5%] Fetching source files for warehouse/fake');
expect(io.stdout()).toContain('Job: raw-missing-viz-run');
expect(io.stdout()).not.toContain('[5%]');
expect(io.stdout()).not.toContain('KTX memory flow');
expect(io.stderr()).toContain(
'Visualization requested but stdin raw mode is unavailable; printing plain output.',

2
packages/cli/src/ingest.test-utils.ts
View file

@ -546,7 +546,7 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync
),
).resolves.toBe(0);
expect(io.stderr()).toBe('');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).toContain(`target=warehouse_a database=1 status=done job=${jobId}`);

38
packages/cli/src/ingest.test.ts
View file

@ -159,7 +159,7 @@ describe('runKtxIngest', () => {
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).toContain('warehouse_a');
expect(io.stdout()).toContain('metabase-child-1');
expect(io.stderr()).toBe('');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
});
it('returns a non-zero code when Metabase fan-out has failed children', async () => {
@ -229,10 +229,10 @@ describe('runKtxIngest', () => {
expect(io.stdout()).toContain('Metabase fan-out: partial_failure');
expect(io.stdout()).toContain('Failed work units: 1');
expect(io.stdout()).toContain('status=error');
expect(io.stderr()).toBe('');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
});
it('prints Metabase fan-out progress before the final summary', async () => {
it('writes metabase fan-out progress to stderr and final result to stdout', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo();
@ -303,12 +303,12 @@ describe('runKtxIngest', () => {
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Metabase ingest: prod-metabase');
expect(io.stdout()).toContain('Targets: 1 mapped database');
expect(io.stdout()).toContain('- database=1 target=warehouse_a status=running job=metabase-child-1');
expect(io.stdout()).toContain('- database=1 target=warehouse_a status=done job=metabase-child-1');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
expect(io.stderr()).toContain('Targets: 1 mapped database');
expect(io.stderr()).toContain('- database=1 target=warehouse_a status=running job=metabase-child-1');
expect(io.stderr()).toContain('- database=1 target=warehouse_a status=done job=metabase-child-1');
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stderr()).toBe('');
expect(io.stdout()).not.toContain('status=running job=metabase-child-1');
});
it('runs Metabase scheduled ingest through the public CLI command path with real fan-out', async () => {
@ -393,7 +393,7 @@ describe('runKtxIngest', () => {
),
).resolves.toBe(0);
expect(io.stderr()).toBe('');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).toContain('Source: prod-metabase');
expect(io.stdout()).toContain('Children: 2');
@ -858,7 +858,7 @@ describe('runKtxIngest', () => {
);
});
it('prints live progress for plain local ingest in interactive terminals', async () => {
it('writes plain TTY ingest progress to stderr and final report to stdout', async () => {
const projectDir = join(tempDir, 'historic-sql-progress-project');
await mkdir(projectDir, { recursive: true });
await writeFile(
@ -944,16 +944,14 @@ describe('runKtxIngest', () => {
),
).resolves.toBe(0);
const stdout = io.stdout();
expect(stdout).toContain('[5%] Fetching source files for warehouse/historic-sql');
expect(stdout).toContain('[15%] Fetched 3 source files from historic-sql');
expect(stdout).toContain('[45%] Planned 1 work unit');
expect(stdout).toContain('[80%] Processed 1/1 work units');
expect(stdout).toContain('[100%] Ingest completed');
expect(stdout.indexOf('[5%] Fetching source files for warehouse/historic-sql')).toBeLessThan(
stdout.indexOf('Report: report-live-1'),
);
expect(io.stderr()).toBe('');
const stderr = io.stderr();
expect(stderr).toContain('[5%] Fetching source files for warehouse/historic-sql');
expect(stderr).toContain('[15%] Fetched 3 source files from historic-sql');
expect(stderr).toContain('[45%] Planned 1 work unit');
expect(stderr).toContain('[80%] Processed 1/1 work units');
expect(stderr).toContain('[100%] Ingest completed');
expect(io.stdout()).toContain('Report: report-live-1');
expect(io.stdout()).not.toContain('[5%]');
});
it('passes local Looker pull-config options and agent runner into scheduled ingest for Looker scheduled ingest', async () => {

14
packages/cli/src/ingest.ts
View file

@ -143,22 +143,22 @@ function createMetabaseFanoutProgress(
connectionId: string,
io: KtxIngestIo,
): LocalMetabaseFanoutProgress {
io.stdout.write(`Metabase ingest: ${connectionId}\n`);
io.stdout.write('Checking mappings and scheduled-pull targets...\n');
io.stderr.write(`Metabase ingest: ${connectionId}\n`);
io.stderr.write('Checking mappings and scheduled-pull targets...\n');
return {
onMetabaseFanoutPlanned(event) {
io.stdout.write(`Targets: ${pluralize(event.children.length, 'mapped database')}\n`);
io.stderr.write(`Targets: ${pluralize(event.children.length, 'mapped database')}\n`);
for (const child of event.children) {
io.stdout.write(`- database=${child.metabaseDatabaseId} target=${child.targetConnectionId} status=queued\n`);
io.stderr.write(`- database=${child.metabaseDatabaseId} target=${child.targetConnectionId} status=queued\n`);
}
},
onMetabaseChildStarted(event) {
io.stdout.write(
io.stderr.write(
`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=running job=${event.jobId}\n`,
);
},
onMetabaseChildCompleted(event) {
io.stdout.write(
io.stderr.write(
`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=${event.status} job=${event.jobId}\n`,
);
},
@ -251,7 +251,7 @@ function createPlainIngestProgressRenderer(
const write = (percent: number, message: string) => {
const nextPercent = Math.max(lastPercent, Math.max(0, Math.min(100, percent)));
lastPercent = nextPercent;
io.stdout.write(`[${nextPercent}%] ${message}\n`);
io.stderr.write(`[${nextPercent}%] ${message}\n`);
};
return {