From de72a10ffb9bedc154ceb499df4f41665b2f9541 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Sun, 17 May 2026 01:04:44 +0200 Subject: [PATCH] fix(cli): build runtime assets during dev setup (#121) --- package.json | 1 + packages/cli/src/public-ingest.test.ts | 37 ++++++++++++++++++++++++++ packages/cli/src/public-ingest.ts | 33 ++++++++++++++++++----- scripts/package-artifacts.mjs | 14 ++++++++++ scripts/setup-dev.mjs | 6 +++++ scripts/setup-dev.test.mjs | 1 + 6 files changed, 85 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 3941644b..ad773b28 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ }, "scripts": { "artifacts:build": "node scripts/package-artifacts.mjs build", + "artifacts:build-runtime": "node scripts/package-artifacts.mjs build-runtime", "artifacts:check": "node scripts/package-artifacts.mjs check", "artifacts:live-db-smoke": "node scripts/installed-live-database-smoke.mjs", "artifacts:verify": "node scripts/package-artifacts.mjs verify", diff --git a/packages/cli/src/public-ingest.test.ts b/packages/cli/src/public-ingest.test.ts index 2e96be4b..1a5ace5f 100644 --- a/packages/cli/src/public-ingest.test.ts +++ b/packages/cli/src/public-ingest.test.ts @@ -829,6 +829,43 @@ describe('runKtxPublicIngest', () => { expect(io.stdout()).not.toContain('historic-sql'); }); + it('prints the runtime artifact build hint for missing query-history runtime assets', async () => { + const io = makeIo(); + const project = deepReadyProject({ + warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + }); + const runScan = vi.fn(async () => 0); + const runIngest = vi.fn(async (_args, ingestIo) => { + ingestIo.stderr.write('Missing bundled Python runtime manifest: /repo/packages/cli/assets/python/manifest.json\n'); + ingestIo.stderr.write('In a source checkout, build the local runtime assets with: pnpm run artifacts:build\n'); + ingestIo.stderr.write('Then retry the runtime-backed KTX command.\n'); + return 1; + }); + + await expect( + runKtxPublicIngest( + { + command: 'run', + projectDir: '/tmp/project', + targetConnectionId: 'warehouse', + all: false, + json: false, + inputMode: 'disabled', + queryHistory: 'enabled', + }, + io.io, + { loadProject: vi.fn(async () => project), runScan, runIngest }, + ), + ).resolves.toBe(1); + + expect(io.stdout()).toContain('Missing bundled Python runtime manifest'); + expect(io.stdout()).toContain( + 'In a source checkout, build the local runtime assets with: pnpm run artifacts:build', + ); + expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history'); + expect(io.stdout()).not.toContain('Then retry the runtime-backed KTX command'); + }); + it('fails deep-readiness targets before work starts while continuing independent --all targets', async () => { const io = makeIo(); const project = projectWithConnections({ diff --git a/packages/cli/src/public-ingest.ts b/packages/cli/src/public-ingest.ts index 5b8a34ed..7a75244b 100644 --- a/packages/cli/src/public-ingest.ts +++ b/packages/cli/src/public-ingest.ts @@ -663,16 +663,35 @@ function createCapturedPublicIngestIo(): CapturedPublicIngestIo { const INTERNAL_STATUS_LINE_RE = /^(Report|Run|Job|Status|Adapter|Connection|Sync|Diff|Tasks|Work units|Failed tasks|Saved memory|Provenance rows):\s*/; +const ACTIONABLE_FAILURE_LINE_RE = + /^(Missing bundled Python runtime manifest|KTX Python runtime is required|KTX managed daemon|Error:|Failed\b|Could not\b|Cannot\b)/; +const RUNTIME_BACKED_RETRY_LINE_RE = /^Then retry the runtime-backed KTX command\.?$/; -function firstCapturedFailureLine(output: string): string | undefined { - return output +function capturedFailureMessage(output: string): string | undefined { + const lines = output .split(/\r?\n/) .map((line) => line.trim()) .filter((line) => line.length > 0) .filter((line) => !line.startsWith('KTX scan completed')) .filter((line) => !INTERNAL_STATUS_LINE_RE.test(line)) - .map(publicIngestOutputLine) - .find((line) => line.length > 0); + .map(publicIngestOutputLine); + + const actionableIndex = lines.findIndex((line) => ACTIONABLE_FAILURE_LINE_RE.test(line)); + if (actionableIndex < 0) { + return lines.find((line) => line.length > 0); + } + + const firstLine = lines[actionableIndex]; + if (!firstLine?.startsWith('Missing bundled Python runtime manifest')) { + return firstLine; + } + + const followupLines = lines + .slice(actionableIndex + 1) + .filter((line) => !RUNTIME_BACKED_RETRY_LINE_RE.test(line)) + .filter((line) => !/\bRetry:\s/.test(line)) + .filter((line) => line.startsWith('In a source checkout, build the local runtime assets with:')); + return [firstLine, ...followupLines].join('\n'); } export async function executePublicIngestTarget( @@ -737,7 +756,7 @@ export async function executePublicIngestTarget( args, 'failed', 'database-schema', - capturedScanIo ? firstCapturedFailureLine(capturedScanIo.capturedOutput()) : undefined, + capturedScanIo ? capturedFailureMessage(capturedScanIo.capturedOutput()) : undefined, ); } deps.onPhaseEnd?.('database-schema', 'done'); @@ -779,7 +798,7 @@ export async function executePublicIngestTarget( args, 'failed', 'query-history', - capturedIngestIo ? firstCapturedFailureLine(capturedIngestIo.capturedOutput()) : undefined, + capturedIngestIo ? capturedFailureMessage(capturedIngestIo.capturedOutput()) : undefined, ); } deps.onPhaseEnd?.('query-history', 'done'); @@ -819,7 +838,7 @@ export async function executePublicIngestTarget( args, exitCode === 0 ? 'done' : 'failed', 'source-ingest', - capturedIngestIo ? firstCapturedFailureLine(capturedIngestIo.capturedOutput()) : undefined, + capturedIngestIo ? capturedFailureMessage(capturedIngestIo.capturedOutput()) : undefined, ); } diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 1a43ddf2..8e1f174d 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -942,6 +942,16 @@ async function buildArtifacts(layout) { await assertPathExists(artifactManifestPath(layout), 'artifact manifest'); } +async function buildRuntimeWheelAssets(layout) { + await rm(layout.pythonDir, { recursive: true, force: true }); + await mkdir(layout.pythonDir, { recursive: true }); + + const [, wheelCommand] = buildArtifactCommands(layout); + await runCommand(wheelCommand.command, wheelCommand.args, { cwd: wheelCommand.cwd }); + const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); + await copyRuntimeWheelAssets(layout, pythonArtifacts); +} + async function verifyNpmArtifacts(layout, tmpRoot) { for (const packageInfo of NPM_ARTIFACT_PACKAGES) { await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`); @@ -1011,6 +1021,10 @@ async function main() { await buildArtifacts(layout); return; } + if (command === 'build-runtime') { + await buildRuntimeWheelAssets(layout); + return; + } if (command === 'verify') { await verifyArtifacts(layout); return; diff --git a/scripts/setup-dev.mjs b/scripts/setup-dev.mjs index 67a2e4bd..11a690ff 100644 --- a/scripts/setup-dev.mjs +++ b/scripts/setup-dev.mjs @@ -41,6 +41,12 @@ export async function runSetupDev(options = {}) { args: ['run', 'build'], retry: 'pnpm run build', }, + { + name: 'runtime wheel assets', + command: 'pnpm', + args: ['run', 'artifacts:build-runtime'], + retry: 'pnpm run artifacts:build-runtime', + }, { name: 'doctor setup', command: process.execPath, diff --git a/scripts/setup-dev.test.mjs b/scripts/setup-dev.test.mjs index e50e8b52..3323f1d8 100644 --- a/scripts/setup-dev.test.mjs +++ b/scripts/setup-dev.test.mjs @@ -22,6 +22,7 @@ test('runSetupDev runs phased setup without global linking', async () => { ['pnpm', ['install', '--frozen-lockfile']], ['pnpm', ['run', 'native:rebuild']], ['pnpm', ['run', 'build']], + ['pnpm', ['run', 'artifacts:build-runtime']], [process.execPath, ['packages/cli/dist/bin.js', 'status', '--no-input']], ], );