From c6f3759a249e829d81ed90f9a38996fdbf4a65ff Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Wed, 13 May 2026 15:41:29 +0200 Subject: [PATCH] Recover setup scan from SQLite ABI mismatch --- packages/cli/src/setup-databases.test.ts | 55 +++++++++++ packages/cli/src/setup-databases.ts | 117 +++++++++++++++++++++-- 2 files changed, 166 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index 45c85c67..95d1e3fb 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -1253,6 +1253,7 @@ describe('setup databases step', () => { io.io, { testConnection: vi.fn(async () => 0), + rebuildNativeSqlite: vi.fn(async () => 1), scanConnection: vi.fn(async (_projectDir: string, _connectionId: string, commandIo: KtxCliIo) => { commandIo.stderr.write( [ @@ -1278,6 +1279,60 @@ describe('setup databases step', () => { expect(io.stderr()).not.toMatch(/^Native SQLite is built for a different Node.js ABI\./m); }); + it('rebuilds native SQLite once and retries setup scanning after a Node ABI mismatch', async () => { + const io = makeIo(); + const scanConnection = vi.fn(async (_projectDir: string, _connectionId: string, commandIo: KtxCliIo) => { + if (scanConnection.mock.calls.length === 1) { + commandIo.stderr.write( + [ + "The module '/workspace/node_modules/better-sqlite3/build/Release/better_sqlite3.node'", + 'was compiled against a different Node.js version using', + 'NODE_MODULE_VERSION 147. This version of Node.js requires', + 'NODE_MODULE_VERSION 137. Please try re-compiling or re-installing', + 'the module (for instance, using `npm rebuild` or `npm install`).', + '', + ].join('\n'), + ); + return 1; + } + + commandIo.stdout.write('What changed\n'); + commandIo.stdout.write(' Semantic layer comparison found 0 changes across 56 tables\n'); + commandIo.stdout.write(' New tables: 0\n'); + commandIo.stdout.write(' Changed tables: 0\n'); + commandIo.stdout.write(' Removed tables: 0\n'); + commandIo.stdout.write(' Unchanged tables: 56\n'); + return 0; + }); + const rebuildNativeSqlite = vi.fn(async () => 0); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + databaseDrivers: ['postgres'], + databaseConnectionId: 'warehouse', + databaseUrl: 'env:DATABASE_URL', + databaseSchemas: [], + skipDatabases: false, + }, + io.io, + { + testConnection: vi.fn(async () => 0), + scanConnection, + rebuildNativeSqlite, + }, + ); + + expect(result.status).toBe('ready'); + expect(rebuildNativeSqlite).toHaveBeenCalledOnce(); + expect(rebuildNativeSqlite).toHaveBeenCalledWith(expect.anything()); + expect(scanConnection).toHaveBeenCalledTimes(2); + expect(io.stderr()).toContain('Native SQLite is built for a different Node.js ABI.'); + expect(io.stderr()).toContain('Rebuilding Native SQLite with pnpm run native:rebuild…'); + expect(io.stdout()).toContain('◇ Scan complete for warehouse'); + }); + it('writes Historic SQL config for supported Snowflake databases after validation succeeds', async () => { const io = makeIo(); const result = await runKtxSetupDatabasesStep( diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 02157e41..f697dd75 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -1,4 +1,8 @@ -import { writeFile } from 'node:fs/promises'; +import { execFile as execFileCallback } from 'node:child_process'; +import { readFile, writeFile } from 'node:fs/promises'; +import { delimiter, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; import { cancel, confirm, isCancel, multiselect, password, select, text } from '@clack/prompts'; import type { HistoricSqlDialect } from '@ktx/context/ingest'; import { @@ -17,6 +21,7 @@ import { withSetupInterruptConfirmation } from './setup-interrupt.js'; import { writeProjectLocalSecretReference } from './setup-secrets.js'; const HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY = 6; +const execFileAsync = promisify(execFileCallback); export type KtxSetupDatabaseDriver = | 'sqlite' @@ -81,6 +86,7 @@ export interface KtxSetupDatabasesDeps { prompts?: KtxSetupDatabasesPromptAdapter; testConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise; scanConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise; + rebuildNativeSqlite?: (io: KtxCliIo) => Promise; listSchemas?: (projectDir: string, connectionId: string) => Promise; listTables?: (projectDir: string, connectionId: string) => Promise; historicSqlProbe?: KtxSetupHistoricSqlProbe; @@ -957,6 +963,81 @@ function writePrefixedLines(write: (chunk: string) => void, output: string): voi } } +function envWithCurrentNodeFirst(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv { + return { + ...env, + PATH: `${dirname(process.execPath)}${delimiter}${env.PATH ?? ''}`, + }; +} + +function errorTextProperty(error: unknown, property: 'stderr' | 'stdout'): string { + if (typeof error !== 'object' || error === null || !(property in error)) { + return ''; + } + const value = (error as Record)[property]; + return typeof value === 'string' ? value : ''; +} + +function commandFailureOutput(error: unknown): string { + const stderr = errorTextProperty(error, 'stderr'); + const stdout = errorTextProperty(error, 'stdout'); + const message = error instanceof Error ? error.message : String(error); + return [stderr.trim(), stdout.trim(), message.trim()].filter((line) => line.length > 0).join('\n'); +} + +type PackageJsonScriptStatus = 'has-script' | 'exists' | 'missing'; + +async function packageJsonScriptStatus( + packageJsonPath: string, + scriptName: string, +): Promise { + try { + const parsed = JSON.parse(await readFile(packageJsonPath, 'utf-8')) as unknown; + if (typeof parsed !== 'object' || parsed === null || !('scripts' in parsed)) { + return 'exists'; + } + const scripts = (parsed as { scripts?: unknown }).scripts; + return typeof scripts === 'object' && scripts !== null && scriptName in scripts ? 'has-script' : 'exists'; + } catch { + return 'missing'; + } +} + +async function nativeSqliteRebuildCommand(): Promise<{ cwd: string; args: string[] }> { + let dir = dirname(fileURLToPath(import.meta.url)); + let packageRoot: string | undefined; + while (true) { + const status = await packageJsonScriptStatus(join(dir, 'package.json'), 'native:rebuild'); + if (status === 'has-script') { + return { cwd: dir, args: ['run', 'native:rebuild'] }; + } + if (status === 'exists') { + packageRoot ??= dir; + } + + const parent = dirname(dir); + if (parent === dir) { + return { cwd: packageRoot ?? process.cwd(), args: ['rebuild', 'better-sqlite3'] }; + } + dir = parent; + } +} + +async function defaultRebuildNativeSqlite(io: KtxCliIo): Promise { + const command = await nativeSqliteRebuildCommand(); + try { + await execFileAsync('pnpm', command.args, { + cwd: command.cwd, + env: envWithCurrentNodeFirst(), + maxBuffer: 1024 * 1024 * 16, + }); + return 0; + } catch (error) { + writePrefixedLines((chunk) => io.stderr.write(chunk), commandFailureOutput(error)); + return typeof (error as { code?: unknown })?.code === 'number' ? (error as { code: number }).code : 1; + } +} + function flushPrefixedBufferedCommandOutput(io: KtxCliIo, bufferedIo: BufferedCommandIo): void { writePrefixedLines((chunk) => io.stdout.write(chunk), bufferedIo.stdoutText()); writePrefixedLines((chunk) => io.stderr.write(chunk), bufferedIo.stderrText()); @@ -1470,8 +1551,8 @@ async function validateAndScanConnection(input: { writeSetupSection(input.io, `Scanning ${input.connectionId}`, [ 'Running structural scan…', ]); - const scanIo = createBufferedCommandIo(); - const scanCode = await scanConnection(input.projectDir, input.connectionId, scanIo); + let scanIo = createBufferedCommandIo(); + let scanCode = await scanConnection(input.projectDir, input.connectionId, scanIo); if (scanCode !== 0) { const nativeSqliteDetail = nativeSqliteAbiMismatchDetail(`${scanIo.stderrText()}\n${scanIo.stdoutText()}`); if (nativeSqliteDetail) { @@ -1481,10 +1562,32 @@ async function validateAndScanConnection(input: { `Structural scan failed for ${input.connectionId}.`, 'Native SQLite is built for a different Node.js ABI.', `Detail: ${nativeSqliteDetail}`, - 'Fix: pnpm run native:rebuild', - `Retry: ktx scan --project-dir ${input.projectDir} ${input.connectionId}`, + 'Rebuilding Native SQLite with pnpm run native:rebuild…', ].join('\n'), ); + const rebuildNativeSqlite = input.deps.rebuildNativeSqlite ?? defaultRebuildNativeSqlite; + const rebuildCode = await rebuildNativeSqlite(input.io); + if (rebuildCode === 0) { + writePrefixedLines( + (chunk) => input.io.stderr.write(chunk), + 'Native SQLite rebuild complete. Retrying structural scan…', + ); + const retryScanIo = createBufferedCommandIo(); + scanCode = await scanConnection(input.projectDir, input.connectionId, retryScanIo); + scanIo = retryScanIo; + } + if (scanCode !== 0) { + writePrefixedLines( + (chunk) => input.io.stderr.write(chunk), + [ + rebuildCode === 0 + ? `Structural scan still failed for ${input.connectionId} after rebuilding Native SQLite.` + : `Native SQLite rebuild failed for ${input.connectionId}.`, + 'Fix: pnpm run native:rebuild', + `Retry: ktx scan --project-dir ${input.projectDir} ${input.connectionId}`, + ].join('\n'), + ); + } } else { flushPrefixedBufferedCommandOutput(input.io, scanIo); writePrefixedLines( @@ -1495,7 +1598,9 @@ async function validateAndScanConnection(input: { ].join('\n'), ); } - return false; + if (scanCode !== 0) { + return false; + } } const scanOutput = scanIo.stdoutText(); const reportPath = readOutputValue(scanOutput, 'Report');