From 131b904229937171e6a9951e8df33a529823f8ca Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Mon, 11 May 2026 09:54:20 +0200 Subject: [PATCH] build: add bundled python runtime wheel builder --- scripts/build-python-runtime-wheel.mjs | 144 ++++++++++++++++++++ scripts/build-python-runtime-wheel.test.mjs | 115 ++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 scripts/build-python-runtime-wheel.mjs create mode 100644 scripts/build-python-runtime-wheel.test.mjs diff --git a/scripts/build-python-runtime-wheel.mjs b/scripts/build-python-runtime-wheel.mjs new file mode 100644 index 00000000..9623b48a --- /dev/null +++ b/scripts/build-python-runtime-wheel.mjs @@ -0,0 +1,144 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +export const RUNTIME_WHEEL_DISTRIBUTION_NAME = 'kaelio-ktx'; +export const RUNTIME_WHEEL_NORMALIZED_NAME = 'kaelio_ktx'; +export const RUNTIME_WHEEL_PACKAGE_VERSION = '0.1.0'; + +function scriptRootDir() { + return resolve(dirname(fileURLToPath(import.meta.url)), '..'); +} + +export function runtimeWheelLayout(rootDir = scriptRootDir()) { + return { + rootDir, + semanticLayerSourceDir: join(rootDir, 'python', 'ktx-sl', 'semantic_layer'), + daemonSourceDir: join(rootDir, 'python', 'ktx-daemon', 'src', 'ktx_daemon'), + buildRoot: join(rootDir, 'dist', 'runtime-wheel-src'), + outputDir: join(rootDir, 'dist', 'artifacts', 'python'), + }; +} + +export function runtimeWheelPyproject() { + return `[project] +name = "${RUNTIME_WHEEL_DISTRIBUTION_NAME}" +version = "${RUNTIME_WHEEL_PACKAGE_VERSION}" +description = "Bundled Python runtime payload for the KTX npm package" +readme = "README.md" +requires-python = ">=3.13" +license = "Apache-2.0" +dependencies = [ + "fastapi>=0.115.0", + "lkml>=1.3.7", + "numpy>=2.2.6", + "orjson>=3.11.4", + "pandas>=2.2.3", + "psycopg[binary]>=3.2.0", + "pydantic>=2.9.0", + "pyyaml>=6", + "requests>=2.32.0", + "sqlglot>=26", + "uvicorn[standard]>=0.32.0", +] + +[project.optional-dependencies] +local-embeddings = [ + "sentence-transformers>=5.1.1", + "torch>=2.2.0", +] + +[project.scripts] +ktx-daemon = "ktx_daemon.__main__:main" + +[project.urls] +Homepage = "https://github.com/kaelio/ktx" +Repository = "https://github.com/kaelio/ktx" +Issues = "https://github.com/kaelio/ktx/issues" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["semantic_layer", "ktx_daemon"] +`; +} + +export function runtimeWheelReadme() { + return `# kaelio-ktx Python runtime + +Bundled Python runtime wheel for KTX. + +This wheel is built from the repository's \`semantic_layer\` and +\`ktx_daemon\` source trees for inclusion in the npm package. It is not a +separate public PyPI release artifact. +`; +} + +export async function createRuntimeWheelBuildTree(layout = runtimeWheelLayout()) { + await rm(layout.buildRoot, { recursive: true, force: true }); + await mkdir(layout.buildRoot, { recursive: true }); + await cp(layout.semanticLayerSourceDir, join(layout.buildRoot, 'semantic_layer'), { + recursive: true, + }); + await cp(layout.daemonSourceDir, join(layout.buildRoot, 'ktx_daemon'), { + recursive: true, + }); + await writeFile(join(layout.buildRoot, 'pyproject.toml'), runtimeWheelPyproject()); + await writeFile(join(layout.buildRoot, 'README.md'), runtimeWheelReadme()); +} + +export function runtimeWheelBuildCommand(layout = runtimeWheelLayout()) { + return { + command: 'uv', + args: ['build', '--wheel', '--out-dir', layout.outputDir, layout.buildRoot], + cwd: layout.rootDir, + }; +} + +async function runCommand(command, args, options) { + const result = await execFileAsync(command, args, { + cwd: options.cwd, + encoding: 'utf8', + maxBuffer: 1024 * 1024 * 20, + }); + if (result.stdout) { + process.stdout.write(result.stdout); + } + if (result.stderr) { + process.stderr.write(result.stderr); + } +} + +export async function buildRuntimeWheel(layout = runtimeWheelLayout()) { + await mkdir(layout.outputDir, { recursive: true }); + await createRuntimeWheelBuildTree(layout); + const command = runtimeWheelBuildCommand(layout); + await runCommand(command.command, command.args, { cwd: command.cwd }); + const pyproject = await readFile(join(layout.buildRoot, 'pyproject.toml'), 'utf8'); + return { + buildRoot: layout.buildRoot, + outputDir: layout.outputDir, + pyproject, + }; +} + +async function main() { + await buildRuntimeWheel(runtimeWheelLayout()); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + try { + await main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/build-python-runtime-wheel.test.mjs b/scripts/build-python-runtime-wheel.test.mjs new file mode 100644 index 00000000..2fbb3fbc --- /dev/null +++ b/scripts/build-python-runtime-wheel.test.mjs @@ -0,0 +1,115 @@ +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, + createRuntimeWheelBuildTree, + runtimeWheelBuildCommand, + runtimeWheelLayout, + runtimeWheelPyproject, +} from './build-python-runtime-wheel.mjs'; + +async function writeRuntimeSourceFixture(root) { + await mkdir(join(root, 'python', 'ktx-sl', 'semantic_layer'), { + recursive: true, + }); + await mkdir(join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon'), { + recursive: true, + }); + + await writeFile( + join(root, 'python', 'ktx-sl', 'semantic_layer', '__init__.py'), + 'SEMANTIC_LAYER_FIXTURE = True\n', + ); + await writeFile( + join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon', '__init__.py'), + 'KTX_DAEMON_FIXTURE = True\n', + ); + await writeFile( + join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon', '__main__.py'), + 'def main():\n return 0\n', + ); +} + +describe('runtimeWheelLayout', () => { + it('uses stable source, build, and output paths', () => { + const layout = runtimeWheelLayout('/repo/ktx'); + + assert.equal(layout.rootDir, '/repo/ktx'); + assert.equal(layout.semanticLayerSourceDir, '/repo/ktx/python/ktx-sl/semantic_layer'); + assert.equal(layout.daemonSourceDir, '/repo/ktx/python/ktx-daemon/src/ktx_daemon'); + assert.equal(layout.buildRoot, '/repo/ktx/dist/runtime-wheel-src'); + assert.equal(layout.outputDir, '/repo/ktx/dist/artifacts/python'); + }); +}); + +describe('runtimeWheelPyproject', () => { + it('describes one kaelio-ktx wheel with lazy local embeddings', () => { + const pyproject = runtimeWheelPyproject(); + + assert.match(pyproject, /name = "kaelio-ktx"/); + assert.match(pyproject, /version = "0\.1\.0"/); + assert.match(pyproject, /ktx-daemon = "ktx_daemon\.__main__:main"/); + assert.match(pyproject, /packages = \["semantic_layer", "ktx_daemon"\]/); + assert.match(pyproject, /\[project\.optional-dependencies\]/); + assert.match(pyproject, /local-embeddings = \[/); + assert.match(pyproject, /"sentence-transformers>=5\.1\.1"/); + assert.match(pyproject, /"torch>=2\.2\.0"/); + assert.doesNotMatch( + pyproject.match(/dependencies = \[[\s\S]*?\]/)?.[0] ?? '', + /sentence-transformers|torch/, + ); + }); +}); + +describe('createRuntimeWheelBuildTree', () => { + it('copies KTX-owned Python packages into the build tree', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-wheel-test-')); + try { + await writeRuntimeSourceFixture(root); + const layout = runtimeWheelLayout(root); + + await createRuntimeWheelBuildTree(layout); + + assert.equal( + await readFile(join(layout.buildRoot, 'semantic_layer', '__init__.py'), 'utf8'), + 'SEMANTIC_LAYER_FIXTURE = True\n', + ); + assert.equal( + await readFile(join(layout.buildRoot, 'ktx_daemon', '__main__.py'), 'utf8'), + 'def main():\n return 0\n', + ); + const pyproject = await readFile(join(layout.buildRoot, 'pyproject.toml'), 'utf8'); + assert.match(pyproject, /name = "kaelio-ktx"/); + assert.match(pyproject, /local-embeddings = \[/); + const readme = await readFile(join(layout.buildRoot, 'README.md'), 'utf8'); + assert.match(readme, /Bundled Python runtime wheel for KTX/); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); + +describe('runtimeWheelBuildCommand', () => { + it('runs uv build against the generated build tree', () => { + const layout = runtimeWheelLayout('/repo/ktx'); + + assert.deepEqual(runtimeWheelBuildCommand(layout), { + command: 'uv', + args: [ + 'build', + '--wheel', + '--out-dir', + '/repo/ktx/dist/artifacts/python', + '/repo/ktx/dist/runtime-wheel-src', + ], + cwd: '/repo/ktx', + }); + assert.equal(RUNTIME_WHEEL_DISTRIBUTION_NAME, 'kaelio-ktx'); + assert.equal(RUNTIME_WHEEL_PACKAGE_VERSION, '0.1.0'); + }); +});