diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 635af69d..f4a4f8f6 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -4,7 +4,7 @@ import { createHash } from 'node:crypto'; import { execFile } from 'node:child_process'; import { access, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { @@ -18,9 +18,6 @@ import { publicNpmPackageTarballName, } from './build-public-npm-package.mjs'; -const PACKAGE_VERSION = '0.0.0-private'; -const PYTHON_PACKAGE_VERSION = '0.1.0'; - export { RUNTIME_WHEEL_DISTRIBUTION_NAME, RUNTIME_WHEEL_NORMALIZED_NAME, @@ -51,28 +48,15 @@ const CONNECTOR_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES const NPM_ARTIFACT_BUILD_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli']; -const ordersSource = { - name: 'orders', - table: 'public.orders', - grain: ['id'], - columns: [ - { name: 'id', type: 'number' }, - { name: 'status', type: 'string' }, - { name: 'amount', type: 'number' }, - ], - measures: [{ name: 'order_count', expr: 'count(*)' }], - joins: [], -}; - function scriptRootDir() { return resolve(dirname(fileURLToPath(import.meta.url)), '..'); } function npmPackageTarballName(packageName) { - if (packageName === PUBLIC_NPM_PACKAGE_NAME) { - return publicNpmPackageTarballName(PUBLIC_NPM_PACKAGE_VERSION); + if (packageName !== PUBLIC_NPM_PACKAGE_NAME) { + throw new Error(`Unsupported npm artifact package: ${packageName}`); } - return `${packageName.replace('@ktx/', 'ktx-')}-${PACKAGE_VERSION}.tgz`; + return publicNpmPackageTarballName(PUBLIC_NPM_PACKAGE_VERSION); } function npmPackageTarballs(npmDir) { @@ -126,16 +110,6 @@ export function buildArtifactCommands(layout) { args: ['scripts/build-python-runtime-wheel.mjs'], cwd: layout.rootDir, }, - { - command: 'uv', - args: ['build', '--package', 'ktx-sl', '--out-dir', layout.pythonDir], - cwd: layout.rootDir, - }, - { - command: 'uv', - args: ['build', '--package', 'ktx-daemon', '--out-dir', layout.pythonDir], - cwd: layout.rootDir, - }, publicPackageCommand, ]; } @@ -159,7 +133,7 @@ function normalizePythonDistributionName(name) { return name.replaceAll('-', '_'); } -function findOne(files, distributionName, suffix, label, pythonDir, version = PYTHON_PACKAGE_VERSION) { +function findOne(files, distributionName, suffix, label, pythonDir, version) { const normalized = normalizePythonDistributionName(distributionName); const found = files.find((file) => file.startsWith(`${normalized}-${version}`) && file.endsWith(suffix)); if (!found) { @@ -180,10 +154,6 @@ export async function findPythonArtifacts(pythonDir) { pythonDir, RUNTIME_WHEEL_PACKAGE_VERSION, ), - ktxSlWheel: findOne(files, 'ktx-sl', '.whl', 'ktx-sl wheel', pythonDir), - ktxSlSdist: findOne(files, 'ktx-sl', '.tar.gz', 'ktx-sl source distribution', pythonDir), - ktxDaemonWheel: findOne(files, 'ktx-daemon', '.whl', 'ktx-daemon wheel', pythonDir), - ktxDaemonSdist: findOne(files, 'ktx-daemon', '.tar.gz', 'ktx-daemon source distribution', pythonDir), }; } @@ -195,47 +165,6 @@ async function readJson(path) { return JSON.parse(await readFile(path, 'utf-8')); } -function readProjectBlock(toml, sourcePath) { - const lines = toml.split(/\r?\n/); - const block = []; - let inProject = false; - - for (const line of lines) { - if (/^\[project\]\s*$/.test(line)) { - inProject = true; - continue; - } - if (inProject && /^\[.*\]\s*$/.test(line)) { - break; - } - if (inProject) { - block.push(line); - } - } - - if (!inProject) { - throw new Error(`Missing [project] table in ${sourcePath}`); - } - return block.join('\n'); -} - -function readTomlStringField(projectBlock, fieldName, sourcePath) { - const match = projectBlock.match(new RegExp(`^${fieldName}\\s*=\\s*"([^"]+)"\\s*$`, 'm')); - if (!match) { - throw new Error(`Missing project.${fieldName} in ${sourcePath}`); - } - return match[1]; -} - -async function readPyprojectMetadata(path) { - const toml = await readFile(path, 'utf-8'); - const projectBlock = readProjectBlock(toml, path); - return { - name: readTomlStringField(projectBlock, 'name', path), - version: readTomlStringField(projectBlock, 'version', path), - }; -} - function releaseMetadataEntry({ ecosystem, packageName, packageRoot, packageVersion, privatePackage }) { return { ecosystem, @@ -269,25 +198,9 @@ export async function packageReleaseMetadata(rootDir = scriptRootDir()) { const npmPackages = await Promise.all( NPM_ARTIFACT_PACKAGES.map((packageInfo) => readNpmPackageMetadata(rootDir, packageInfo)), ); - const ktxSlPackage = await readPyprojectMetadata(join(rootDir, 'python', 'ktx-sl', 'pyproject.toml')); - const ktxDaemonPackage = await readPyprojectMetadata(join(rootDir, 'python', 'ktx-daemon', 'pyproject.toml')); return [ ...npmPackages, - releaseMetadataEntry({ - ecosystem: 'python', - packageName: ktxSlPackage.name, - packageRoot: 'python/ktx-sl', - packageVersion: ktxSlPackage.version, - privatePackage: false, - }), - releaseMetadataEntry({ - ecosystem: 'python', - packageName: ktxDaemonPackage.name, - packageRoot: 'python/ktx-daemon', - packageVersion: ktxDaemonPackage.version, - privatePackage: false, - }), releaseMetadataEntry({ ecosystem: 'python', packageName: RUNTIME_WHEEL_DISTRIBUTION_NAME, @@ -325,26 +238,6 @@ function artifactPackageRecords(layout, pythonArtifacts, packages) { artifactPath: pythonArtifacts.runtimeWheel, metadata: requirePackageMetadata(packagesByName, RUNTIME_WHEEL_DISTRIBUTION_NAME), }, - { - artifactKind: 'wheel', - artifactPath: pythonArtifacts.ktxSlWheel, - metadata: requirePackageMetadata(packagesByName, 'ktx-sl'), - }, - { - artifactKind: 'sdist', - artifactPath: pythonArtifacts.ktxSlSdist, - metadata: requirePackageMetadata(packagesByName, 'ktx-sl'), - }, - { - artifactKind: 'wheel', - artifactPath: pythonArtifacts.ktxDaemonWheel, - metadata: requirePackageMetadata(packagesByName, 'ktx-daemon'), - }, - { - artifactKind: 'sdist', - artifactPath: pythonArtifacts.ktxDaemonSdist, - metadata: requirePackageMetadata(packagesByName, 'ktx-daemon'), - }, ]; } @@ -524,10 +417,6 @@ export async function copyRuntimeWheelAssets(layout, pythonArtifacts) { return { assetDir, wheelPath, manifestPath }; } -export function pythonArtifactInstallArgs(python, pythonArtifacts) { - return ['pip', 'install', '--python', python, pythonArtifacts.runtimeWheel]; -} - function runCommand(command, args, options = {}) { const cwd = options.cwd ?? process.cwd(); process.stdout.write(`$ ${command} ${args.join(' ')}\n`); @@ -1227,36 +1116,6 @@ try { `; } -export function pythonVerifySource() { - return ` -import importlib.metadata - -import semantic_layer -import ktx_daemon - -assert importlib.metadata.version("kaelio-ktx") == "0.1.0" -assert semantic_layer is not None -assert ktx_daemon.PACKAGE_NAME == "ktx-daemon" -`; -} - -function pythonExecutable(projectDir) { - if (process.platform === 'win32') { - return join(projectDir, '.venv', 'Scripts', 'python.exe'); - } - return join(projectDir, '.venv', 'bin', 'python'); -} - -export function npmSmokePythonEnv(projectDir, baseEnv = process.env) { - const binDir = process.platform === 'win32' ? join(projectDir, '.venv', 'Scripts') : join(projectDir, '.venv', 'bin'); - const existingPath = baseEnv.PATH ?? ''; - - return { - ...baseEnv, - PATH: existingPath ? `${binDir}${delimiter}${existingPath}` : binDir, - }; -} - async function buildArtifacts(layout) { await rm(layout.artifactDir, { recursive: true, force: true }); await mkdir(layout.npmDir, { recursive: true }); @@ -1322,32 +1181,12 @@ async function verifyNpmDemoArtifacts(layout, tmpRoot) { await runCommand('node', ['verify-installed-demo.mjs'], { cwd: projectDir }); } -async function verifyPythonArtifacts(layout, tmpRoot) { - const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); - - const projectDir = join(tmpRoot, 'python-clean-install'); - await mkdir(projectDir, { recursive: true }); - const python = pythonExecutable(projectDir); - await writeFile(join(projectDir, 'verify_python.py'), pythonVerifySource()); - - await runCommand('uv', ['venv', '.venv'], { cwd: projectDir }); - await runCommand('uv', pythonArtifactInstallArgs(python, pythonArtifacts), { - cwd: projectDir, - }); - await runCommand(python, ['verify_python.py'], { cwd: projectDir }); - await runCommand(python, ['-m', 'ktx_daemon', 'semantic-validate'], { - cwd: projectDir, - input: `${JSON.stringify({ sources: [ordersSource], dialect: 'postgres' })}\n`, - }); -} - async function verifyArtifacts(layout) { await verifyArtifactManifest(layout); const tmpRoot = await mkdtemp(join(tmpdir(), 'ktx-artifacts-')); try { await verifyNpmArtifacts(layout, tmpRoot); - await verifyPythonArtifacts(layout, tmpRoot); } finally { await rm(tmpRoot, { recursive: true, force: true }); } diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index de051d59..08070464 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -22,8 +22,6 @@ import { npmVerifySource, packageArtifactLayout, packageReleaseMetadata, - pythonArtifactInstallArgs, - pythonVerifySource, verifyArtifactManifest, writeArtifactManifest, } from './package-artifacts.mjs'; @@ -49,17 +47,6 @@ async function writeReleaseMetadataInputs(root) { private: true, }); } - - await mkdir(join(root, 'python', 'ktx-sl'), { recursive: true }); - await mkdir(join(root, 'python', 'ktx-daemon'), { recursive: true }); - await writeFile( - join(root, 'python', 'ktx-sl', 'pyproject.toml'), - ['[project]', 'name = "ktx-sl"', 'version = "0.1.0"', ''].join('\n'), - ); - await writeFile( - join(root, 'python', 'ktx-daemon', 'pyproject.toml'), - ['[project]', 'name = "ktx-daemon"', 'version = "0.1.0"', ''].join('\n'), - ); } async function writeUploadableArtifactFixtures(layout) { @@ -75,10 +62,6 @@ async function writeUploadableArtifactFixtures(layout) { join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), 'kaelio-ktx-runtime-wheel', ], - [join(layout.pythonDir, 'ktx_sl-0.1.0-py3-none-any.whl'), 'ktx-sl-wheel'], - [join(layout.pythonDir, 'ktx_sl-0.1.0.tar.gz'), 'ktx-sl-sdist'], - [join(layout.pythonDir, 'ktx_daemon-0.1.0-py3-none-any.whl'), 'ktx-daemon-wheel'], - [join(layout.pythonDir, 'ktx_daemon-0.1.0.tar.gz'), 'ktx-daemon-sdist'], ]); for (const [path, contents] of fileContents) { @@ -99,7 +82,7 @@ describe('packageArtifactLayout', () => { }); describe('buildArtifactCommands', () => { - it('builds TypeScript packages in dependency order before packing npm artifacts and builds Python packages', () => { + it('builds TypeScript packages and the runtime wheel before packing npm artifacts', () => { const layout = packageArtifactLayout('/repo/ktx'); const commands = buildArtifactCommands(layout); @@ -108,18 +91,14 @@ describe('buildArtifactCommands', () => { NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]), ); assert.deepEqual( - commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 3).map((command) => [ + commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [ command.command, command.args, ]), - [ - [process.execPath, ['scripts/build-python-runtime-wheel.mjs']], - ['uv', ['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python']], - ['uv', ['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python']], - ], + [[process.execPath, ['scripts/build-python-runtime-wheel.mjs']]], ); assert.deepEqual( - commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 3).map((command) => [command.command, command.args]), + commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [command.command, command.args]), [[process.execPath, ['scripts/build-public-npm-package.mjs']]], ); }); @@ -140,22 +119,6 @@ describe('packageReleaseMetadata', () => { private: false, releaseMode: 'ci-artifact-only', }, - { - ecosystem: 'python', - packageName: 'ktx-sl', - packageRoot: 'python/ktx-sl', - packageVersion: '0.1.0', - private: false, - releaseMode: 'ci-artifact-only', - }, - { - ecosystem: 'python', - packageName: 'ktx-daemon', - packageRoot: 'python/ktx-daemon', - packageVersion: '0.1.0', - private: false, - releaseMode: 'ci-artifact-only', - }, { ecosystem: 'python', packageName: 'kaelio-ktx', @@ -172,21 +135,13 @@ describe('packageReleaseMetadata', () => { }); describe('findPythonArtifacts', () => { - it('finds one wheel and one source distribution for each Python package', async () => { + it('finds the bundled runtime wheel only', async () => { const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-')); try { await writeFile(join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), ''); - await writeFile(join(root, 'ktx_sl-0.1.0-py3-none-any.whl'), ''); - await writeFile(join(root, 'ktx_sl-0.1.0.tar.gz'), ''); - await writeFile(join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'), ''); - await writeFile(join(root, 'ktx_daemon-0.1.0.tar.gz'), ''); assert.deepEqual(await findPythonArtifacts(root), { runtimeWheel: join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), - ktxSlWheel: join(root, 'ktx_sl-0.1.0-py3-none-any.whl'), - ktxSlSdist: join(root, 'ktx_sl-0.1.0.tar.gz'), - ktxDaemonWheel: join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'), - ktxDaemonSdist: join(root, 'ktx_daemon-0.1.0.tar.gz'), }); } finally { await rm(root, { recursive: true, force: true }); @@ -235,22 +190,6 @@ describe('artifact manifest', () => { assert.deepEqual( manifest.packages.filter((entry) => entry.ecosystem === 'python'), [ - { - ecosystem: 'python', - packageName: 'ktx-sl', - packageRoot: 'python/ktx-sl', - packageVersion: '0.1.0', - private: false, - releaseMode: 'ci-artifact-only', - }, - { - ecosystem: 'python', - packageName: 'ktx-daemon', - packageRoot: 'python/ktx-daemon', - packageVersion: '0.1.0', - private: false, - releaseMode: 'ci-artifact-only', - }, { ecosystem: 'python', packageName: 'kaelio-ktx', @@ -300,34 +239,6 @@ describe('artifact manifest', () => { packageVersion: '0.1.0', path: 'python/kaelio_ktx-0.1.0-py3-none-any.whl', }, - { - artifactKind: 'wheel', - ecosystem: 'python', - packageName: 'ktx-daemon', - packageVersion: '0.1.0', - path: 'python/ktx_daemon-0.1.0-py3-none-any.whl', - }, - { - artifactKind: 'sdist', - ecosystem: 'python', - packageName: 'ktx-daemon', - packageVersion: '0.1.0', - path: 'python/ktx_daemon-0.1.0.tar.gz', - }, - { - artifactKind: 'wheel', - ecosystem: 'python', - packageName: 'ktx-sl', - packageVersion: '0.1.0', - path: 'python/ktx_sl-0.1.0-py3-none-any.whl', - }, - { - artifactKind: 'sdist', - ecosystem: 'python', - packageName: 'ktx-sl', - packageVersion: '0.1.0', - path: 'python/ktx_sl-0.1.0.tar.gz', - }, ], ); @@ -361,7 +272,7 @@ describe('verifyArtifactManifest', () => { assert.equal(manifest.schemaVersion, 2); assert.equal(manifest.sourceRevision, 'abc123'); - assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 5); + assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 1); } finally { await rm(root, { recursive: true, force: true }); } @@ -471,29 +382,6 @@ describe('copyRuntimeWheelAssets', () => { }); }); -describe('pythonArtifactInstallArgs', () => { - it('installs the built Python wheels by artifact path', () => { - const args = pythonArtifactInstallArgs('/tmp/smoke/.venv/bin/python', { - runtimeWheel: '/repo/ktx/dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl', - ktxSlWheel: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0-py3-none-any.whl', - ktxSlSdist: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0.tar.gz', - ktxDaemonWheel: '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0-py3-none-any.whl', - ktxDaemonSdist: '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0.tar.gz', - }); - - assert.deepEqual(args, [ - 'pip', - 'install', - '--python', - '/tmp/smoke/.venv/bin/python', - '/repo/ktx/dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl', - ]); - assert.equal(args.includes('ktx-daemon'), false); - assert.equal(args.includes('ktx-sl'), false); - assert.equal(args.includes('--find-links'), false); - }); -}); - describe('verifyNpmArtifacts', () => { it('does not prepare an external Python environment for the npm smoke', async () => { const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8'); @@ -509,6 +397,20 @@ describe('verifyNpmArtifacts', () => { }); }); +describe('standalone Python artifact cleanup', () => { + it('does not build or verify standalone Python package artifacts', async () => { + const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8'); + + assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-sl'/); + assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-daemon'/); + assert.doesNotMatch(source, /async function verifyPythonArtifacts/); + assert.doesNotMatch(source, /pythonArtifactInstallArgs/); + assert.doesNotMatch(source, /pythonVerifySource/); + assert.doesNotMatch(source, /ktx_sl-0\.1\.0/); + assert.doesNotMatch(source, /ktx_daemon-0\.1\.0/); + }); +}); + describe('verification snippets', () => { it('pins the smoke project to the public package artifact', () => { const layout = packageArtifactLayout('/repo/ktx'); @@ -624,12 +526,4 @@ describe('verification snippets', () => { assert.match(source, /'@kaelio\/ktx'/); }); }); - - it('asserts the Python modules that clean installs must expose', () => { - const source = pythonVerifySource(); - - assert.match(source, /semantic_layer/); - assert.match(source, /ktx_daemon/); - assert.match(source, /importlib.metadata/); - }); });