diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 52d49470..545b6fe8 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -7,9 +7,21 @@ import { tmpdir } from 'node:os'; import { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; +import { + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_NORMALIZED_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, +} from './build-python-runtime-wheel.mjs'; + const PACKAGE_VERSION = '0.0.0-private'; const PYTHON_PACKAGE_VERSION = '0.1.0'; +export { + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_NORMALIZED_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, +}; + export const NPM_ARTIFACT_PACKAGES = [ { name: '@ktx/context', packageRoot: 'packages/context' }, { name: '@ktx/llm', packageRoot: 'packages/llm' }, @@ -24,6 +36,8 @@ export const NPM_ARTIFACT_PACKAGES = [ { name: '@ktx/cli', packageRoot: 'packages/cli' }, ]; +export const CLI_PYTHON_ASSET_MANIFEST = 'manifest.json'; + const CONNECTOR_PACKAGE_NAMES = NPM_ARTIFACT_PACKAGES .map((packageInfo) => packageInfo.name) .filter((packageName) => packageName.startsWith('@ktx/connector-')); @@ -90,7 +104,11 @@ export function buildArtifactCommands(layout) { return [ ...npmBuildCommands, - ...npmPackCommands, + { + command: process.execPath, + args: ['scripts/build-python-runtime-wheel.mjs'], + cwd: layout.rootDir, + }, { command: 'uv', args: ['build', '--package', 'ktx-sl', '--out-dir', layout.pythonDir], @@ -101,6 +119,7 @@ export function buildArtifactCommands(layout) { args: ['build', '--package', 'ktx-daemon', '--out-dir', layout.pythonDir], cwd: layout.rootDir, }, + ...npmPackCommands, ]; } @@ -123,9 +142,9 @@ function normalizePythonDistributionName(name) { return name.replaceAll('-', '_'); } -function findOne(files, distributionName, suffix, label, pythonDir) { +function findOne(files, distributionName, suffix, label, pythonDir, version = PYTHON_PACKAGE_VERSION) { const normalized = normalizePythonDistributionName(distributionName); - const found = files.find((file) => file.startsWith(`${normalized}-${PYTHON_PACKAGE_VERSION}`) && file.endsWith(suffix)); + const found = files.find((file) => file.startsWith(`${normalized}-${version}`) && file.endsWith(suffix)); if (!found) { throw new Error(`Missing Python artifact: ${label}`); } @@ -136,6 +155,14 @@ export async function findPythonArtifacts(pythonDir) { const files = await readdir(pythonDir); return { + runtimeWheel: findOne( + files, + RUNTIME_WHEEL_DISTRIBUTION_NAME, + '.whl', + 'kaelio-ktx runtime wheel', + pythonDir, + RUNTIME_WHEEL_PACKAGE_VERSION, + ), ktxSlWheel: findOne(files, 'ktx-sl', '.whl', 'ktx-sl wheel', pythonDir), ktxSlSdist: findOne(files, 'ktx-sl', '.tar.gz', 'ktx-sl source distribution', pythonDir), ktxDaemonWheel: findOne(files, 'ktx-daemon', '.whl', 'ktx-daemon wheel', pythonDir), @@ -242,6 +269,13 @@ export async function packageReleaseMetadata(rootDir = scriptRootDir()) { packageVersion: ktxDaemonPackage.version, privatePackage: false, }), + releaseMetadataEntry({ + ecosystem: 'python', + packageName: RUNTIME_WHEEL_DISTRIBUTION_NAME, + packageRoot: 'python/runtime-wheel', + packageVersion: RUNTIME_WHEEL_PACKAGE_VERSION, + privatePackage: false, + }), ]; } @@ -267,6 +301,11 @@ function artifactPackageRecords(layout, pythonArtifacts, packages) { return [ ...npmRecords, + { + artifactKind: 'wheel', + artifactPath: pythonArtifacts.runtimeWheel, + metadata: requirePackageMetadata(packagesByName, RUNTIME_WHEEL_DISTRIBUTION_NAME), + }, { artifactKind: 'wheel', artifactPath: pythonArtifacts.ktxSlWheel, @@ -429,15 +468,45 @@ export async function verifyArtifactManifest(layout, options = {}) { return manifest; } +function runtimeWheelAssetName(runtimeWheelPath) { + return runtimeWheelPath.split(sep).at(-1); +} + +export async function copyRuntimeWheelAssets(layout, pythonArtifacts) { + const assetDir = join(layout.rootDir, 'packages', 'cli', 'assets', 'python'); + const wheelFile = runtimeWheelAssetName(pythonArtifacts.runtimeWheel); + if (!wheelFile) { + throw new Error(`Unable to determine runtime wheel filename: ${pythonArtifacts.runtimeWheel}`); + } + const wheelContents = await readFile(pythonArtifacts.runtimeWheel); + await rm(assetDir, { recursive: true, force: true }); + await mkdir(assetDir, { recursive: true }); + const wheelPath = join(assetDir, wheelFile); + const manifestPath = join(assetDir, CLI_PYTHON_ASSET_MANIFEST); + await writeFile(wheelPath, wheelContents); + await writeFile( + manifestPath, + `${JSON.stringify( + { + schemaVersion: 1, + distributionName: RUNTIME_WHEEL_DISTRIBUTION_NAME, + normalizedName: RUNTIME_WHEEL_NORMALIZED_NAME, + version: RUNTIME_WHEEL_PACKAGE_VERSION, + wheel: { + file: wheelFile, + sha256: createHash('sha256').update(wheelContents).digest('hex'), + bytes: wheelContents.byteLength, + }, + }, + null, + 2, + )}\n`, + ); + return { assetDir, wheelPath, manifestPath }; +} + export function pythonArtifactInstallArgs(python, pythonArtifacts) { - return [ - 'pip', - 'install', - '--python', - python, - pythonArtifacts.ktxSlWheel, - pythonArtifacts.ktxDaemonWheel, - ]; + return ['pip', 'install', '--python', python, pythonArtifacts.runtimeWheel]; } function runCommand(command, args, options = {}) { @@ -1513,11 +1582,11 @@ try { export function pythonVerifySource() { return ` import importlib.metadata -import ktx_daemon -import semantic_layer -assert importlib.metadata.version("ktx-sl") == "0.1.0" -assert importlib.metadata.version("ktx-daemon") == "0.1.0" +import semantic_layer +import ktx_daemon + +assert importlib.metadata.version("kaelio-ktx") == "0.1.0" assert semantic_layer is not None assert ktx_daemon.PACKAGE_NAME == "ktx-daemon" `; @@ -1544,14 +1613,25 @@ async function buildArtifacts(layout) { await mkdir(layout.npmDir, { recursive: true }); await mkdir(layout.pythonDir, { recursive: true }); - for (const command of buildArtifactCommands(layout)) { + const commands = buildArtifactCommands(layout); + const npmBuildCount = NPM_ARTIFACT_PACKAGES.length; + const npmPackStart = commands.length - NPM_ARTIFACT_PACKAGES.length; + + for (const command of commands.slice(0, npmBuildCount)) { + await runCommand(command.command, command.args, { cwd: command.cwd }); + } + for (const command of commands.slice(npmBuildCount, npmPackStart)) { + await runCommand(command.command, command.args, { cwd: command.cwd }); + } + const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); + await copyRuntimeWheelAssets(layout, pythonArtifacts); + for (const command of commands.slice(npmPackStart)) { await runCommand(command.command, command.args, { cwd: command.cwd }); } for (const packageInfo of NPM_ARTIFACT_PACKAGES) { await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`); } - await findPythonArtifacts(layout.pythonDir); await writeArtifactManifest(layout); await assertPathExists(artifactManifestPath(layout), 'artifact manifest'); } diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index 4aec3c6e..946538d1 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -6,8 +6,13 @@ import { join } from 'node:path'; import { describe, it } from 'node:test'; import { + CLI_PYTHON_ASSET_MANIFEST, + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_NORMALIZED_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, artifactManifestPath, buildArtifactCommands, + copyRuntimeWheelAssets, findPythonArtifacts, NPM_ARTIFACT_PACKAGES, npmDemoSmokeSource, @@ -82,6 +87,10 @@ async function writeUploadableArtifactFixtures(layout) { layout.npmTarballs[packageInfo.name], `${packageInfo.name}-tarball`, ]), + [ + join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + 'kaelio-ktx-runtime-wheel', + ], [join(layout.pythonDir, 'ktx_sl-0.1.0-py3-none-any.whl'), 'ktx-sl-wheel'], [join(layout.pythonDir, 'ktx_sl-0.1.0.tar.gz'), 'ktx-sl-sdist'], [join(layout.pythonDir, 'ktx_daemon-0.1.0-py3-none-any.whl'), 'ktx-daemon-wheel'], @@ -128,20 +137,30 @@ describe('buildArtifactCommands', () => { ); assert.deepEqual( commands - .slice(NPM_ARTIFACT_PACKAGES.length, NPM_ARTIFACT_PACKAGES.length * 2) + .slice(NPM_ARTIFACT_PACKAGES.length, NPM_ARTIFACT_PACKAGES.length + 3) .map((command) => [command.command, command.args]), + [ + [ + process.execPath, + ['scripts/build-python-runtime-wheel.mjs'], + ], + [ + 'uv', + ['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python'], + ], + [ + 'uv', + ['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python'], + ], + ], + ); + assert.deepEqual( + commands.slice(NPM_ARTIFACT_PACKAGES.length + 3).map((command) => [command.command, command.args]), NPM_ARTIFACT_PACKAGES.map((packageInfo) => [ 'pnpm', ['--filter', packageInfo.name, 'pack', '--out', layout.npmTarballs[packageInfo.name]], ]), ); - assert.deepEqual( - commands.slice(NPM_ARTIFACT_PACKAGES.length * 2).map((command) => [command.command, command.args]), - [ - ['uv', ['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python']], - ['uv', ['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python']], - ], - ); }); }); @@ -176,6 +195,14 @@ describe('packageReleaseMetadata', () => { private: false, releaseMode: 'ci-artifact-only', }, + { + ecosystem: 'python', + packageName: 'kaelio-ktx', + packageRoot: 'python/runtime-wheel', + packageVersion: '0.1.0', + private: false, + releaseMode: 'ci-artifact-only', + }, ]); } finally { await rm(root, { recursive: true, force: true }); @@ -187,12 +214,14 @@ describe('findPythonArtifacts', () => { it('finds one wheel and one source distribution for each Python package', async () => { const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-')); try { + await writeFile(join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), ''); await writeFile(join(root, 'ktx_sl-0.1.0-py3-none-any.whl'), ''); await writeFile(join(root, 'ktx_sl-0.1.0.tar.gz'), ''); await writeFile(join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'), ''); await writeFile(join(root, 'ktx_daemon-0.1.0.tar.gz'), ''); assert.deepEqual(await findPythonArtifacts(root), { + runtimeWheel: join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), ktxSlWheel: join(root, 'ktx_sl-0.1.0-py3-none-any.whl'), ktxSlSdist: join(root, 'ktx_sl-0.1.0.tar.gz'), ktxDaemonWheel: join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'), @@ -206,7 +235,7 @@ describe('findPythonArtifacts', () => { it('throws when a required Python artifact is missing', async () => { const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-')); try { - await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: ktx-sl wheel/); + await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: kaelio-ktx runtime wheel/); } finally { await rm(root, { recursive: true, force: true }); } @@ -259,6 +288,14 @@ describe('artifact manifest', () => { private: false, releaseMode: 'ci-artifact-only', }, + { + ecosystem: 'python', + packageName: 'kaelio-ktx', + packageRoot: 'python/runtime-wheel', + packageVersion: '0.1.0', + private: false, + releaseMode: 'ci-artifact-only', + }, ], ); assert.deepEqual( @@ -291,6 +328,13 @@ describe('artifact manifest', () => { path: file.path, })), [ + { + artifactKind: 'wheel', + ecosystem: 'python', + packageName: 'kaelio-ktx', + packageVersion: '0.1.0', + path: 'python/kaelio_ktx-0.1.0-py3-none-any.whl', + }, { artifactKind: 'wheel', ecosystem: 'python', @@ -352,7 +396,7 @@ describe('verifyArtifactManifest', () => { assert.equal(manifest.schemaVersion, 2); assert.equal(manifest.sourceRevision, 'abc123'); - assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 4); + assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 5); } finally { await rm(root, { recursive: true, force: true }); } @@ -419,9 +463,53 @@ describe('verifyArtifactManifest', () => { }); }); +describe('copyRuntimeWheelAssets', () => { + it('copies the runtime wheel and checksum manifest into CLI assets', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-assets-test-')); + const layout = packageArtifactLayout(root); + try { + await mkdir(layout.pythonDir, { recursive: true }); + await writeFile( + join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + 'kaelio-ktx-runtime-wheel', + ); + + const assets = await copyRuntimeWheelAssets(layout, { + runtimeWheel: join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + }); + + assert.equal( + assets.wheelPath, + join(root, 'packages', 'cli', 'assets', 'python', 'kaelio_ktx-0.1.0-py3-none-any.whl'), + ); + assert.equal( + assets.manifestPath, + join(root, 'packages', 'cli', 'assets', 'python', CLI_PYTHON_ASSET_MANIFEST), + ); + const manifest = JSON.parse(await readFile(assets.manifestPath, 'utf8')); + assert.deepEqual(manifest, { + schemaVersion: 1, + distributionName: RUNTIME_WHEEL_DISTRIBUTION_NAME, + normalizedName: RUNTIME_WHEEL_NORMALIZED_NAME, + version: RUNTIME_WHEEL_PACKAGE_VERSION, + wheel: { + file: 'kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: createHash('sha256') + .update('kaelio-ktx-runtime-wheel') + .digest('hex'), + bytes: Buffer.byteLength('kaelio-ktx-runtime-wheel'), + }, + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); + describe('pythonArtifactInstallArgs', () => { it('installs the built Python wheels by artifact path', () => { const args = pythonArtifactInstallArgs('/tmp/smoke/.venv/bin/python', { + runtimeWheel: '/repo/ktx/dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl', ktxSlWheel: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0-py3-none-any.whl', ktxSlSdist: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0.tar.gz', ktxDaemonWheel: '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0-py3-none-any.whl', @@ -433,10 +521,10 @@ describe('pythonArtifactInstallArgs', () => { 'install', '--python', '/tmp/smoke/.venv/bin/python', - '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0-py3-none-any.whl', - '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0-py3-none-any.whl', + '/repo/ktx/dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl', ]); assert.equal(args.includes('ktx-daemon'), false); + assert.equal(args.includes('ktx-sl'), false); assert.equal(args.includes('--find-links'), false); }); });