apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-10 08:05:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

build: bundle python runtime wheel in cli artifacts

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-11 09:55:43 +02:00
parent 5461b53f89
commit 6ce740220e
2 changed files with 197 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

114
scripts/package-artifacts.mjs
View file

@ -7,9 +7,21 @@ import { tmpdir } from 'node:os';
import { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import {
RUNTIME_WHEEL_DISTRIBUTION_NAME,
RUNTIME_WHEEL_NORMALIZED_NAME,
RUNTIME_WHEEL_PACKAGE_VERSION,
} from './build-python-runtime-wheel.mjs';
const PACKAGE_VERSION = '0.0.0-private';
const PYTHON_PACKAGE_VERSION = '0.1.0';
export {
RUNTIME_WHEEL_DISTRIBUTION_NAME,
RUNTIME_WHEEL_NORMALIZED_NAME,
RUNTIME_WHEEL_PACKAGE_VERSION,
};
export const NPM_ARTIFACT_PACKAGES = [
{ name: '@ktx/context', packageRoot: 'packages/context' },
{ name: '@ktx/llm', packageRoot: 'packages/llm' },
@ -24,6 +36,8 @@ export const NPM_ARTIFACT_PACKAGES = [
{ name: '@ktx/cli', packageRoot: 'packages/cli' },
];
export const CLI_PYTHON_ASSET_MANIFEST = 'manifest.json';
const CONNECTOR_PACKAGE_NAMES = NPM_ARTIFACT_PACKAGES
.map((packageInfo) => packageInfo.name)
.filter((packageName) => packageName.startsWith('@ktx/connector-'));
@ -90,7 +104,11 @@ export function buildArtifactCommands(layout) {
return [
...npmBuildCommands,
...npmPackCommands,
{
command: process.execPath,
args: ['scripts/build-python-runtime-wheel.mjs'],
cwd: layout.rootDir,
},
{
command: 'uv',
args: ['build', '--package', 'ktx-sl', '--out-dir', layout.pythonDir],
@ -101,6 +119,7 @@ export function buildArtifactCommands(layout) {
args: ['build', '--package', 'ktx-daemon', '--out-dir', layout.pythonDir],
cwd: layout.rootDir,
},
...npmPackCommands,
];
}
@ -123,9 +142,9 @@ function normalizePythonDistributionName(name) {
return name.replaceAll('-', '_');
}
function findOne(files, distributionName, suffix, label, pythonDir) {
function findOne(files, distributionName, suffix, label, pythonDir, version = PYTHON_PACKAGE_VERSION) {
const normalized = normalizePythonDistributionName(distributionName);
const found = files.find((file) => file.startsWith(`${normalized}-${PYTHON_PACKAGE_VERSION}`) && file.endsWith(suffix));
const found = files.find((file) => file.startsWith(`${normalized}-${version}`) && file.endsWith(suffix));
if (!found) {
throw new Error(`Missing Python artifact: ${label}`);
}
@ -136,6 +155,14 @@ export async function findPythonArtifacts(pythonDir) {
const files = await readdir(pythonDir);
return {
runtimeWheel: findOne(
files,
RUNTIME_WHEEL_DISTRIBUTION_NAME,
'.whl',
'kaelio-ktx runtime wheel',
pythonDir,
RUNTIME_WHEEL_PACKAGE_VERSION,
),
ktxSlWheel: findOne(files, 'ktx-sl', '.whl', 'ktx-sl wheel', pythonDir),
ktxSlSdist: findOne(files, 'ktx-sl', '.tar.gz', 'ktx-sl source distribution', pythonDir),
ktxDaemonWheel: findOne(files, 'ktx-daemon', '.whl', 'ktx-daemon wheel', pythonDir),
@ -242,6 +269,13 @@ export async function packageReleaseMetadata(rootDir = scriptRootDir()) {
packageVersion: ktxDaemonPackage.version,
privatePackage: false,
}),
releaseMetadataEntry({
ecosystem: 'python',
packageName: RUNTIME_WHEEL_DISTRIBUTION_NAME,
packageRoot: 'python/runtime-wheel',
packageVersion: RUNTIME_WHEEL_PACKAGE_VERSION,
privatePackage: false,
}),
];
}
@ -267,6 +301,11 @@ function artifactPackageRecords(layout, pythonArtifacts, packages) {
return [
...npmRecords,
{
artifactKind: 'wheel',
artifactPath: pythonArtifacts.runtimeWheel,
metadata: requirePackageMetadata(packagesByName, RUNTIME_WHEEL_DISTRIBUTION_NAME),
},
{
artifactKind: 'wheel',
artifactPath: pythonArtifacts.ktxSlWheel,
@ -429,15 +468,45 @@ export async function verifyArtifactManifest(layout, options = {}) {
return manifest;
}
function runtimeWheelAssetName(runtimeWheelPath) {
return runtimeWheelPath.split(sep).at(-1);
}
export async function copyRuntimeWheelAssets(layout, pythonArtifacts) {
const assetDir = join(layout.rootDir, 'packages', 'cli', 'assets', 'python');
const wheelFile = runtimeWheelAssetName(pythonArtifacts.runtimeWheel);
if (!wheelFile) {
throw new Error(`Unable to determine runtime wheel filename: ${pythonArtifacts.runtimeWheel}`);
}
const wheelContents = await readFile(pythonArtifacts.runtimeWheel);
await rm(assetDir, { recursive: true, force: true });
await mkdir(assetDir, { recursive: true });
const wheelPath = join(assetDir, wheelFile);
const manifestPath = join(assetDir, CLI_PYTHON_ASSET_MANIFEST);
await writeFile(wheelPath, wheelContents);
await writeFile(
manifestPath,
`${JSON.stringify(
{
schemaVersion: 1,
distributionName: RUNTIME_WHEEL_DISTRIBUTION_NAME,
normalizedName: RUNTIME_WHEEL_NORMALIZED_NAME,
version: RUNTIME_WHEEL_PACKAGE_VERSION,
wheel: {
file: wheelFile,
sha256: createHash('sha256').update(wheelContents).digest('hex'),
bytes: wheelContents.byteLength,
},
},
null,
2,
)}\n`,
);
return { assetDir, wheelPath, manifestPath };
}
export function pythonArtifactInstallArgs(python, pythonArtifacts) {
return [
'pip',
'install',
'--python',
python,
pythonArtifacts.ktxSlWheel,
pythonArtifacts.ktxDaemonWheel,
];
return ['pip', 'install', '--python', python, pythonArtifacts.runtimeWheel];
}
function runCommand(command, args, options = {}) {
@ -1513,11 +1582,11 @@ try {
export function pythonVerifySource() {
return `
import importlib.metadata
import ktx_daemon
import semantic_layer
assert importlib.metadata.version("ktx-sl") == "0.1.0"
assert importlib.metadata.version("ktx-daemon") == "0.1.0"
import semantic_layer
import ktx_daemon
assert importlib.metadata.version("kaelio-ktx") == "0.1.0"
assert semantic_layer is not None
assert ktx_daemon.PACKAGE_NAME == "ktx-daemon"
`;
@ -1544,14 +1613,25 @@ async function buildArtifacts(layout) {
await mkdir(layout.npmDir, { recursive: true });
await mkdir(layout.pythonDir, { recursive: true });
for (const command of buildArtifactCommands(layout)) {
const commands = buildArtifactCommands(layout);
const npmBuildCount = NPM_ARTIFACT_PACKAGES.length;
const npmPackStart = commands.length - NPM_ARTIFACT_PACKAGES.length;
for (const command of commands.slice(0, npmBuildCount)) {
await runCommand(command.command, command.args, { cwd: command.cwd });
}
for (const command of commands.slice(npmBuildCount, npmPackStart)) {
await runCommand(command.command, command.args, { cwd: command.cwd });
}
const pythonArtifacts = await findPythonArtifacts(layout.pythonDir);
await copyRuntimeWheelAssets(layout, pythonArtifacts);
for (const command of commands.slice(npmPackStart)) {
await runCommand(command.command, command.args, { cwd: command.cwd });
}
for (const packageInfo of NPM_ARTIFACT_PACKAGES) {
await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`);
}
await findPythonArtifacts(layout.pythonDir);
await writeArtifactManifest(layout);
await assertPathExists(artifactManifestPath(layout), 'artifact manifest');
}

112
scripts/package-artifacts.test.mjs
View file

@ -6,8 +6,13 @@ import { join } from 'node:path';
import { describe, it } from 'node:test';
import {
CLI_PYTHON_ASSET_MANIFEST,
RUNTIME_WHEEL_DISTRIBUTION_NAME,
RUNTIME_WHEEL_NORMALIZED_NAME,
RUNTIME_WHEEL_PACKAGE_VERSION,
artifactManifestPath,
buildArtifactCommands,
copyRuntimeWheelAssets,
findPythonArtifacts,
NPM_ARTIFACT_PACKAGES,
npmDemoSmokeSource,
@ -82,6 +87,10 @@ async function writeUploadableArtifactFixtures(layout) {
layout.npmTarballs[packageInfo.name],
`${packageInfo.name}-tarball`,
]),
[
join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
'kaelio-ktx-runtime-wheel',
],
[join(layout.pythonDir, 'ktx_sl-0.1.0-py3-none-any.whl'), 'ktx-sl-wheel'],
[join(layout.pythonDir, 'ktx_sl-0.1.0.tar.gz'), 'ktx-sl-sdist'],
[join(layout.pythonDir, 'ktx_daemon-0.1.0-py3-none-any.whl'), 'ktx-daemon-wheel'],
@ -128,20 +137,30 @@ describe('buildArtifactCommands', () => {
);
assert.deepEqual(
commands
.slice(NPM_ARTIFACT_PACKAGES.length, NPM_ARTIFACT_PACKAGES.length * 2)
.slice(NPM_ARTIFACT_PACKAGES.length, NPM_ARTIFACT_PACKAGES.length + 3)
.map((command) => [command.command, command.args]),
[
[
process.execPath,
['scripts/build-python-runtime-wheel.mjs'],
],
[
'uv',
['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python'],
],
[
'uv',
['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python'],
],
],
);
assert.deepEqual(
commands.slice(NPM_ARTIFACT_PACKAGES.length + 3).map((command) => [command.command, command.args]),
NPM_ARTIFACT_PACKAGES.map((packageInfo) => [
'pnpm',
['--filter', packageInfo.name, 'pack', '--out', layout.npmTarballs[packageInfo.name]],
]),
);
assert.deepEqual(
commands.slice(NPM_ARTIFACT_PACKAGES.length * 2).map((command) => [command.command, command.args]),
[
['uv', ['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python']],
['uv', ['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python']],
],
);
});
});
@ -176,6 +195,14 @@ describe('packageReleaseMetadata', () => {
private: false,
releaseMode: 'ci-artifact-only',
},
{
ecosystem: 'python',
packageName: 'kaelio-ktx',
packageRoot: 'python/runtime-wheel',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
},
]);
} finally {
await rm(root, { recursive: true, force: true });
@ -187,12 +214,14 @@ describe('findPythonArtifacts', () => {
it('finds one wheel and one source distribution for each Python package', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-'));
try {
await writeFile(join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), '');
await writeFile(join(root, 'ktx_sl-0.1.0-py3-none-any.whl'), '');
await writeFile(join(root, 'ktx_sl-0.1.0.tar.gz'), '');
await writeFile(join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'), '');
await writeFile(join(root, 'ktx_daemon-0.1.0.tar.gz'), '');
assert.deepEqual(await findPythonArtifacts(root), {
runtimeWheel: join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
ktxSlWheel: join(root, 'ktx_sl-0.1.0-py3-none-any.whl'),
ktxSlSdist: join(root, 'ktx_sl-0.1.0.tar.gz'),
ktxDaemonWheel: join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'),
@ -206,7 +235,7 @@ describe('findPythonArtifacts', () => {
it('throws when a required Python artifact is missing', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-'));
try {
await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: ktx-sl wheel/);
await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: kaelio-ktx runtime wheel/);
} finally {
await rm(root, { recursive: true, force: true });
}
@ -259,6 +288,14 @@ describe('artifact manifest', () => {
private: false,
releaseMode: 'ci-artifact-only',
},
{
ecosystem: 'python',
packageName: 'kaelio-ktx',
packageRoot: 'python/runtime-wheel',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
},
],
);
assert.deepEqual(
@ -291,6 +328,13 @@ describe('artifact manifest', () => {
path: file.path,
})),
[
{
artifactKind: 'wheel',
ecosystem: 'python',
packageName: 'kaelio-ktx',
packageVersion: '0.1.0',
path: 'python/kaelio_ktx-0.1.0-py3-none-any.whl',
},
{
artifactKind: 'wheel',
ecosystem: 'python',
@ -352,7 +396,7 @@ describe('verifyArtifactManifest', () => {
assert.equal(manifest.schemaVersion, 2);
assert.equal(manifest.sourceRevision, 'abc123');
assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 4);
assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 5);
} finally {
await rm(root, { recursive: true, force: true });
}
@ -419,9 +463,53 @@ describe('verifyArtifactManifest', () => {
});
});
describe('copyRuntimeWheelAssets', () => {
it('copies the runtime wheel and checksum manifest into CLI assets', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-assets-test-'));
const layout = packageArtifactLayout(root);
try {
await mkdir(layout.pythonDir, { recursive: true });
await writeFile(
join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
'kaelio-ktx-runtime-wheel',
);
const assets = await copyRuntimeWheelAssets(layout, {
runtimeWheel: join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
});
assert.equal(
assets.wheelPath,
join(root, 'packages', 'cli', 'assets', 'python', 'kaelio_ktx-0.1.0-py3-none-any.whl'),
);
assert.equal(
assets.manifestPath,
join(root, 'packages', 'cli', 'assets', 'python', CLI_PYTHON_ASSET_MANIFEST),
);
const manifest = JSON.parse(await readFile(assets.manifestPath, 'utf8'));
assert.deepEqual(manifest, {
schemaVersion: 1,
distributionName: RUNTIME_WHEEL_DISTRIBUTION_NAME,
normalizedName: RUNTIME_WHEEL_NORMALIZED_NAME,
version: RUNTIME_WHEEL_PACKAGE_VERSION,
wheel: {
file: 'kaelio_ktx-0.1.0-py3-none-any.whl',
sha256: createHash('sha256')
.update('kaelio-ktx-runtime-wheel')
.digest('hex'),
bytes: Buffer.byteLength('kaelio-ktx-runtime-wheel'),
},
});
} finally {
await rm(root, { recursive: true, force: true });
}
});
});
describe('pythonArtifactInstallArgs', () => {
it('installs the built Python wheels by artifact path', () => {
const args = pythonArtifactInstallArgs('/tmp/smoke/.venv/bin/python', {
runtimeWheel: '/repo/ktx/dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl',
ktxSlWheel: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0-py3-none-any.whl',
ktxSlSdist: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0.tar.gz',
ktxDaemonWheel: '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0-py3-none-any.whl',
@ -433,10 +521,10 @@ describe('pythonArtifactInstallArgs', () => {
'install',
'--python',
'/tmp/smoke/.venv/bin/python',
'/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0-py3-none-any.whl',
'/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0-py3-none-any.whl',
'/repo/ktx/dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl',
]);
assert.equal(args.includes('ktx-daemon'), false);
assert.equal(args.includes('ktx-sl'), false);
assert.equal(args.includes('--find-links'), false);
});
});