apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-10 08:05:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor: limit release artifacts to public package runtime

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-11 13:37:12 +02:00
parent 1cb941a0e5
commit 9fb3d33f2f
2 changed files with 25 additions and 292 deletions
Download patch file Download diff file Expand all files Collapse all files

171
scripts/package-artifacts.mjs
View file

@ -4,7 +4,7 @@ import { createHash } from 'node:crypto';
import { execFile } from 'node:child_process';
import { access, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import {
@ -18,9 +18,6 @@ import {
publicNpmPackageTarballName,
} from './build-public-npm-package.mjs';
const PACKAGE_VERSION = '0.0.0-private';
const PYTHON_PACKAGE_VERSION = '0.1.0';
export {
RUNTIME_WHEEL_DISTRIBUTION_NAME,
RUNTIME_WHEEL_NORMALIZED_NAME,
@ -51,28 +48,15 @@ const CONNECTOR_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES
const NPM_ARTIFACT_BUILD_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli'];
const ordersSource = {
name: 'orders',
table: 'public.orders',
grain: ['id'],
columns: [
{ name: 'id', type: 'number' },
{ name: 'status', type: 'string' },
{ name: 'amount', type: 'number' },
],
measures: [{ name: 'order_count', expr: 'count(*)' }],
joins: [],
};
function scriptRootDir() {
return resolve(dirname(fileURLToPath(import.meta.url)), '..');
}
function npmPackageTarballName(packageName) {
if (packageName === PUBLIC_NPM_PACKAGE_NAME) {
return publicNpmPackageTarballName(PUBLIC_NPM_PACKAGE_VERSION);
if (packageName !== PUBLIC_NPM_PACKAGE_NAME) {
throw new Error(`Unsupported npm artifact package: ${packageName}`);
}
return `${packageName.replace('@ktx/', 'ktx-')}-${PACKAGE_VERSION}.tgz`;
return publicNpmPackageTarballName(PUBLIC_NPM_PACKAGE_VERSION);
}
function npmPackageTarballs(npmDir) {
@ -126,16 +110,6 @@ export function buildArtifactCommands(layout) {
args: ['scripts/build-python-runtime-wheel.mjs'],
cwd: layout.rootDir,
},
{
command: 'uv',
args: ['build', '--package', 'ktx-sl', '--out-dir', layout.pythonDir],
cwd: layout.rootDir,
},
{
command: 'uv',
args: ['build', '--package', 'ktx-daemon', '--out-dir', layout.pythonDir],
cwd: layout.rootDir,
},
publicPackageCommand,
];
}
@ -159,7 +133,7 @@ function normalizePythonDistributionName(name) {
return name.replaceAll('-', '_');
}
function findOne(files, distributionName, suffix, label, pythonDir, version = PYTHON_PACKAGE_VERSION) {
function findOne(files, distributionName, suffix, label, pythonDir, version) {
const normalized = normalizePythonDistributionName(distributionName);
const found = files.find((file) => file.startsWith(`${normalized}-${version}`) && file.endsWith(suffix));
if (!found) {
@ -180,10 +154,6 @@ export async function findPythonArtifacts(pythonDir) {
pythonDir,
RUNTIME_WHEEL_PACKAGE_VERSION,
),
ktxSlWheel: findOne(files, 'ktx-sl', '.whl', 'ktx-sl wheel', pythonDir),
ktxSlSdist: findOne(files, 'ktx-sl', '.tar.gz', 'ktx-sl source distribution', pythonDir),
ktxDaemonWheel: findOne(files, 'ktx-daemon', '.whl', 'ktx-daemon wheel', pythonDir),
ktxDaemonSdist: findOne(files, 'ktx-daemon', '.tar.gz', 'ktx-daemon source distribution', pythonDir),
};
}
@ -195,47 +165,6 @@ async function readJson(path) {
return JSON.parse(await readFile(path, 'utf-8'));
}
function readProjectBlock(toml, sourcePath) {
const lines = toml.split(/\r?\n/);
const block = [];
let inProject = false;
for (const line of lines) {
if (/^\[project\]\s*$/.test(line)) {
inProject = true;
continue;
}
if (inProject && /^\[.*\]\s*$/.test(line)) {
break;
}
if (inProject) {
block.push(line);
}
}
if (!inProject) {
throw new Error(`Missing [project] table in ${sourcePath}`);
}
return block.join('\n');
}
function readTomlStringField(projectBlock, fieldName, sourcePath) {
const match = projectBlock.match(new RegExp(`^${fieldName}\\s*=\\s*"([^"]+)"\\s*$`, 'm'));
if (!match) {
throw new Error(`Missing project.${fieldName} in ${sourcePath}`);
}
return match[1];
}
async function readPyprojectMetadata(path) {
const toml = await readFile(path, 'utf-8');
const projectBlock = readProjectBlock(toml, path);
return {
name: readTomlStringField(projectBlock, 'name', path),
version: readTomlStringField(projectBlock, 'version', path),
};
}
function releaseMetadataEntry({ ecosystem, packageName, packageRoot, packageVersion, privatePackage }) {
return {
ecosystem,
@ -269,25 +198,9 @@ export async function packageReleaseMetadata(rootDir = scriptRootDir()) {
const npmPackages = await Promise.all(
NPM_ARTIFACT_PACKAGES.map((packageInfo) => readNpmPackageMetadata(rootDir, packageInfo)),
);
const ktxSlPackage = await readPyprojectMetadata(join(rootDir, 'python', 'ktx-sl', 'pyproject.toml'));
const ktxDaemonPackage = await readPyprojectMetadata(join(rootDir, 'python', 'ktx-daemon', 'pyproject.toml'));
return [
...npmPackages,
releaseMetadataEntry({
ecosystem: 'python',
packageName: ktxSlPackage.name,
packageRoot: 'python/ktx-sl',
packageVersion: ktxSlPackage.version,
privatePackage: false,
}),
releaseMetadataEntry({
ecosystem: 'python',
packageName: ktxDaemonPackage.name,
packageRoot: 'python/ktx-daemon',
packageVersion: ktxDaemonPackage.version,
privatePackage: false,
}),
releaseMetadataEntry({
ecosystem: 'python',
packageName: RUNTIME_WHEEL_DISTRIBUTION_NAME,
@ -325,26 +238,6 @@ function artifactPackageRecords(layout, pythonArtifacts, packages) {
artifactPath: pythonArtifacts.runtimeWheel,
metadata: requirePackageMetadata(packagesByName, RUNTIME_WHEEL_DISTRIBUTION_NAME),
},
{
artifactKind: 'wheel',
artifactPath: pythonArtifacts.ktxSlWheel,
metadata: requirePackageMetadata(packagesByName, 'ktx-sl'),
},
{
artifactKind: 'sdist',
artifactPath: pythonArtifacts.ktxSlSdist,
metadata: requirePackageMetadata(packagesByName, 'ktx-sl'),
},
{
artifactKind: 'wheel',
artifactPath: pythonArtifacts.ktxDaemonWheel,
metadata: requirePackageMetadata(packagesByName, 'ktx-daemon'),
},
{
artifactKind: 'sdist',
artifactPath: pythonArtifacts.ktxDaemonSdist,
metadata: requirePackageMetadata(packagesByName, 'ktx-daemon'),
},
];
}
@ -524,10 +417,6 @@ export async function copyRuntimeWheelAssets(layout, pythonArtifacts) {
return { assetDir, wheelPath, manifestPath };
}
export function pythonArtifactInstallArgs(python, pythonArtifacts) {
return ['pip', 'install', '--python', python, pythonArtifacts.runtimeWheel];
}
function runCommand(command, args, options = {}) {
const cwd = options.cwd ?? process.cwd();
process.stdout.write(`$ ${command} ${args.join(' ')}\n`);
@ -1227,36 +1116,6 @@ try {
`;
}
export function pythonVerifySource() {
return `
import importlib.metadata
import semantic_layer
import ktx_daemon
assert importlib.metadata.version("kaelio-ktx") == "0.1.0"
assert semantic_layer is not None
assert ktx_daemon.PACKAGE_NAME == "ktx-daemon"
`;
}
function pythonExecutable(projectDir) {
if (process.platform === 'win32') {
return join(projectDir, '.venv', 'Scripts', 'python.exe');
}
return join(projectDir, '.venv', 'bin', 'python');
}
export function npmSmokePythonEnv(projectDir, baseEnv = process.env) {
const binDir = process.platform === 'win32' ? join(projectDir, '.venv', 'Scripts') : join(projectDir, '.venv', 'bin');
const existingPath = baseEnv.PATH ?? '';
return {
...baseEnv,
PATH: existingPath ? `${binDir}${delimiter}${existingPath}` : binDir,
};
}
async function buildArtifacts(layout) {
await rm(layout.artifactDir, { recursive: true, force: true });
await mkdir(layout.npmDir, { recursive: true });
@ -1322,32 +1181,12 @@ async function verifyNpmDemoArtifacts(layout, tmpRoot) {
await runCommand('node', ['verify-installed-demo.mjs'], { cwd: projectDir });
}
async function verifyPythonArtifacts(layout, tmpRoot) {
const pythonArtifacts = await findPythonArtifacts(layout.pythonDir);
const projectDir = join(tmpRoot, 'python-clean-install');
await mkdir(projectDir, { recursive: true });
const python = pythonExecutable(projectDir);
await writeFile(join(projectDir, 'verify_python.py'), pythonVerifySource());
await runCommand('uv', ['venv', '.venv'], { cwd: projectDir });
await runCommand('uv', pythonArtifactInstallArgs(python, pythonArtifacts), {
cwd: projectDir,
});
await runCommand(python, ['verify_python.py'], { cwd: projectDir });
await runCommand(python, ['-m', 'ktx_daemon', 'semantic-validate'], {
cwd: projectDir,
input: `${JSON.stringify({ sources: [ordersSource], dialect: 'postgres' })}\n`,
});
}
async function verifyArtifacts(layout) {
await verifyArtifactManifest(layout);
const tmpRoot = await mkdtemp(join(tmpdir(), 'ktx-artifacts-'));
try {
await verifyNpmArtifacts(layout, tmpRoot);
await verifyPythonArtifacts(layout, tmpRoot);
} finally {
await rm(tmpRoot, { recursive: true, force: true });
}

146
scripts/package-artifacts.test.mjs
View file

@ -22,8 +22,6 @@ import {
npmVerifySource,
packageArtifactLayout,
packageReleaseMetadata,
pythonArtifactInstallArgs,
pythonVerifySource,
verifyArtifactManifest,
writeArtifactManifest,
} from './package-artifacts.mjs';
@ -49,17 +47,6 @@ async function writeReleaseMetadataInputs(root) {
private: true,
});
}
await mkdir(join(root, 'python', 'ktx-sl'), { recursive: true });
await mkdir(join(root, 'python', 'ktx-daemon'), { recursive: true });
await writeFile(
join(root, 'python', 'ktx-sl', 'pyproject.toml'),
['[project]', 'name = "ktx-sl"', 'version = "0.1.0"', ''].join('\n'),
);
await writeFile(
join(root, 'python', 'ktx-daemon', 'pyproject.toml'),
['[project]', 'name = "ktx-daemon"', 'version = "0.1.0"', ''].join('\n'),
);
}
async function writeUploadableArtifactFixtures(layout) {
@ -75,10 +62,6 @@ async function writeUploadableArtifactFixtures(layout) {
join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
'kaelio-ktx-runtime-wheel',
],
[join(layout.pythonDir, 'ktx_sl-0.1.0-py3-none-any.whl'), 'ktx-sl-wheel'],
[join(layout.pythonDir, 'ktx_sl-0.1.0.tar.gz'), 'ktx-sl-sdist'],
[join(layout.pythonDir, 'ktx_daemon-0.1.0-py3-none-any.whl'), 'ktx-daemon-wheel'],
[join(layout.pythonDir, 'ktx_daemon-0.1.0.tar.gz'), 'ktx-daemon-sdist'],
]);
for (const [path, contents] of fileContents) {
@ -99,7 +82,7 @@ describe('packageArtifactLayout', () => {
});
describe('buildArtifactCommands', () => {
it('builds TypeScript packages in dependency order before packing npm artifacts and builds Python packages', () => {
it('builds TypeScript packages and the runtime wheel before packing npm artifacts', () => {
const layout = packageArtifactLayout('/repo/ktx');
const commands = buildArtifactCommands(layout);
@ -108,18 +91,14 @@ describe('buildArtifactCommands', () => {
NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]),
);
assert.deepEqual(
commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 3).map((command) => [
commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [
command.command,
command.args,
]),
[
[process.execPath, ['scripts/build-python-runtime-wheel.mjs']],
['uv', ['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python']],
['uv', ['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python']],
],
[[process.execPath, ['scripts/build-python-runtime-wheel.mjs']]],
);
assert.deepEqual(
commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 3).map((command) => [command.command, command.args]),
commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [command.command, command.args]),
[[process.execPath, ['scripts/build-public-npm-package.mjs']]],
);
});
@ -140,22 +119,6 @@ describe('packageReleaseMetadata', () => {
private: false,
releaseMode: 'ci-artifact-only',
},
{
ecosystem: 'python',
packageName: 'ktx-sl',
packageRoot: 'python/ktx-sl',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
},
{
ecosystem: 'python',
packageName: 'ktx-daemon',
packageRoot: 'python/ktx-daemon',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
},
{
ecosystem: 'python',
packageName: 'kaelio-ktx',
@ -172,21 +135,13 @@ describe('packageReleaseMetadata', () => {
});
describe('findPythonArtifacts', () => {
it('finds one wheel and one source distribution for each Python package', async () => {
it('finds the bundled runtime wheel only', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-'));
try {
await writeFile(join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), '');
await writeFile(join(root, 'ktx_sl-0.1.0-py3-none-any.whl'), '');
await writeFile(join(root, 'ktx_sl-0.1.0.tar.gz'), '');
await writeFile(join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'), '');
await writeFile(join(root, 'ktx_daemon-0.1.0.tar.gz'), '');
assert.deepEqual(await findPythonArtifacts(root), {
runtimeWheel: join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
ktxSlWheel: join(root, 'ktx_sl-0.1.0-py3-none-any.whl'),
ktxSlSdist: join(root, 'ktx_sl-0.1.0.tar.gz'),
ktxDaemonWheel: join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'),
ktxDaemonSdist: join(root, 'ktx_daemon-0.1.0.tar.gz'),
});
} finally {
await rm(root, { recursive: true, force: true });
@ -235,22 +190,6 @@ describe('artifact manifest', () => {
assert.deepEqual(
manifest.packages.filter((entry) => entry.ecosystem === 'python'),
[
{
ecosystem: 'python',
packageName: 'ktx-sl',
packageRoot: 'python/ktx-sl',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
},
{
ecosystem: 'python',
packageName: 'ktx-daemon',
packageRoot: 'python/ktx-daemon',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
},
{
ecosystem: 'python',
packageName: 'kaelio-ktx',
@ -300,34 +239,6 @@ describe('artifact manifest', () => {
packageVersion: '0.1.0',
path: 'python/kaelio_ktx-0.1.0-py3-none-any.whl',
},
{
artifactKind: 'wheel',
ecosystem: 'python',
packageName: 'ktx-daemon',
packageVersion: '0.1.0',
path: 'python/ktx_daemon-0.1.0-py3-none-any.whl',
},
{
artifactKind: 'sdist',
ecosystem: 'python',
packageName: 'ktx-daemon',
packageVersion: '0.1.0',
path: 'python/ktx_daemon-0.1.0.tar.gz',
},
{
artifactKind: 'wheel',
ecosystem: 'python',
packageName: 'ktx-sl',
packageVersion: '0.1.0',
path: 'python/ktx_sl-0.1.0-py3-none-any.whl',
},
{
artifactKind: 'sdist',
ecosystem: 'python',
packageName: 'ktx-sl',
packageVersion: '0.1.0',
path: 'python/ktx_sl-0.1.0.tar.gz',
},
],
);
@ -361,7 +272,7 @@ describe('verifyArtifactManifest', () => {
assert.equal(manifest.schemaVersion, 2);
assert.equal(manifest.sourceRevision, 'abc123');
assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 5);
assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 1);
} finally {
await rm(root, { recursive: true, force: true });
}
@ -471,29 +382,6 @@ describe('copyRuntimeWheelAssets', () => {
});
});
describe('pythonArtifactInstallArgs', () => {
it('installs the built Python wheels by artifact path', () => {
const args = pythonArtifactInstallArgs('/tmp/smoke/.venv/bin/python', {
runtimeWheel: '/repo/ktx/dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl',
ktxSlWheel: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0-py3-none-any.whl',
ktxSlSdist: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0.tar.gz',
ktxDaemonWheel: '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0-py3-none-any.whl',
ktxDaemonSdist: '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0.tar.gz',
});
assert.deepEqual(args, [
'pip',
'install',
'--python',
'/tmp/smoke/.venv/bin/python',
'/repo/ktx/dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl',
]);
assert.equal(args.includes('ktx-daemon'), false);
assert.equal(args.includes('ktx-sl'), false);
assert.equal(args.includes('--find-links'), false);
});
});
describe('verifyNpmArtifacts', () => {
it('does not prepare an external Python environment for the npm smoke', async () => {
const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8');
@ -509,6 +397,20 @@ describe('verifyNpmArtifacts', () => {
});
});
describe('standalone Python artifact cleanup', () => {
it('does not build or verify standalone Python package artifacts', async () => {
const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8');
assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-sl'/);
assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-daemon'/);
assert.doesNotMatch(source, /async function verifyPythonArtifacts/);
assert.doesNotMatch(source, /pythonArtifactInstallArgs/);
assert.doesNotMatch(source, /pythonVerifySource/);
assert.doesNotMatch(source, /ktx_sl-0\.1\.0/);
assert.doesNotMatch(source, /ktx_daemon-0\.1\.0/);
});
});
describe('verification snippets', () => {
it('pins the smoke project to the public package artifact', () => {
const layout = packageArtifactLayout('/repo/ktx');
@ -624,12 +526,4 @@ describe('verification snippets', () => {
assert.match(source, /'@kaelio\/ktx'/);
});
});
it('asserts the Python modules that clean installs must expose', () => {
const source = pythonVerifySource();
assert.match(source, /semantic_layer/);
assert.match(source, /ktx_daemon/);
assert.match(source, /importlib.metadata/);
});
});