apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-07 07:55:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: npm-managed Python runtime for @kaelio/ktx (#7)

Browse source 
* docs: add npm managed python runtime design

* build: add bundled python runtime wheel builder

* build: make local embedding dependencies optional

* build: bundle python runtime wheel in cli artifacts

* build: track bundled python runtime release artifact

* test: verify bundled python runtime wheel

* docs: add plan for bundled python runtime wheel

* test: cover managed python runtime lifecycle

* feat: add managed python runtime installer

* feat: add runtime command runner

* feat: expose runtime management commands

* test: verify managed python runtime commands

* docs: add plan for managed python runtime installer

* feat: add managed python command helper

* feat: use managed runtime for sl query compute

* feat: route sl query managed runtime policy

* docs: add plan for managed runtime sl query integration

* feat: add managed runtime daemon metadata

* feat: manage python daemon lifecycle

* feat: add runtime daemon start stop commands

* fix: verify managed runtime daemon lifecycle

* docs: add plan for managed runtime daemon lifecycle

* feat: add managed local embeddings config marker

* feat: add managed local embeddings daemon helper

* feat: use managed runtime for local embedding setup

* feat: pass managed runtime policy through setup

* docs: add plan for managed local embeddings runtime

* feat: read CLI package metadata dynamically

* feat: assemble public kaelio ktx npm package

* feat: release one public kaelio ktx npm artifact

* test: cover public kaelio ktx package invocations

* chore: verify public kaelio ktx package artifacts

* docs: add plan for public kaelio ktx npm package

* test: verify managed runtime in public package smoke

* test: finalize managed runtime release smoke

* docs: add plan for managed runtime release smoke

* test: specify local embeddings release smoke

* feat: add local embeddings runtime smoke

* chore: register local embeddings smoke

* fix: verify local embeddings smoke

* fix: restore artifact smoke python env helper

* docs: add plan for managed local embeddings release smoke

* refactor: share managed runtime install policy parsing

* feat: use managed runtime for agent semantic queries

* feat: use managed runtime for MCP semantic compute

* docs: add plan for managed agent and MCP semantic runtime

* feat(cli): add managed daemon HTTP helpers

* feat(cli): route local adapters through managed daemon

* feat(cli): use managed daemon for ingest helpers

* feat(cli): pass managed daemon options to scan

* feat(context): pass MCP ingest pull config options

* feat(cli): pass managed daemon options to serve ingest

* test: verify managed local ingest daemon runtime

* docs: add plan for managed local ingest daemon runtime

* docs: align managed runtime examples

* docs: add plan for managed runtime docs cleanup

* test: cover published package runtime smoke commands

* test: validate published package smoke outputs

* docs: add plan for published package runtime smoke

* build: stamp public npm package version

* release: add npm public release policy

* release: add guarded npm publish script

* release: document public npm release handoff

* docs: add plan for public npm release handoff

* test: cover managed runtime prune in package smoke

* docs: document managed runtime prune

* docs: add plan for managed runtime prune smoke and docs

* chore: encode uv runtime prerequisite policy

* fix: clarify missing uv runtime error

* docs: document uv runtime prerequisite

* docs: add plan for uv runtime prerequisite contract

* refactor: limit release artifacts to public package runtime

* chore: align release policy with bundled runtime wheel

* docs: describe single public runtime artifact surface

* test: verify single public runtime artifact contract

* docs: add plan for single public runtime artifact cleanup

* fix: align local embeddings smoke with public version

* docs: add plan for local embeddings smoke public version

* release: soft-launch as @kaelio/ktx@0.1.0-rc.0 on next tag

Publish target moves to the pre-release version 0.1.0-rc.0 under the next
dist-tag so npm install @kaelio/ktx (which resolves to latest) does not
pick up the soft-launch build. Users opt in via @kaelio/ktx@next.

* Fix release script boundary checks

* Remove PostHog from public package bundle
This commit is contained in:
Andrey Avtomonov 2026-05-11 15:50:34 +02:00 • committed by GitHub
parent 075764fe77
commit 9dad936ac7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
99 changed files with 25375 additions and 1538 deletions
Download patch file Download diff file Expand all files Collapse all files

69
.github/workflows/release.yml vendored Normal file
View file

@ -0,0 +1,69 @@
name: KTX Release
on:
workflow_dispatch:
inputs:
publish_live:
description: "Publish @kaelio/ktx to npm instead of running a dry-run"
required: true
type: boolean
default: false
permissions:
contents: read
concurrency:
group: ktx-release-${{ github.ref }}
cancel-in-progress: false
jobs:
npm-public-release:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version: "24"
cache: "pnpm"
cache-dependency-path: "pnpm-lock.yaml"
- name: Install TypeScript dependencies
run: pnpm install --frozen-lockfile
- name: Setup Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.13"
- name: Setup uv
uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0
with:
enable-cache: true
cache-dependency-glob: "uv.lock"
- name: Install Python dependencies
run: uv sync --all-packages
- name: Build and verify artifacts
run: pnpm run artifacts:check
- name: Check release readiness
run: pnpm run release:readiness
- name: Dry-run npm publish
if: ${{ !inputs.publish_live }}
run: pnpm run release:npm-publish
- name: Publish npm package
if: ${{ inputs.publish_live }}
run: pnpm run release:npm-publish -- --publish
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

1
.gitignore vendored
View file

@ -8,6 +8,7 @@ venv/
env/
build/
dist/
packages/cli/assets/python/
*.egg-info/
.pytest_cache/
.coverage

120
README.md
View file

@ -20,13 +20,11 @@ artifacts. You can inspect them, commit them, and serve them to any MCP client.
## Quick start
Run the pre-seeded demo from the repository root:
Run the pre-seeded demo through the public npm package:
```bash
pnpm install
pnpm run setup:dev
pnpm run ktx -- setup demo --no-input
pnpm run ktx -- setup demo inspect
npx @kaelio/ktx setup demo --no-input
npx @kaelio/ktx setup demo inspect
```
The default demo uses packaged sample data and prebuilt context. It does not
@ -35,7 +33,7 @@ require API keys, network access, or an LLM provider.
To replay the packaged ingest run, use:
```bash
pnpm run ktx -- setup demo --mode replay --no-input
npx @kaelio/ktx setup demo --mode replay --no-input
```
To run the full agentic demo with an LLM provider, set a provider key for the
@ -43,22 +41,29 @@ current process:
```bash
ANTHROPIC_API_KEY=$YOUR_ANTHROPIC_API_KEY \
pnpm run ktx -- setup demo --mode full --no-input
npx @kaelio/ktx setup demo --mode full --no-input
```
Interactive full-demo setup can prompt for a provider key without writing the
key to `ktx.yaml`.
## Build a local project
Create a project from the repository root:
You can also install the CLI in a project or globally:
```bash
uv sync --all-packages
source .venv/bin/activate
npm install @kaelio/ktx
npx ktx --help
npm install -g @kaelio/ktx
ktx --help
```
## Build a local project
Create a project from a local workspace:
```bash
npm install @kaelio/ktx
PROJECT_DIR="$(mktemp -d)/ktx-demo"
pnpm run ktx -- init "$PROJECT_DIR" --name ktx-demo
npx ktx init "$PROJECT_DIR" --name ktx-demo
```
Create a SQLite warehouse:
@ -112,7 +117,7 @@ YAML
Write and validate a semantic-layer source:
```bash
pnpm run ktx -- sl write accounts --project-dir "$PROJECT_DIR" \
npx ktx sl write accounts --project-dir "$PROJECT_DIR" \
--connection-id warehouse --yaml 'name: accounts
table: accounts
description: CRM accounts with segmentation attributes.
@ -133,14 +138,14 @@ measures:
joins: []
'
pnpm run ktx -- sl validate accounts --project-dir "$PROJECT_DIR" \
npx ktx sl validate accounts --project-dir "$PROJECT_DIR" \
--connection-id warehouse
```
Generate SQL and execute the query:
```bash
pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \
npx ktx sl query --project-dir "$PROJECT_DIR" \
--connection-id warehouse \
--measure accounts.account_count \
--dimension accounts.segment \
@ -148,7 +153,7 @@ pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \
--limit 5 \
--format sql
pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \
npx ktx sl query --project-dir "$PROJECT_DIR" \
--connection-id warehouse \
--measure accounts.account_count \
--dimension accounts.segment \
@ -161,8 +166,8 @@ pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \
List and test the warehouse connection:
```bash
pnpm run ktx -- connection list --project-dir "$PROJECT_DIR"
pnpm run ktx -- connection test warehouse --project-dir "$PROJECT_DIR"
npx ktx connection list --project-dir "$PROJECT_DIR"
npx ktx connection test warehouse --project-dir "$PROJECT_DIR"
```
The connection test prints the configured driver and discovered table count:
@ -179,34 +184,66 @@ Scan artifacts are written under
```bash
SCAN_OUTPUT="$(pnpm run ktx -- scan warehouse --project-dir "$PROJECT_DIR")"
SCAN_OUTPUT="$(npx ktx scan warehouse --project-dir "$PROJECT_DIR")"
printf '%s\n' "$SCAN_OUTPUT"
SCAN_RUN_ID="$(printf '%s\n' "$SCAN_OUTPUT" | awk '/^Run: / { print $2 }')"
pnpm run ktx -- scan status --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID"
pnpm run ktx -- scan report --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID"
npx ktx scan status --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID"
npx ktx scan report --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID"
```
For non-SQLite drivers, prefer credential references such as `--url env:NAME`
or `--url file:PATH` over literal credential URLs.
## Managed Python runtime
KTX installs its Python runtime only when a Python-backed command needs it.
The runtime lives outside the npm cache, is versioned by the installed CLI
version, and is managed by `ktx runtime` commands.
KTX requires `uv` on `PATH` to create the managed runtime. Install `uv` with
your system package manager or the official installer before running Python-
backed KTX commands. KTX doesn't download `uv` automatically; run
`ktx runtime doctor` if runtime installation fails:
```bash
npx ktx runtime install --yes
npx ktx runtime status
npx ktx runtime doctor
npx ktx runtime start
npx ktx runtime stop
npx ktx runtime prune --dry-run
npx ktx runtime prune --yes
```
Use `runtime prune --dry-run` to preview stale runtime directories from older
CLI versions. Add `--yes` to remove those stale directories after daemon
processes are stopped.
Commands such as `npx @kaelio/ktx sl query ... --yes` can install the core
runtime lazily from the bundled wheel. Local embeddings remain lazy; prepare
them only when you select local `sentence-transformers` embeddings:
```bash
npx ktx runtime install --feature local-embeddings --yes
npx ktx runtime start --feature local-embeddings
```
## Serve MCP
Start the Python compute daemon in one terminal:
Start the stdio MCP server from the project directory:
```bash
source .venv/bin/activate
uv run ktx-daemon serve-http --host 127.0.0.1 --port 8765
```
Start the stdio MCP server in another terminal:
```bash
pnpm run ktx -- serve --mcp stdio --project-dir "$PROJECT_DIR" \
npx ktx serve --mcp stdio --project-dir "$PROJECT_DIR" \
--user-id local \
--semantic-compute-url http://127.0.0.1:8765 \
--execute-queries
--semantic-compute \
--execute-queries \
--yes
```
The `--semantic-compute` flag uses the managed Python runtime when no explicit
semantic compute URL is provided. KTX starts or reuses the managed runtime as
needed.
The MCP server exposes `connection_list`, `knowledge_search`,
`knowledge_read`, `knowledge_write`, `sl_list_sources`, `sl_read_source`,
`sl_write_source`, `sl_validate`, `sl_query`, `ingest_trigger`,
@ -251,19 +288,26 @@ packages.
## Release status
This repository is prepared for source publication. Package publishing is still
disabled by `release-policy.json`; registry names, public versions, package
visibility, and provenance policy must be chosen before publishing artifacts to
npm or Python package indexes.
This repository builds one public npm artifact named `@kaelio/ktx`. The release
artifact manifest contains the public npm tarball and the bundled `kaelio-ktx`
runtime wheel. The first public npm handoff is policy-gated through
`release-policy.json`, which keeps Python package publishing disabled because
KTX-owned Python code ships inside the npm package as a bundled wheel. The
`python/ktx-sl` and `python/ktx-daemon` directories remain source packages for
development, not public release artifacts.
Build local package artifacts with:
Build local package artifacts and verify the guarded dry-run publish path with:
```bash
source .venv/bin/activate
pnpm run artifacts:check
pnpm run release:readiness
pnpm run release:npm-publish
```
Run the live npm publish only from the manual `KTX Release` workflow with the
`publish_live` input enabled after the `NPM_TOKEN` secret is configured.
## License
KTX is licensed under the Apache License, Version 2.0. See `LICENSE`.

1144
docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md Normal file
View file

File diff suppressed because it is too large Load diff

1109
docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md Normal file
View file

File diff suppressed because it is too large Load diff

856
docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md Normal file
View file

@ -0,0 +1,856 @@
# Managed Local Embeddings Release Smoke Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use
> superpowers:subagent-driven-development (recommended) or
> superpowers:executing-plans to implement this plan task-by-task. Steps use
> checkbox (`- [ ]`) syntax for tracking.
**Goal:** Add an opt-in release smoke that proves the public `@kaelio/ktx`
package can install `local-embeddings`, start the managed daemon, compute a real
local embedding, and persist the managed embedding marker through setup.
**Architecture:** Keep the default `artifacts:verify` path lightweight. Add a
separate Node smoke script with an explicit opt-in gate, source-level tests, and
a package script that a release job can run only when large Python and model
downloads are acceptable.
**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, uv, KTX managed Python
runtime assets, FastAPI embedding endpoint, sentence-transformers.
---
## Existing status
This plan is based on
`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`.
The following plans are based on that spec and are already implemented in this
worktree:
- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md`
- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md`
Implementation evidence found before writing this plan includes:
- `scripts/build-python-runtime-wheel.mjs` and matching tests.
- `packages/cli/src/managed-python-runtime.ts`, `runtime.ts`, and
`commands/runtime-commands.ts`.
- `packages/cli/src/managed-python-command.ts` and `ktx sl query` runtime
install policy flags.
- `packages/cli/src/managed-python-daemon.ts` and `ktx runtime start` /
`ktx runtime stop`.
- `packages/cli/src/managed-local-embeddings.ts`,
`packages/context/src/llm/local-config.ts`, and setup embedding wiring.
- `scripts/build-public-npm-package.mjs`, `release-policy.json` listing
`@kaelio/ktx`, and public-package smoke command construction.
- `scripts/package-artifacts.mjs` installed CLI smoke that isolates
`KTX_RUNTIME_ROOT`, lazily installs the core runtime, runs `ktx sl query`,
checks runtime status and doctor output, and starts, reuses, and stops the
core daemon.
The remaining spec gap is the release-check item that permits local embeddings
coverage in a separate job or opt-in check. The default release artifact smoke
must not download `sentence-transformers`, `torch`, or the
`all-MiniLM-L6-v2` model.
## File structure
- Create `scripts/local-embeddings-runtime-smoke.mjs`: an opt-in smoke script
that consumes the built public npm tarball, installs it in a temporary pnpm
project, isolates all runtime and model caches, installs the
`local-embeddings` feature, starts the managed daemon, computes one real
embedding, runs setup with local embeddings, verifies the managed config
marker, and stops the daemon.
- Create `scripts/local-embeddings-runtime-smoke.test.mjs`: fast source-level
tests for opt-in gating, public tarball selection, cache isolation, command
construction, daemon URL parsing, embedding response validation, and package
script registration.
- Modify `package.json`: add `release:local-embeddings-smoke` without adding
it to default `check`, `test`, `artifacts:verify`, or release readiness.
### Task 1: Add failing local embeddings smoke tests
**Files:**
- Create: `scripts/local-embeddings-runtime-smoke.test.mjs`
- Test: `scripts/local-embeddings-runtime-smoke.test.mjs`
- [ ] **Step 1: Write the failing test file**
Create `scripts/local-embeddings-runtime-smoke.test.mjs` with this content:
```javascript
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import { describe, it } from 'node:test';
import {
buildLocalEmbeddingsSmokeEnv,
localEmbeddingsSmokeCommands,
localEmbeddingsSmokeOptIn,
parseDaemonBaseUrl,
publicKtxTarballName,
validateEmbeddingResponse,
} from './local-embeddings-runtime-smoke.mjs';
describe('localEmbeddingsSmokeOptIn', () => {
it('skips unless the smoke is explicitly enabled', () => {
assert.deepEqual(localEmbeddingsSmokeOptIn({}, []), {
run: false,
message: 'Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke.',
});
});
it('runs when the environment opt-in is set', () => {
assert.deepEqual(localEmbeddingsSmokeOptIn({ KTX_RUN_LOCAL_EMBEDDINGS_SMOKE: '1' }, []), {
run: true,
});
});
it('runs when --force is present', () => {
assert.deepEqual(localEmbeddingsSmokeOptIn({}, ['--force']), {
run: true,
});
});
});
describe('publicKtxTarballName', () => {
it('selects the public @kaelio/ktx tarball name', () => {
assert.equal(
publicKtxTarballName(['kaelio-ktx-0.0.0-private.tgz', 'ignore-me.tgz']),
'kaelio-ktx-0.0.0-private.tgz',
);
});
it('fails when the public package tarball is missing', () => {
assert.throws(
() => publicKtxTarballName(['ktx-cli-0.0.0-private.tgz']),
/Expected exactly one @kaelio\/ktx tarball/,
);
});
it('fails when multiple public package tarballs are present', () => {
assert.throws(
() => publicKtxTarballName(['kaelio-ktx-0.1.0.tgz', 'kaelio-ktx-0.2.0.tgz']),
/Expected exactly one @kaelio\/ktx tarball/,
);
});
});
describe('buildLocalEmbeddingsSmokeEnv', () => {
it('isolates the runtime root and model caches inside the smoke root', () => {
const env = buildLocalEmbeddingsSmokeEnv('/tmp/ktx-local-embedding-smoke', {
PATH: '/usr/bin',
});
assert.equal(env.PATH, '/usr/bin');
assert.equal(env.KTX_RUN_LOCAL_EMBEDDINGS_SMOKE, '1');
assert.equal(env.KTX_RUNTIME_ROOT, '/tmp/ktx-local-embedding-smoke/managed-runtime');
assert.equal(env.HF_HOME, '/tmp/ktx-local-embedding-smoke/hf-home');
assert.equal(env.TRANSFORMERS_CACHE, '/tmp/ktx-local-embedding-smoke/transformers-cache');
assert.equal(env.SENTENCE_TRANSFORMERS_HOME, '/tmp/ktx-local-embedding-smoke/sentence-transformers-home');
assert.equal(env.TORCH_HOME, '/tmp/ktx-local-embedding-smoke/torch-home');
});
});
describe('localEmbeddingsSmokeCommands', () => {
it('describes the installed-package commands needed for the smoke', () => {
const commands = localEmbeddingsSmokeCommands({
projectDir: '/tmp/ktx-local-embedding-smoke/project',
});
assert.deepEqual(commands.map((command) => command.label), [
'ktx public package version',
'ktx runtime status missing',
'ktx runtime install local embeddings',
'ktx runtime status local embeddings ready',
'ktx runtime start local embeddings',
'ktx setup local embeddings',
'ktx runtime stop local embeddings',
]);
assert.deepEqual(commands[2], {
label: 'ktx runtime install local embeddings',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'],
timeoutMs: 1_200_000,
});
assert.deepEqual(commands[4], {
label: 'ktx runtime start local embeddings',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'start', '--feature', 'local-embeddings'],
timeoutMs: 300_000,
});
assert.deepEqual(commands[5].args, [
'exec',
'ktx',
'setup',
'--project-dir',
'/tmp/ktx-local-embedding-smoke/project',
'--new',
'--no-input',
'--yes',
'--skip-llm',
'--embedding-backend',
'sentence-transformers',
'--skip-databases',
'--skip-sources',
'--skip-agents',
]);
});
});
describe('parseDaemonBaseUrl', () => {
it('extracts the daemon URL from runtime start output', () => {
assert.equal(
parseDaemonBaseUrl('Started KTX Python daemon\nurl: http://127.0.0.1:61234\nfeatures: local-embeddings\n'),
'http://127.0.0.1:61234',
);
});
it('rejects output without a daemon URL', () => {
assert.throws(() => parseDaemonBaseUrl('Started KTX Python daemon\n'), /Daemon URL was not printed/);
});
});
describe('validateEmbeddingResponse', () => {
it('accepts a finite embedding vector with the expected dimensions', () => {
validateEmbeddingResponse({ embedding: [0.1, -0.2, 0.3] }, 3);
});
it('rejects a vector with the wrong dimensions', () => {
assert.throws(
() => validateEmbeddingResponse({ embedding: [0.1, 0.2] }, 3),
/Expected embedding dimension 3, got 2/,
);
});
it('rejects non-finite embedding values', () => {
assert.throws(
() => validateEmbeddingResponse({ embedding: [0.1, Number.NaN, 0.3] }, 3),
/Embedding value at index 1 is not a finite number/,
);
});
});
describe('package script', () => {
it('registers the opt-in local embeddings smoke command', async () => {
const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'));
assert.equal(
packageJson.scripts['release:local-embeddings-smoke'],
'node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in',
);
});
});
```
- [ ] **Step 2: Run the failing test**
Run:
```bash
node --test scripts/local-embeddings-runtime-smoke.test.mjs
```
Expected: FAIL with an import error for
`./local-embeddings-runtime-smoke.mjs`.
- [ ] **Step 3: Commit the failing tests**
Run:
```bash
git add scripts/local-embeddings-runtime-smoke.test.mjs
git commit -m "test: specify local embeddings release smoke"
```
### Task 2: Implement the opt-in smoke script
**Files:**
- Create: `scripts/local-embeddings-runtime-smoke.mjs`
- Test: `scripts/local-embeddings-runtime-smoke.test.mjs`
- [ ] **Step 1: Create the smoke script**
Create `scripts/local-embeddings-runtime-smoke.mjs` with this content:
```javascript
import { execFile } from 'node:child_process';
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
const execFileAsync = promisify(execFile);
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
const DEFAULT_ROOT_DIR = resolve(SCRIPT_DIR, '..');
const PUBLIC_NPM_ARTIFACT_DIR = join('dist', 'artifacts', 'npm');
const OPT_IN_MESSAGE =
'Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke.';
export function localEmbeddingsSmokeOptIn(env = process.env, args = process.argv.slice(2)) {
if (env.KTX_RUN_LOCAL_EMBEDDINGS_SMOKE === '1' || args.includes('--force')) {
return { run: true };
}
return { run: false, message: OPT_IN_MESSAGE };
}
export function publicKtxTarballName(files) {
const matches = files.filter((file) => /^kaelio-ktx-.+\.tgz$/.test(file)).sort();
if (matches.length !== 1) {
throw new Error(
`Expected exactly one @kaelio/ktx tarball in ${PUBLIC_NPM_ARTIFACT_DIR}, found ${matches.length}: ${
matches.join(', ') || 'none'
}. Run pnpm run artifacts:build first.`,
);
}
return matches[0];
}
export async function selectPublicKtxTarball(rootDir = DEFAULT_ROOT_DIR) {
const npmArtifactDir = join(rootDir, PUBLIC_NPM_ARTIFACT_DIR);
const files = await readdir(npmArtifactDir);
return join(npmArtifactDir, publicKtxTarballName(files));
}
export function buildLocalEmbeddingsSmokeEnv(root, baseEnv = process.env) {
return {
...baseEnv,
KTX_RUN_LOCAL_EMBEDDINGS_SMOKE: '1',
KTX_RUNTIME_ROOT: join(root, 'managed-runtime'),
HF_HOME: join(root, 'hf-home'),
TRANSFORMERS_CACHE: join(root, 'transformers-cache'),
SENTENCE_TRANSFORMERS_HOME: join(root, 'sentence-transformers-home'),
TORCH_HOME: join(root, 'torch-home'),
};
}
export function localEmbeddingsSmokeCommands(input) {
return [
{
label: 'ktx public package version',
command: 'pnpm',
args: ['exec', 'ktx', '--version'],
timeoutMs: 60_000,
},
{
label: 'ktx runtime status missing',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'status', '--json'],
timeoutMs: 60_000,
},
{
label: 'ktx runtime install local embeddings',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'],
timeoutMs: 1_200_000,
},
{
label: 'ktx runtime status local embeddings ready',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'status', '--json'],
timeoutMs: 60_000,
},
{
label: 'ktx runtime start local embeddings',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'start', '--feature', 'local-embeddings'],
timeoutMs: 300_000,
},
{
label: 'ktx setup local embeddings',
command: 'pnpm',
args: [
'exec',
'ktx',
'setup',
'--project-dir',
input.projectDir,
'--new',
'--no-input',
'--yes',
'--skip-llm',
'--embedding-backend',
'sentence-transformers',
'--skip-databases',
'--skip-sources',
'--skip-agents',
],
timeoutMs: 900_000,
},
{
label: 'ktx runtime stop local embeddings',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'stop'],
timeoutMs: 60_000,
},
];
}
export function parseDaemonBaseUrl(stdout) {
const match = stdout.match(/^url: (http:\/\/127\.0\.0\.1:\d+)$/m);
if (!match) {
throw new Error(`Daemon URL was not printed by runtime start:\n${stdout}`);
}
return match[1];
}
export function validateEmbeddingResponse(raw, expectedDimensions) {
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
throw new Error('Embedding response must be a JSON object');
}
const embedding = raw.embedding;
if (!Array.isArray(embedding)) {
throw new Error('Embedding response must include an embedding array');
}
if (embedding.length !== expectedDimensions) {
throw new Error(`Expected embedding dimension ${expectedDimensions}, got ${embedding.length}`);
}
for (const [index, value] of embedding.entries()) {
if (typeof value !== 'number' || !Number.isFinite(value)) {
throw new Error(`Embedding value at index ${index} is not a finite number`);
}
}
}
async function run(command, args, options = {}) {
process.stdout.write(`$ ${command} ${args.join(' ')}\n`);
try {
const result = await execFileAsync(command, args, {
cwd: options.cwd,
env: { ...process.env, ...options.env },
encoding: 'utf8',
maxBuffer: 1024 * 1024 * 20,
timeout: options.timeoutMs ?? 120_000,
});
if (result.stdout) {
process.stdout.write(result.stdout);
}
if (result.stderr) {
process.stderr.write(result.stderr);
}
return { code: 0, stdout: result.stdout, stderr: result.stderr };
} catch (error) {
const stdout = typeof error.stdout === 'string' ? error.stdout : '';
const stderr = typeof error.stderr === 'string' ? error.stderr : error.message;
if (stdout) {
process.stdout.write(stdout);
}
if (stderr) {
process.stderr.write(stderr);
}
return {
code: typeof error.code === 'number' ? error.code : 1,
stdout,
stderr,
};
}
}
function requireSuccess(label, result, options = {}) {
if (result.code !== 0) {
throw new Error(`${label} failed with code ${result.code}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`);
}
if (options.stderrPattern && !options.stderrPattern.test(result.stderr)) {
throw new Error(`${label} stderr did not match ${options.stderrPattern}\nstderr:\n${result.stderr}`);
}
}
function parseJsonStdout(label, result) {
requireSuccess(label, result);
try {
return JSON.parse(result.stdout);
} catch (error) {
throw new Error(`${label} did not write JSON stdout: ${error.message}\nstdout:\n${result.stdout}`);
}
}
function requireOutput(label, result, pattern) {
if (!pattern.test(result.stdout)) {
throw new Error(`${label} stdout did not match ${pattern}\nstdout:\n${result.stdout}`);
}
}
async function postJson(baseUrl, path, payload, timeoutMs) {
const response = await fetch(new URL(path, baseUrl), {
method: 'POST',
headers: {
accept: 'application/json',
'content-type': 'application/json',
},
body: JSON.stringify(payload),
signal: AbortSignal.timeout(timeoutMs),
});
const text = await response.text();
if (!response.ok) {
throw new Error(`POST ${path} failed with ${response.status}: ${text}`);
}
try {
return JSON.parse(text);
} catch (error) {
throw new Error(`POST ${path} returned non-JSON response: ${error.message}\n${text}`);
}
}
async function writeSmokePackage(projectDir, tarballPath) {
await mkdir(projectDir, { recursive: true });
await writeFile(
join(projectDir, 'package.json'),
`${JSON.stringify(
{
name: 'ktx-local-embeddings-runtime-smoke',
version: '0.0.0',
private: true,
type: 'module',
dependencies: {
'@kaelio/ktx': `file:${tarballPath}`,
},
},
null,
2,
)}\n`,
);
}
export async function runLocalEmbeddingsRuntimeSmoke(options = {}) {
const rootDir = options.rootDir ?? DEFAULT_ROOT_DIR;
const tarballPath = options.tarballPath ?? (await selectPublicKtxTarball(rootDir));
const root = await mkdtemp(join(tmpdir(), 'ktx-local-embeddings-smoke-'));
const keepTemp = options.keepTemp ?? process.env.KTX_KEEP_LOCAL_EMBEDDINGS_SMOKE === '1';
const installDir = join(root, 'installed-package');
const projectDir = join(root, 'project');
const smokeEnv = buildLocalEmbeddingsSmokeEnv(root);
const commands = localEmbeddingsSmokeCommands({ projectDir });
let daemonStarted = false;
try {
await writeSmokePackage(installDir, tarballPath);
requireSuccess(
'pnpm install public package',
await run('pnpm', ['install', '--ignore-scripts=false'], {
cwd: installDir,
env: smokeEnv,
timeoutMs: 300_000,
}),
);
const version = await run(commands[0].command, commands[0].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[0].timeoutMs,
});
requireSuccess(commands[0].label, version);
requireOutput(commands[0].label, version, /@kaelio\/ktx 0\.0\.0-private/);
const missingStatus = parseJsonStdout(
commands[1].label,
await run(commands[1].command, commands[1].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[1].timeoutMs,
}),
);
if (missingStatus.kind !== 'missing') {
throw new Error(`Expected missing runtime before install, got ${JSON.stringify(missingStatus)}`);
}
const install = await run(commands[2].command, commands[2].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[2].timeoutMs,
});
requireSuccess(commands[2].label, install);
requireOutput(commands[2].label, install, /Installed KTX Python runtime/);
requireOutput(commands[2].label, install, /features: core, local-embeddings/);
const readyStatus = parseJsonStdout(
commands[3].label,
await run(commands[3].command, commands[3].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[3].timeoutMs,
}),
);
if (readyStatus.kind !== 'ready') {
throw new Error(`Expected ready runtime after install, got ${JSON.stringify(readyStatus)}`);
}
if (!readyStatus.manifest?.features?.includes('local-embeddings')) {
throw new Error(`Runtime manifest did not include local-embeddings: ${JSON.stringify(readyStatus.manifest)}`);
}
const start = await run(commands[4].command, commands[4].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[4].timeoutMs,
});
requireSuccess(commands[4].label, start);
daemonStarted = true;
const baseUrl = parseDaemonBaseUrl(start.stdout);
const embeddingResponse = await postJson(
baseUrl,
'/embeddings/compute',
{ text: 'KTX local embeddings release smoke' },
900_000,
);
validateEmbeddingResponse(embeddingResponse, 384);
process.stdout.write('KTX local embeddings daemon computed a 384-dimensional embedding\n');
const setup = await run(commands[5].command, commands[5].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[5].timeoutMs,
});
requireSuccess(commands[5].label, setup);
requireOutput(commands[5].label, setup, /Embeddings ready: yes \(all-MiniLM-L6-v2\)/);
const config = await readFile(join(projectDir, 'ktx.yaml'), 'utf8');
if (!config.includes('base_url: managed:local-embeddings')) {
throw new Error(`ktx.yaml did not contain managed local embeddings marker:\n${config}`);
}
process.stdout.write('KTX setup persisted managed local embeddings marker\n');
const stop = await run(commands[6].command, commands[6].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[6].timeoutMs,
});
requireSuccess(commands[6].label, stop);
daemonStarted = false;
requireOutput(commands[6].label, stop, /Stopped KTX Python daemon/);
process.stdout.write('KTX local embeddings runtime smoke verified\n');
} finally {
if (daemonStarted) {
await run('pnpm', ['exec', 'ktx', 'runtime', 'stop'], {
cwd: installDir,
env: smokeEnv,
timeoutMs: 60_000,
});
}
if (!keepTemp) {
await rm(root, { recursive: true, force: true });
} else {
process.stdout.write(`Kept local embeddings smoke root: ${root}\n`);
}
}
}
async function main() {
const args = process.argv.slice(2);
const optIn = localEmbeddingsSmokeOptIn(process.env, args);
if (!optIn.run) {
process.stdout.write(`Skipping KTX local embeddings runtime smoke. ${optIn.message}\n`);
if (args.includes('--require-opt-in')) {
process.exitCode = 1;
}
return;
}
await runLocalEmbeddingsRuntimeSmoke();
}
if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`);
process.exitCode = 1;
});
}
```
- [ ] **Step 2: Run the smoke test**
Run:
```bash
node --test scripts/local-embeddings-runtime-smoke.test.mjs
```
Expected: FAIL only in the package script test because
`release:local-embeddings-smoke` is not registered yet.
- [ ] **Step 3: Commit the smoke script**
Run:
```bash
git add scripts/local-embeddings-runtime-smoke.mjs
git commit -m "feat: add local embeddings runtime smoke"
```
### Task 3: Register the opt-in package script
**Files:**
- Modify: `package.json`
- Test: `scripts/local-embeddings-runtime-smoke.test.mjs`
- [ ] **Step 1: Add the package script**
In `package.json`, add this script immediately after
`"release:published-smoke"`:
```json
"release:local-embeddings-smoke": "node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in",
```
The surrounding `scripts` section must contain this sequence after the edit:
```json
"release:published-smoke": "node scripts/published-package-smoke.mjs --require-config",
"release:local-embeddings-smoke": "node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in",
"release:readiness": "node scripts/release-readiness.mjs",
```
- [ ] **Step 2: Run the focused test**
Run:
```bash
node --test scripts/local-embeddings-runtime-smoke.test.mjs
```
Expected: PASS.
- [ ] **Step 3: Verify the script stays opt-in**
Run:
```bash
pnpm run release:local-embeddings-smoke
```
Expected: FAIL with:
```text
Skipping KTX local embeddings runtime smoke. Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke.
```
The command must exit non-zero because `--require-opt-in` is present. This
protects local and CI runs from downloading large dependencies by accident.
- [ ] **Step 4: Commit the package script**
Run:
```bash
git add package.json
git commit -m "chore: register local embeddings smoke"
```
### Task 4: Verify the opt-in smoke path
**Files:**
- Verify: `scripts/local-embeddings-runtime-smoke.mjs`
- Verify: `scripts/local-embeddings-runtime-smoke.test.mjs`
- Verify: `package.json`
- [ ] **Step 1: Run fast script tests**
Run:
```bash
node --test scripts/local-embeddings-runtime-smoke.test.mjs scripts/package-artifacts.test.mjs
```
Expected: PASS. Existing package artifact tests must still prove that the
default npm artifact smoke does not prepare an external Python environment or
run local embeddings downloads.
- [ ] **Step 2: Build release artifacts for the smoke**
Run:
```bash
pnpm run artifacts:build
```
Expected: PASS and `dist/artifacts/npm/` contains exactly one
`kaelio-ktx-*.tgz` tarball.
- [ ] **Step 3: Run the opt-in local embeddings smoke**
Run this only in an environment where downloading `sentence-transformers`,
`torch`, and `all-MiniLM-L6-v2` is acceptable:
```bash
KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 pnpm run release:local-embeddings-smoke
```
Expected: PASS with output containing:
```text
KTX local embeddings daemon computed a 384-dimensional embedding
KTX setup persisted managed local embeddings marker
KTX local embeddings runtime smoke verified
```
- [ ] **Step 4: Run release readiness**
Run:
```bash
pnpm run release:readiness
```
Expected: PASS. The readiness report must not require
`release:local-embeddings-smoke`; that smoke remains a separately triggered
release job.
- [ ] **Step 5: Run pre-commit for changed files when configured**
Run:
```bash
uv run pre-commit run --files scripts/local-embeddings-runtime-smoke.mjs scripts/local-embeddings-runtime-smoke.test.mjs package.json
```
Expected: PASS. If pre-commit is unavailable in the environment, record the
tooling failure and keep the previous verification output.
- [ ] **Step 6: Commit verification fixes if needed**
If verification required edits, run:
```bash
git add scripts/local-embeddings-runtime-smoke.mjs scripts/local-embeddings-runtime-smoke.test.mjs package.json
git commit -m "fix: verify local embeddings smoke"
```
Skip this commit when no files changed after the previous commits.
## Acceptance criteria
- `node --test scripts/local-embeddings-runtime-smoke.test.mjs` passes.
- `pnpm run release:local-embeddings-smoke` fails fast without the opt-in
environment variable and prints the exact opt-in guidance.
- `KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 pnpm run release:local-embeddings-smoke`
installs the public `@kaelio/ktx` tarball into a clean project, isolates
`KTX_RUNTIME_ROOT` and model caches, installs `local-embeddings`, starts the
managed daemon, computes a 384-dimensional embedding through
`/embeddings/compute`, runs setup with `--embedding-backend
sentence-transformers`, verifies `base_url: managed:local-embeddings` in
`ktx.yaml`, and stops the daemon.
- The default `pnpm run artifacts:verify`, `pnpm run release:readiness`, and
`pnpm run check` paths do not run the local embeddings smoke.
## Self-review
- Spec coverage: this plan covers the remaining release-check item for local
embeddings in a separate job or opt-in check. Earlier implemented plans cover
the bundled wheel, managed runtime installer, `sl query` command integration,
daemon lifecycle, managed local embeddings runtime behavior, public npm
package assembly, and default core runtime release smoke.
- Placeholder scan: no steps contain placeholder implementation language.
- Type consistency: runtime feature names are consistently `core` and
`local-embeddings`; the public npm package name is `@kaelio/ktx`; the opt-in
environment variable is `KTX_RUN_LOCAL_EMBEDDINGS_SMOKE`; the managed local
embedding marker remains `managed:local-embeddings`.

1122
docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md Normal file
View file

File diff suppressed because it is too large Load diff

239
docs/superpowers/plans/2026-05-11-managed-local-embeddings-smoke-public-version.md Normal file
View file

@ -0,0 +1,239 @@
# Managed Local Embeddings Smoke Public Version Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Make the opt-in local embeddings release smoke validate the public
`@kaelio/ktx` package version instead of the private workspace version.
**Architecture:** Reuse the public package constants from
`scripts/build-public-npm-package.mjs` inside the local embeddings smoke. Add a
small exported RegExp helper so the unit test can lock the version expectation
without running the expensive model-download smoke.
**Tech Stack:** Node.js ESM scripts, `node:test`, pnpm release scripts.
---
## Current State
The npm-managed Python runtime spec is
`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`.
The current branch already contains implementation commits for each existing
plan derived from that spec.
Implemented spec-derived plans:
- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md`
- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md`
- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md`
- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md`
- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md`
- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md`
- `docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md`
- `docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md`
- `docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md`
The remaining gap is in
`scripts/local-embeddings-runtime-smoke.mjs`. The script selects and installs a
public tarball named `kaelio-ktx-*.tgz` and writes a smoke package dependency on
`@kaelio/ktx`, but line 267 still expects `@kaelio/ktx 0.0.0-private`. The
public package builder defines `PUBLIC_NPM_PACKAGE_VERSION = '0.1.0'`, and the
main packed-package smoke already expects `@kaelio/ktx 0.1.0`.
## File Structure
This change keeps the release version source of truth in one script and reuses
it from the opt-in smoke.
- Modify `scripts/local-embeddings-runtime-smoke.mjs`: import the public package
constants, export `expectedPublicKtxVersionPattern()`, and use that pattern
for the smoke version assertion.
- Modify `scripts/local-embeddings-runtime-smoke.test.mjs`: import
`expectedPublicKtxVersionPattern()` and assert that it accepts
`@kaelio/ktx 0.1.0` and rejects `@kaelio/ktx 0.0.0-private`.
### Task 1: Align the local embeddings smoke version assertion
**Files:**
- Modify: `scripts/local-embeddings-runtime-smoke.mjs:1-267`
- Modify: `scripts/local-embeddings-runtime-smoke.test.mjs:5-118`
- Test: `scripts/local-embeddings-runtime-smoke.test.mjs`
- [ ] **Step 1: Write the failing version-pattern test**
In `scripts/local-embeddings-runtime-smoke.test.mjs`, update the import block
to include `expectedPublicKtxVersionPattern`:
```js
import {
buildLocalEmbeddingsSmokeEnv,
expectedPublicKtxVersionPattern,
localEmbeddingsSmokeCommands,
localEmbeddingsSmokeOptIn,
parseDaemonBaseUrl,
publicKtxTarballName,
validateEmbeddingResponse,
} from './local-embeddings-runtime-smoke.mjs';
```
Then add this test after the `publicKtxTarballName` describe block:
```js
describe('expectedPublicKtxVersionPattern', () => {
it('matches the public package version and rejects the private workspace version', () => {
const pattern = expectedPublicKtxVersionPattern();
assert.match('@kaelio/ktx 0.1.0\n', pattern);
assert.doesNotMatch('@kaelio/ktx 0.0.0-private\n', pattern);
});
});
```
- [ ] **Step 2: Run the test to verify it fails**
Run:
```bash
node --test scripts/local-embeddings-runtime-smoke.test.mjs
```
Expected: FAIL with an ESM export error that says
`expectedPublicKtxVersionPattern` is not exported from
`./local-embeddings-runtime-smoke.mjs`.
- [ ] **Step 3: Import the public package constants**
In `scripts/local-embeddings-runtime-smoke.mjs`, add this import after the
existing Node imports:
```js
import {
PUBLIC_NPM_PACKAGE_NAME,
PUBLIC_NPM_PACKAGE_VERSION,
} from './build-public-npm-package.mjs';
```
The top of the file becomes:
```js
import { execFile } from 'node:child_process';
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import {
PUBLIC_NPM_PACKAGE_NAME,
PUBLIC_NPM_PACKAGE_VERSION,
} from './build-public-npm-package.mjs';
```
- [ ] **Step 4: Add the version-pattern helper**
In `scripts/local-embeddings-runtime-smoke.mjs`, add these functions after the
`OPT_IN_MESSAGE` constant:
```js
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
export function expectedPublicKtxVersionPattern() {
return new RegExp(
`${escapeRegExp(PUBLIC_NPM_PACKAGE_NAME)} ${escapeRegExp(PUBLIC_NPM_PACKAGE_VERSION)}`,
);
}
```
- [ ] **Step 5: Use the helper in the smoke**
In `scripts/local-embeddings-runtime-smoke.mjs`, replace this line:
```js
requireOutput(commands[0].label, version, /@kaelio\/ktx 0\.0\.0-private/);
```
with:
```js
requireOutput(commands[0].label, version, expectedPublicKtxVersionPattern());
```
- [ ] **Step 6: Run the focused test**
Run:
```bash
node --test scripts/local-embeddings-runtime-smoke.test.mjs
```
Expected: PASS. The new test proves the smoke accepts `@kaelio/ktx 0.1.0` and
rejects `@kaelio/ktx 0.0.0-private`.
- [ ] **Step 7: Run related release-script tests**
Run:
```bash
node --test scripts/local-embeddings-runtime-smoke.test.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs
```
Expected: PASS. These tests cover the public package constants, tarball name,
artifact smoke source, and local embeddings smoke helpers.
- [ ] **Step 8: Run a stale-expectation search**
Run:
```bash
rg -n "@kaelio/ktx 0\\.0\\.0-private|0\\\\\\.0\\\\\\.0-private" scripts/local-embeddings-runtime-smoke.mjs
```
Expected: no output. The opt-in local embeddings smoke no longer contains the
private package version expectation. The test file still uses
`@kaelio/ktx 0.0.0-private` as a negative fixture.
- [ ] **Step 9: Commit**
Run:
```bash
git add scripts/local-embeddings-runtime-smoke.mjs scripts/local-embeddings-runtime-smoke.test.mjs
git commit -m "fix: align local embeddings smoke with public version"
```
## Verification
Run these checks before marking the plan complete:
```bash
node --test scripts/local-embeddings-runtime-smoke.test.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs
rg -n "@kaelio/ktx 0\\.0\\.0-private|0\\\\\\.0\\\\\\.0-private" scripts/local-embeddings-runtime-smoke.mjs
```
Expected results:
- `node --test ...` exits with code 0.
- `rg ...` prints no matches.
- No Python files changed, so the repository Python pre-commit requirement does
not apply.
## Self-Review
- Spec coverage: this plan fixes the opt-in local embeddings release smoke from
the npm-managed runtime spec so it validates the public npm package produced
by the current release artifact flow.
- Placeholder scan: the plan contains concrete file paths, code blocks,
commands, and expected outcomes.
- Type consistency: the helper name is consistently
`expectedPublicKtxVersionPattern`, and it uses
`PUBLIC_NPM_PACKAGE_NAME` plus `PUBLIC_NPM_PACKAGE_VERSION` from the public
package builder.

1650
docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md Normal file
View file

File diff suppressed because it is too large Load diff

935
docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md Normal file
View file

@ -0,0 +1,935 @@
# Managed Python Runtime Command Integration Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use
> superpowers:subagent-driven-development (recommended) or
> superpowers:executing-plans to implement this plan task-by-task. Steps use
> checkbox (`- [ ]`) syntax for tracking.
**Goal:** Make `ktx sl query` use the KTX-managed bundled Python runtime
instead of relying on a user-provided `python -m ktx_daemon`.
**Architecture:** Add a small CLI helper that resolves the managed runtime,
installs the `core` feature when policy permits it, and creates the existing
`@ktx/context/daemon` one-shot semantic-layer compute port with the managed
`ktx-daemon` executable. Wire `ktx sl query` to pass an explicit runtime
install policy from `--yes`, `--no-input`, or the default interactive mode.
**Tech Stack:** TypeScript, Commander, Vitest, `@clack/prompts`,
`@ktx/context/daemon`, existing KTX managed runtime installer.
---
## Existing status
This plan is based on
`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`.
Existing plans based on the spec:
- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` is
implemented. The worktree contains
`scripts/build-python-runtime-wheel.mjs`,
`scripts/build-python-runtime-wheel.test.mjs`, runtime-wheel packaging in
`scripts/package-artifacts.mjs`, release-policy coverage, and matching
artifact tests. The targeted verification passes:
`node --test scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs`.
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` is
implemented. The worktree contains
`packages/cli/src/managed-python-runtime.ts`,
`packages/cli/src/runtime.ts`,
`packages/cli/src/commands/runtime-commands.ts`, CLI registration, and
matching Vitest coverage. The targeted CLI verification passes:
`pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts src/runtime.test.ts src/index.test.ts`.
Spec requirements still outside this plan:
- `ktx runtime start` and `ktx runtime stop`.
- Managed HTTP daemon state, health checks, reuse, and stale daemon repair.
- Lazy `local-embeddings` installation and local embedding daemon reuse.
- Public npm package rename from `@ktx/cli` to `@kaelio/ktx`.
This plan implements the next runnable user path: `ktx sl query` installs or
uses the managed `core` Python runtime according to the command's input policy.
## File structure
- Create `packages/cli/src/managed-python-command.ts`: CLI helper for managed
runtime policy, optional prompt, runtime install, and managed semantic-layer
compute port creation.
- Create `packages/cli/src/managed-python-command.test.ts`: unit tests for
ready runtime reuse, `--no-input` failure, `--yes` installation, and
interactive prompt acceptance.
- Modify `packages/cli/src/sl.ts`: extend `KtxSlArgs` with CLI version and
runtime install policy for `query`, and use the managed helper when no test
compute port is injected.
- Modify `packages/cli/src/sl.test.ts`: update existing `query` arguments and
assert `runKtxSl` delegates default compute creation to the managed helper.
- Modify `packages/cli/src/commands/sl-commands.ts`: add `--yes` and
`--no-input` to `sl query`, derive the runtime install policy, and pass the
CLI package version.
- Modify `packages/cli/src/command-schemas.ts`: validate `cliVersion` and
`runtimeInstallPolicy` on parsed `sl query` arguments.
- Modify `packages/cli/src/index.test.ts`: assert Commander routes the new
`sl query` runtime policy flags.
### Task 1: Add failing managed Python command helper tests
**Files:**
- Create: `packages/cli/src/managed-python-command.test.ts`
- Test: `packages/cli/src/managed-python-command.test.ts`
- [ ] **Step 1: Write the failing test file**
Create `packages/cli/src/managed-python-command.test.ts` with this content:
```typescript
import { describe, expect, it, vi } from 'vitest';
import {
createManagedPythonSemanticLayerComputePort,
managedRuntimeInstallCommand,
} from './managed-python-command.js';
import type {
InstalledKtxRuntimeManifest,
KtxRuntimeFeature,
ManagedPythonRuntimeInstallResult,
ManagedPythonRuntimeLayout,
ManagedPythonRuntimeStatus,
} from './managed-python-runtime.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
function layout(): ManagedPythonRuntimeLayout {
return {
cliVersion: '0.2.0',
runtimeRoot: '/runtime',
versionDir: '/runtime/0.2.0',
venvDir: '/runtime/0.2.0/.venv',
manifestPath: '/runtime/0.2.0/manifest.json',
installLogPath: '/runtime/0.2.0/install.log',
assetDir: '/assets/python',
assetManifestPath: '/assets/python/manifest.json',
pythonPath: '/runtime/0.2.0/.venv/bin/python',
daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon',
};
}
function manifest(features: KtxRuntimeFeature[] = ['core']): InstalledKtxRuntimeManifest {
return {
schemaVersion: 1,
cliVersion: '0.2.0',
installedAt: '2026-05-11T00:00:00.000Z',
asset: {
schemaVersion: 1,
distributionName: 'kaelio-ktx',
normalizedName: 'kaelio_ktx',
version: '0.2.0',
wheel: {
file: 'kaelio_ktx-0.2.0-py3-none-any.whl',
sha256: 'a'.repeat(64),
bytes: 123,
},
},
features,
python: {
executable: '/runtime/0.2.0/.venv/bin/python',
daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon',
},
installLog: '/runtime/0.2.0/install.log',
};
}
function readyStatus(features: KtxRuntimeFeature[] = ['core']): ManagedPythonRuntimeStatus {
return {
kind: 'ready',
detail: 'Runtime ready at /runtime/0.2.0',
layout: layout(),
manifest: manifest(features),
};
}
function missingStatus(): ManagedPythonRuntimeStatus {
return {
kind: 'missing',
detail: 'No runtime manifest at /runtime/0.2.0/manifest.json',
layout: layout(),
};
}
function installResult(features: KtxRuntimeFeature[] = ['core']): ManagedPythonRuntimeInstallResult {
const installedManifest = manifest(features);
return {
status: 'installed',
layout: layout(),
asset: {
manifest: installedManifest.asset,
wheelPath: '/assets/python/kaelio_ktx-0.2.0-py3-none-any.whl',
},
manifest: installedManifest,
};
}
describe('managedRuntimeInstallCommand', () => {
it('prints the exact command for each managed runtime feature', () => {
expect(managedRuntimeInstallCommand('core')).toBe('ktx runtime install --yes');
expect(managedRuntimeInstallCommand('local-embeddings')).toBe(
'ktx runtime install --feature local-embeddings --yes',
);
});
});
describe('createManagedPythonSemanticLayerComputePort', () => {
it('uses the managed ktx-daemon executable when the runtime is ready', async () => {
const io = makeIo();
const compute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };
const createPythonCompute = vi.fn(() => compute);
await expect(
createManagedPythonSemanticLayerComputePort({
cliVersion: '0.2.0',
installPolicy: 'never',
io: io.io,
readStatus: vi.fn(async () => readyStatus()),
installRuntime: vi.fn(),
createPythonCompute,
}),
).resolves.toBe(compute);
expect(createPythonCompute).toHaveBeenCalledWith({
command: '/runtime/0.2.0/.venv/bin/ktx-daemon',
args: [],
});
expect(io.stderr()).toBe('');
});
it('fails with a preparation command when input is disabled and the runtime is missing', async () => {
const io = makeIo();
const installRuntime = vi.fn();
await expect(
createManagedPythonSemanticLayerComputePort({
cliVersion: '0.2.0',
installPolicy: 'never',
io: io.io,
readStatus: vi.fn(async () => missingStatus()),
installRuntime,
}),
).rejects.toThrow('KTX Python runtime is required for this command. Run: ktx runtime install --yes');
expect(installRuntime).not.toHaveBeenCalled();
});
it('installs the core runtime without prompting when policy is auto', async () => {
const io = makeIo();
const compute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };
const createPythonCompute = vi.fn(() => compute);
const installRuntime = vi.fn(async () => installResult());
await expect(
createManagedPythonSemanticLayerComputePort({
cliVersion: '0.2.0',
installPolicy: 'auto',
io: io.io,
readStatus: vi.fn(async () => missingStatus()),
installRuntime,
createPythonCompute,
}),
).resolves.toBe(compute);
expect(installRuntime).toHaveBeenCalledWith({
cliVersion: '0.2.0',
features: ['core'],
force: false,
});
expect(io.stderr()).toContain('Installing KTX Python runtime (core) with uv');
expect(io.stderr()).toContain('KTX Python runtime ready: /runtime/0.2.0');
});
it('prompts before installing when policy is prompt', async () => {
const io = makeIo();
const confirmInstall = vi.fn(async () => true);
const installRuntime = vi.fn(async () => installResult());
await createManagedPythonSemanticLayerComputePort({
cliVersion: '0.2.0',
installPolicy: 'prompt',
io: io.io,
readStatus: vi.fn(async () => missingStatus()),
installRuntime,
createPythonCompute: vi.fn(() => ({ query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() })),
confirmInstall,
});
expect(confirmInstall).toHaveBeenCalledWith(
'KTX needs to install the core Python runtime. This downloads Python dependencies with uv. Continue?',
);
expect(installRuntime).toHaveBeenCalledWith({
cliVersion: '0.2.0',
features: ['core'],
force: false,
});
});
});
```
- [ ] **Step 2: Run the failing test**
Run:
```bash
pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts
```
Expected: FAIL with an import error for
`./managed-python-command.js`.
### Task 2: Implement the managed Python command helper
**Files:**
- Create: `packages/cli/src/managed-python-command.ts`
- Test: `packages/cli/src/managed-python-command.test.ts`
- [ ] **Step 1: Create the helper**
Create `packages/cli/src/managed-python-command.ts` with this content:
```typescript
import { cancel, confirm, isCancel } from '@clack/prompts';
import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon';
import type { KtxCliIo } from './cli-runtime.js';
import {
installManagedPythonRuntime,
readManagedPythonRuntimeStatus,
type InstalledKtxRuntimeManifest,
type KtxRuntimeFeature,
type ManagedPythonRuntimeInstallOptions,
type ManagedPythonRuntimeInstallResult,
type ManagedPythonRuntimeLayout,
type ManagedPythonRuntimeLayoutOptions,
type ManagedPythonRuntimeStatus,
} from './managed-python-runtime.js';
export type KtxManagedPythonInstallPolicy = 'prompt' | 'auto' | 'never';
export interface ManagedPythonCommandRuntime {
layout: ManagedPythonRuntimeLayout;
manifest: InstalledKtxRuntimeManifest;
}
export interface ManagedPythonCommandDeps {
readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise<ManagedPythonRuntimeStatus>;
installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise<ManagedPythonRuntimeInstallResult>;
confirmInstall?: (message: string) => Promise<boolean>;
}
export interface ManagedPythonCommandOptions extends ManagedPythonCommandDeps {
cliVersion: string;
installPolicy: KtxManagedPythonInstallPolicy;
io: KtxCliIo;
feature?: KtxRuntimeFeature;
}
export interface ManagedPythonSemanticLayerComputeOptions extends ManagedPythonCommandOptions {
createPythonCompute?: typeof createPythonSemanticLayerComputePort;
}
export function managedRuntimeInstallCommand(feature: KtxRuntimeFeature): string {
return feature === 'local-embeddings'
? 'ktx runtime install --feature local-embeddings --yes'
: 'ktx runtime install --yes';
}
function installPrompt(feature: KtxRuntimeFeature): string {
const label = feature === 'local-embeddings' ? 'local embeddings Python runtime' : 'core Python runtime';
return `KTX needs to install the ${label}. This downloads Python dependencies with uv. Continue?`;
}
function runtimeRequiredMessage(feature: KtxRuntimeFeature): string {
return `KTX Python runtime is required for this command. Run: ${managedRuntimeInstallCommand(feature)}`;
}
function hasFeature(manifest: InstalledKtxRuntimeManifest, feature: KtxRuntimeFeature): boolean {
return manifest.features.includes(feature);
}
async function defaultConfirmInstall(message: string): Promise<boolean> {
if (process.stdin.isTTY !== true || process.stdout.isTTY !== true) {
return false;
}
const response = await confirm({ message, initialValue: true });
if (isCancel(response)) {
cancel('Runtime installation cancelled.');
return false;
}
return response === true;
}
export async function ensureManagedPythonCommandRuntime(
options: ManagedPythonCommandOptions,
): Promise<ManagedPythonCommandRuntime> {
const feature = options.feature ?? 'core';
const readStatus = options.readStatus ?? readManagedPythonRuntimeStatus;
const installRuntime = options.installRuntime ?? installManagedPythonRuntime;
const status = await readStatus({ cliVersion: options.cliVersion });
if (status.kind === 'ready' && status.manifest && hasFeature(status.manifest, feature)) {
return { layout: status.layout, manifest: status.manifest };
}
if (options.installPolicy === 'never') {
throw new Error(runtimeRequiredMessage(feature));
}
if (options.installPolicy === 'prompt') {
const confirmInstall = options.confirmInstall ?? defaultConfirmInstall;
const confirmed = await confirmInstall(installPrompt(feature));
if (!confirmed) {
throw new Error(`KTX Python runtime installation was cancelled. Run: ${managedRuntimeInstallCommand(feature)}`);
}
}
options.io.stderr.write(`Installing KTX Python runtime (${feature}) with uv...\n`);
const installed = await installRuntime({
cliVersion: options.cliVersion,
features: [feature],
force: false,
});
options.io.stderr.write(`KTX Python runtime ready: ${installed.layout.versionDir}\n`);
return { layout: installed.layout, manifest: installed.manifest };
}
export async function createManagedPythonSemanticLayerComputePort(
options: ManagedPythonSemanticLayerComputeOptions,
): Promise<KtxSemanticLayerComputePort> {
const runtime = await ensureManagedPythonCommandRuntime({
cliVersion: options.cliVersion,
installPolicy: options.installPolicy,
io: options.io,
feature: 'core',
...(options.readStatus ? { readStatus: options.readStatus } : {}),
...(options.installRuntime ? { installRuntime: options.installRuntime } : {}),
...(options.confirmInstall ? { confirmInstall: options.confirmInstall } : {}),
});
const createPythonCompute = options.createPythonCompute ?? createPythonSemanticLayerComputePort;
return createPythonCompute({
command: runtime.manifest.python.daemonExecutable,
args: [],
});
}
```
- [ ] **Step 2: Run the helper test**
Run:
```bash
pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts
```
Expected: PASS.
- [ ] **Step 3: Commit**
Run:
```bash
git add packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts
git commit -m "feat: add managed python command helper"
```
Expected: commit succeeds.
### Task 3: Add failing `runKtxSl` managed runtime tests
**Files:**
- Modify: `packages/cli/src/sl.test.ts`
- Test: `packages/cli/src/sl.test.ts`
- [ ] **Step 1: Add runtime fields to existing `query` test args**
In each existing `runKtxSl` call whose argument object has
`command: 'query'`, add these properties:
```typescript
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
```
For example, the first `query` argument object becomes:
```typescript
{
command: 'query',
projectDir: '/tmp/project',
connectionId: 'warehouse',
query: { measures: ['orders.order_count'], dimensions: [] },
format: 'sql',
execute: false,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
}
```
- [ ] **Step 2: Add the managed helper delegation test**
In `packages/cli/src/sl.test.ts`, add this test inside
`describe('runKtxSl', () => { ... })` after the existing
`runs sl query and prints SQL output` test:
```typescript
it('creates default sl query compute through the managed runtime helper', async () => {
const projectDir = join(tempDir, 'project');
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
`name: orders
table: public.orders
grain: [id]
columns:
- name: id
type: number
measures:
- name: order_count
expr: count(*)
joins: []
`,
'ktx',
'ktx@example.com',
'Add orders source',
);
const stdout = { write: vi.fn() };
const stderr = { write: vi.fn() };
const compute = {
query: vi.fn(async () => ({
sql: 'select count(*) as order_count from public.orders',
dialect: 'postgres',
columns: [{ name: 'orders.order_count' }],
plan: {},
})),
validateSources: vi.fn(),
generateSources: vi.fn(),
};
const createManagedSemanticLayerCompute = vi.fn(async () => compute);
await expect(
runKtxSl(
{
command: 'query',
projectDir,
connectionId: 'warehouse',
query: { measures: ['orders.order_count'], dimensions: [] },
format: 'sql',
execute: false,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
},
{ stdout, stderr },
{ createManagedSemanticLayerCompute },
),
).resolves.toBe(0);
expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({
cliVersion: '0.2.0',
installPolicy: 'auto',
io: { stdout, stderr },
});
expect(stdout.write).toHaveBeenCalledWith('select count(*) as order_count from public.orders\n');
});
```
- [ ] **Step 3: Run the failing `sl` test**
Run:
```bash
pnpm --filter @ktx/cli run test -- src/sl.test.ts
```
Expected: FAIL with a TypeScript/Vitest error because `runKtxSl` does not
accept `createManagedSemanticLayerCompute` yet.
### Task 4: Wire `runKtxSl` to the managed helper
**Files:**
- Modify: `packages/cli/src/sl.ts`
- Test: `packages/cli/src/sl.test.ts`
- [ ] **Step 1: Add the managed helper imports**
In `packages/cli/src/sl.ts`, add this import after the existing imports:
```typescript
import {
createManagedPythonSemanticLayerComputePort,
type KtxManagedPythonInstallPolicy,
} from './managed-python-command.js';
```
- [ ] **Step 2: Extend the `query` args type**
In the `KtxSlArgs` union, replace the current `query` object type with this
shape:
```typescript
| {
command: 'query';
projectDir: string;
connectionId?: string;
query: SemanticLayerQueryInput;
format: SlQueryFormat;
execute: boolean;
maxRows?: number;
cliVersion: string;
runtimeInstallPolicy: KtxManagedPythonInstallPolicy;
};
```
- [ ] **Step 3: Extend `KtxSlDeps`**
In `packages/cli/src/sl.ts`, replace `KtxSlDeps` with this interface:
```typescript
interface KtxSlDeps {
loadProject?: typeof loadKtxProject;
createSemanticLayerCompute?: () => KtxSemanticLayerComputePort;
createManagedSemanticLayerCompute?: (options: {
cliVersion: string;
installPolicy: KtxManagedPythonInstallPolicy;
io: KtxSlIo;
}) => Promise<KtxSemanticLayerComputePort>;
createQueryExecutor?: () => KtxSqlQueryExecutorPort;
}
```
- [ ] **Step 4: Use the managed helper in the `query` branch**
In the `args.command === 'query'` branch, replace:
```typescript
const compute = (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)();
```
with:
```typescript
const compute = deps.createSemanticLayerCompute
? deps.createSemanticLayerCompute()
: await (deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort)({
cliVersion: args.cliVersion,
installPolicy: args.runtimeInstallPolicy,
io,
});
```
- [ ] **Step 5: Run the `sl` test**
Run:
```bash
pnpm --filter @ktx/cli run test -- src/sl.test.ts
```
Expected: PASS.
- [ ] **Step 6: Commit**
Run:
```bash
git add packages/cli/src/sl.ts packages/cli/src/sl.test.ts
git commit -m "feat: use managed runtime for sl query compute"
```
Expected: commit succeeds.
### Task 5: Add failing Commander routing tests for `sl query`
**Files:**
- Modify: `packages/cli/src/index.test.ts`
- Test: `packages/cli/src/index.test.ts`
- [ ] **Step 1: Add routing tests**
In `packages/cli/src/index.test.ts`, add this test near the other command
routing tests:
```typescript
it('routes sl query managed runtime install policies', async () => {
const sl = vi.fn(async () => 0);
const promptIo = makeIo();
await expect(
runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count'], promptIo.io, { sl }),
).resolves.toBe(0);
expect(sl).toHaveBeenLastCalledWith(
expect.objectContaining({
command: 'query',
projectDir: tempDir,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'prompt',
query: expect.objectContaining({ measures: ['orders.order_count'], dimensions: [] }),
}),
promptIo.io,
);
const autoIo = makeIo();
await expect(
runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes'], autoIo.io, {
sl,
}),
).resolves.toBe(0);
expect(sl).toHaveBeenLastCalledWith(
expect.objectContaining({
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'auto',
}),
autoIo.io,
);
const noInputIo = makeIo();
await expect(
runKtxCli(
['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--no-input'],
noInputIo.io,
{ sl },
),
).resolves.toBe(0);
expect(sl).toHaveBeenLastCalledWith(
expect.objectContaining({
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'never',
}),
noInputIo.io,
);
});
it('rejects conflicting sl query runtime install flags', async () => {
const io = makeIo();
const sl = vi.fn(async () => 0);
await expect(
runKtxCli(
['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes', '--no-input'],
io.io,
{ sl },
),
).resolves.toBe(1);
expect(sl).not.toHaveBeenCalled();
expect(io.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
});
```
- [ ] **Step 2: Run the failing routing tests**
Run:
```bash
pnpm --filter @ktx/cli run test -- src/index.test.ts
```
Expected: FAIL because `sl query` does not accept `--yes` or `--no-input`
and does not pass runtime policy fields yet.
### Task 6: Wire `sl query` flags and schema validation
**Files:**
- Modify: `packages/cli/src/commands/sl-commands.ts`
- Modify: `packages/cli/src/command-schemas.ts`
- Test: `packages/cli/src/index.test.ts`
- [ ] **Step 1: Add the runtime policy type import**
In `packages/cli/src/commands/sl-commands.ts`, add this import:
```typescript
import type { KtxManagedPythonInstallPolicy } from '../managed-python-command.js';
```
- [ ] **Step 2: Add the runtime policy parser**
In `packages/cli/src/commands/sl-commands.ts`, add this function near the
other option parsers:
```typescript
function runtimeInstallPolicy(options: { yes?: boolean; input?: boolean }): KtxManagedPythonInstallPolicy {
if (options.yes === true && options.input === false) {
throw new Error('Choose only one runtime install mode: --yes or --no-input');
}
if (options.yes === true) {
return 'auto';
}
return options.input === false ? 'never' : 'prompt';
}
```
- [ ] **Step 3: Add the command options**
In the `sl.command('query')` option chain, add these options after
`.option('--execute', 'Execute the compiled query', false)`:
```typescript
.option('--yes', 'Install the managed Python runtime without prompting when required', false)
.option('--no-input', 'Disable interactive managed runtime installation')
```
- [ ] **Step 4: Pass runtime fields into `slQueryCommandSchema.parse`**
In the `sl.command('query')` action, add these properties to the parsed object:
```typescript
cliVersion: context.packageInfo.version,
runtimeInstallPolicy: runtimeInstallPolicy(options),
```
The parsed object must include these fields next to `execute` and `format`:
```typescript
const args = slQueryCommandSchema.parse({
command: 'query',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
query: {
measures: options.measure,
dimensions: options.dimension,
...(options.filter.length > 0 ? { filters: options.filter } : {}),
...(options.segment.length > 0 ? { segments: options.segment } : {}),
...(options.orderBy.length > 0 ? { order_by: options.orderBy } : {}),
...(options.limit !== undefined ? { limit: options.limit } : {}),
...(options.includeEmpty === true ? { include_empty: true } : {}),
},
format: options.format,
execute: options.execute === true,
cliVersion: context.packageInfo.version,
runtimeInstallPolicy: runtimeInstallPolicy(options),
...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}),
});
```
- [ ] **Step 5: Extend the command schema**
In `packages/cli/src/command-schemas.ts`, add these fields to
`slQueryCommandSchema` after `execute: z.boolean()`:
```typescript
cliVersion: z.string().min(1),
runtimeInstallPolicy: z.enum(['prompt', 'auto', 'never']),
```
- [ ] **Step 6: Run the routing tests**
Run:
```bash
pnpm --filter @ktx/cli run test -- src/index.test.ts
```
Expected: PASS.
- [ ] **Step 7: Commit**
Run:
```bash
git add packages/cli/src/commands/sl-commands.ts packages/cli/src/command-schemas.ts packages/cli/src/index.test.ts
git commit -m "feat: route sl query managed runtime policy"
```
Expected: commit succeeds.
### Task 7: Verify the full changed surface
**Files:**
- Verify: `packages/cli/src/managed-python-command.test.ts`
- Verify: `packages/cli/src/sl.test.ts`
- Verify: `packages/cli/src/index.test.ts`
- Verify: `packages/cli/src/managed-python-command.ts`
- Verify: `packages/cli/src/sl.ts`
- Verify: `packages/cli/src/commands/sl-commands.ts`
- Verify: `packages/cli/src/command-schemas.ts`
- [ ] **Step 1: Run focused CLI tests**
Run:
```bash
pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts src/sl.test.ts src/index.test.ts
```
Expected: PASS.
- [ ] **Step 2: Run CLI type checking**
Run:
```bash
pnpm --filter @ktx/cli run type-check
```
Expected: PASS.
- [ ] **Step 3: Run pre-commit for changed TypeScript files**
Run:
```bash
uv run pre-commit run --files packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts packages/cli/src/sl.ts packages/cli/src/sl.test.ts packages/cli/src/commands/sl-commands.ts packages/cli/src/command-schemas.ts packages/cli/src/index.test.ts
```
Expected: PASS. If pre-commit is unavailable because the local `uv` version
does not satisfy `pyproject.toml`, record the version mismatch and run the
focused CLI tests plus type checking from Steps 1 and 2.
- [ ] **Step 4: Commit verification fixes when needed**
If Step 1, Step 2, or Step 3 changes files through formatting hooks, run:
```bash
git add packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts packages/cli/src/sl.ts packages/cli/src/sl.test.ts packages/cli/src/commands/sl-commands.ts packages/cli/src/command-schemas.ts packages/cli/src/index.test.ts
git commit -m "test: verify managed runtime sl query integration"
```
Expected: commit succeeds only when verification changed files. If no files
changed, leave the branch with the commits from Tasks 2, 4, and 6.
## Acceptance criteria
When this plan is complete:
- `ktx sl query` uses the managed runtime's installed `ktx-daemon` executable
for semantic-layer compilation when no test compute dependency is injected.
- `ktx sl query --yes` installs the `core` runtime feature without prompting
when the managed runtime is missing.
- `ktx sl query --no-input` fails with
`KTX Python runtime is required for this command. Run: ktx runtime install --yes`
when the managed runtime is missing.
- `ktx sl query` prompts before first managed runtime installation in an
interactive terminal.
- Existing injected-compute tests still bypass runtime installation.

1546
docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md Normal file
View file

File diff suppressed because it is too large Load diff

1750
docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md Normal file
View file

File diff suppressed because it is too large Load diff

585
docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md Normal file
View file

@ -0,0 +1,585 @@
# Managed Python Runtime Release Smoke Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use
> superpowers:subagent-driven-development (recommended) or
> superpowers:executing-plans to implement this plan task-by-task. Steps use
> checkbox (`- [ ]`) syntax for tracking.
**Goal:** Make the public `@kaelio/ktx` artifact smoke prove that the npm
package installs and uses its own managed Python runtime without an externally
prepared Python environment.
**Architecture:** Keep the release smoke black-box: install the packed public
npm tarball into a clean project, isolate `KTX_RUNTIME_ROOT`, and exercise the
installed `ktx` binary. The first `ktx sl query --yes` performs the lazy core
runtime install from bundled package assets, then the smoke verifies
`runtime status`, `runtime doctor`, daemon start/reuse, and daemon stop.
**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, uv, KTX CLI managed
Python runtime assets.
---
## Existing status
This plan is based on
`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`.
Existing plans based on the spec:
- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md`
- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md`
All six are implemented in this worktree. Evidence found before writing this
plan includes:
- `scripts/build-python-runtime-wheel.mjs` and
`scripts/build-python-runtime-wheel.test.mjs`.
- `packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl` and
`packages/cli/assets/python/manifest.json`.
- `packages/cli/src/managed-python-runtime.ts`,
`packages/cli/src/runtime.ts`, and
`packages/cli/src/commands/runtime-commands.ts`.
- `packages/cli/src/managed-python-command.ts` and `ktx sl query` runtime
install policy flags.
- `packages/cli/src/managed-python-daemon.ts`, daemon state paths, and
`ktx runtime start` / `ktx runtime stop`.
- `packages/cli/src/managed-local-embeddings.ts`,
`packages/context/src/llm/local-config.ts` managed marker constants, and
setup wiring in `packages/cli/src/setup-embeddings.ts`.
- `scripts/build-public-npm-package.mjs`,
`scripts/build-public-npm-package.test.mjs`, `release-policy.json` listing
`@kaelio/ktx`, and published smoke command construction for the required
`@kaelio/ktx` invocation modes.
The remaining release-smoke gap is in `scripts/package-artifacts.mjs`:
- `verifyNpmArtifacts()` creates a smoke `.venv`, installs the built Python
runtime wheel into it, and runs installed CLI smoke scripts with that venv at
the front of `PATH`.
- The installed CLI smoke does run `ktx sl query --yes`, but it does not
isolate `KTX_RUNTIME_ROOT`, does not assert that the first query installed
the managed runtime from bundled npm assets, and does not exercise
`ktx runtime status`, `doctor`, `start`, reuse, and `stop`.
This plan closes that release-flow gap without changing the separate Python
artifact smoke. `verifyPythonArtifacts()` must continue to install the built
Python wheel directly because it verifies the Python artifact itself.
## File structure
- Modify `scripts/package-artifacts.test.mjs`: remove the npm-smoke venv test,
add a source-level guard that npm artifact verification does not prepare an
external Python venv, and assert that the installed CLI smoke exercises the
managed runtime lifecycle.
- Modify `scripts/package-artifacts.mjs`: remove npm-smoke Python venv PATH
setup, isolate `KTX_RUNTIME_ROOT` inside `npmRuntimeSmokeSource()`, assert
first-run lazy install, and add runtime status/doctor/start/reuse/stop smoke
commands.
### Task 1: Add failing release-smoke tests
**Files:**
- Modify: `scripts/package-artifacts.test.mjs`
- Test: `scripts/package-artifacts.test.mjs`
- [ ] **Step 1: Remove the stale npm-smoke venv import**
In `scripts/package-artifacts.test.mjs`, delete `npmSmokePythonEnv` from the
import list. The surrounding import block must contain this sequence after the
edit:
```javascript
npmDemoSmokeSource,
npmRuntimeSmokeSource,
npmSmokePackageJson,
npmVerifySource,
```
- [ ] **Step 2: Replace the npm-smoke venv test with a source guard**
Delete this entire test block:
```javascript
describe('npmSmokePythonEnv', () => {
it('prepends the npm smoke virtualenv bin directory to PATH', () => {
const env = npmSmokePythonEnv('/tmp/ktx-npm-smoke', { PATH: '/usr/bin' });
assert.match(env.PATH, /^\/tmp\/ktx-npm-smoke\/\.venv\/(bin|Scripts)/);
assert.match(env.PATH, /\/usr\/bin$/);
});
});
```
Insert this block in the same location:
```javascript
describe('verifyNpmArtifacts', () => {
it('does not prepare an external Python environment for the npm smoke', async () => {
const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8');
const start = source.indexOf('async function verifyNpmArtifacts');
const end = source.indexOf('async function verifyNpmDemoArtifacts');
assert.ok(start > 0, 'verifyNpmArtifacts function must exist');
assert.ok(end > start, 'verifyNpmDemoArtifacts must follow verifyNpmArtifacts');
const body = source.slice(start, end);
assert.doesNotMatch(body, /uv', \['venv', '\.venv'\]/);
assert.doesNotMatch(body, /pythonArtifactInstallArgs/);
assert.doesNotMatch(body, /npmSmokePythonEnv/);
});
});
```
- [ ] **Step 3: Extend the installed CLI smoke assertions**
In the `it('runs installed CLI commands through the public package runtime',
...)` test, add these assertions after the existing
`assert.match(source, /ktx sl query sqlite execute/);` assertion:
```javascript
assert.match(source, /import Database from 'better-sqlite3'/);
assert.doesNotMatch(source, /run\('python'/);
assert.match(source, /KTX_RUNTIME_ROOT/);
assert.match(source, /managed-runtime/);
assert.match(source, /ktx runtime status missing/);
assert.match(source, /runtimeStatusBefore\.kind, 'missing'/);
assert.match(source, /Installing KTX Python runtime \(core\) with uv/);
assert.match(source, /KTX Python runtime ready:/);
assert.match(source, /ktx runtime status ready/);
assert.match(source, /runtimeStatusAfter\.kind, 'ready'/);
assert.match(source, /runtimeStatusAfter\.manifest\.features/);
assert.match(source, /ktx runtime doctor/);
assert.match(source, /PASS Managed Python runtime/);
assert.match(source, /ktx runtime start/);
assert.match(source, /ktx runtime start reuse/);
assert.match(source, /Using existing KTX Python daemon/);
assert.match(source, /ktx runtime stop/);
```
- [ ] **Step 4: Run the failing package artifact tests**
Run:
```bash
node --test scripts/package-artifacts.test.mjs
```
Expected: FAIL. The guard fails because `verifyNpmArtifacts()` still creates
the npm-smoke `.venv`, and the installed CLI smoke assertions fail because
`npmRuntimeSmokeSource()` does not yet isolate or verify the managed runtime.
### Task 2: Make the npm smoke use only the managed runtime
**Files:**
- Modify: `scripts/package-artifacts.mjs`
- Modify: `scripts/package-artifacts.test.mjs`
- Test: `scripts/package-artifacts.test.mjs`
- [ ] **Step 1: Remove the npm-smoke PATH helper**
In `scripts/package-artifacts.mjs`, change the path import from:
```javascript
import { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
```
to:
```javascript
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
```
Then delete this exported helper:
```javascript
export function npmSmokePythonEnv(projectDir, baseEnv = process.env) {
const binDir = process.platform === 'win32' ? join(projectDir, '.venv', 'Scripts') : join(projectDir, '.venv', 'bin');
const existingPath = baseEnv.PATH ?? '';
return Object.assign({}, baseEnv, {
PATH: existingPath ? `${binDir}${delimiter}${existingPath}` : binDir,
});
}
```
- [ ] **Step 2: Add runtime-smoke helpers to `npmRuntimeSmokeSource()`**
Inside the template string returned by `npmRuntimeSmokeSource()`, add this
helper immediately after `requireSuccess()`:
```javascript
function requireSuccessWithStderr(label, result, stderrPattern) {
assert.equal(
result.code,
0,
label + ' failed with code ' + result.code + '\\nstdout:\\n' + result.stdout + '\\nstderr:\\n' + result.stderr,
);
assert.match(result.stderr, stderrPattern, label + ' stderr did not match ' + stderrPattern);
}
```
Then replace the smoke root setup:
```javascript
const root = await mkdtemp(join(tmpdir(), 'ktx-installed-cli-smoke-'));
try {
const projectDir = join(root, 'project');
const sourceDir = join(root, 'source');
```
with:
```javascript
const root = await mkdtemp(join(tmpdir(), 'ktx-installed-cli-smoke-'));
const previousRuntimeRoot = process.env.KTX_RUNTIME_ROOT;
process.env.KTX_RUNTIME_ROOT = join(root, 'managed-runtime');
let daemonStarted = false;
try {
const projectDir = join(root, 'project');
const sourceDir = join(root, 'source');
```
Finally replace the existing `finally` block at the end of
`npmRuntimeSmokeSource()`:
```javascript
} finally {
await rm(root, { recursive: true, force: true });
}
```
with:
```javascript
} finally {
if (daemonStarted) {
await run('pnpm', ['exec', 'ktx', 'runtime', 'stop']);
}
if (previousRuntimeRoot === undefined) {
delete process.env.KTX_RUNTIME_ROOT;
} else {
process.env.KTX_RUNTIME_ROOT = previousRuntimeRoot;
}
await rm(root, { recursive: true, force: true });
}
```
- [ ] **Step 3: Create the sqlite smoke warehouse without Python**
Inside the template string returned by `npmRuntimeSmokeSource()`, add this
import after the `assert` import:
```javascript
import Database from 'better-sqlite3';
```
Then replace the current `writeSqliteWarehouse()` function:
```javascript
async function writeSqliteWarehouse(projectDir) {
const createDb = await run('python', [
'-c',
[
'import sqlite3',
'import sys',
'db_path = sys.argv[1]',
'conn = sqlite3.connect(db_path)',
'conn.executescript("""',
'DROP TABLE IF EXISTS orders;',
'CREATE TABLE orders (',
' id INTEGER PRIMARY KEY,',
' status TEXT NOT NULL,',
' amount INTEGER NOT NULL',
');',
"INSERT INTO orders (status, amount) VALUES ('paid', 20), ('paid', 30), ('open', 10);",
'""")',
'conn.close()',
].join('\\n'),
join(projectDir, 'warehouse.db'),
]);
requireSuccess('create sqlite warehouse', createDb);
}
```
with:
```javascript
async function writeSqliteWarehouse(projectDir) {
const database = new Database(join(projectDir, 'warehouse.db'));
try {
database.exec(`
DROP TABLE IF EXISTS orders;
CREATE TABLE orders (
id INTEGER PRIMARY KEY,
status TEXT NOT NULL,
amount INTEGER NOT NULL
);
INSERT INTO orders (status, amount) VALUES ('paid', 20), ('paid', 30), ('open', 10);
`);
} finally {
database.close();
}
}
```
- [ ] **Step 4: Assert the isolated runtime is initially missing**
In `npmRuntimeSmokeSource()`, insert this block immediately after the public
package version assertion:
```javascript
const runtimeStatusBefore = parseJsonResult(
'ktx runtime status missing',
await run('pnpm', ['exec', 'ktx', 'runtime', 'status', '--json']),
);
assert.equal(runtimeStatusBefore.kind, 'missing');
assert.equal(runtimeStatusBefore.layout.runtimeRoot, process.env.KTX_RUNTIME_ROOT);
process.stdout.write('ktx managed runtime starts missing in isolated root\\n');
```
- [ ] **Step 5: Assert first `sl query --yes` performs lazy managed install**
In `npmRuntimeSmokeSource()`, replace the current `slQuery` verification block:
```javascript
const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query',
'--connection-id',
'warehouse',
'--measure',
'orders.order_count',
'--format',
'json',
'--yes',
'--project-dir',
projectDir,
]);
requireSuccess('ktx sl query', slQuery);
requireOutput('ktx sl query', slQuery, /"mode": "compile_only"/);
requireOutput('ktx sl query', slQuery, /orders/);
```
with:
```javascript
const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query',
'--connection-id',
'warehouse',
'--measure',
'orders.order_count',
'--format',
'json',
'--yes',
'--project-dir',
projectDir,
]);
requireSuccessWithStderr(
'ktx sl query first managed runtime install',
slQuery,
/Installing KTX Python runtime \(core\) with uv[\s\S]*KTX Python runtime ready:/,
);
requireOutput('ktx sl query first managed runtime install', slQuery, /"mode": "compile_only"/);
requireOutput('ktx sl query first managed runtime install', slQuery, /orders/);
const runtimeStatusAfter = parseJsonResult(
'ktx runtime status ready',
await run('pnpm', ['exec', 'ktx', 'runtime', 'status', '--json']),
);
assert.equal(runtimeStatusAfter.kind, 'ready');
assert.deepEqual(runtimeStatusAfter.manifest.features, ['core']);
assert.equal(runtimeStatusAfter.layout.runtimeRoot, process.env.KTX_RUNTIME_ROOT);
process.stdout.write('ktx managed runtime lazy install verified\\n');
```
- [ ] **Step 6: Add runtime doctor and daemon lifecycle smoke**
In `npmRuntimeSmokeSource()`, insert this block immediately after the
`sqliteSlQuery` verification block:
```javascript
const runtimeDoctor = await run('pnpm', ['exec', 'ktx', 'runtime', 'doctor']);
requireSuccess('ktx runtime doctor', runtimeDoctor);
requireOutput('ktx runtime doctor', runtimeDoctor, /PASS uv/);
requireOutput('ktx runtime doctor', runtimeDoctor, /PASS Bundled Python wheel/);
requireOutput('ktx runtime doctor', runtimeDoctor, /PASS Managed Python runtime/);
process.stdout.write('ktx runtime doctor verified\\n');
const runtimeStart = await run('pnpm', ['exec', 'ktx', 'runtime', 'start']);
requireSuccess('ktx runtime start', runtimeStart);
daemonStarted = true;
requireOutput('ktx runtime start', runtimeStart, /Started KTX Python daemon/);
requireOutput('ktx runtime start', runtimeStart, /url: http:\/\/127\.0\.0\.1:\d+/);
requireOutput('ktx runtime start', runtimeStart, /features: core/);
const runtimeStartReuse = await run('pnpm', ['exec', 'ktx', 'runtime', 'start']);
requireSuccess('ktx runtime start reuse', runtimeStartReuse);
requireOutput('ktx runtime start reuse', runtimeStartReuse, /Using existing KTX Python daemon/);
requireOutput('ktx runtime start reuse', runtimeStartReuse, /features: core/);
const runtimeStop = await run('pnpm', ['exec', 'ktx', 'runtime', 'stop']);
requireSuccess('ktx runtime stop', runtimeStop);
daemonStarted = false;
requireOutput('ktx runtime stop', runtimeStop, /Stopped KTX Python daemon/);
process.stdout.write('ktx runtime daemon lifecycle verified\\n');
```
- [ ] **Step 7: Remove npm-smoke Python preparation from artifact verification**
In `scripts/package-artifacts.mjs`, replace `verifyNpmArtifacts()` with this
implementation:
```javascript
async function verifyNpmArtifacts(layout, tmpRoot) {
for (const packageInfo of NPM_ARTIFACT_PACKAGES) {
await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`);
}
const projectDir = join(tmpRoot, 'npm-clean-install');
await mkdir(projectDir, { recursive: true });
await writeFile(
join(projectDir, 'package.json'),
`${JSON.stringify(npmSmokePackageJson(layout), null, 2)}\n`,
);
await writeFile(join(projectDir, 'verify-npm.mjs'), npmVerifySource());
await writeFile(join(projectDir, 'verify-installed-cli.mjs'), npmRuntimeSmokeSource());
await writeFile(join(projectDir, 'verify-installed-demo.mjs'), npmDemoSmokeSource());
await runCommand('pnpm', ['install'], { cwd: projectDir });
await runCommand('pnpm', ['rebuild', 'better-sqlite3'], { cwd: projectDir });
await runCommand('node', ['verify-npm.mjs'], { cwd: projectDir });
await runCommand('pnpm', ['exec', 'ktx', '--version'], { cwd: projectDir });
await runCommand('node', ['verify-installed-cli.mjs'], { cwd: projectDir });
await runCommand('node', ['verify-installed-demo.mjs'], { cwd: projectDir });
}
```
- [ ] **Step 8: Run the focused package artifact tests**
Run:
```bash
node --test scripts/package-artifacts.test.mjs
```
Expected: PASS.
- [ ] **Step 9: Commit the release-smoke implementation**
Run:
```bash
git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs
git commit -m "test: verify managed runtime in public package smoke"
```
### Task 3: Verify the release-smoke surface
**Files:**
- Test: `scripts/package-artifacts.test.mjs`
- Test: `scripts/package-artifacts.mjs`
- [ ] **Step 1: Run script unit tests that cover artifact packaging**
Run:
```bash
node --test scripts/build-python-runtime-wheel.test.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs
```
Expected: PASS.
- [ ] **Step 2: Run the public package artifact smoke**
Run:
```bash
pnpm run artifacts:verify
```
Expected: PASS. The `verify-installed-cli.mjs` output must include:
```text
ktx managed runtime starts missing in isolated root
ktx managed runtime lazy install verified
ktx runtime doctor verified
ktx runtime daemon lifecycle verified
```
- [ ] **Step 3: Run release readiness**
Run:
```bash
pnpm run release:readiness
```
Expected: PASS. The report must still list `@kaelio/ktx` as the only npm
package and must still report registry publishing as disabled by
`release-policy.json`.
- [ ] **Step 4: Run pre-commit for changed files**
Run:
```bash
if [ -d .venv ]; then source .venv/bin/activate; fi
uv run pre-commit run --files scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs
```
Expected: PASS. If pre-commit cannot run because the local environment lacks a
compatible hook version, record the exact failure and keep the passing
`node --test` and artifact smoke results.
- [ ] **Step 5: Commit verification fixes if needed**
If Step 1, Step 2, Step 3, or Step 4 required edits, run:
```bash
git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs
git commit -m "test: finalize managed runtime release smoke"
```
If no files changed after Task 2, do not create an empty commit.
## Acceptance criteria
- `verifyNpmArtifacts()` no longer creates a Python `.venv`, no longer calls
`pythonArtifactInstallArgs()`, and no longer runs npm smoke scripts with a
custom Python venv at the front of `PATH`.
- The installed public npm smoke creates its sqlite warehouse with
`better-sqlite3` and does not shell out to `python`.
- The installed public npm smoke sets an isolated `KTX_RUNTIME_ROOT` and
confirms that `ktx runtime status --json` starts as `missing`.
- The first installed `ktx sl query --yes` installs the `core` managed Python
runtime from bundled npm package assets and still returns compile-only SQL.
- A second semantic query executes against sqlite using the installed managed
runtime.
- `ktx runtime doctor` passes after lazy install.
- `ktx runtime start` starts a core daemon, a second `ktx runtime start` reuses
the daemon, and `ktx runtime stop` stops it.
- The separate Python artifact verification still installs and tests the
Python wheel directly.
- Focused script tests, `pnpm run artifacts:verify`, release readiness, and
pre-commit pass or have explicitly recorded environment blockers.
## Self-review
- Spec coverage: the previous six plans cover the bundled wheel, runtime
installer, `sl query` command integration, daemon lifecycle, local embeddings,
and public npm package surface. This plan covers release-flow checks for clean
install of the packed npm package, first-run managed runtime install from the
bundled wheel, one-shot semantic-layer query through the managed runtime,
runtime status and doctor output, and daemon start/reuse/stop.
- Remaining intentional gap: optional `local-embeddings` smoke remains outside
the default release artifact smoke because the spec permits it in a separate
job or opt-in check and the dependency downloads are large.
- Placeholder scan: no steps contain placeholder implementation language.
- Type consistency: runtime feature names remain `core` and
`local-embeddings`; the public npm package name remains `@kaelio/ktx`; the
runtime root environment variable is `KTX_RUNTIME_ROOT`.

657
docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md Normal file
View file

@ -0,0 +1,657 @@
# Managed Runtime Docs and Postgres Smoke Cleanup Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use
> superpowers:subagent-driven-development (recommended) or
> superpowers:executing-plans to implement this plan task-by-task. Steps use
> checkbox (`- [ ]`) syntax for tracking.
**Goal:** Remove the remaining manual Python service guidance from the Postgres
historic SQL smoke and update public docs so the npm-managed Python runtime is
the documented path.
**Architecture:** Keep the existing managed-runtime code unchanged. Add source
and docs guards first, then make the Postgres historic smoke use the
CLI-managed core daemon through `createKtxCliLocalIngestAdapters()`, and update
the README files that still describe internal package artifacts, manual
`ktx-daemon` startup, or `python-service/`.
**Tech Stack:** Bash, Node 22 ESM, `node:test`, Markdown, pnpm, uv, KTX CLI
managed Python runtime.
---
## Existing status
This plan is based on
`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`.
The following plans are based on that spec and are already implemented in this
worktree:
- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md`
- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md`
- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md`
Implementation evidence found before writing this plan includes:
- `scripts/build-python-runtime-wheel.mjs` and
`packages/cli/assets/python/manifest.json`.
- `packages/cli/src/managed-python-runtime.ts`,
`packages/cli/src/runtime.ts`, and
`packages/cli/src/commands/runtime-commands.ts`.
- `packages/cli/src/managed-python-command.ts` and managed `ktx sl query`
runtime policy flags.
- `packages/cli/src/managed-python-daemon.ts` and `ktx runtime start` /
`ktx runtime stop`.
- `packages/cli/src/managed-local-embeddings.ts` and local embeddings setup
wiring.
- `scripts/build-public-npm-package.mjs`, release policy updates, release
smoke coverage, and opt-in local embeddings smoke coverage.
- `packages/cli/src/agent-runtime.ts` and `packages/cli/src/serve.ts` now
create managed semantic-layer compute when no explicit semantic HTTP URL is
provided.
- `packages/cli/src/managed-python-http.ts`,
`packages/cli/src/local-adapters.ts`, `packages/cli/src/ingest.ts`,
`packages/cli/src/scan.ts`, and `packages/cli/src/serve.ts` wire local ingest
helper paths to the managed core daemon.
The remaining drift is documentation and one example smoke script:
- `examples/postgres-historic/scripts/smoke.sh` still checks for
`python-service/.venv`, starts `uvicorn app.main:app`, and exports
`KTX_SQL_ANALYSIS_URL`.
- `examples/postgres-historic/README.md` still documents
`python-service/.venv` or `KTX_SQL_ANALYSIS_URL` as a prerequisite.
- `examples/package-artifacts/README.md` still says the npm smoke installs
generated `@ktx/context` and `@ktx/cli` tarballs.
- `README.md` still presents source-tree `pnpm run ktx -- ...` commands as the
quick start and tells users to start `ktx-daemon` manually for MCP.
This plan closes that drift. It does not rename internal workspace packages and
does not remove explicit daemon URL override behavior from production code.
## File structure
- Modify `scripts/examples-docs.test.mjs`: add regression coverage for managed
runtime docs, public npm package docs, and the Postgres smoke script.
- Modify `examples/postgres-historic/scripts/smoke.sh`: remove
`python-service/` startup and pass managed daemon options into stage-only
historic SQL ingest.
- Modify `examples/postgres-historic/README.md`: document the managed runtime
and remove old SQL-analysis service instructions.
- Modify `examples/package-artifacts/README.md`: describe the single public
`@kaelio/ktx` npm artifact and managed runtime smoke.
- Modify `README.md`: make public `@kaelio/ktx` invocation modes and managed
runtime commands visible while keeping source-tree development commands in
the development section.
### Task 1: Add failing docs and smoke guards
**Files:**
- Modify: `scripts/examples-docs.test.mjs`
- Test: `scripts/examples-docs.test.mjs`
- [ ] **Step 1: Add public runtime README assertions**
In `scripts/examples-docs.test.mjs`, insert this test after the existing
`walks through ktx connection list and ktx connection test in the README
quickstart` test:
```javascript
it('documents public npm and managed runtime usage in the README', async () => {
const rootReadme = await readText('README.md');
assert.match(rootReadme, /npx @kaelio\/ktx setup demo --no-input/);
assert.match(rootReadme, /npx @kaelio\/ktx sl query/);
assert.match(rootReadme, /npm install @kaelio\/ktx/);
assert.match(rootReadme, /npm install -g @kaelio\/ktx/);
assert.match(rootReadme, /ktx runtime install/);
assert.match(rootReadme, /ktx runtime status/);
assert.match(rootReadme, /ktx runtime doctor/);
assert.match(rootReadme, /ktx runtime start/);
assert.match(rootReadme, /ktx runtime stop/);
assert.match(rootReadme, /ktx serve --mcp stdio/);
assert.doesNotMatch(rootReadme, /uv run ktx-daemon serve-http/);
assert.doesNotMatch(rootReadme, /--semantic-compute-url http:\/\/127\.0\.0\.1:8765/);
});
```
- [ ] **Step 2: Add package artifact README assertions**
In `scripts/examples-docs.test.mjs`, insert this test after the new public
runtime README test:
```javascript
it('documents the public package artifact smoke shape', async () => {
const readme = await readText('examples/package-artifacts/README.md');
assert.match(readme, /@kaelio\/ktx/);
assert.match(readme, /managed Python runtime/);
assert.match(readme, /ktx runtime status/);
assert.match(readme, /ktx runtime doctor/);
assert.doesNotMatch(readme, /@ktx\/context/);
assert.doesNotMatch(readme, /@ktx\/cli/);
assert.doesNotMatch(readme, /python -m ktx_daemon semantic-validate/);
});
```
- [ ] **Step 3: Extend Postgres smoke assertions**
In the existing `documents the Postgres historic SQL smoke example` test in
`scripts/examples-docs.test.mjs`, add these assertions after
`assert.match(smoke, /pg_stat_statements_reset/);`:
```javascript
assert.match(smoke, /KTX_RUNTIME_ROOT/);
assert.match(smoke, /managedDaemon/);
assert.match(smoke, /installPolicy: 'auto'/);
assert.match(smoke, /getKtxCliPackageInfo/);
assert.doesNotMatch(smoke, /python-service/);
assert.doesNotMatch(smoke, /PYTHON_SERVICE/);
assert.doesNotMatch(smoke, /uvicorn app\.main:app/);
assert.doesNotMatch(smoke, /export KTX_SQL_ANALYSIS_URL/);
assert.doesNotMatch(readme, /python-service/);
assert.doesNotMatch(readme, /KTX_SQL_ANALYSIS_URL/);
```
- [ ] **Step 4: Run the docs test to verify it fails**
Run:
```bash
node --test scripts/examples-docs.test.mjs
```
Expected: FAIL. The failure includes missing `@kaelio/ktx` README matches and
the existing `python-service` / `KTX_SQL_ANALYSIS_URL` references in the
Postgres smoke files.
### Task 2: Move the Postgres historic smoke to the managed runtime
**Files:**
- Modify: `examples/postgres-historic/scripts/smoke.sh`
- Test: `scripts/examples-docs.test.mjs`
- [ ] **Step 1: Remove Python service process state**
In `examples/postgres-historic/scripts/smoke.sh`, replace the variable block:
```bash
KTX_BIN="$KTX_ROOT/packages/cli/dist/bin.js"
PYTHON_SERVICE_LOG="$PROJECT_PARENT/python-service.log"
PYTHON_SERVICE_PID=""
```
with:
```bash
KTX_BIN="$KTX_ROOT/packages/cli/dist/bin.js"
export KTX_RUNTIME_ROOT="$PROJECT_PARENT/managed-runtime"
unset KTX_DAEMON_URL
unset KTX_SQL_ANALYSIS_URL
```
- [ ] **Step 2: Replace cleanup**
In `examples/postgres-historic/scripts/smoke.sh`, replace the `cleanup()`
function with:
```bash
cleanup() {
if [[ -f "$KTX_BIN" ]]; then
node "$KTX_BIN" runtime stop >/dev/null 2>&1 || true
fi
if [[ "${KTX_POSTGRES_HISTORIC_KEEP_DOCKER:-0}" != "1" ]]; then
docker compose -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT
```
- [ ] **Step 3: Delete the old SQL analysis service starter**
Delete the entire `start_sql_analysis_if_needed()` function from
`examples/postgres-historic/scripts/smoke.sh`. The deleted function begins with
this line:
```bash
start_sql_analysis_if_needed() {
```
and ends with this line:
```bash
}
```
immediately before the `latest_manifest()` function.
- [ ] **Step 4: Pass managed daemon options to stage-only ingest**
In the Node heredoc inside `run_historic_stage_only()`, replace this block:
```javascript
const { createKtxCliLocalIngestAdapters } = await import(join(ktxRoot, 'packages/cli/dist/local-adapters.js'));
const project = await loadKtxProject({ projectDir });
const adapters = createKtxCliLocalIngestAdapters(project, { historicSqlConnectionId: 'warehouse' });
```
with:
```javascript
const { createKtxCliLocalIngestAdapters } = await import(join(ktxRoot, 'packages/cli/dist/local-adapters.js'));
const { getKtxCliPackageInfo } = await import(join(ktxRoot, 'packages/cli/dist/index.js'));
const project = await loadKtxProject({ projectDir });
const cliVersion = getKtxCliPackageInfo().version;
const managedRuntimeIo = { stdout: process.stdout, stderr: process.stderr };
const adapters = createKtxCliLocalIngestAdapters(project, {
historicSqlConnectionId: 'warehouse',
managedDaemon: {
cliVersion,
installPolicy: 'auto',
io: managedRuntimeIo,
},
});
```
- [ ] **Step 5: Remove the old starter call**
Delete this line from the bottom half of
`examples/postgres-historic/scripts/smoke.sh`:
```bash
start_sql_analysis_if_needed
```
- [ ] **Step 6: Run the docs test to verify the script guards pass**
Run:
```bash
node --test scripts/examples-docs.test.mjs
```
Expected: FAIL remains because README files have not been updated yet. The
Postgres smoke script assertions now pass.
### Task 3: Update Postgres historic and artifact docs
**Files:**
- Modify: `examples/postgres-historic/README.md`
- Modify: `examples/package-artifacts/README.md`
- Test: `scripts/examples-docs.test.mjs`
- [ ] **Step 1: Replace Postgres prerequisites**
In `examples/postgres-historic/README.md`, replace the `## Prerequisites`
section with:
```markdown
## Prerequisites
- Docker with Compose v2
- Node and pnpm matching the KTX workspace
- `uv` on `PATH` so the KTX-managed Python runtime can install the bundled
runtime wheel
```
- [ ] **Step 2: Replace the smoke run description**
In `examples/postgres-historic/README.md`, replace the paragraph after the
`examples/postgres-historic/scripts/smoke.sh` command with:
```markdown
The smoke creates a temporary KTX project, isolates the managed Python runtime
under the temporary project parent, starts Postgres on `127.0.0.1:55432`, and
uses this connection URL:
```
- [ ] **Step 3: Update the full ingest command**
In `examples/postgres-historic/README.md`, replace the manual ingest command:
```bash
node packages/cli/dist/bin.js --project-dir /tmp/ktx-postgres-historic dev ingest run \
--connection-id warehouse \
--adapter historic-sql \
--plain \
--no-input
```
with:
```bash
pnpm run ktx -- dev ingest run --project-dir /tmp/ktx-postgres-historic \
--connection-id warehouse \
--adapter historic-sql \
--plain \
--yes \
--no-input
```
- [ ] **Step 4: Replace SQL-analysis troubleshooting**
In `examples/postgres-historic/README.md`, replace the final troubleshooting
bullet:
```markdown
- SQL-analysis failures: set `KTX_SQL_ANALYSIS_URL` to the running service URL
or create `python-service/.venv` before running `scripts/smoke.sh`.
```
with:
```markdown
- SQL-analysis failures: run `pnpm run ktx -- runtime doctor` from the KTX
repository root and confirm `uv`, the bundled Python wheel, and the managed
runtime all pass.
```
- [ ] **Step 5: Replace package artifact README body**
Replace the full contents of `examples/package-artifacts/README.md` with:
````markdown
# Package artifact smoke checks
The package artifact smoke checks create temporary projects instead of storing
sample projects in this directory. Run the checks from `ktx/`:
```bash
pnpm run artifacts:check
```
The npm smoke project installs the generated public `@kaelio/ktx` tarball,
imports the package entry point, and runs installed `ktx` commands against a
generated local project.
The managed runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies
`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from
the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed
daemon, and stops it.
The Python smoke project still installs the Python artifacts directly because
it verifies the standalone Python distributions that feed the bundled runtime
wheel.
````
- [ ] **Step 6: Run the docs test to verify these docs pass**
Run:
```bash
node --test scripts/examples-docs.test.mjs
```
Expected: FAIL remains because `README.md` still lacks the public npm managed
runtime documentation. The Postgres and package artifact assertions now pass.
### Task 4: Update the root README public runtime path
**Files:**
- Modify: `README.md`
- Test: `scripts/examples-docs.test.mjs`
- [ ] **Step 1: Replace quick start**
In `README.md`, replace the `## Quick start` section through the end of the
full-demo paragraph with:
````markdown
## Quick start
Run the pre-seeded demo through the public npm package:
```bash
npx @kaelio/ktx setup demo --no-input
npx @kaelio/ktx setup demo inspect
```
The default demo uses packaged sample data and prebuilt context. It does not
require API keys, network access, or an LLM provider.
To replay the packaged ingest run, use:
```bash
npx @kaelio/ktx setup demo --mode replay --no-input
```
To run the full agentic demo with an LLM provider, set a provider key for the
current process:
```bash
ANTHROPIC_API_KEY=$YOUR_ANTHROPIC_API_KEY \
npx @kaelio/ktx setup demo --mode full --no-input
```
Interactive full-demo setup can prompt for a provider key without writing the
key to `ktx.yaml`.
You can also install the CLI in a project or globally:
```bash
npm install @kaelio/ktx
npx ktx --help
npm install -g @kaelio/ktx
ktx --help
```
````
- [ ] **Step 2: Replace local project setup command**
In the `## Build a local project` section of `README.md`, replace:
```bash
uv sync --all-packages
source .venv/bin/activate
PROJECT_DIR="$(mktemp -d)/ktx-demo"
pnpm run ktx -- init "$PROJECT_DIR" --name ktx-demo
```
with:
```bash
npm install @kaelio/ktx
PROJECT_DIR="$(mktemp -d)/ktx-demo"
npx ktx init "$PROJECT_DIR" --name ktx-demo
```
- [ ] **Step 3: Replace README command prefixes**
In `README.md`, replace the source-tree command prefix `pnpm run ktx --` with
`npx ktx` in all user workflow commands under `## Build a local project`,
`### Scan the demo warehouse`, and `## Serve MCP`. Keep `pnpm run ktx --` in
the `## Development` section.
For example, this command:
```bash
pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \
```
becomes:
```bash
npx ktx sl query --project-dir "$PROJECT_DIR" \
```
- [ ] **Step 4: Add managed runtime section**
Insert this section after the scan walkthrough in `README.md`:
````markdown
## Managed Python runtime
KTX installs its Python runtime only when a Python-backed command needs it.
The runtime lives outside the npm cache, is versioned by the installed CLI
version, and is managed by `ktx runtime` commands:
```bash
npx ktx runtime install --yes
npx ktx runtime status
npx ktx runtime doctor
npx ktx runtime start
npx ktx runtime stop
```
Commands such as `npx @kaelio/ktx sl query ... --yes` can install the core
runtime lazily from the bundled wheel. Local embeddings remain lazy; prepare
them only when you select local `sentence-transformers` embeddings:
```bash
npx ktx runtime install --feature local-embeddings --yes
npx ktx runtime start --feature local-embeddings
```
````
- [ ] **Step 5: Replace Serve MCP section**
In `README.md`, replace the full `## Serve MCP` section with:
````markdown
## Serve MCP
Start the stdio MCP server from the project directory:
```bash
npx ktx serve --mcp stdio --project-dir "$PROJECT_DIR" \
--user-id local \
--semantic-compute \
--execute-queries \
--yes
```
The `--semantic-compute` flag uses the managed Python runtime when no explicit
semantic compute URL is provided. KTX starts or reuses the managed runtime as
needed.
The MCP server exposes `connection_list`, `knowledge_search`,
`knowledge_read`, `knowledge_write`, `sl_list_sources`, `sl_read_source`,
`sl_write_source`, `sl_validate`, `sl_query`, `ingest_trigger`,
`ingest_status`, `ingest_report`, and `ingest_replay`.
````
- [ ] **Step 6: Update release status wording**
In `README.md`, replace this sentence in `## Release status`:
```markdown
This repository is prepared for source publication. Package publishing is still
disabled by `release-policy.json`; registry names, public versions, package
visibility, and provenance policy must be chosen before publishing artifacts to
npm or Python package indexes.
```
with:
```markdown
This repository builds a single public npm artifact named `@kaelio/ktx`.
Package publishing is still disabled by `release-policy.json`; registry
credentials, public versions, release tags, and provenance policy must be
chosen before publishing artifacts to npm or Python package indexes.
```
- [ ] **Step 7: Run the docs test to verify the README passes**
Run:
```bash
node --test scripts/examples-docs.test.mjs
```
Expected: PASS.
### Task 5: Final verification and commit
**Files:**
- Verify: `scripts/examples-docs.test.mjs`
- Verify: `examples/postgres-historic/scripts/smoke.sh`
- Verify: `examples/postgres-historic/README.md`
- Verify: `examples/package-artifacts/README.md`
- Verify: `README.md`
- [ ] **Step 1: Run the script test suite affected by docs**
Run:
```bash
node --test scripts/examples-docs.test.mjs scripts/check-boundaries.test.mjs
```
Expected: PASS.
- [ ] **Step 2: Run the boundary check**
Run:
```bash
node scripts/check-boundaries.mjs
```
Expected:
```text
ktx boundary check passed
```
- [ ] **Step 3: Search for removed external runtime references**
Run:
```bash
rg -n "python-service|uvicorn app\\.main:app|export KTX_SQL_ANALYSIS_URL|uv run ktx-daemon serve-http|@ktx/context.*@ktx/cli" README.md examples/postgres-historic/README.md examples/postgres-historic/scripts/smoke.sh examples/package-artifacts/README.md
```
Expected: no matches.
- [ ] **Step 4: Commit**
```bash
git add scripts/examples-docs.test.mjs \
examples/postgres-historic/scripts/smoke.sh \
examples/postgres-historic/README.md \
examples/package-artifacts/README.md \
README.md
git commit -m "docs: align managed runtime examples"
```
## Acceptance criteria
- The Postgres historic SQL smoke no longer references `python-service/`,
`uvicorn app.main:app`, or `export KTX_SQL_ANALYSIS_URL`.
- The stage-only Postgres historic smoke uses `createKtxCliLocalIngestAdapters`
with managed daemon options and `installPolicy: 'auto'`.
- The root README documents `npx @kaelio/ktx`, local `npx ktx`, global `ktx`,
`ktx runtime ...`, and MCP `--semantic-compute --yes` managed-runtime usage.
- Package artifact docs describe the single public `@kaelio/ktx` tarball and
the managed runtime smoke.
- `node --test scripts/examples-docs.test.mjs scripts/check-boundaries.test.mjs`
passes.
- `node scripts/check-boundaries.mjs` passes.
## Self-review
- Spec coverage: This plan covers the remaining user-facing drift from the
npm-managed runtime spec by removing manual Python service guidance,
documenting public `@kaelio/ktx` invocation modes, and making the Postgres
example smoke use the managed core daemon.
- Placeholder scan: The plan contains exact files, edits, commands, expected
outcomes, and commit instructions.
- Type consistency: The plan uses the existing `managedDaemon` option shape
from `packages/cli/src/local-adapters.ts` and the existing
`installPolicy: 'auto'` value from `packages/cli/src/managed-python-command.ts`.

377
docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md Normal file
View file

@ -0,0 +1,377 @@
# Managed Runtime Prune Smoke and Docs Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Prove and document `ktx runtime prune` as part of the npm-managed
Python runtime release contract.
**Architecture:** The prune command already exists in the CLI runtime layer, so
this plan adds black-box package smoke coverage and public documentation only.
The smoke creates an isolated stale versioned runtime directory, previews it,
verifies confirmation is required, and removes it through the installed
`@kaelio/ktx` package.
**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, Markdown, KTX CLI
managed Python runtime.
---
## Current state
This plan follows
`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`.
The following plan files are based on that spec and are implemented in the
current tree:
- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md`
- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md`
- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md`
- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md`
- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md`
- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md`
Implementation evidence found before writing this plan includes:
- `packages/cli/assets/python/manifest.json` and
`packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl`.
- `packages/cli/src/managed-python-runtime.ts`, including
`installManagedPythonRuntime()`, `doctorManagedPythonRuntime()`, and
`pruneManagedPythonRuntimes()`.
- `packages/cli/src/runtime.ts`, including the `install`, `status`,
`doctor`, `start`, `stop`, and `prune` runtime command runner branches.
- `packages/cli/src/commands/runtime-commands.ts`, including the
`runtime prune --dry-run` and `runtime prune --yes` Commander wiring.
- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`,
`scripts/published-package-smoke.mjs`, `scripts/local-embeddings-runtime-smoke.mjs`,
`scripts/publish-public-npm-package.mjs`, `release-policy.json`, and
`.github/workflows/release.yml`.
- `README.md` and `examples/package-artifacts/README.md` document the managed
runtime but do not mention `ktx runtime prune`.
The remaining gap is narrow: the spec lists `ktx runtime prune` as part of the
runtime management command family, but public docs and installed package smoke
coverage only prove `install`, `status`, `doctor`, `start`, and `stop`.
## File structure
- Modify `scripts/package-artifacts.test.mjs`: assert that the generated
installed npm smoke covers `ktx runtime prune --dry-run`, confirmation
failure, and confirmed deletion.
- Modify `scripts/package-artifacts.mjs`: extend `npmRuntimeSmokeSource()` to
create a stale runtime directory and exercise `ktx runtime prune`.
- Modify `scripts/examples-docs.test.mjs`: require public docs to mention
`ktx runtime prune --dry-run` and `ktx runtime prune --yes`.
- Modify `README.md`: add prune commands and one sentence describing preview
and confirmed deletion.
- Modify `examples/package-artifacts/README.md`: describe prune coverage in the
package artifact smoke.
### Task 1: Add installed package prune smoke coverage
**Files:**
- Modify: `scripts/package-artifacts.test.mjs`
- Modify: `scripts/package-artifacts.mjs`
- [ ] **Step 1: Add failing smoke-source assertions**
In `scripts/package-artifacts.test.mjs`, inside
`it('runs installed CLI commands through the public package runtime', () => {`
and immediately after the existing assertions for `ktx runtime stop`, add:
```javascript
assert.match(source, /ktx runtime prune dry run/);
assert.match(source, /0\.0\.0/);
assert.match(source, /ktx runtime prune needs confirmation/);
assert.match(source, /Refusing to prune without --yes/);
assert.match(source, /ktx runtime prune confirmed/);
assert.match(source, /Removed stale KTX Python runtimes/);
assert.match(source, /assert\.rejects\(\(\) => access\(staleRuntimeDir\)\)/);
```
- [ ] **Step 2: Run the package artifact test and verify failure**
Run:
```bash
node --test scripts/package-artifacts.test.mjs
```
Expected: FAIL in the installed CLI smoke source test because
`npmRuntimeSmokeSource()` does not yet contain the prune labels, confirmation
guard, or stale runtime removal assertion.
- [ ] **Step 3: Extend the generated installed CLI smoke**
In `scripts/package-artifacts.mjs`, inside `npmRuntimeSmokeSource()`, add this
block immediately after:
```javascript
process.stdout.write('ktx runtime daemon lifecycle verified\n');
```
Add:
```javascript
const staleRuntimeDir = join(process.env.KTX_RUNTIME_ROOT, '0.0.0');
await mkdir(staleRuntimeDir, { recursive: true });
const runtimePruneDryRun = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune', '--dry-run']);
requireSuccess('ktx runtime prune dry run', runtimePruneDryRun);
requireOutput('ktx runtime prune dry run', runtimePruneDryRun, /Stale KTX Python runtimes/);
requireOutput('ktx runtime prune dry run', runtimePruneDryRun, /0\.0\.0/);
await access(staleRuntimeDir);
const runtimePruneNeedsConfirmation = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune']);
assert.equal(runtimePruneNeedsConfirmation.code, 1, 'ktx runtime prune without --yes must fail');
assert.equal(runtimePruneNeedsConfirmation.stdout, '', 'ktx runtime prune confirmation failure wrote stdout');
assert.match(runtimePruneNeedsConfirmation.stderr, /Refusing to prune without --yes/);
const runtimePruneConfirmed = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune', '--yes']);
requireSuccess('ktx runtime prune confirmed', runtimePruneConfirmed);
requireOutput('ktx runtime prune confirmed', runtimePruneConfirmed, /Removed stale KTX Python runtimes/);
requireOutput('ktx runtime prune confirmed', runtimePruneConfirmed, /0\.0\.0/);
await assert.rejects(() => access(staleRuntimeDir));
process.stdout.write('ktx runtime prune verified\n');
```
No import changes are needed because the generated smoke already imports
`assert`, `access`, `mkdir`, and `join`.
- [ ] **Step 4: Run the package artifact test and verify pass**
Run:
```bash
node --test scripts/package-artifacts.test.mjs
```
Expected: PASS. The source assertions now find prune dry-run coverage,
confirmation failure coverage, confirmed prune coverage, and stale directory
deletion verification.
- [ ] **Step 5: Commit the smoke coverage**
Run:
```bash
git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs
git commit -m "test: cover managed runtime prune in package smoke"
```
### Task 2: Document runtime prune in public docs
**Files:**
- Modify: `scripts/examples-docs.test.mjs`
- Modify: `README.md`
- Modify: `examples/package-artifacts/README.md`
- [ ] **Step 1: Add failing docs assertions**
In `scripts/examples-docs.test.mjs`, inside
`it('documents public npm and managed runtime usage in the README', async () => {`
and immediately after:
```javascript
assert.match(rootReadme, /ktx runtime stop/);
```
Add:
```javascript
assert.match(rootReadme, /ktx runtime prune --dry-run/);
assert.match(rootReadme, /ktx runtime prune --yes/);
```
In the same file, inside
`it('documents the public package artifact smoke shape', async () => {` and
immediately after:
```javascript
assert.match(readme, /ktx runtime doctor/);
```
Add:
```javascript
assert.match(readme, /ktx runtime prune --dry-run/);
assert.match(readme, /ktx runtime prune --yes/);
```
- [ ] **Step 2: Run the docs test and verify failure**
Run:
```bash
node --test scripts/examples-docs.test.mjs
```
Expected: FAIL because `README.md` and
`examples/package-artifacts/README.md` do not yet mention `ktx runtime prune`.
- [ ] **Step 3: Update the root README runtime section**
In `README.md`, in the `## Managed Python runtime` command block, replace:
```bash
npx ktx runtime install --yes
npx ktx runtime status
npx ktx runtime doctor
npx ktx runtime start
npx ktx runtime stop
```
with:
```bash
npx ktx runtime install --yes
npx ktx runtime status
npx ktx runtime doctor
npx ktx runtime start
npx ktx runtime stop
npx ktx runtime prune --dry-run
npx ktx runtime prune --yes
```
Immediately after that command block, add:
```markdown
Use `runtime prune --dry-run` to preview stale runtime directories from older
CLI versions. Add `--yes` to remove those stale directories after daemon
processes are stopped.
```
- [ ] **Step 4: Update package artifact smoke docs**
In `examples/package-artifacts/README.md`, replace:
```markdown
The managed Python runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies
`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from
the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed
daemon, and stops it.
```
with:
```markdown
The managed Python runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies
`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from
the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed
daemon, stops it, previews a stale runtime with `ktx runtime prune --dry-run`,
verifies confirmation is required, and removes the stale runtime with
`ktx runtime prune --yes`.
```
- [ ] **Step 5: Run the docs test and verify pass**
Run:
```bash
node --test scripts/examples-docs.test.mjs
```
Expected: PASS. The public README and package artifact README now document
runtime prune alongside the other managed runtime commands.
- [ ] **Step 6: Commit the docs coverage**
Run:
```bash
git add scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md
git commit -m "docs: document managed runtime prune"
```
### Task 3: Verify the completed prune release surface
**Files:**
- Verify: `scripts/package-artifacts.mjs`
- Verify: `scripts/package-artifacts.test.mjs`
- Verify: `scripts/examples-docs.test.mjs`
- Verify: `README.md`
- Verify: `examples/package-artifacts/README.md`
- [ ] **Step 1: Run focused tests**
Run:
```bash
node --test scripts/package-artifacts.test.mjs scripts/examples-docs.test.mjs
```
Expected: PASS. The source-level tests cover generated package smoke behavior
and docs assertions.
- [ ] **Step 2: Run the installed package artifact smoke**
Run:
```bash
pnpm run artifacts:check
```
Expected: PASS. The generated installed CLI smoke prints:
```text
ktx runtime prune verified
```
and removes the temporary `0.0.0` directory from the isolated
`KTX_RUNTIME_ROOT`.
- [ ] **Step 3: Inspect git status**
Run:
```bash
git status --short
```
Expected: only the five planned files are modified before the final commit, or
no modified files remain after the task commits.
- [ ] **Step 4: Commit verification fixes if needed**
If verification required small corrections, commit only those intended files:
```bash
git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md
git commit -m "test: verify managed runtime prune release surface"
```
## Acceptance criteria
- The generated installed npm package smoke creates a stale versioned runtime
directory under the isolated `KTX_RUNTIME_ROOT`.
- `ktx runtime prune --dry-run` lists the stale runtime and leaves it on disk.
- `ktx runtime prune` without `--yes` exits nonzero and prints the existing
confirmation guidance.
- `ktx runtime prune --yes` removes the stale runtime directory.
- `README.md` lists `ktx runtime prune --dry-run` and
`ktx runtime prune --yes` with the other managed runtime commands.
- `examples/package-artifacts/README.md` describes prune coverage in the
package artifact smoke.
## Self-review
- Spec coverage: this plan covers the remaining visible gap for the runtime
management command family in the npm-managed Python runtime spec. The prune
implementation already exists, and this plan adds release smoke and public
docs coverage.
- Placeholder scan: no placeholder steps, deferred implementation notes, or
unspecified behavior gaps remain.
- Type consistency: the plan uses existing labels and functions:
`npmRuntimeSmokeSource()`, `requireSuccess()`, `requireOutput()`,
`KTX_RUNTIME_ROOT`, `ktx runtime prune --dry-run`, and
`ktx runtime prune --yes`.

647
docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md Normal file
View file

@ -0,0 +1,647 @@
# Managed Runtime uv Prerequisite Contract Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use
> superpowers:subagent-driven-development (recommended) or
> superpowers:executing-plans to implement this plan task-by-task. Steps use
> checkbox (`- [ ]`) syntax for tracking.
**Goal:** Close the remaining npm-managed Python runtime open decision by
making `uv` a documented, release-policy-checked prerequisite.
**Architecture:** Keep the runtime installer behavior simple: the CLI locates
`uv` on `PATH` and prints a focused error when it is missing. Encode that
decision in `release-policy.json`, validate it during release readiness, use one
shared runtime error message, and document the prerequisite in public docs.
**Tech Stack:** Node 22 ESM scripts, `node:test`, TypeScript, Vitest, JSON
release policy, Markdown.
---
## Existing status
This plan is based on
`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`.
The following plan files are based on that spec and are already implemented in
this worktree:
- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md`
- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md`
- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md`
- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md`
- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md`
- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md`
- `docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md`
Implementation evidence found before writing this plan includes:
- `packages/cli/assets/python/manifest.json` and the bundled
`kaelio_ktx-0.1.0-py3-none-any.whl`.
- `packages/cli/src/managed-python-runtime.ts`, including runtime roots,
bundled wheel verification, install, status, doctor, and prune behavior.
- `packages/cli/src/managed-python-command.ts`,
`packages/cli/src/managed-python-daemon.ts`,
`packages/cli/src/managed-local-embeddings.ts`, and
`packages/cli/src/managed-python-http.ts`.
- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`,
`scripts/published-package-smoke.mjs`,
`scripts/local-embeddings-runtime-smoke.mjs`, and
`scripts/publish-public-npm-package.mjs`.
- `release-policy.json` is already in `npm-public-release-ready` mode for
`@kaelio/ktx` `0.1.0` and keeps Python package publishing disabled.
- `README.md` and `examples/package-artifacts/README.md` document the managed
runtime command family, including `runtime prune`.
The remaining spec gap is the open decision in
`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`:
```text
KTX still needs a final decision on whether uv is a hard prerequisite or a
bootstrap dependency that KTX downloads automatically.
```
This plan chooses the hard-prerequisite path for the first public release. KTX
will not download `uv` automatically in this release.
## File structure
- Modify `release-policy.json`: add a `runtimeInstaller` policy section that
records the hard `uv` prerequisite decision.
- Modify `scripts/release-readiness.mjs`: validate the runtime installer
policy, include it in readiness reports, and print it in text output.
- Modify `scripts/release-readiness.test.mjs`: cover the accepted policy and
rejection paths for missing or bootstrap-style `uv` policies.
- Modify `packages/cli/src/managed-python-runtime.ts`: export one shared
missing-`uv` message and use it for install and doctor output.
- Modify `packages/cli/src/managed-python-runtime.test.ts`: cover install and
doctor behavior when `uv` is missing.
- Modify `scripts/examples-docs.test.mjs`: require public docs to state the
hard `uv` prerequisite.
- Modify `README.md`: document that `uv` must be on `PATH` and KTX does not
download it automatically.
- Modify `examples/package-artifacts/README.md`: document the artifact smoke
`uv` prerequisite.
### Task 1: Encode the runtime installer policy
**Files:**
- Modify: `release-policy.json`
- Modify: `scripts/release-readiness.test.mjs`
- Modify: `scripts/release-readiness.mjs`
- Test: `scripts/release-readiness.test.mjs`
- [ ] **Step 1: Add failing release policy tests**
In `scripts/release-readiness.test.mjs`, inside the `releasePolicy()` helper
return value, add the `runtimeInstaller` object immediately after
`publishedPackageSmoke`:
```javascript
runtimeInstaller: {
uvStrategy: 'path-prerequisite',
bootstrapUv: false,
missingUvBehavior: 'focused-error',
},
```
In the three `assert.deepEqual(report, { ... })` expectations, add this field
immediately after `publishedPackageSmokeGate`:
```javascript
runtimeInstaller: {
uvStrategy: 'path-prerequisite',
bootstrapUv: false,
missingUvBehavior: 'focused-error',
},
```
Add these tests immediately after the
`it('accepts the npm public release ready policy', async () => { ... })` block:
```javascript
it('rejects npm public release ready mode without a runtime installer policy', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-policy-missing-test-'));
try {
await writeReadyFixture(root, {
policy: releasePolicy({
releaseMode: 'npm-public-release-ready',
npm: {
publish: true,
registry: null,
access: 'public',
tag: 'latest',
},
publishedPackageSmoke: {
packageName: '@kaelio/ktx',
version: PUBLIC_NPM_PACKAGE_VERSION,
registry: null,
},
runtimeInstaller: undefined,
requiredBeforePublishing: [],
}),
});
await assert.rejects(
() => releaseReadinessReport(root),
/Release policy runtimeInstaller must be a JSON object/,
);
} finally {
await rm(root, { recursive: true, force: true });
}
});
it('rejects uv bootstrap download policy for the first public npm release', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-policy-bootstrap-test-'));
try {
await writeReadyFixture(root, {
policy: releasePolicy({
releaseMode: 'npm-public-release-ready',
npm: {
publish: true,
registry: null,
access: 'public',
tag: 'latest',
},
publishedPackageSmoke: {
packageName: '@kaelio/ktx',
version: PUBLIC_NPM_PACKAGE_VERSION,
registry: null,
},
runtimeInstaller: {
uvStrategy: 'bootstrap-download',
bootstrapUv: true,
missingUvBehavior: 'download',
},
requiredBeforePublishing: [],
}),
});
await assert.rejects(
() => releaseReadinessReport(root),
/Release policy runtimeInstaller\.uvStrategy must be path-prerequisite/,
);
} finally {
await rm(root, { recursive: true, force: true });
}
});
```
- [ ] **Step 2: Run the release readiness tests and verify failure**
Run:
```bash
node --test scripts/release-readiness.test.mjs
```
Expected: FAIL because `releaseReadinessReport()` does not include
`runtimeInstaller`, and `validateReleasePolicy()` does not validate the new
policy section.
- [ ] **Step 3: Validate the runtime installer policy**
In `scripts/release-readiness.mjs`, add this function immediately after the
`assertRequiredBeforePublishing(policy)` function definition:
```javascript
function assertRuntimeInstallerPolicy(policy) {
assertPlainObject(policy.runtimeInstaller, 'Release policy runtimeInstaller');
assertString(policy.runtimeInstaller.uvStrategy, 'Release policy runtimeInstaller.uvStrategy');
assertBoolean(policy.runtimeInstaller.bootstrapUv, 'Release policy runtimeInstaller.bootstrapUv');
assertString(
policy.runtimeInstaller.missingUvBehavior,
'Release policy runtimeInstaller.missingUvBehavior',
);
if (policy.runtimeInstaller.uvStrategy !== 'path-prerequisite') {
throw new Error('Release policy runtimeInstaller.uvStrategy must be path-prerequisite');
}
if (policy.runtimeInstaller.bootstrapUv !== false) {
throw new Error('Release policy runtimeInstaller.bootstrapUv must be false');
}
if (policy.runtimeInstaller.missingUvBehavior !== 'focused-error') {
throw new Error('Release policy runtimeInstaller.missingUvBehavior must be focused-error');
}
}
```
In `validateReleasePolicy(policy)`, add this call immediately after
`assertRequiredBeforePublishing(policy);`:
```javascript
assertRuntimeInstallerPolicy(policy);
```
In `releaseReadinessReport(rootDir = scriptRootDir())`, add
`runtimeInstaller` to the returned object immediately after
`publishedPackageSmokeGate`:
```javascript
runtimeInstaller: policy.runtimeInstaller,
```
In `main()`, add these lines immediately after the published package smoke
registry line:
```javascript
process.stdout.write(`Runtime uv strategy: ${report.runtimeInstaller.uvStrategy}\n`);
process.stdout.write(
`Runtime uv bootstrap: ${report.runtimeInstaller.bootstrapUv ? 'enabled' : 'disabled'}\n`,
);
```
- [ ] **Step 4: Encode the policy in `release-policy.json`**
Replace `release-policy.json` with this exact content:
```json
{
"schemaVersion": 1,
"releaseMode": "npm-public-release-ready",
"npm": {
"publish": true,
"registry": null,
"access": "public",
"tag": "latest",
"packages": ["@kaelio/ktx"]
},
"python": {
"publish": false,
"repository": null,
"packages": ["ktx-sl", "ktx-daemon", "kaelio-ktx"]
},
"publishedPackageSmoke": {
"packageName": "@kaelio/ktx",
"version": "0.1.0",
"registry": null
},
"runtimeInstaller": {
"uvStrategy": "path-prerequisite",
"bootstrapUv": false,
"missingUvBehavior": "focused-error"
},
"requiredBeforePublishing": []
}
```
- [ ] **Step 5: Run the release readiness tests and verify success**
Run:
```bash
node --test scripts/release-readiness.test.mjs
```
Expected: PASS.
- [ ] **Step 6: Commit the release policy contract**
```bash
git add release-policy.json scripts/release-readiness.mjs scripts/release-readiness.test.mjs
git commit -m "chore: encode uv runtime prerequisite policy"
```
### Task 2: Centralize missing-uv runtime output
**Files:**
- Modify: `packages/cli/src/managed-python-runtime.test.ts`
- Modify: `packages/cli/src/managed-python-runtime.ts`
- Test: `packages/cli/src/managed-python-runtime.test.ts`
- [ ] **Step 1: Add failing missing-uv runtime tests**
In `packages/cli/src/managed-python-runtime.test.ts`, add
`MISSING_UV_RUNTIME_INSTALL_MESSAGE` to the import from
`./managed-python-runtime.js`:
```typescript
import {
MISSING_UV_RUNTIME_INSTALL_MESSAGE,
doctorManagedPythonRuntime,
installManagedPythonRuntime,
managedPythonRuntimeLayout,
pruneManagedPythonRuntimes,
readManagedPythonRuntimeStatus,
verifyRuntimeAsset,
type ManagedPythonRuntimeExec,
} from './managed-python-runtime.js';
```
Inside `describe('installManagedPythonRuntime', () => { ... })`, add this test
after the local embeddings test:
```typescript
it('fails with the hard-prerequisite message when uv is missing', async () => {
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
const commands: Array<{ command: string; args: string[] }> = [];
const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => {
commands.push({ command, args });
throw new Error('spawn uv ENOENT');
});
await expect(
installManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
features: ['core'],
exec,
}),
).rejects.toThrow(MISSING_UV_RUNTIME_INSTALL_MESSAGE);
expect(commands).toEqual([{ command: 'uv', args: ['--version'] }]);
});
```
Inside `describe('doctorManagedPythonRuntime', () => { ... })`, add this test
after the existing doctor test:
```typescript
it('reports uv as a hard prerequisite when uv is missing', async () => {
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
const exec: ManagedPythonRuntimeExec = vi.fn(async () => {
throw new Error('spawn uv ENOENT');
});
const checks = await doctorManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
exec,
});
expect(checks[0]).toEqual({
id: 'uv',
label: 'uv',
status: 'fail',
detail: MISSING_UV_RUNTIME_INSTALL_MESSAGE,
fix: 'Install uv, make sure it is on PATH, and run: ktx runtime install --yes',
});
});
```
- [ ] **Step 2: Run the runtime tests and verify failure**
Run:
```bash
pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts
```
Expected: FAIL because the shared message constant does not exist and the
doctor fix text still uses the older message.
- [ ] **Step 3: Add the shared missing-uv message**
In `packages/cli/src/managed-python-runtime.ts`, add this export immediately
after the `ManagedPythonRuntimePruneResult` interface:
```typescript
export const MISSING_UV_RUNTIME_INSTALL_MESSAGE =
'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx runtime install --yes';
```
Replace the body of the `catch` block in `ensureUv()` with:
```typescript
throw new Error(MISSING_UV_RUNTIME_INSTALL_MESSAGE);
```
In `doctorManagedPythonRuntime()`, replace the `fix` value for the `uv` check
with:
```typescript
fix: 'Install uv, make sure it is on PATH, and run: ktx runtime install --yes',
```
- [ ] **Step 4: Run the runtime tests and verify success**
Run:
```bash
pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts
```
Expected: PASS.
- [ ] **Step 5: Commit the runtime output contract**
```bash
git add packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts
git commit -m "fix: clarify missing uv runtime error"
```
### Task 3: Document the hard uv prerequisite
**Files:**
- Modify: `scripts/examples-docs.test.mjs`
- Modify: `README.md`
- Modify: `examples/package-artifacts/README.md`
- Test: `scripts/examples-docs.test.mjs`
- [ ] **Step 1: Add failing docs assertions**
In `scripts/examples-docs.test.mjs`, inside
`it('documents public npm and managed runtime usage in the README', ... )`, add
these assertions immediately after the existing `ktx runtime prune --yes`
assertion:
```javascript
assert.match(rootReadme, /KTX requires `uv` on `PATH`/);
assert.match(rootReadme, /KTX doesn't download `uv` automatically/);
```
Inside `it('documents the public package artifact smoke shape', ... )`, add
this assertion immediately after the `managed Python runtime` assertion:
```javascript
assert.match(readme, /requires `uv` on `PATH`/);
```
- [ ] **Step 2: Run the docs test and verify failure**
Run:
```bash
node --test scripts/examples-docs.test.mjs
```
Expected: FAIL because the README files do not state the hard `uv`
prerequisite.
- [ ] **Step 3: Update the root README runtime section**
In `README.md`, in the `## Managed Python runtime` section, replace this
paragraph:
```markdown
KTX installs its Python runtime only when a Python-backed command needs it.
The runtime lives outside the npm cache, is versioned by the installed CLI
version, and is managed by `ktx runtime` commands:
```
With:
```markdown
KTX installs its Python runtime only when a Python-backed command needs it.
The runtime lives outside the npm cache, is versioned by the installed CLI
version, and is managed by `ktx runtime` commands.
KTX requires `uv` on `PATH` to create the managed runtime. Install `uv` with
your system package manager or the official installer before running Python-
backed KTX commands. KTX doesn't download `uv` automatically; run
`ktx runtime doctor` if runtime installation fails:
```
- [ ] **Step 4: Update the package artifact smoke README**
In `examples/package-artifacts/README.md`, replace this paragraph:
```markdown
The managed Python runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies
`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from
the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed
daemon, stops it, previews a stale runtime with `ktx runtime prune --dry-run`,
verifies confirmation is required, and removes the stale runtime with
`ktx runtime prune --yes`.
```
With:
```markdown
The managed Python runtime smoke requires `uv` on `PATH`, isolates
`KTX_RUNTIME_ROOT`, verifies `ktx runtime status`, runs `ktx sl query --yes` to
install the core runtime from the bundled wheel, checks `ktx runtime doctor`,
starts and reuses the managed daemon, stops it, previews a stale runtime with
`ktx runtime prune --dry-run`, verifies confirmation is required, and removes
the stale runtime with `ktx runtime prune --yes`.
```
- [ ] **Step 5: Run the docs test and verify success**
Run:
```bash
node --test scripts/examples-docs.test.mjs
```
Expected: PASS.
- [ ] **Step 6: Commit the public docs update**
```bash
git add README.md examples/package-artifacts/README.md scripts/examples-docs.test.mjs
git commit -m "docs: document uv runtime prerequisite"
```
### Task 4: Verify the completed contract
**Files:**
- Verify: `release-policy.json`
- Verify: `scripts/release-readiness.mjs`
- Verify: `scripts/release-readiness.test.mjs`
- Verify: `packages/cli/src/managed-python-runtime.ts`
- Verify: `packages/cli/src/managed-python-runtime.test.ts`
- Verify: `scripts/examples-docs.test.mjs`
- Verify: `README.md`
- Verify: `examples/package-artifacts/README.md`
- [ ] **Step 1: Run focused verification**
Run:
```bash
node --test scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs
pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts
```
Expected: PASS.
- [ ] **Step 2: Verify release readiness text output**
Run:
```bash
pnpm run release:readiness
```
Expected output includes:
```text
KTX release mode: npm-public-release-ready
Runtime uv strategy: path-prerequisite
Runtime uv bootstrap: disabled
NPM publish target: @kaelio/ktx@0.1.0 (latest)
```
- [ ] **Step 3: Verify no pre-commit config is required**
Run:
```bash
rg --files -g '.pre-commit-config.yaml' -g 'pre-commit-config.yaml'
```
Expected: no output and exit code 1. No Python files changed, so the repository
Python pre-commit requirement does not apply.
- [ ] **Step 4: Review the final diff**
Run:
```bash
git diff --stat
git diff -- release-policy.json scripts/release-readiness.mjs scripts/release-readiness.test.mjs packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md
```
Expected: only the runtime installer policy, missing-`uv` message/tests, and
public docs changed.
- [ ] **Step 5: Commit final verification notes if needed**
If Task 4 produces only verification output and no file changes, skip this
step. If a correction was made during verification, commit it:
```bash
git add release-policy.json scripts/release-readiness.mjs scripts/release-readiness.test.mjs packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md
git commit -m "chore: finish uv prerequisite release contract"
```
## Self-review
Spec coverage:
- The earlier implemented plans cover the single public npm package, bundled
Python wheel, managed runtime installer, runtime commands, daemon lifecycle,
local embeddings, Python-backed command integration, release smoke, published
smoke, docs cleanup, release handoff, and prune coverage.
- This plan closes the spec's remaining `uv` open decision by choosing
`path-prerequisite`, recording that decision in release policy, validating it
in release readiness, using one CLI error message, and documenting it.
- The plan keeps Python package publication disabled and keeps KTX-owned Python
code bundled in the npm package.
Placeholder scan:
- No task contains deferred implementation markers.
- Each code-changing step names exact files and includes the concrete code to
add or replace.
Type consistency:
- The release policy field is consistently named `runtimeInstaller`.
- The chosen strategy is consistently `path-prerequisite`.
- The shared CLI message constant is consistently
`MISSING_UV_RUNTIME_INSTALL_MESSAGE`.

1904
docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md Normal file
View file

File diff suppressed because it is too large Load diff

1332
docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md Normal file
View file

File diff suppressed because it is too large Load diff

602
docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md Normal file
View file

@ -0,0 +1,602 @@
# Published Package Managed Runtime Smoke Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use
> superpowers:subagent-driven-development (recommended) or
> superpowers:executing-plans to implement this plan task-by-task. Steps use
> checkbox (`- [ ]`) syntax for tracking.
**Goal:** Make the post-publication smoke prove that the published
`@kaelio/ktx` package uses the same isolated managed Python runtime across
`npx @kaelio/ktx`, local `npx ktx`, and global `ktx` invocation modes.
**Architecture:** Keep the smoke black-box and network-gated. Strengthen the
command builder so every Python-backed published-package command receives the
same temporary `KTX_RUNTIME_ROOT`, then run a real semantic-layer query through
the direct `npx`, local install, and global install paths instead of checking
only `--version` for local and global binaries.
**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, npx, KTX managed Python
runtime, published `@kaelio/ktx` package smoke.
---
## Existing status
This plan is based on
`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`.
The following plans are based on that spec and are implemented in this
worktree:
- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md`
- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md`
- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md`
- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md`
Implementation evidence found before writing this plan includes:
- `scripts/build-python-runtime-wheel.mjs` and
`packages/cli/assets/python/manifest.json`.
- `packages/cli/src/managed-python-runtime.ts`,
`packages/cli/src/runtime.ts`,
`packages/cli/src/commands/runtime-commands.ts`,
`packages/cli/src/managed-python-command.ts`,
`packages/cli/src/managed-python-daemon.ts`,
`packages/cli/src/managed-local-embeddings.ts`, and
`packages/cli/src/managed-python-http.ts`.
- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`,
`scripts/local-embeddings-runtime-smoke.mjs`, and
`scripts/published-package-smoke.mjs`.
- `packages/cli/src/agent-runtime.ts`, `packages/cli/src/serve.ts`,
`packages/cli/src/ingest.ts`, and `packages/cli/src/scan.ts` thread managed
runtime policy through the Python-backed CLI paths.
- `examples/postgres-historic/scripts/smoke.sh`,
`examples/postgres-historic/README.md`,
`examples/package-artifacts/README.md`, and `README.md` now document the
managed runtime instead of a manual `python-service/` process.
The remaining release-confidence gap is in the post-publication smoke:
- `scripts/published-package-smoke-config.mjs` runs `npx @kaelio/ktx setup
demo` and `npx @kaelio/ktx sl query ... --yes`, but it does not isolate
`KTX_RUNTIME_ROOT` for those commands.
- The same smoke installs `@kaelio/ktx` locally and globally, but local and
global verification only run `--version`.
- The design spec requires the direct `npx @kaelio/ktx`, local `npx ktx`, and
global `ktx` modes to work for real KTX commands. A semantic-layer query is
the smallest Python-backed command that proves the bundled managed runtime is
usable in each mode.
## File structure
- Modify `scripts/published-package-smoke.test.mjs`: expect a shared
`KTX_RUNTIME_ROOT` in the published smoke commands, expect local and global
semantic query commands, and cover label classification used by the runner.
- Modify `scripts/published-package-smoke-config.mjs`: derive a temporary
runtime root from the smoke project directory, merge it with registry
environment settings, and add local and global `sl query` commands.
- Modify `scripts/published-package-smoke.mjs`: validate the renamed version
labels and semantic query labels when the smoke runs.
### Task 1: Isolate runtime roots and add real local/global command coverage
**Files:**
- Modify: `scripts/published-package-smoke.test.mjs`
- Modify: `scripts/published-package-smoke-config.mjs`
- Test: `scripts/published-package-smoke.test.mjs`
- [ ] **Step 1: Write the failing command-list test**
In `scripts/published-package-smoke.test.mjs`, replace the existing
`it('builds the full public package smoke command list', ...)` block with this
test:
```javascript
it('builds the full public package smoke command list', () => {
assert.deepEqual(
buildPublishedPackageSmokeCommands(
config,
'/tmp/ktx-smoke/demo',
'/tmp/ktx-smoke/managed-runtime',
),
[
{
label: 'published package npx version',
command: 'npx',
args: ['--yes', '@kaelio/ktx@latest', '--version'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package setup demo',
command: 'npx',
args: [
'--yes',
'@kaelio/ktx@latest',
'setup',
'demo',
'--project-dir',
'/tmp/ktx-smoke/demo',
'--no-input',
'--plain',
],
env: {
npm_config_registry: 'https://registry.npmjs.org/',
KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime',
},
},
{
label: 'published package npx sl query',
command: 'npx',
args: [
'--yes',
'@kaelio/ktx@latest',
'sl',
'query',
'--project-dir',
'/tmp/ktx-smoke/demo',
'--connection-id',
'orbit_demo',
'--measure',
'contracts.contract_count',
'--format',
'sql',
'--yes',
],
env: {
npm_config_registry: 'https://registry.npmjs.org/',
KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime',
},
},
{
label: 'published package local install',
command: 'pnpm',
args: ['add', '@kaelio/ktx@latest'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package local version',
command: 'npx',
args: ['ktx', '--version'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package local sl query',
command: 'npx',
args: [
'ktx',
'sl',
'query',
'--project-dir',
'/tmp/ktx-smoke/demo',
'--connection-id',
'orbit_demo',
'--measure',
'contracts.contract_count',
'--format',
'sql',
'--yes',
],
env: {
npm_config_registry: 'https://registry.npmjs.org/',
KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime',
},
},
{
label: 'published package global install',
command: 'pnpm',
args: ['add', '--global', '@kaelio/ktx@latest'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package global version',
command: 'ktx',
args: ['--version'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package global sl query',
command: 'ktx',
args: [
'sl',
'query',
'--project-dir',
'/tmp/ktx-smoke/demo',
'--connection-id',
'orbit_demo',
'--measure',
'contracts.contract_count',
'--format',
'sql',
'--yes',
],
env: {
npm_config_registry: 'https://registry.npmjs.org/',
KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime',
},
},
],
);
});
```
- [ ] **Step 2: Run the test and verify it fails**
Run:
```bash
node --test scripts/published-package-smoke.test.mjs
```
Expected: FAIL with an `AssertionError` showing that the actual command list
still uses `published package version`, lacks `KTX_RUNTIME_ROOT`, and lacks the
local/global `sl query` commands.
- [ ] **Step 3: Implement the command builder changes**
In `scripts/published-package-smoke-config.mjs`, add this import before the
existing `node:assert/strict` import:
```javascript
import { dirname, join } from 'node:path';
```
In the same file, add these helper functions after
`assertHttpRegistry(registry, label)`:
```javascript
function registryEnv(config) {
return config.registry ? { npm_config_registry: config.registry } : {};
}
function runtimeCommandEnv(config, runtimeRoot) {
return { ...registryEnv(config), KTX_RUNTIME_ROOT: runtimeRoot };
}
function semanticQueryArgs(projectDir) {
return [
'sl',
'query',
'--project-dir',
projectDir,
'--connection-id',
'orbit_demo',
'--measure',
'contracts.contract_count',
'--format',
'sql',
'--yes',
];
}
```
Replace `buildPublishedPackageNpxCommand()` and
`buildPublishedPackageSmokeCommands()` with this implementation:
```javascript
export function buildPublishedPackageNpxCommand(config, args, label = 'published package command', extraEnv = {}) {
return {
label,
command: 'npx',
args: ['--yes', publishedPackageSpec(config), ...args],
env: { ...registryEnv(config), ...extraEnv },
};
}
export function buildPublishedPackageSmokeCommands(
config,
projectDir,
runtimeRoot = join(dirname(projectDir), 'managed-runtime'),
) {
const runtimeEnv = runtimeCommandEnv(config, runtimeRoot);
const packageEnv = registryEnv(config);
const queryArgs = semanticQueryArgs(projectDir);
return [
buildPublishedPackageNpxCommand(config, ['--version'], 'published package npx version'),
buildPublishedPackageNpxCommand(
config,
['setup', 'demo', '--project-dir', projectDir, '--no-input', '--plain'],
'published package setup demo',
{ KTX_RUNTIME_ROOT: runtimeRoot },
),
buildPublishedPackageNpxCommand(config, queryArgs, 'published package npx sl query', {
KTX_RUNTIME_ROOT: runtimeRoot,
}),
{
label: 'published package local install',
command: 'pnpm',
args: ['add', publishedPackageSpec(config)],
env: packageEnv,
},
{
label: 'published package local version',
command: 'npx',
args: ['ktx', '--version'],
env: packageEnv,
},
{
label: 'published package local sl query',
command: 'npx',
args: ['ktx', ...queryArgs],
env: runtimeEnv,
},
{
label: 'published package global install',
command: 'pnpm',
args: ['add', '--global', publishedPackageSpec(config)],
env: packageEnv,
},
{
label: 'published package global version',
command: 'ktx',
args: ['--version'],
env: packageEnv,
},
{
label: 'published package global sl query',
command: 'ktx',
args: queryArgs,
env: runtimeEnv,
},
];
}
```
- [ ] **Step 4: Run the command-list test and verify it passes**
Run:
```bash
node --test scripts/published-package-smoke.test.mjs
```
Expected: PASS for the command construction tests, with remaining failures only
if the runner label validation test from Task 2 has already been added.
- [ ] **Step 5: Commit the command-builder change**
Run:
```bash
git add scripts/published-package-smoke-config.mjs scripts/published-package-smoke.test.mjs
git commit -m "test: cover published package runtime smoke commands"
```
### Task 2: Validate smoke runner labels for the new command list
**Files:**
- Modify: `scripts/published-package-smoke.test.mjs`
- Modify: `scripts/published-package-smoke.mjs`
- Test: `scripts/published-package-smoke.test.mjs`
- [ ] **Step 1: Write the failing label classification test**
In `scripts/published-package-smoke.test.mjs`, replace the import from
`./published-package-smoke.mjs` with this import:
```javascript
import {
buildPublishedPackageNpxCommand,
buildPublishedPackageSmokeCommands,
isPublishedPackageSemanticQueryLabel,
isPublishedPackageVersionLabel,
publishedPackageSpec,
readPublishedPackageSmokeConfig,
} from './published-package-smoke.mjs';
```
Add this test after the `describe('published package smoke command
construction', ...)` block:
```javascript
describe('published package smoke output validation labels', () => {
it('classifies version and semantic query commands', () => {
assert.equal(isPublishedPackageVersionLabel('published package npx version'), true);
assert.equal(isPublishedPackageVersionLabel('published package local version'), true);
assert.equal(isPublishedPackageVersionLabel('published package global version'), true);
assert.equal(isPublishedPackageVersionLabel('published package setup demo'), false);
assert.equal(isPublishedPackageSemanticQueryLabel('published package npx sl query'), true);
assert.equal(isPublishedPackageSemanticQueryLabel('published package local sl query'), true);
assert.equal(isPublishedPackageSemanticQueryLabel('published package global sl query'), true);
assert.equal(isPublishedPackageSemanticQueryLabel('published package local install'), false);
});
});
```
- [ ] **Step 2: Run the test and verify it fails**
Run:
```bash
node --test scripts/published-package-smoke.test.mjs
```
Expected: FAIL with an import error because
`isPublishedPackageSemanticQueryLabel` and `isPublishedPackageVersionLabel` are
not exported yet.
- [ ] **Step 3: Implement label classification and runner validation**
In `scripts/published-package-smoke.mjs`, add these constants and exports after
`const SMOKE_TIMEOUT_MS = 180_000;`:
```javascript
const VERSION_LABELS = new Set([
'published package npx version',
'published package local version',
'published package global version',
]);
const SEMANTIC_QUERY_LABELS = new Set([
'published package npx sl query',
'published package local sl query',
'published package global sl query',
]);
export function isPublishedPackageVersionLabel(label) {
return VERSION_LABELS.has(label);
}
export function isPublishedPackageSemanticQueryLabel(label) {
return SEMANTIC_QUERY_LABELS.has(label);
}
```
In `runPublishedPackageSmoke(config)`, replace this block:
```javascript
if (
command.label === 'published package version' ||
command.label === 'published package local binary' ||
command.label === 'published package global binary'
) {
assert.match(result.stdout, /@kaelio\/ktx /);
}
if (command.label === 'published package sl query') {
assert.match(result.stdout, /SELECT/i);
assert.match(result.stdout, /contracts/i);
}
```
with this block:
```javascript
if (isPublishedPackageVersionLabel(command.label)) {
assert.match(result.stdout, /@kaelio\/ktx /);
}
if (isPublishedPackageSemanticQueryLabel(command.label)) {
assert.match(result.stdout, /SELECT/i);
assert.match(result.stdout, /contracts/i);
}
```
- [ ] **Step 4: Run the label tests and verify they pass**
Run:
```bash
node --test scripts/published-package-smoke.test.mjs
```
Expected: PASS.
- [ ] **Step 5: Commit the runner-label change**
Run:
```bash
git add scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs
git commit -m "test: validate published package smoke outputs"
```
### Task 3: Verify release-script compatibility
**Files:**
- Verify: `scripts/published-package-smoke-config.mjs`
- Verify: `scripts/published-package-smoke.mjs`
- Verify: `scripts/published-package-smoke.test.mjs`
- Verify: `scripts/release-readiness.test.mjs`
- Verify: `package.json`
- [ ] **Step 1: Run the focused Node tests**
Run:
```bash
node --test scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs
```
Expected: PASS. The release-readiness tests must continue to report the
published package smoke gate without executing the network smoke.
- [ ] **Step 2: Run release readiness**
Run:
```bash
pnpm run release:readiness
```
Expected: PASS and output containing these lines:
```text
Release mode: ci-artifact-only
NPM publish enabled: false
Published package smoke: pending
Published package smoke script: pnpm run release:published-smoke
```
- [ ] **Step 3: Confirm the network smoke stays explicit**
Run:
```bash
rg -n '"release:published-smoke": "node scripts/published-package-smoke.mjs --require-config"' package.json
```
Expected: PASS with one match in `package.json`. Do not run
`pnpm run release:published-smoke` in normal CI before the package is published
to the configured registry.
- [ ] **Step 4: Check pre-commit availability**
Run:
```bash
test ! -f .pre-commit-config.yaml
```
Expected: PASS in the current worktree. If a pre-commit config exists when this
plan is executed, run this instead after activating `.venv`:
```bash
source .venv/bin/activate
uv run pre-commit run --files scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs
```
- [ ] **Step 5: Commit verification-only fixes if needed**
If Step 1 or Step 2 required additional source changes, commit them with:
```bash
git add scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs package.json
git commit -m "chore: verify published package runtime smoke"
```
If no files changed after Task 2, do not create an empty commit.
## Acceptance criteria
- `buildPublishedPackageSmokeCommands()` derives
`<smoke root>/managed-runtime` from the demo project directory by default.
- Direct `npx @kaelio/ktx`, local `npx ktx`, and global `ktx` semantic query
commands all receive the same `KTX_RUNTIME_ROOT`.
- Local and global post-publication smoke coverage runs `sl query ... --yes`,
not only `--version`.
- `runPublishedPackageSmoke()` validates version output for all version labels
and validates generated SQL output for all semantic query labels.
- `node --test scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs`
passes.
- `pnpm run release:readiness` still reports the published-package smoke as a
pending explicit release gate while registry publishing is disabled.
## Self-review notes
- Spec coverage: this plan covers the remaining invocation-mode confidence gap
from the spec by proving the published package uses an isolated managed
runtime across direct `npx`, local binary, and global binary paths.
- Placeholder scan: the plan contains concrete file paths, exact code blocks,
exact commands, and exact expected outcomes.
- Type consistency: the command label strings are consistent across tests,
command construction, and smoke-runner output validation.

978
docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md Normal file
View file

@ -0,0 +1,978 @@
# Single Public Runtime Artifact Cleanup Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use
> superpowers:subagent-driven-development (recommended) or
> superpowers:executing-plans to implement this plan task-by-task. Steps use
> checkbox (`- [ ]`) syntax for tracking.
**Goal:** Make release artifacts match the npm-managed Python runtime design:
one public `@kaelio/ktx` npm tarball plus one bundled `kaelio-ktx` runtime
wheel, with no standalone `ktx-sl` or `ktx-daemon` release artifacts.
**Architecture:** Keep `python/ktx-sl` and `python/ktx-daemon` as source
packages used to assemble the bundled runtime wheel. Remove direct standalone
Python wheel and source-distribution builds from the release artifact path,
manifest, readiness policy, and artifact smoke docs. The packed npm package
remains the only user-visible package; Python-backed verification continues
through the managed runtime installed from the bundled wheel.
**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, uv-built bundled
runtime wheel, JSON release policy, Markdown.
---
## Current state
This plan follows
`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`.
The following plan files are based on that spec and are implemented in the
current tree:
- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md`
- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md`
- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md`
- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md`
- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md`
- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md`
- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md`
- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md`
- `docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md`
- `docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md`
Implementation evidence found before writing this plan includes:
- `packages/cli/assets/python/manifest.json` and
`packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl`.
- `packages/cli/src/managed-python-runtime.ts`,
`packages/cli/src/managed-python-command.ts`,
`packages/cli/src/managed-python-daemon.ts`,
`packages/cli/src/managed-local-embeddings.ts`,
`packages/cli/src/managed-python-http.ts`, and `packages/cli/src/runtime.ts`.
- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`,
`scripts/published-package-smoke.mjs`,
`scripts/local-embeddings-runtime-smoke.mjs`,
`scripts/publish-public-npm-package.mjs`, and
`.github/workflows/release.yml`.
- `release-policy.json` is in `npm-public-release-ready` mode, publishes
`@kaelio/ktx`, disables Python package publishing, and encodes the hard
`uv` prerequisite.
- `README.md` and `examples/package-artifacts/README.md` document public npm
usage, managed runtime commands, `runtime prune`, and the `uv` prerequisite.
The remaining mismatch is in the artifact release surface:
- `scripts/package-artifacts.mjs` still runs `uv build --package ktx-sl` and
`uv build --package ktx-daemon`.
- `scripts/package-artifacts.mjs` still adds `ktx-sl` and `ktx-daemon` wheel
and source-distribution files to the artifact manifest.
- `scripts/package-artifacts.mjs` still runs a direct Python clean-install
smoke, even though the npm artifact smoke already proves Python-backed
commands through the managed runtime.
- `release-policy.json` still lists `ktx-sl` and `ktx-daemon` under
`python.packages`.
- `examples/package-artifacts/README.md` says the Python smoke installs
standalone Python artifacts directly.
This plan removes those release artifacts. It does not delete the Python source
packages because the bundled runtime wheel builder still copies from
`python/ktx-sl/semantic_layer` and `python/ktx-daemon/src/ktx_daemon`.
## File structure
- Modify `scripts/package-artifacts.test.mjs`: make artifact tests expect only
`@kaelio/ktx` plus the `kaelio-ktx` bundled runtime wheel, and add a guard
that direct standalone Python artifact smoke code is gone.
- Modify `scripts/package-artifacts.mjs`: stop building standalone Python
artifacts, stop looking for their wheel and source-distribution files, remove
their release metadata, and remove the direct Python artifact verification
path.
- Modify `scripts/release-readiness.test.mjs`: update release policy fixtures
and readiness reports so the only Python release metadata is `kaelio-ktx`.
- Modify `release-policy.json`: set `python.packages` to `["kaelio-ktx"]`.
- Modify `scripts/examples-docs.test.mjs`: require docs to describe the single
npm tarball plus runtime wheel artifact shape and reject the old direct
Python-artifact smoke wording.
- Modify `README.md`: clarify that `python/ktx-sl` and `python/ktx-daemon` are
source packages, not release artifacts for the first npm release.
- Modify `examples/package-artifacts/README.md`: replace the stale standalone
Python smoke paragraph with the managed-runtime artifact contract.
### Task 1: Make package artifact tests expect one runtime wheel
**Files:**
- Modify: `scripts/package-artifacts.test.mjs`
- Test: `scripts/package-artifacts.test.mjs`
- [ ] **Step 1: Update package artifact imports**
In `scripts/package-artifacts.test.mjs`, replace the import from
`./package-artifacts.mjs` with this import:
```javascript
import {
CLI_PYTHON_ASSET_MANIFEST,
INTERNAL_NPM_WORKSPACE_PACKAGES,
RUNTIME_WHEEL_DISTRIBUTION_NAME,
RUNTIME_WHEEL_NORMALIZED_NAME,
RUNTIME_WHEEL_PACKAGE_VERSION,
artifactManifestPath,
buildArtifactCommands,
copyRuntimeWheelAssets,
findPythonArtifacts,
NPM_ARTIFACT_PACKAGES,
npmDemoSmokeSource,
npmRuntimeSmokeSource,
npmSmokePackageJson,
npmVerifySource,
packageArtifactLayout,
packageReleaseMetadata,
verifyArtifactManifest,
writeArtifactManifest,
} from './package-artifacts.mjs';
```
- [ ] **Step 2: Remove standalone Python fixture setup**
In `scripts/package-artifacts.test.mjs`, replace `writeReleaseMetadataInputs`
with this function:
```javascript
async function writeReleaseMetadataInputs(root) {
for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) {
await mkdir(join(root, packageInfo.packageRoot), { recursive: true });
await writeJson(join(root, packageInfo.packageRoot, 'package.json'), {
name: packageInfo.name,
version: '0.0.0-private',
private: true,
});
}
}
```
Replace `writeUploadableArtifactFixtures` with this function:
```javascript
async function writeUploadableArtifactFixtures(layout) {
await mkdir(layout.npmDir, { recursive: true });
await mkdir(layout.pythonDir, { recursive: true });
const fileContents = new Map([
...NPM_ARTIFACT_PACKAGES.map((packageInfo) => [
layout.npmTarballs[packageInfo.name],
`${packageInfo.name}-tarball`,
]),
[
join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
'kaelio-ktx-runtime-wheel',
],
]);
for (const [path, contents] of fileContents) {
await writeFile(path, contents);
}
}
```
- [ ] **Step 3: Change build command expectations**
In the `buildArtifactCommands` test, replace the body with this code:
```javascript
it('builds TypeScript packages and the runtime wheel before packing npm artifacts', () => {
const layout = packageArtifactLayout('/repo/ktx');
const commands = buildArtifactCommands(layout);
assert.deepEqual(
commands.slice(0, NPM_BUILD_PACKAGE_ORDER.length).map((command) => [command.command, command.args]),
NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]),
);
assert.deepEqual(
commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [
command.command,
command.args,
]),
[[process.execPath, ['scripts/build-python-runtime-wheel.mjs']]],
);
assert.deepEqual(
commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [command.command, command.args]),
[[process.execPath, ['scripts/build-public-npm-package.mjs']]],
);
});
```
- [ ] **Step 4: Change release metadata expectations**
In the `packageReleaseMetadata` test, replace the expected array with this
array:
```javascript
assert.deepEqual(await packageReleaseMetadata(root), [
{
ecosystem: 'npm',
packageName: '@kaelio/ktx',
packageRoot: 'packages/cli',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
},
{
ecosystem: 'python',
packageName: 'kaelio-ktx',
packageRoot: 'python/runtime-wheel',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
},
]);
```
- [ ] **Step 5: Change Python artifact discovery expectations**
Replace the `findPythonArtifacts` success test with this test:
```javascript
it('finds the bundled runtime wheel only', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-'));
try {
await writeFile(join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), '');
assert.deepEqual(await findPythonArtifacts(root), {
runtimeWheel: join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
});
} finally {
await rm(root, { recursive: true, force: true });
}
});
```
- [ ] **Step 6: Change artifact manifest expectations**
Inside the artifact manifest test, replace the Python package assertion with:
```javascript
assert.deepEqual(
manifest.packages.filter((entry) => entry.ecosystem === 'python'),
[
{
ecosystem: 'python',
packageName: 'kaelio-ktx',
packageRoot: 'python/runtime-wheel',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
},
],
);
```
Replace the Python file assertion with:
```javascript
assert.deepEqual(
manifest.files
.filter((file) => file.ecosystem === 'python')
.map((file) => ({
artifactKind: file.artifactKind,
ecosystem: file.ecosystem,
packageName: file.packageName,
packageVersion: file.packageVersion,
path: file.path,
})),
[
{
artifactKind: 'wheel',
ecosystem: 'python',
packageName: 'kaelio-ktx',
packageVersion: '0.1.0',
path: 'python/kaelio_ktx-0.1.0-py3-none-any.whl',
},
],
);
```
In the `verifyArtifactManifest` success test, replace the file-count assertion
with:
```javascript
assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 1);
```
- [ ] **Step 7: Replace direct Python smoke tests with a dead-code guard**
Remove the whole `describe('pythonArtifactInstallArgs', ...)` block.
In `describe('verification snippets', ...)`, remove the test named
`asserts the Python modules that clean installs must expose`.
Add this test after the `verifyNpmArtifacts` test:
```javascript
describe('standalone Python artifact cleanup', () => {
it('does not build or verify standalone Python package artifacts', async () => {
const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8');
assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-sl'/);
assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-daemon'/);
assert.doesNotMatch(source, /async function verifyPythonArtifacts/);
assert.doesNotMatch(source, /pythonArtifactInstallArgs/);
assert.doesNotMatch(source, /pythonVerifySource/);
assert.doesNotMatch(source, /ktx_sl-0\.1\.0/);
assert.doesNotMatch(source, /ktx_daemon-0\.1\.0/);
});
});
```
- [ ] **Step 8: Run package artifact tests and verify failure**
Run:
```bash
node --test scripts/package-artifacts.test.mjs
```
Expected: FAIL. The failures mention the extra `ktx-sl` and `ktx-daemon`
artifact commands, metadata entries, manifest files, or direct Python smoke
helpers.
### Task 2: Remove standalone Python artifacts from package artifacts
**Files:**
- Modify: `scripts/package-artifacts.mjs`
- Test: `scripts/package-artifacts.test.mjs`
- [ ] **Step 1: Remove dead constants and imports**
In `scripts/package-artifacts.mjs`, replace the `node:path` import with this
import:
```javascript
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
```
Remove these constants:
```javascript
const PACKAGE_VERSION = '0.0.0-private';
const PYTHON_PACKAGE_VERSION = '0.1.0';
```
Remove the whole `ordersSource` constant block.
- [ ] **Step 2: Make npm artifact names public-package only**
Replace `npmPackageTarballName` with this function:
```javascript
function npmPackageTarballName(packageName) {
if (packageName !== PUBLIC_NPM_PACKAGE_NAME) {
throw new Error(`Unsupported npm artifact package: ${packageName}`);
}
return publicNpmPackageTarballName(PUBLIC_NPM_PACKAGE_VERSION);
}
```
- [ ] **Step 3: Remove standalone Python build commands**
Replace `buildArtifactCommands` with this function:
```javascript
export function buildArtifactCommands(layout) {
const packagesByName = new Map(INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => [packageInfo.name, packageInfo]));
const npmBuildCommands = NPM_ARTIFACT_BUILD_ORDER.map((packageName) => {
const packageInfo = packagesByName.get(packageName);
if (!packageInfo) {
throw new Error(`Unknown npm artifact build package: ${packageName}`);
}
return {
command: 'pnpm',
args: ['--filter', packageInfo.name, 'run', 'build'],
cwd: layout.rootDir,
};
});
const publicPackageCommand = {
command: process.execPath,
args: ['scripts/build-public-npm-package.mjs'],
cwd: layout.rootDir,
};
return [
...npmBuildCommands,
{
command: process.execPath,
args: ['scripts/build-python-runtime-wheel.mjs'],
cwd: layout.rootDir,
},
publicPackageCommand,
];
}
```
- [ ] **Step 4: Discover only the bundled runtime wheel**
Replace `findOne` and `findPythonArtifacts` with these functions:
```javascript
function findOne(files, distributionName, suffix, label, pythonDir, version) {
const normalized = normalizePythonDistributionName(distributionName);
const found = files.find((file) => file.startsWith(`${normalized}-${version}`) && file.endsWith(suffix));
if (!found) {
throw new Error(`Missing Python artifact: ${label}`);
}
return join(pythonDir, found);
}
export async function findPythonArtifacts(pythonDir) {
const files = await readdir(pythonDir);
return {
runtimeWheel: findOne(
files,
RUNTIME_WHEEL_DISTRIBUTION_NAME,
'.whl',
'kaelio-ktx runtime wheel',
pythonDir,
RUNTIME_WHEEL_PACKAGE_VERSION,
),
};
}
```
- [ ] **Step 5: Emit release metadata only for npm and runtime wheel**
Replace `packageReleaseMetadata` with this function:
```javascript
export async function packageReleaseMetadata(rootDir = scriptRootDir()) {
const npmPackages = await Promise.all(
NPM_ARTIFACT_PACKAGES.map((packageInfo) => readNpmPackageMetadata(rootDir, packageInfo)),
);
return [
...npmPackages,
releaseMetadataEntry({
ecosystem: 'python',
packageName: RUNTIME_WHEEL_DISTRIBUTION_NAME,
packageRoot: 'python/runtime-wheel',
packageVersion: RUNTIME_WHEEL_PACKAGE_VERSION,
privatePackage: false,
}),
];
}
```
- [ ] **Step 6: Remove dead TOML metadata helpers**
Delete these helper functions from `scripts/package-artifacts.mjs` because
release metadata no longer reads standalone Python `pyproject.toml` files:
```javascript
function readProjectBlock(toml, sourcePath) {
const lines = toml.split(/\r?\n/);
const block = [];
let inProject = false;
for (const line of lines) {
if (/^\[project\]\s*$/.test(line)) {
inProject = true;
continue;
}
if (inProject && /^\[.*\]\s*$/.test(line)) {
break;
}
if (inProject) {
block.push(line);
}
}
if (!inProject) {
throw new Error(`Missing [project] table in ${sourcePath}`);
}
return block.join('\n');
}
```
```javascript
function readTomlStringField(projectBlock, fieldName, sourcePath) {
const match = projectBlock.match(new RegExp(`^${fieldName}\\s*=\\s*"([^"]+)"\\s*$`, 'm'));
if (!match) {
throw new Error(`Missing project.${fieldName} in ${sourcePath}`);
}
return match[1];
}
```
```javascript
async function readPyprojectMetadata(path) {
const toml = await readFile(path, 'utf-8');
const projectBlock = readProjectBlock(toml, path);
return {
name: readTomlStringField(projectBlock, 'name', path),
version: readTomlStringField(projectBlock, 'version', path),
};
}
```
- [ ] **Step 7: Emit manifest records only for npm and runtime wheel**
Replace `artifactPackageRecords` with this function:
```javascript
function artifactPackageRecords(layout, pythonArtifacts, packages) {
const packagesByName = packageMetadataByName(packages);
const npmRecords = NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({
artifactKind: 'tarball',
artifactPath: layout.npmTarballs[packageInfo.name],
metadata: requirePackageMetadata(packagesByName, packageInfo.name),
}));
return [
...npmRecords,
{
artifactKind: 'wheel',
artifactPath: pythonArtifacts.runtimeWheel,
metadata: requirePackageMetadata(packagesByName, RUNTIME_WHEEL_DISTRIBUTION_NAME),
},
];
}
```
- [ ] **Step 8: Remove direct Python artifact verification helpers**
Delete these exports and functions from `scripts/package-artifacts.mjs`:
```javascript
export function pythonArtifactInstallArgs(python, pythonArtifacts) {
return ['pip', 'install', '--python', python, pythonArtifacts.runtimeWheel];
}
```
```javascript
export function pythonVerifySource() {
return `
import importlib.metadata
import semantic_layer
import ktx_daemon
assert importlib.metadata.version("kaelio-ktx") == "0.1.0"
assert semantic_layer is not None
assert ktx_daemon.PACKAGE_NAME == "ktx-daemon"
`;
}
```
```javascript
function pythonExecutable(projectDir) {
if (process.platform === 'win32') {
return join(projectDir, '.venv', 'Scripts', 'python.exe');
}
return join(projectDir, '.venv', 'bin', 'python');
}
```
```javascript
export function npmSmokePythonEnv(projectDir, baseEnv = process.env) {
const binDir = process.platform === 'win32' ? join(projectDir, '.venv', 'Scripts') : join(projectDir, '.venv', 'bin');
const existingPath = baseEnv.PATH ?? '';
return {
...baseEnv,
PATH: existingPath ? `${binDir}${delimiter}${existingPath}` : binDir,
};
}
```
```javascript
async function verifyPythonArtifacts(layout, tmpRoot) {
const pythonArtifacts = await findPythonArtifacts(layout.pythonDir);
const projectDir = join(tmpRoot, 'python-clean-install');
await mkdir(projectDir, { recursive: true });
const python = pythonExecutable(projectDir);
await writeFile(join(projectDir, 'verify_python.py'), pythonVerifySource());
await runCommand('uv', ['venv', '.venv'], { cwd: projectDir });
await runCommand('uv', pythonArtifactInstallArgs(python, pythonArtifacts), {
cwd: projectDir,
});
await runCommand(python, ['verify_python.py'], { cwd: projectDir });
await runCommand(python, ['-m', 'ktx_daemon', 'semantic-validate'], {
cwd: projectDir,
input: `${JSON.stringify({ sources: [ordersSource], dialect: 'postgres' })}\n`,
});
}
```
- [ ] **Step 9: Verify artifacts through npm only**
Replace `verifyArtifacts` with this function:
```javascript
async function verifyArtifacts(layout) {
await verifyArtifactManifest(layout);
const tmpRoot = await mkdtemp(join(tmpdir(), 'ktx-artifacts-'));
try {
await verifyNpmArtifacts(layout, tmpRoot);
} finally {
await rm(tmpRoot, { recursive: true, force: true });
}
}
```
- [ ] **Step 10: Run package artifact tests and verify pass**
Run:
```bash
node --test scripts/package-artifacts.test.mjs
```
Expected: PASS. The output includes `# fail 0`.
- [ ] **Step 11: Commit package artifact cleanup**
Run:
```bash
git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs
git commit -m "refactor: limit release artifacts to public package runtime"
```
### Task 3: Align release policy and readiness reports
**Files:**
- Modify: `release-policy.json`
- Modify: `scripts/release-readiness.test.mjs`
- Test: `scripts/release-readiness.test.mjs`
- [ ] **Step 1: Update release readiness fixtures**
In `scripts/release-readiness.test.mjs`, replace
`writeReleaseMetadataInputs` with:
```javascript
async function writeReleaseMetadataInputs(root) {
for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) {
await mkdir(join(root, packageInfo.packageRoot), { recursive: true });
await writeJson(join(root, packageInfo.packageRoot, 'package.json'), {
name: packageInfo.name,
version: '0.0.0-private',
private: true,
});
}
}
```
Replace `writeUploadableArtifactFixtures` with:
```javascript
async function writeUploadableArtifactFixtures(layout) {
await mkdir(layout.npmDir, { recursive: true });
await mkdir(layout.pythonDir, { recursive: true });
const fileContents = new Map([
...NPM_ARTIFACT_PACKAGES.map((packageInfo) => [
layout.npmTarballs[packageInfo.name],
`${packageInfo.name}-tarball`,
]),
[join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), 'kaelio-ktx-runtime-wheel'],
]);
for (const [path, contents] of fileContents) {
await writeFile(path, contents);
}
}
```
In `releasePolicy`, replace the `python` object with:
```javascript
python: {
publish: false,
repository: null,
packages: ['kaelio-ktx'],
...pythonOverrides,
},
```
- [ ] **Step 2: Update readiness report expectations**
In `scripts/release-readiness.test.mjs`, replace every expected
`packageNames` array with:
```javascript
packageNames: ['@kaelio/ktx', 'kaelio-ktx'],
```
There are three report assertions to update:
- `accepts the current ci-artifact-only policy, package metadata, and artifact manifest`
- `reports required published package smoke when release mode requires it`
- `accepts the npm public release ready policy`
- [ ] **Step 3: Update checked release policy**
In `release-policy.json`, replace the `python.packages` value with:
```json
"packages": ["kaelio-ktx"]
```
- [ ] **Step 4: Run readiness tests and verify pass**
Run:
```bash
node --test scripts/release-readiness.test.mjs
```
Expected: PASS. The output includes `# fail 0`.
- [ ] **Step 5: Commit release policy cleanup**
Run:
```bash
git add release-policy.json scripts/release-readiness.test.mjs
git commit -m "chore: align release policy with bundled runtime wheel"
```
### Task 4: Document the single release artifact surface
**Files:**
- Modify: `scripts/examples-docs.test.mjs`
- Modify: `README.md`
- Modify: `examples/package-artifacts/README.md`
- Test: `scripts/examples-docs.test.mjs`
- [ ] **Step 1: Add failing docs assertions**
In `scripts/examples-docs.test.mjs`, inside
`it('documents the public package artifact smoke shape', ...)`, add these
assertions after the existing `assert.match(readme, /managed Python runtime/);`
line:
```javascript
assert.match(readme, /public `@kaelio\/ktx` npm tarball and the bundled `kaelio-ktx` runtime wheel/);
assert.match(readme, /does not install standalone Python packages directly/);
assert.doesNotMatch(readme, /standalone Python distributions/);
assert.doesNotMatch(readme, /installs the Python artifacts directly/);
```
In `it('documents public npm and managed runtime usage in the README', ...)`,
add these assertions after the existing `uv` assertions:
```javascript
assert.match(rootReadme, /release artifact manifest contains the public npm tarball and the bundled `kaelio-ktx` runtime wheel/);
assert.match(rootReadme, /source packages for development, not public release artifacts/);
```
- [ ] **Step 2: Run docs tests and verify failure**
Run:
```bash
node --test scripts/examples-docs.test.mjs
```
Expected: FAIL. The failure mentions the missing single-artifact wording in
`README.md` or `examples/package-artifacts/README.md`.
- [ ] **Step 3: Update the package artifact example README**
In `examples/package-artifacts/README.md`, replace:
```markdown
The Python smoke project still installs the Python artifacts directly because
it verifies the standalone Python distributions that feed the bundled runtime
wheel.
```
with:
```markdown
The artifact manifest contains the public `@kaelio/ktx` npm tarball and the
bundled `kaelio-ktx` runtime wheel. The smoke does not install standalone
Python packages directly; Python-backed behavior is verified through the
managed runtime installed from the npm package.
```
- [ ] **Step 4: Update the root README release status**
In `README.md`, in the `## Release status` section, replace this paragraph:
```markdown
This repository builds one public npm artifact named `@kaelio/ktx`. The first
public npm handoff is policy-gated through `release-policy.json`, which keeps
Python package publishing disabled because KTX-owned Python code ships inside
the npm package as a bundled wheel.
```
with:
```markdown
This repository builds one public npm artifact named `@kaelio/ktx`. The release
artifact manifest contains the public npm tarball and the bundled `kaelio-ktx`
runtime wheel. The first public npm handoff is policy-gated through
`release-policy.json`, which keeps Python package publishing disabled because
KTX-owned Python code ships inside the npm package as a bundled wheel. The
`python/ktx-sl` and `python/ktx-daemon` directories remain source packages for
development, not public release artifacts.
```
- [ ] **Step 5: Run docs tests and verify pass**
Run:
```bash
node --test scripts/examples-docs.test.mjs
```
Expected: PASS. The output includes `# fail 0`.
- [ ] **Step 6: Commit docs cleanup**
Run:
```bash
git add README.md examples/package-artifacts/README.md scripts/examples-docs.test.mjs
git commit -m "docs: describe single public runtime artifact surface"
```
### Task 5: Verify the cleaned release artifact contract
**Files:**
- Verify: `scripts/package-artifacts.mjs`
- Verify: `scripts/package-artifacts.test.mjs`
- Verify: `scripts/release-readiness.test.mjs`
- Verify: `scripts/examples-docs.test.mjs`
- Verify: `release-policy.json`
- Verify: `README.md`
- Verify: `examples/package-artifacts/README.md`
- [ ] **Step 1: Run focused tests**
Run:
```bash
node --test scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs
```
Expected: PASS. The output includes `# fail 0`.
- [ ] **Step 2: Verify stale artifact strings are gone from production/docs files**
Run (scans only production and docs files, not test files — test files keep guard assertions that reference the removed strings):
```bash
rg -n "uv', \\['build', '--package', 'ktx-sl'|uv', \\['build', '--package', 'ktx-daemon'|ktx_sl-0\\.1\\.0|ktx_daemon-0\\.1\\.0|pythonArtifactInstallArgs|pythonVerifySource|verifyPythonArtifacts|standalone Python distributions|installs the Python artifacts directly" scripts/package-artifacts.mjs scripts/release-readiness.mjs README.md examples/package-artifacts/README.md release-policy.json
```
Expected: no matches.
- [ ] **Step 3: Verify release readiness against the current artifact manifest**
Run:
```bash
pnpm run release:readiness -- --json
```
Expected: PASS when `dist/artifacts/manifest.json` has been rebuilt after this
change. The JSON output contains:
```json
{
"releaseMode": "npm-public-release-ready",
"packageNames": ["@kaelio/ktx", "kaelio-ktx"],
"pythonPublishEnabled": false
}
```
If this command fails because the local artifact manifest was generated before
the cleanup, run:
```bash
pnpm run artifacts:check
pnpm run release:readiness -- --json
```
Expected: both commands pass. The rebuilt manifest contains only
`npm/kaelio-ktx-0.1.0.tgz` and
`python/kaelio_ktx-0.1.0-py3-none-any.whl` under `files`.
- [ ] **Step 4: Run pre-commit on changed files when configured**
Run:
```bash
uv run pre-commit run --files scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs release-policy.json README.md examples/package-artifacts/README.md
```
Expected: PASS. If pre-commit is not installed or no pre-commit config exists,
record the exact error and keep the focused Node test output from Step 1.
- [ ] **Step 5: Commit final verification fixes if needed**
If Step 1, Step 2, Step 3, or Step 4 required code or docs fixes, commit them:
```bash
git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs release-policy.json README.md examples/package-artifacts/README.md
git commit -m "test: verify single public runtime artifact contract"
```
If no fixes were required after the previous commits, do not create an empty
commit.
## Acceptance criteria
- `scripts/package-artifacts.mjs` builds TypeScript packages, builds the
bundled `kaelio-ktx` runtime wheel, copies it into CLI assets, and packs the
public `@kaelio/ktx` npm tarball.
- `scripts/package-artifacts.mjs` no longer builds `ktx-sl` or `ktx-daemon`
standalone wheel or source-distribution artifacts.
- Artifact manifests contain release metadata for `@kaelio/ktx` and
`kaelio-ktx` only.
- `release-policy.json` lists only `@kaelio/ktx` under `npm.packages` and only
`kaelio-ktx` under `python.packages`.
- The artifact smoke verifies Python-backed behavior through the installed
public npm package and managed runtime, not by installing standalone Python
artifacts directly.
- Public docs state that `python/ktx-sl` and `python/ktx-daemon` remain source
packages for development, not public release artifacts.
## Self-review
Spec coverage:
- The plan preserves the single public npm package requirement.
- The plan preserves the bundled KTX-owned Python wheel requirement.
- The plan keeps Python package publishing disabled.
- The plan removes the only remaining artifact path that treated KTX-owned
Python source packages as standalone release artifacts.
Placeholder scan:
- No steps contain placeholder implementation text.
- Every code-changing step names exact files and provides concrete replacement
snippets.
Type and name consistency:
- Public npm package name remains `@kaelio/ktx`.
- Bundled runtime distribution name remains `kaelio-ktx`.
- Runtime wheel filename remains `kaelio_ktx-0.1.0-py3-none-any.whl`.
- Removed standalone Python artifact names are consistently `ktx-sl` and
`ktx-daemon`.

234
docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md Normal file
View file

@ -0,0 +1,234 @@
# npm-managed Python runtime design
This spec defines how KTX ships as one visible npm package while still using
Python for sqlglot, semantic-layer planning, database-agent compute, and local
embeddings. The goal is a user experience where users install or run only
`@kaelio/ktx`, and KTX manages its Python runtime automatically when a command
needs it.
## Goals
KTX must be usable through the npm package `@kaelio/ktx` with a `ktx` binary.
Users can run KTX without learning about the Python packages that power parts of
the system.
The first release must support these invocation modes:
- `npx @kaelio/ktx setup demo`
- `npx @kaelio/ktx sl query ...`
- `npm install @kaelio/ktx`, followed by `npx ktx ...`
- `npm install -g @kaelio/ktx`, followed by `ktx ...`
KTX-owned Python code must ship inside the npm package as a bundled wheel. KTX
doesn't need to publish its own Python code to PyPI for this release.
## Non-goals
This release does not need to provide a public TypeScript SDK split across
multiple npm packages. The internal workspace package layout can remain useful
for development, but the public npm surface is a single package.
This release does not need a fully offline install. KTX's own Python wheel is
bundled, but third-party Python dependencies can come from PyPI through `uv`.
This release does not install local embedding dependencies by default. Local
embeddings remain lazy because `sentence-transformers`, `torch`, and model
downloads are large.
## Package model
KTX publishes one public npm package:
```text
@kaelio/ktx
```
That package exposes one binary:
```json
{
"bin": {
"ktx": "./dist/bin.js"
}
}
```
The npm package includes these assets:
- Bundled JavaScript CLI output.
- Packaged demo assets.
- One KTX-owned Python wheel, for example
`python/kaelio_ktx-0.1.0-py3-none-any.whl`.
- A wheel checksum or runtime manifest that lets the CLI verify the bundled
Python payload before installation.
The Python wheel contains the current `semantic_layer` and `ktx_daemon`
modules. It exposes at least the `ktx-daemon` console script.
## Runtime installation
KTX creates a managed Python runtime only when a command needs Python-backed
behavior. The runtime lives outside the npm cache so it survives `npx` runs.
The runtime root is platform-specific:
- macOS: `~/Library/Application Support/kaelio/ktx/runtime`
- Linux: `${XDG_DATA_HOME:-~/.local/share}/kaelio/ktx/runtime`
- Windows: `%LOCALAPPDATA%/Kaelio/KTX/runtime`
The runtime is versioned by the npm package version. A versioned runtime avoids
mixing JavaScript and Python code from incompatible releases.
The installer performs these steps:
1. Locate `uv`.
2. Create a virtual environment under the versioned runtime directory.
3. Install the bundled KTX wheel into that environment.
4. Write a runtime manifest with the CLI version, wheel checksum, Python
executable, daemon executable, and installed feature set.
For lightweight Python support, the install command uses the bundled wheel's
default dependency set. For local embeddings, the installer adds the embeddings
extra only when selected:
```bash
uv pip install "/path/to/kaelio_ktx-0.1.0-py3-none-any.whl"
uv pip install "/path/to/kaelio_ktx-0.1.0-py3-none-any.whl[local-embeddings]"
```
## Feature installation levels
KTX manages Python runtime features in levels so first use stays fast.
`core` includes:
- `sqlglot`
- `pydantic`
- `pyyaml`
- `fastapi`
- `uvicorn`
- lightweight daemon dependencies
`local-embeddings` adds:
- `sentence-transformers`
- `torch`
- model download support for `all-MiniLM-L6-v2`
Commands that only need semantic-layer SQL generation require `core`.
Commands that need local embeddings require `local-embeddings`.
## Command behavior
Pure TypeScript commands run without the managed Python runtime.
Python-backed one-shot operations use the managed `ktx-daemon` executable
directly. Examples include semantic query compilation, semantic validation,
semantic source generation, and sqlglot-backed table identifier parsing.
Repeated or expensive operations use a managed HTTP daemon. Local embeddings use
the daemon because loading the model for every one-shot process is too slow.
KTX provides runtime management commands:
```bash
ktx runtime install
ktx runtime status
ktx runtime start
ktx runtime stop
ktx runtime doctor
ktx runtime prune
```
Normal commands can install the runtime lazily. Runtime commands make that
behavior inspectable and debuggable.
## Daemon lifecycle
The daemon binds to `127.0.0.1` on an available random port. KTX writes daemon
state to the runtime manifest or an adjacent state file:
```json
{
"pid": 12345,
"port": 58731,
"version": "0.1.0",
"features": ["core", "local-embeddings"],
"startedAt": "2026-05-11T00:00:00Z"
}
```
Before reusing a daemon, KTX checks that the process is alive, the port responds
to `/health`, and the daemon version matches the CLI version. If any check
fails, KTX treats the daemon as stale and starts a new one.
KTX uses one-shot Python for short operations by default. It starts the daemon
only when a command benefits from process reuse.
## Interactive and CI behavior
In an interactive terminal, KTX prompts before installing the managed runtime
for the first time. The prompt states that Python dependencies will be
downloaded.
With `--yes`, KTX installs the required runtime features without prompting.
With `--no-input`, KTX fails if a required runtime feature is missing and no
explicit auto-install flag is present. The error prints the exact command to
prepare the runtime.
For local embeddings, KTX prompts separately because the dependency and model
downloads are larger than the core runtime.
## Error handling
If `uv` is missing, KTX prints a focused error that explains how to install it
and how to retry. A later release can add a bundled or downloaded `uv` strategy.
If Python runtime installation fails, KTX preserves install logs in the runtime
directory and prints the log path.
If the daemon fails to start, KTX prints the captured daemon stdout and stderr
path. It falls back to one-shot mode only when the requested operation supports
one-shot execution.
If JavaScript and Python versions don't match, KTX reinstalls the managed
runtime for the current npm package version.
## Release flow
The release builds the Python wheel before packing npm artifacts. The npm pack
step includes the wheel as an asset.
Release checks must cover:
1. Clean install of the packed npm package.
2. `npx` execution of the packed package.
3. First-run managed runtime install from the bundled wheel.
4. One-shot semantic-layer query through the managed runtime.
5. Runtime status and doctor output.
6. Daemon start, health check, reuse, and stop.
7. Optional local embeddings smoke in a separate job or opt-in check.
## Open decisions
KTX still needs a final decision on whether `uv` is a hard prerequisite or a
bootstrap dependency that KTX downloads automatically.
KTX also needs the final Python distribution name. This spec uses
`kaelio-ktx` as the distribution name and `kaelio_ktx` in wheel filenames.
## Success criteria
Users can run `npx @kaelio/ktx ...` and complete Python-backed KTX operations
without manually installing a KTX Python package.
Users who install `@kaelio/ktx` locally can run `npx ktx ...` through the local
project's npm binary resolution.
The first Python-backed command installs only the core runtime. Local embedding
dependencies install only after the user selects local embeddings or explicitly
requests the `local-embeddings` runtime feature.
KTX can diagnose and repair stale or mismatched managed runtimes without asking
users to delete directories manually.

21
examples/package-artifacts/README.md
View file

@ -4,14 +4,21 @@ The package artifact smoke checks create temporary projects instead of storing
sample projects in this directory. Run the checks from `ktx/`:
```bash
source .venv/bin/activate
pnpm run artifacts:check
```
The npm smoke project installs the generated `@ktx/context` and `@ktx/cli`
tarballs, imports public package entry points, and runs installed `ktx`
commands against a generated local project.
The npm smoke project installs the generated public `@kaelio/ktx` tarball,
imports the package entry point, and runs installed `ktx` commands against a
generated local project.
The Python smoke project installs `ktx-daemon` through the local artifact
directory, imports `semantic_layer` and `ktx_daemon`, and runs
`python -m ktx_daemon semantic-validate`.
The managed Python runtime smoke requires `uv` on `PATH`, isolates
`KTX_RUNTIME_ROOT`, verifies `ktx runtime status`, runs `ktx sl query --yes` to
install the core runtime from the bundled wheel, checks `ktx runtime doctor`,
starts and reuses the managed daemon, stops it, previews a stale runtime with
`ktx runtime prune --dry-run`, verifies confirmation is required, and removes
the stale runtime with `ktx runtime prune --yes`.
The artifact manifest contains the public `@kaelio/ktx` npm tarball and the
bundled `kaelio-ktx` runtime wheel. The smoke does not install standalone
Python packages directly; Python-backed behavior is verified through the
managed runtime installed from the npm package.

17
examples/postgres-historic/README.md
View file

@ -13,8 +13,8 @@ generates query workload under separate users, runs `ktx setup` with
- Docker with Compose v2
- Node and pnpm matching the KTX workspace
- `KTX_SQL_ANALYSIS_URL` or `KTX_DAEMON_URL` pointing at a running SQL-analysis
service that exposes `/api/sql/analyze-for-fingerprint`
- `uv` on `PATH` so the KTX-managed Python runtime can install the bundled
runtime wheel
## Run
@ -24,8 +24,9 @@ From the KTX repository root:
examples/postgres-historic/scripts/smoke.sh
```
The smoke creates a temporary KTX project, starts Postgres on
`127.0.0.1:55432`, and uses this connection URL:
The smoke creates a temporary KTX project, isolates the managed Python runtime
under the temporary project parent, starts Postgres on `127.0.0.1:55432`, and
uses this connection URL:
```bash
postgresql://ktx_reader:ktx_reader@127.0.0.1:55432/analytics # pragma: allowlist secret
@ -83,10 +84,11 @@ Historic SQL (warehouse)` when `pg_stat_statements` is installed,
Run local historic-SQL ingest:
```bash
node packages/cli/dist/bin.js --project-dir /tmp/ktx-postgres-historic dev ingest run \
pnpm run ktx -- dev ingest run --project-dir /tmp/ktx-postgres-historic \
--connection-id warehouse \
--adapter historic-sql \
--plain \
--yes \
--no-input
```
@ -111,5 +113,6 @@ The manifest should have `dialect: "postgres"`, `degraded: true`,
- Missing grants: confirm `GRANT pg_read_all_stats TO ktx_reader;`.
- Empty templates: rerun `scripts/generate-workload.sh base` and keep
`--historic-sql-min-calls 2` for the smoke.
- SQL-analysis failures: set `KTX_SQL_ANALYSIS_URL` or `KTX_DAEMON_URL` to a
running service URL before running `scripts/smoke.sh`.
- SQL-analysis failures: run `pnpm run ktx -- runtime doctor` from the KTX
repository root and confirm `uv`, the bundled Python wheel, and the managed
runtime all pass.

27
examples/postgres-historic/scripts/smoke.sh
View file

@ -8,22 +8,20 @@ COMPOSE_FILE="$EXAMPLE_DIR/docker-compose.yml"
PROJECT_PARENT="${KTX_POSTGRES_HISTORIC_PROJECT_PARENT:-$(mktemp -d)}"
PROJECT_DIR="$PROJECT_PARENT/postgres-historic-ktx"
KTX_BIN="$KTX_ROOT/packages/cli/dist/bin.js"
export KTX_RUNTIME_ROOT="$PROJECT_PARENT/managed-runtime"
unset KTX_DAEMON_URL
unset KTX_SQL_ANALYSIS_URL
cleanup() {
if [[ -f "$KTX_BIN" ]]; then
node "$KTX_BIN" runtime stop >/dev/null 2>&1 || true
fi
if [[ "${KTX_POSTGRES_HISTORIC_KEEP_DOCKER:-0}" != "1" ]]; then
docker compose -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT
require_sql_analysis_url() {
if [[ -n "${KTX_SQL_ANALYSIS_URL:-}" || -n "${KTX_DAEMON_URL:-}" ]]; then
return
fi
echo "Set KTX_SQL_ANALYSIS_URL or KTX_DAEMON_URL before running this smoke." >&2
exit 1
}
latest_manifest() {
find "$PROJECT_DIR/raw-sources/warehouse/historic-sql" -name manifest.json | sort | tail -n 1
}
@ -60,9 +58,19 @@ const jobId = process.argv[4];
const { loadKtxProject } = await import(join(ktxRoot, 'packages/context/dist/project/index.js'));
const { runLocalStageOnlyIngest } = await import(join(ktxRoot, 'packages/context/dist/ingest/index.js'));
const { createKtxCliLocalIngestAdapters } = await import(join(ktxRoot, 'packages/cli/dist/local-adapters.js'));
const { getKtxCliPackageInfo } = await import(join(ktxRoot, 'packages/cli/dist/index.js'));
const project = await loadKtxProject({ projectDir });
const adapters = createKtxCliLocalIngestAdapters(project, { historicSqlConnectionId: 'warehouse' });
const cliVersion = getKtxCliPackageInfo().version;
const managedRuntimeIo = { stdout: process.stdout, stderr: process.stderr };
const adapters = createKtxCliLocalIngestAdapters(project, {
historicSqlConnectionId: 'warehouse',
managedDaemon: {
cliVersion,
installPolicy: 'auto',
io: managedRuntimeIo,
},
});
const adapter = adapters.find((candidate) => candidate.source === 'historic-sql');
if (!adapter) throw new Error('historic-sql adapter was not registered for local run');
const record = await runLocalStageOnlyIngest({
@ -88,7 +96,6 @@ NODE
cd "$KTX_ROOT"
pnpm --filter @ktx/context run build
pnpm --filter @ktx/cli run build
require_sql_analysis_url
docker compose -f "$COMPOSE_FILE" up -d --wait
"$EXAMPLE_DIR/scripts/generate-workload.sh" base

2
package.json
View file

@ -23,6 +23,8 @@
"native:rebuild": "pnpm -r rebuild better-sqlite3",
"setup:dev": "node scripts/setup-dev.mjs",
"release:published-smoke": "node scripts/published-package-smoke.mjs --require-config",
"release:local-embeddings-smoke": "node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in",
"release:npm-publish": "node scripts/publish-public-npm-package.mjs",
"release:readiness": "node scripts/release-readiness.mjs",
"relationships:acquire-public-fixtures": "node scripts/acquire-public-benchmark-fixtures.mjs",
"relationships:rebuild-public-snapshots": "node scripts/build-benchmark-snapshot.mjs --rebuild-all",

44
packages/cli/src/agent-runtime.test.ts
View file

@ -105,4 +105,48 @@ describe('agent runtime helpers', () => {
queryExecutor,
});
});
it('creates managed semantic compute when no test override is injected', async () => {
const project = {
projectDir: tempDir,
configPath: join(tempDir, 'ktx.yaml'),
config: { project: 'revenue', connections: {} },
coreConfig: {},
git: {},
fileStore: {},
} as never;
const ports = { semanticLayer: {} } as never;
const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };
const loadProject = vi.fn(async () => project);
const createContextTools = vi.fn(() => ports);
const createManagedSemanticLayerCompute = vi.fn(async () => semanticLayerCompute);
const { io } = makeIo();
await expect(
createKtxAgentRuntime(
{
projectDir: tempDir,
enableSemanticCompute: true,
enableQueryExecution: false,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
io,
},
{
loadProject,
createContextTools,
createManagedSemanticLayerCompute,
},
),
).resolves.toMatchObject({ project, ports, semanticLayerCompute });
expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({
cliVersion: '0.2.0',
installPolicy: 'auto',
io,
});
expect(createContextTools).toHaveBeenCalledWith(project, {
semanticLayerCompute,
});
});
});

36
packages/cli/src/agent-runtime.ts
View file

@ -1,9 +1,13 @@
import { readFile } from 'node:fs/promises';
import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from '@ktx/context/connections';
import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon';
import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon';
import { createLocalProjectMcpContextPorts, type KtxMcpContextPorts } from '@ktx/context/mcp';
import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import {
createManagedPythonSemanticLayerComputePort,
type KtxManagedPythonInstallPolicy,
} from './managed-python-command.js';
export const KTX_AGENT_MAX_ROWS_CAP = 1000;
@ -11,6 +15,9 @@ export interface KtxAgentRuntimeOptions {
projectDir: string;
enableSemanticCompute: boolean;
enableQueryExecution: boolean;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
io?: KtxCliIo;
}
export interface KtxAgentRuntime {
@ -24,6 +31,7 @@ export interface KtxAgentRuntimeDeps {
loadProject?: typeof loadKtxProject;
createContextTools?: typeof createLocalProjectMcpContextPorts;
createSemanticLayerCompute?: () => KtxSemanticLayerComputePort;
createManagedSemanticLayerCompute?: typeof createManagedPythonSemanticLayerComputePort;
createQueryExecutor?: () => KtxSqlQueryExecutorPort;
}
@ -57,14 +65,34 @@ export function parseAgentMaxRows(value: number | undefined): number {
return value;
}
async function createAgentSemanticLayerCompute(
options: KtxAgentRuntimeOptions,
deps: KtxAgentRuntimeDeps,
): Promise<KtxSemanticLayerComputePort | undefined> {
if (!options.enableSemanticCompute) {
return undefined;
}
if (deps.createSemanticLayerCompute) {
return deps.createSemanticLayerCompute();
}
if (!options.cliVersion || !options.runtimeInstallPolicy || !options.io) {
throw new Error('Managed Python semantic compute requires cliVersion, runtimeInstallPolicy, and io.');
}
const createManagedSemanticLayerCompute =
deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort;
return createManagedSemanticLayerCompute({
cliVersion: options.cliVersion,
installPolicy: options.runtimeInstallPolicy,
io: options.io,
});
}
export async function createKtxAgentRuntime(
options: KtxAgentRuntimeOptions,
deps: KtxAgentRuntimeDeps = {},
): Promise<KtxAgentRuntime> {
const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: options.projectDir });
const semanticLayerCompute = options.enableSemanticCompute
? (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)()
: undefined;
const semanticLayerCompute = await createAgentSemanticLayerCompute(options, deps);
const queryExecutor = options.enableQueryExecution
? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)()
: undefined;

35
packages/cli/src/agent.test.ts
View file

@ -231,6 +231,8 @@ describe('runKtxAgent', () => {
queryFile,
execute: true,
maxRows: 100,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'never',
},
io.io,
{ createRuntime: async () => runtime() },
@ -240,6 +242,39 @@ describe('runKtxAgent', () => {
expect(JSON.parse(io.stdout())).toMatchObject({ sql: 'select 1', rows: [[1]] });
});
it('passes managed runtime options into default SL query runtime creation', async () => {
const queryFile = join(tempDir, 'sl-query.json');
const io = makeIo();
const createRuntime = vi.fn(async () => runtime());
await writeFile(queryFile, '{"measures":["total_revenue"],"dimensions":[]}', 'utf-8');
await expect(
runKtxAgent(
{
command: 'sl-query',
projectDir: tempDir,
json: true,
connectionId: 'warehouse',
queryFile,
execute: false,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
},
io.io,
{ createRuntime },
),
).resolves.toBe(0);
expect(createRuntime).toHaveBeenCalledWith({
projectDir: tempDir,
enableSemanticCompute: true,
enableQueryExecution: false,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
io: io.io,
});
});
it('executes read-only SQL from a SQL file with an explicit row limit', async () => {
const sqlFile = join(tempDir, 'query.sql');
const fakeRuntime = runtime();

37
packages/cli/src/agent.ts
View file

@ -17,6 +17,7 @@ import {
noIndexedSourcesSlSearchReadiness,
type KtxAgentSlSearchReadinessDetail,
} from './agent-search-readiness.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
import { readKtxSetupStatus, type KtxSetupStatus } from './setup.js';
export type KtxAgentArgs =
@ -32,6 +33,8 @@ export type KtxAgentArgs =
queryFile: string;
execute: boolean;
maxRows?: number;
cliVersion: string;
runtimeInstallPolicy: KtxManagedPythonInstallPolicy;
}
| { command: 'wiki-search'; projectDir: string; json: true; query: string; limit: number }
| { command: 'wiki-read'; projectDir: string; json: true; pageId: string }
@ -42,6 +45,9 @@ export interface KtxAgentDeps extends KtxAgentRuntimeDeps {
projectDir: string;
enableSemanticCompute: boolean;
enableQueryExecution: boolean;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
io?: KtxCliIo;
}) => Promise<KtxAgentRuntime>;
readSetupStatus?: (
projectDir: string,
@ -68,23 +74,22 @@ function writeAgentSlSearchReadinessError(io: KtxCliIo, detail: KtxAgentSlSearch
writeAgentJsonError(io, detail.message, { code: detail.code, nextSteps: detail.nextSteps });
}
async function runtimeFor(args: KtxAgentArgs, deps: KtxAgentDeps): Promise<KtxAgentRuntime> {
async function runtimeFor(args: KtxAgentArgs, deps: KtxAgentDeps, io: KtxCliIo): Promise<KtxAgentRuntime> {
const needsSemanticCompute = args.command === 'sl-query';
const needsQueryExecution = args.command === 'sql-execute' || (args.command === 'sl-query' && args.execute);
return deps.createRuntime
? deps.createRuntime({
projectDir: args.projectDir,
enableSemanticCompute: needsSemanticCompute,
enableQueryExecution: needsQueryExecution,
})
: createKtxAgentRuntime(
{
projectDir: args.projectDir,
enableSemanticCompute: needsSemanticCompute,
enableQueryExecution: needsQueryExecution,
},
deps,
);
const runtimeOptions = {
projectDir: args.projectDir,
enableSemanticCompute: needsSemanticCompute,
enableQueryExecution: needsQueryExecution,
...(args.command === 'sl-query'
? {
cliVersion: args.cliVersion,
runtimeInstallPolicy: args.runtimeInstallPolicy,
io,
}
: {}),
};
return deps.createRuntime ? deps.createRuntime(runtimeOptions) : createKtxAgentRuntime(runtimeOptions, deps);
}
function connectionIdForSource(runtime: KtxAgentRuntime, requested: string | undefined): string {
@ -101,7 +106,7 @@ export async function runKtxAgent(args: KtxAgentArgs, io: KtxCliIo, deps: KtxAge
return 0;
}
const runtime = await runtimeFor(args, deps);
const runtime = await runtimeFor(args, deps, io);
if (args.command === 'context') {
const [status, connections, semanticLayer] = await Promise.all([

7
packages/cli/src/cli-program.ts
View file

@ -4,6 +4,7 @@ import { registerAgentCommands } from './commands/agent-commands.js';
import { registerConnectionCommands } from './commands/connection-commands.js';
import { registerWikiCommands } from './commands/knowledge-commands.js';
import { registerPublicIngestCommands } from './commands/public-ingest-commands.js';
import { registerRuntimeCommands } from './commands/runtime-commands.js';
import { registerServeCommands } from './commands/serve-commands.js';
import { registerSetupCommands } from './commands/setup-commands.js';
import { registerSlCommands } from './commands/sl-commands.js';
@ -17,6 +18,7 @@ profileMark('module:cli-program');
export interface KtxCliCommandContext {
io: KtxCliIo;
deps: KtxCliDeps;
packageInfo: KtxCliPackageInfo;
setExitCode: (code: number) => void;
runInit: (args: { projectDir: string; projectName?: string; force: boolean }, io: KtxCliIo) => Promise<number>;
writeDebug?: (command: string, commandContext: CommandWithGlobalOptions) => void;
@ -177,6 +179,7 @@ async function runBareInteractiveCommand(
skipAgents: false,
inputMode: 'auto',
yes: false,
cliVersion: context.packageInfo.version,
skipLlm: false,
skipEmbeddings: false,
databaseSchemas: [],
@ -205,6 +208,7 @@ export async function runCommanderKtxCli(
const context: KtxCliCommandContext = {
io,
deps,
packageInfo: info,
setExitCode: (code: number) => {
exitCode = code;
},
@ -229,6 +233,9 @@ export async function runCommanderKtxCli(
registerSlCommands(program, context);
profileMark('commander:register-sl');
registerRuntimeCommands(program, context);
profileMark('commander:register-runtime');
registerServeCommands(program, context);
profileMark('commander:register-serve');

29
packages/cli/src/cli-runtime.ts
View file

@ -1,3 +1,5 @@
import { createRequire } from 'node:module';
import type { KtxConnectionMetabaseSetupArgs } from './commands/connection-metabase-setup.js';
import type { KtxConnectionNotionArgs } from './commands/connection-notion.js';
import type { KtxAgentArgs } from './agent.js';
@ -7,6 +9,7 @@ import type { KtxDoctorArgs } from './doctor.js';
import type { KtxIngestArgs } from './ingest.js';
import type { KtxKnowledgeArgs } from './knowledge.js';
import type { KtxPublicIngestArgs } from './public-ingest.js';
import type { KtxRuntimeArgs } from './runtime.js';
import type { KtxScanArgs } from './scan.js';
import type { KtxServeArgs } from './serve.js';
import type { KtxSetupArgs } from './setup.js';
@ -15,9 +18,11 @@ import { profileMark, profileSpan } from './startup-profile.js';
profileMark('module:cli-runtime');
const requirePackageJson = createRequire(import.meta.url);
export interface KtxCliPackageInfo {
name: '@ktx/cli';
version: '0.0.0-private';
name: string;
version: string;
contextPackageName: '@ktx/context';
}
@ -37,15 +42,31 @@ export interface KtxCliDeps {
doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise<number>;
ingest?: (args: KtxIngestArgs, io: KtxCliIo) => Promise<number>;
publicIngest?: (args: KtxPublicIngestArgs, io: KtxCliIo) => Promise<number>;
runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise<number>;
scan?: (args: KtxScanArgs, io: KtxCliIo) => Promise<number>;
knowledge?: (args: KtxKnowledgeArgs, io: KtxCliIo) => Promise<number>;
sl?: (args: KtxSlArgs, io: KtxCliIo) => Promise<number>;
}
export function getKtxCliPackageInfo(): KtxCliPackageInfo {
return packageInfoFromJson(requirePackageJson('../package.json'));
}
export function packageInfoFromJson(packageJson: unknown): KtxCliPackageInfo {
if (
typeof packageJson !== 'object' ||
packageJson === null ||
!('name' in packageJson) ||
!('version' in packageJson) ||
typeof packageJson.name !== 'string' ||
typeof packageJson.version !== 'string'
) {
throw new Error('Invalid KTX CLI package metadata');
}
return {
name: '@ktx/cli',
version: '0.0.0-private',
name: packageJson.name,
version: packageJson.version,
contextPackageName: '@ktx/context',
};
}

2
packages/cli/src/command-schemas.ts
View file

@ -64,6 +64,8 @@ export const slQueryCommandSchema = z.object({
}),
format: z.enum(['json', 'sql']),
execute: z.boolean(),
cliVersion: z.string().min(1),
runtimeInstallPolicy: z.enum(['prompt', 'auto', 'never']),
maxRows: z.number().int().positive().optional(),
});

14
packages/cli/src/commands/agent-commands.ts
View file

@ -2,6 +2,7 @@ import { Option, type Command } from '@commander-js/extra-typings';
import type { KtxAgentArgs } from '../agent.js';
import type { KtxCliCommandContext } from '../cli-program.js';
import { parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js';
import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
async function runAgent(context: KtxCliCommandContext, args: KtxAgentArgs): Promise<void> {
const runner = context.deps.agent ?? (await import('../agent.js')).runKtxAgent;
@ -73,10 +74,19 @@ export function registerAgentCommands(program: Command, context: KtxCliCommandCo
.requiredOption('--connection-id <id>', 'Connection id for execution')
.requiredOption('--query-file <path>', 'JSON semantic-layer query file')
.option('--execute', 'Execute the compiled query against the connection', false)
.option('--yes', 'Install the managed Python runtime without prompting when required', false)
.option('--no-input', 'Disable interactive managed runtime installation')
.option('--max-rows <number>', 'Maximum rows to return when executing', parsePositiveIntegerOption)
.action(
async (
options: { connectionId: string; queryFile: string; execute: boolean; maxRows?: number },
options: {
connectionId: string;
queryFile: string;
execute: boolean;
maxRows?: number;
yes?: boolean;
input?: boolean;
},
command,
) => {
await runAgent(context, {
@ -86,6 +96,8 @@ export function registerAgentCommands(program: Command, context: KtxCliCommandCo
connectionId: options.connectionId,
queryFile: options.queryFile,
execute: options.execute,
cliVersion: context.packageInfo.version,
runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options),
...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}),
});
},

4
packages/cli/src/commands/ingest-commands.ts
View file

@ -3,6 +3,7 @@ import { type Command, Option } from '@commander-js/extra-typings';
import { type KtxCliCommandContext, type OutputModeOptions, resolveCommandProjectDir } from '../cli-program.js';
import type { KtxCliDeps, KtxCliIo } from '../index.js';
import type { KtxIngestArgs, KtxIngestOutputMode } from '../ingest.js';
import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/ingest-commands');
@ -75,6 +76,7 @@ export function registerIngestCommands(
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
.option('--yes', 'Install the managed Python runtime without prompting when required', false)
.option('--no-input', 'Disable interactive terminal input for visualization')
.action(async (options, command) => {
if (options.reportFile) {
@ -89,6 +91,8 @@ export function registerIngestCommands(
adapter: options.adapter,
sourceDir: options.sourceDir ? resolve(options.sourceDir) : undefined,
databaseIntrospectionUrl: options.databaseIntrospectionUrl || undefined,
cliVersion: context.packageInfo.version,
runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options),
...(options.debugLlmRequestFile ? { debugLlmRequestFile: resolve(options.debugLlmRequestFile) } : {}),
outputMode: outputMode(options),
...inputMode(options),

100
packages/cli/src/commands/runtime-commands.ts Normal file
View file

@ -0,0 +1,100 @@
import { type Command, Option } from '@commander-js/extra-typings';
import type { KtxCliCommandContext } from '../cli-program.js';
import type { KtxRuntimeArgs } from '../runtime.js';
type RuntimeFeature = Extract<KtxRuntimeArgs, { command: 'install' }>['feature'];
function createRuntimeFeatureOption() {
return new Option('--feature <feature>', 'Runtime feature level')
.choices(['core', 'local-embeddings'])
.default('core');
}
async function runRuntimeArgs(context: KtxCliCommandContext, args: KtxRuntimeArgs): Promise<void> {
const runner = context.deps.runtime ?? (await import('../runtime.js')).runKtxRuntime;
context.setExitCode(await runner(args, context.io));
}
export function registerRuntimeCommands(program: Command, context: KtxCliCommandContext): void {
const runtime = program
.command('runtime')
.description('Install, inspect, and prune the KTX-managed Python runtime')
.showHelpAfterError();
runtime
.command('install')
.description('Install the bundled Python runtime wheel into the managed runtime')
.addOption(createRuntimeFeatureOption())
.option('--yes', 'Accept runtime installation without prompting', false)
.option('--force', 'Reinstall even when the runtime already looks ready', false)
.action(async (options: { feature: RuntimeFeature; yes?: boolean; force?: boolean }) => {
await runRuntimeArgs(context, {
command: 'install',
cliVersion: context.packageInfo.version,
feature: options.feature,
force: options.force === true,
});
});
runtime
.command('start')
.description('Start the KTX-managed Python HTTP daemon')
.addOption(createRuntimeFeatureOption())
.option('--force', 'Restart even when a matching daemon is already running', false)
.action(async (options: { feature: RuntimeFeature; force?: boolean }) => {
await runRuntimeArgs(context, {
command: 'start',
cliVersion: context.packageInfo.version,
feature: options.feature,
force: options.force === true,
});
});
runtime
.command('stop')
.description('Stop the KTX-managed Python HTTP daemon')
.action(async () => {
await runRuntimeArgs(context, {
command: 'stop',
cliVersion: context.packageInfo.version,
});
});
runtime
.command('status')
.description('Show managed Python runtime status')
.option('--json', 'Print JSON output', false)
.action(async (options: { json?: boolean }) => {
await runRuntimeArgs(context, {
command: 'status',
cliVersion: context.packageInfo.version,
json: options.json === true,
});
});
runtime
.command('doctor')
.description('Check managed Python runtime prerequisites and installation')
.option('--json', 'Print JSON output', false)
.action(async (options: { json?: boolean }) => {
await runRuntimeArgs(context, {
command: 'doctor',
cliVersion: context.packageInfo.version,
json: options.json === true,
});
});
runtime
.command('prune')
.description('Remove stale managed Python runtimes for older CLI versions')
.option('--dry-run', 'List stale runtimes without deleting them', false)
.option('--yes', 'Confirm deletion of stale runtime directories', false)
.action(async (options: { dryRun?: boolean; yes?: boolean }) => {
await runRuntimeArgs(context, {
command: 'prune',
cliVersion: context.packageInfo.version,
dryRun: options.dryRun === true,
yes: options.yes === true,
});
});
}

5
packages/cli/src/commands/scan-commands.ts
View file

@ -1,5 +1,6 @@
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
import { type KtxCliCommandContext, parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js';
import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
import type { KtxScanArgs } from '../scan.js';
import { profileMark } from '../startup-profile.js';
@ -102,6 +103,8 @@ export function registerScanCommands(program: Command, context: KtxCliCommandCon
)
.option('--dry-run', 'Run without writing scan results', false)
.option('--database-introspection-url <url>', 'Daemon URL for live-database introspection')
.option('--yes', 'Install the managed Python runtime without prompting when required', false)
.option('--no-input', 'Disable interactive managed runtime installation')
.showHelpAfterError()
.addHelpText(
'after',
@ -126,6 +129,8 @@ export function registerScanCommands(program: Command, context: KtxCliCommandCon
detectRelationships: mode === 'relationships',
dryRun: options.dryRun === true,
databaseIntrospectionUrl: options.databaseIntrospectionUrl,
cliVersion: context.packageInfo.version,
runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options),
});
});

5
packages/cli/src/commands/serve-commands.ts
View file

@ -1,5 +1,6 @@
import { type Command, InvalidArgumentError } from '@commander-js/extra-typings';
import { type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
import type { KtxServeArgs } from '../serve.js';
import { profileMark } from '../startup-profile.js';
@ -20,6 +21,8 @@ export function registerServeCommands(program: Command, context: KtxCliCommandCo
.option('--user-id <id>', 'Local user id', 'local')
.option('--semantic-compute', 'Enable semantic-layer compute', false)
.option('--semantic-compute-url <url>', 'HTTP semantic-layer compute URL')
.option('--yes', 'Install the managed Python runtime without prompting when required', false)
.option('--no-input', 'Disable interactive managed runtime installation')
.option('--database-introspection-url <url>', 'Daemon URL for live-database introspection')
.option('--execute-queries', 'Allow semantic-layer query execution', false)
.option('--memory-capture', 'Enable memory capture', false)
@ -40,6 +43,8 @@ export function registerServeCommands(program: Command, context: KtxCliCommandCo
executeQueries: options.executeQueries === true,
memoryCapture: options.memoryCapture === true,
memoryModel: options.memoryModel,
cliVersion: context.packageInfo.version,
runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options),
};
const runner = context.deps.serveStdio ?? (await import('../serve.js')).runKtxServeStdio;
context.setExitCode(await runner(args));

1
packages/cli/src/commands/setup-commands.ts
View file

@ -371,6 +371,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
skipAgents: options.skipAgents === true,
inputMode: options.input === false ? 'disabled' : 'auto',
yes: options.yes === true,
cliVersion: context.packageInfo.version,
...(options.anthropicApiKeyEnv ? { anthropicApiKeyEnv: options.anthropicApiKeyEnv } : {}),
...(options.anthropicApiKeyFile ? { anthropicApiKeyFile: options.anthropicApiKeyFile } : {}),
...(options.anthropicModel ? { anthropicModel: options.anthropicModel } : {}),

5
packages/cli/src/commands/sl-commands.ts
View file

@ -6,6 +6,7 @@ import {
resolveCommandProjectDir,
} from '../cli-program.js';
import { slQueryCommandSchema } from '../command-schemas.js';
import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
import type { KtxSlArgs } from '../sl.js';
import { profileMark } from '../startup-profile.js';
@ -121,6 +122,8 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte
.option('--include-empty', 'Include empty rows', false)
.addOption(new Option('--format <format>', 'json or sql').choices(['json', 'sql']).default('json'))
.option('--execute', 'Execute the compiled query', false)
.option('--yes', 'Install the managed Python runtime without prompting when required', false)
.option('--no-input', 'Disable interactive managed runtime installation')
.option('--max-rows <n>', 'Maximum rows to return when executing', parsePositiveIntegerOption)
.action(async (options, command) => {
if (options.measure.length === 0) {
@ -141,6 +144,8 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte
},
format: options.format,
execute: options.execute === true,
cliVersion: context.packageInfo.version,
runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options),
...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}),
});
await runSlArgs(context, args);

6
packages/cli/src/dev.test.ts
View file

@ -234,6 +234,8 @@ describe('dev Commander tree', () => {
detectRelationships: false,
dryRun: true,
databaseIntrospectionUrl: undefined,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'prompt',
},
scanIo.io,
);
@ -259,6 +261,8 @@ describe('dev Commander tree', () => {
detectRelationships: true,
dryRun: false,
databaseIntrospectionUrl: undefined,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'prompt',
},
io.io,
);
@ -661,6 +665,8 @@ describe('dev Commander tree', () => {
adapter: 'metabase',
sourceDir: undefined,
databaseIntrospectionUrl: undefined,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'prompt',
outputMode: 'json',
},
io.io,

414
packages/cli/src/index.test.ts
View file

@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
getKtxCliPackageInfo,
packageInfoFromJson,
rendererUnavailableVizFallback,
renderMemoryFlowTui,
resolveVizFallback,
@ -56,6 +57,19 @@ describe('getKtxCliPackageInfo', () => {
version: '0.0.0-private',
});
});
it('normalizes public package metadata from package.json contents', () => {
expect(
packageInfoFromJson({
name: '@kaelio/ktx',
version: '0.1.0',
}),
).toEqual({
name: '@kaelio/ktx',
version: '0.1.0',
contextPackageName: '@ktx/context',
});
});
});
describe('memory-flow renderer exports', () => {
@ -108,7 +122,7 @@ describe('runKtxCli', () => {
await expect(runKtxCli(['--help'], testIo.io)).resolves.toBe(0);
expect(testIo.stdout()).toContain('Usage: ktx [options] [command]');
for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'serve', 'status']) {
for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'runtime', 'serve', 'status']) {
expect(testIo.stdout()).toContain(`${command}`);
}
for (const removed of ['demo', 'init', 'connect', 'scan', 'ask', 'knowledge', 'agent', 'completion']) {
@ -124,6 +138,151 @@ describe('runKtxCli', () => {
expect(testIo.stderr()).toBe('');
});
it('routes runtime management commands with the CLI package version', async () => {
const runtime = vi.fn(async () => 0);
const installIo = makeIo();
const startIo = makeIo();
const stopIo = makeIo();
const statusIo = makeIo();
const doctorIo = makeIo();
const pruneIo = makeIo();
await expect(
runKtxCli(['runtime', 'install', '--feature', 'local-embeddings', '--force', '--yes'], installIo.io, {
runtime,
}),
).resolves.toBe(0);
await expect(
runKtxCli(['runtime', 'start', '--feature', 'local-embeddings', '--force'], startIo.io, { runtime }),
).resolves.toBe(0);
await expect(runKtxCli(['runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0);
await expect(runKtxCli(['runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0);
await expect(runKtxCli(['runtime', 'doctor'], doctorIo.io, { runtime })).resolves.toBe(0);
await expect(runKtxCli(['runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(0);
expect(runtime).toHaveBeenNthCalledWith(
1,
{
command: 'install',
cliVersion: '0.0.0-private',
feature: 'local-embeddings',
force: true,
},
installIo.io,
);
expect(runtime).toHaveBeenNthCalledWith(
2,
{
command: 'start',
cliVersion: '0.0.0-private',
feature: 'local-embeddings',
force: true,
},
startIo.io,
);
expect(runtime).toHaveBeenNthCalledWith(
3,
{
command: 'stop',
cliVersion: '0.0.0-private',
},
stopIo.io,
);
expect(runtime).toHaveBeenNthCalledWith(
4,
{
command: 'status',
cliVersion: '0.0.0-private',
json: true,
},
statusIo.io,
);
expect(runtime).toHaveBeenNthCalledWith(
5,
{
command: 'doctor',
cliVersion: '0.0.0-private',
json: false,
},
doctorIo.io,
);
expect(runtime).toHaveBeenNthCalledWith(
6,
{
command: 'prune',
cliVersion: '0.0.0-private',
dryRun: true,
yes: false,
},
pruneIo.io,
);
});
it('routes sl query managed runtime install policies', async () => {
const sl = vi.fn(async () => 0);
const promptIo = makeIo();
await expect(
runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count'], promptIo.io, { sl }),
).resolves.toBe(0);
expect(sl).toHaveBeenLastCalledWith(
expect.objectContaining({
command: 'query',
projectDir: tempDir,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'prompt',
query: expect.objectContaining({ measures: ['orders.order_count'], dimensions: [] }),
}),
promptIo.io,
);
const autoIo = makeIo();
await expect(
runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes'], autoIo.io, {
sl,
}),
).resolves.toBe(0);
expect(sl).toHaveBeenLastCalledWith(
expect.objectContaining({
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'auto',
}),
autoIo.io,
);
const noInputIo = makeIo();
await expect(
runKtxCli(
['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--no-input'],
noInputIo.io,
{ sl },
),
).resolves.toBe(0);
expect(sl).toHaveBeenLastCalledWith(
expect.objectContaining({
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'never',
}),
noInputIo.io,
);
});
it('rejects conflicting sl query runtime install flags', async () => {
const io = makeIo();
const sl = vi.fn(async () => 0);
await expect(
runKtxCli(
['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes', '--no-input'],
io.io,
{ sl },
),
).resolves.toBe(1);
expect(sl).not.toHaveBeenCalled();
expect(io.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
});
it('exposes demo under setup help instead of root help', async () => {
const testIo = makeIo();
@ -179,6 +338,7 @@ describe('runKtxCli', () => {
skipAgents: false,
inputMode: 'auto',
yes: false,
cliVersion: '0.0.0-private',
skipLlm: false,
skipEmbeddings: false,
databaseSchemas: [],
@ -439,6 +599,8 @@ describe('runKtxCli', () => {
executeQueries: false,
memoryCapture: false,
memoryModel: undefined,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'prompt',
});
});
@ -757,6 +919,8 @@ describe('runKtxCli', () => {
adapter: 'fake',
sourceDir: tempDir,
databaseIntrospectionUrl: undefined,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'never',
debugLlmRequestFile: `${tempDir}/debug.jsonl`,
outputMode: 'json',
inputMode: 'disabled',
@ -770,6 +934,60 @@ describe('runKtxCli', () => {
expect(ingestReplayHelpIo.stderr()).toBe('');
});
it('routes ingest managed runtime install policies', async () => {
const autoIo = makeIo();
const conflictIo = makeIo();
const ingest = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'dev',
'ingest',
'run',
'--project-dir',
tempDir,
'--connection-id',
'warehouse',
'--adapter',
'looker',
'--yes',
],
autoIo.io,
{ ingest },
),
).resolves.toBe(0);
await expect(
runKtxCli(
[
'dev',
'ingest',
'run',
'--project-dir',
tempDir,
'--connection-id',
'warehouse',
'--adapter',
'looker',
'--yes',
'--no-input',
],
conflictIo.io,
{ ingest },
),
).resolves.toBe(1);
expect(ingest).toHaveBeenCalledWith(
expect.objectContaining({
command: 'run',
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'auto',
}),
autoIo.io,
);
expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
});
it('dispatches public connection through the existing connection implementation', async () => {
const testIo = makeIo();
const connection = vi.fn(async () => 0);
@ -870,6 +1088,7 @@ describe('runKtxCli', () => {
command: 'run',
projectDir: tempDir,
inputMode: 'disabled',
cliVersion: '0.0.0-private',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
@ -977,6 +1196,7 @@ describe('runKtxCli', () => {
projectDir: '/tmp/project',
inputMode: 'disabled',
yes: true,
cliVersion: '0.0.0-private',
skipLlm: true,
skipEmbeddings: true,
databaseDrivers: ['postgres'],
@ -1239,6 +1459,8 @@ describe('runKtxCli', () => {
queryFile: '/tmp/query.json',
execute: true,
maxRows: 100,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'prompt',
},
},
{
@ -1287,6 +1509,104 @@ describe('runKtxCli', () => {
expect(helpIo.stdout()).not.toContain('agent ');
});
it('routes hidden agent SL query managed runtime policies', async () => {
const autoIo = makeIo();
const neverIo = makeIo();
const conflictIo = makeIo();
const agent = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'--project-dir',
tempDir,
'agent',
'sl',
'query',
'--json',
'--connection-id',
'warehouse',
'--query-file',
'/tmp/query.json',
'--yes',
],
autoIo.io,
{ agent },
),
).resolves.toBe(0);
await expect(
runKtxCli(
[
'--project-dir',
tempDir,
'agent',
'sl',
'query',
'--json',
'--connection-id',
'warehouse',
'--query-file',
'/tmp/query.json',
'--no-input',
],
neverIo.io,
{ agent },
),
).resolves.toBe(0);
await expect(
runKtxCli(
[
'--project-dir',
tempDir,
'agent',
'sl',
'query',
'--json',
'--connection-id',
'warehouse',
'--query-file',
'/tmp/query.json',
'--yes',
'--no-input',
],
conflictIo.io,
{ agent },
),
).resolves.toBe(1);
expect(agent).toHaveBeenNthCalledWith(
1,
{
command: 'sl-query',
projectDir: tempDir,
json: true,
connectionId: 'warehouse',
queryFile: '/tmp/query.json',
execute: false,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'auto',
},
autoIo.io,
);
expect(agent).toHaveBeenNthCalledWith(
2,
{
command: 'sl-query',
projectDir: tempDir,
json: true,
connectionId: 'warehouse',
queryFile: '/tmp/query.json',
execute: false,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'never',
},
neverIo.io,
);
expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
});
it('prints semantic-layer hybrid search metadata from the hidden agent sl list command', async () => {
const agent = vi.fn(async (args, io) => {
expect(args).toEqual({
@ -1797,11 +2117,48 @@ describe('runKtxCli', () => {
detectRelationships: false,
dryRun: false,
databaseIntrospectionUrl: undefined,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'prompt',
},
testIo.io,
);
});
it('routes scan managed runtime install policies', async () => {
const autoIo = makeIo();
const neverIo = makeIo();
const conflictIo = makeIo();
const scan = vi.fn().mockResolvedValue(0);
await expect(runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--yes'], autoIo.io, { scan }))
.resolves.toBe(0);
await expect(runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--no-input'], neverIo.io, { scan }))
.resolves.toBe(0);
await expect(
runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--yes', '--no-input'], conflictIo.io, {
scan,
}),
).resolves.toBe(1);
expect(scan).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
command: 'run',
runtimeInstallPolicy: 'auto',
}),
autoIo.io,
);
expect(scan).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
command: 'run',
runtimeInstallPolicy: 'never',
}),
neverIo.io,
);
expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
});
it('dispatches serve public command options through Commander', async () => {
const serveIo = makeIo();
const serveStdio = vi.fn(async () => 0);
@ -1836,10 +2193,65 @@ describe('runKtxCli', () => {
executeQueries: true,
memoryCapture: true,
memoryModel: 'openai/gpt-5.2',
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'prompt',
});
expect(serveIo.stderr()).toBe('');
});
it('routes serve managed runtime install policies', async () => {
const autoIo = makeIo();
const neverIo = makeIo();
const conflictIo = makeIo();
const serveStdio = vi.fn(async () => 0);
await expect(
runKtxCli(['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--yes'], autoIo.io, {
serveStdio,
}),
).resolves.toBe(0);
await expect(
runKtxCli(['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--no-input'], neverIo.io, {
serveStdio,
}),
).resolves.toBe(0);
await expect(
runKtxCli(
['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--yes', '--no-input'],
conflictIo.io,
{ serveStdio },
),
).resolves.toBe(1);
expect(serveStdio).toHaveBeenNthCalledWith(1, {
mcp: 'stdio',
projectDir: tempDir,
userId: 'local',
semanticCompute: true,
semanticComputeUrl: undefined,
databaseIntrospectionUrl: undefined,
executeQueries: false,
memoryCapture: false,
memoryModel: undefined,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'auto',
});
expect(serveStdio).toHaveBeenNthCalledWith(2, {
mcp: 'stdio',
projectDir: tempDir,
userId: 'local',
semanticCompute: true,
semanticComputeUrl: undefined,
databaseIntrospectionUrl: undefined,
executeQueries: false,
memoryCapture: false,
memoryModel: undefined,
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'never',
});
expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
});
it('prints dev help for bare dev commands', async () => {
const testIo = makeIo();

21
packages/cli/src/index.ts
View file

@ -2,6 +2,7 @@ import { profileMark } from './startup-profile.js';
export {
getKtxCliPackageInfo,
packageInfoFromJson,
runInitForCommander,
runKtxCli,
type KtxCliDeps,
@ -42,6 +43,26 @@ export type {
KtxSetupSourceType,
} from './setup-sources.js';
export { runKtxSetupSourcesStep } from './setup-sources.js';
export { runKtxRuntime, type KtxRuntimeArgs, type KtxRuntimeDeps } from './runtime.js';
export {
allocateDaemonPort,
readManagedPythonDaemonStatus,
startManagedPythonDaemon,
stopManagedPythonDaemon,
} from './managed-python-daemon.js';
export type {
ManagedPythonDaemonStartResult,
ManagedPythonDaemonState,
ManagedPythonDaemonStatus,
ManagedPythonDaemonStopResult,
} from './managed-python-daemon.js';
export {
ensureManagedLocalEmbeddingsDaemon,
managedLocalEmbeddingHealthConfig,
managedLocalEmbeddingProjectConfig,
type ManagedLocalEmbeddingsDaemon,
type ManagedLocalEmbeddingsOptions,
} from './managed-local-embeddings.js';
export type { KtxMemoryFlowTuiIo, MemoryFlowTuiLiveSession } from './memory-flow-tui.js';
export {
renderMemoryFlowTui,

55
packages/cli/src/ingest.test.ts
View file

@ -11,7 +11,7 @@ import {
type RunLocalIngestOptions,
type SourceAdapter,
} from '@ktx/context/ingest';
import { ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project';
import { initKtxProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { type KtxIngestArgs, runKtxIngest } from './ingest.js';
import {
@ -644,6 +644,59 @@ describe('runKtxIngest', () => {
adapters: createdAdapters,
adapter: 'fake',
connectionId: 'warehouse',
pullConfigOptions: {
databaseIntrospectionUrl: 'http://127.0.0.1:8765',
},
}),
);
});
it('passes managed daemon options to adapters and pull-config options when no explicit daemon URL is set', async () => {
const projectDir = join(tempDir, 'managed-daemon-ingest-project');
await initKtxProject({ projectDir, projectName: 'managed-daemon-ingest-project' });
await writeWarehouseConfig(projectDir);
const createdAdapters: SourceAdapter[] = [
{ source: 'fake', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) },
];
const createAdapters = vi.fn(() => createdAdapters as never);
const runLocal = vi.fn(async (input: RunLocalIngestOptions) =>
completedLocalBundleRun(input, input.jobId ?? 'local-job-1'),
);
const io = makeIo();
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'warehouse',
adapter: 'fake',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
outputMode: 'plain',
} satisfies KtxIngestArgs,
io.io,
{
createAdapters,
runLocalIngest: runLocal,
jobIdFactory: () => 'local-job-1',
},
),
).resolves.toBe(0);
const expectedManagedDaemon = {
cliVersion: '0.2.0',
installPolicy: 'auto',
io: io.io,
};
expect(createAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), {
managedDaemon: expectedManagedDaemon,
});
expect(runLocal).toHaveBeenCalledWith(
expect.objectContaining({
pullConfigOptions: {
managedDaemon: expectedManagedDaemon,
},
}),
);
});

20
packages/cli/src/ingest.ts
View file

@ -17,6 +17,7 @@ import {
import { loadKtxProject } from '@ktx/context/project';
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
import { type KtxMemoryFlowStdin, renderMemoryFlowInteractively } from './memory-flow-interactive.js';
import {
type KtxMemoryFlowTuiIo,
@ -40,6 +41,8 @@ export type KtxIngestArgs =
adapter: string;
sourceDir?: string;
databaseIntrospectionUrl?: string;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
debugLlmRequestFile?: string;
outputMode: KtxIngestOutputMode;
inputMode?: KtxIngestInputMode;
@ -256,6 +259,20 @@ function initialRunMemoryFlowInput(
};
}
function managedDaemonOptionsForIngestRun(
args: Extract<KtxIngestArgs, { command: 'run' }>,
io: KtxIngestIo,
) {
if (args.databaseIntrospectionUrl || !args.cliVersion || !args.runtimeInstallPolicy) {
return undefined;
}
return {
cliVersion: args.cliVersion,
installPolicy: args.runtimeInstallPolicy,
io,
};
}
async function writeReportRecord(
report: IngestReportSnapshot,
outputMode: KtxIngestOutputMode,
@ -311,9 +328,11 @@ export async function runKtxIngest(
const createAdapters = deps.createAdapters ?? createKtxCliLocalIngestAdapters;
const executeLocalIngest = deps.runLocalIngest ?? runLocalIngest;
const localIngestOptions = deps.localIngestOptions ?? {};
const managedDaemon = managedDaemonOptionsForIngestRun(args, io);
const adapterOptions = {
...(localIngestOptions.pullConfigOptions ?? {}),
...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}),
...(managedDaemon ? { managedDaemon } : {}),
...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}),
};
if (args.adapter === 'metabase' && args.sourceDir) {
@ -380,6 +399,7 @@ export async function runKtxIngest(
trigger: 'manual_resync',
jobId,
...localIngestOptions,
pullConfigOptions: adapterOptions,
...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}),
...(memoryFlow ? { memoryFlow } : {}),
});

65
packages/cli/src/local-adapters.ts
View file

@ -20,6 +20,12 @@ import {
} from '@ktx/context/ingest';
import type { KtxLocalProject } from '@ktx/context/project';
import { createHttpSqlAnalysisPort } from '@ktx/context/sql-analysis';
import {
createManagedDaemonLookerTableIdentifierParser,
createManagedDaemonSqlAnalysisPort,
managedDaemonDatabaseIntrospectionOptions,
type ManagedPythonCoreDaemonOptions,
} from './managed-python-http.js';
function hasSnowflakeDriver(connection: unknown): boolean {
return (
@ -29,13 +35,55 @@ function hasSnowflakeDriver(connection: unknown): boolean {
);
}
function ktxCliDaemonDatabaseIntrospectionOptions(
options: KtxCliLocalIngestAdaptersOptions,
): DefaultLocalIngestAdaptersOptions['databaseIntrospection'] {
if (options.databaseIntrospectionUrl || options.databaseIntrospection?.requestJson || !options.managedDaemon) {
return options.databaseIntrospection;
}
return {
...(options.databaseIntrospection ?? {}),
...managedDaemonDatabaseIntrospectionOptions(options.managedDaemon),
};
}
function ktxCliLookerOptions(
options: KtxCliLocalIngestAdaptersOptions,
): DefaultLocalIngestAdaptersOptions['looker'] {
const looker = options.looker;
if (looker?.parser || looker?.daemonBaseUrl || process.env.KTX_DAEMON_URL || !options.managedDaemon) {
return looker;
}
return {
...(looker ?? {}),
parser: createManagedDaemonLookerTableIdentifierParser(options.managedDaemon),
};
}
function ktxCliHistoricSqlAnalysis(options: KtxCliLocalIngestAdaptersOptions) {
if (options.sqlAnalysisUrl) {
return createHttpSqlAnalysisPort({ baseUrl: options.sqlAnalysisUrl });
}
if (process.env.KTX_SQL_ANALYSIS_URL) {
return createHttpSqlAnalysisPort({ baseUrl: process.env.KTX_SQL_ANALYSIS_URL });
}
if (process.env.KTX_DAEMON_URL) {
return createHttpSqlAnalysisPort({ baseUrl: process.env.KTX_DAEMON_URL });
}
if (options.managedDaemon) {
return createManagedDaemonSqlAnalysisPort(options.managedDaemon);
}
return createHttpSqlAnalysisPort({ baseUrl: 'http://127.0.0.1:8765' });
}
function createKtxCliLiveDatabaseIntrospection(
project: KtxLocalProject,
options: DefaultLocalIngestAdaptersOptions = {},
options: KtxCliLocalIngestAdaptersOptions = {},
): LiveDatabaseIntrospectionPort {
const databaseIntrospection = ktxCliDaemonDatabaseIntrospectionOptions(options);
const daemon = createDaemonLiveDatabaseIntrospection({
connections: project.config.connections,
...options.databaseIntrospection,
...databaseIntrospection,
...(options.databaseIntrospectionUrl ? { baseUrl: options.databaseIntrospectionUrl } : {}),
});
const sqlite = createSqliteLiveDatabaseIntrospection({
@ -95,9 +143,10 @@ function createKtxCliLiveDatabaseIntrospection(
};
}
interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions {
export interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions {
historicSqlConnectionId?: string;
sqlAnalysisUrl?: string;
managedDaemon?: ManagedPythonCoreDaemonOptions;
}
function isEnabledPostgresHistoricSqlConnection(connection: KtxPostgresConnectionConfig | undefined): boolean {
@ -145,13 +194,7 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli
return undefined;
}
return {
sqlAnalysis: createHttpSqlAnalysisPort({
baseUrl:
options.sqlAnalysisUrl ??
process.env.KTX_SQL_ANALYSIS_URL ??
process.env.KTX_DAEMON_URL ??
'http://127.0.0.1:8765',
}),
sqlAnalysis: ktxCliHistoricSqlAnalysis(options),
postgresQueryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId),
postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'),
};
@ -164,6 +207,8 @@ export function createKtxCliLocalIngestAdapters(
const historicSql = historicSqlOptionsForLocalRun(project, options);
const base = createDefaultLocalIngestAdapters(project, {
...options,
databaseIntrospection: ktxCliDaemonDatabaseIntrospectionOptions(options),
looker: ktxCliLookerOptions(options),
...(historicSql ? { historicSql } : {}),
});
const liveDatabase = new LiveDatabaseSourceAdapter({

180
packages/cli/src/managed-local-embeddings.test.ts Normal file
View file

@ -0,0 +1,180 @@
import { describe, expect, it, vi } from 'vitest';
import {
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV,
} from '@ktx/context';
import {
ensureManagedLocalEmbeddingsDaemon,
managedLocalEmbeddingHealthConfig,
managedLocalEmbeddingProjectConfig,
} from './managed-local-embeddings.js';
import type { ManagedPythonCommandRuntime } from './managed-python-command.js';
import type { ManagedPythonDaemonStartResult } from './managed-python-daemon.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
function runtime(): ManagedPythonCommandRuntime {
return {
layout: {
cliVersion: '0.2.0',
runtimeRoot: '/runtime',
versionDir: '/runtime/0.2.0',
venvDir: '/runtime/0.2.0/.venv',
manifestPath: '/runtime/0.2.0/manifest.json',
installLogPath: '/runtime/0.2.0/install.log',
assetDir: '/assets/python',
assetManifestPath: '/assets/python/manifest.json',
pythonPath: '/runtime/0.2.0/.venv/bin/python',
daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon',
daemonStatePath: '/runtime/0.2.0/daemon.json',
daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log',
daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log',
},
manifest: {
schemaVersion: 1,
cliVersion: '0.2.0',
installedAt: '2026-05-11T00:00:00.000Z',
asset: {
schemaVersion: 1,
distributionName: 'kaelio-ktx',
normalizedName: 'kaelio_ktx',
version: '0.2.0',
wheel: {
file: 'kaelio_ktx-0.2.0-py3-none-any.whl',
sha256: 'a'.repeat(64),
bytes: 123,
},
},
features: ['core', 'local-embeddings'],
python: {
executable: '/runtime/0.2.0/.venv/bin/python',
daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon',
},
installLog: '/runtime/0.2.0/install.log',
},
};
}
function daemonResult(status: 'started' | 'reused' = 'reused'): ManagedPythonDaemonStartResult {
return {
status,
layout: runtime().layout,
baseUrl: 'http://127.0.0.1:61234',
state: {
schemaVersion: 1,
pid: 12345,
host: '127.0.0.1',
port: 61234,
version: '0.2.0',
features: ['core', 'local-embeddings'],
startedAt: '2026-05-11T00:00:00.000Z',
stdoutLog: '/runtime/0.2.0/daemon.stdout.log',
stderrLog: '/runtime/0.2.0/daemon.stderr.log',
},
};
}
describe('managedLocalEmbeddingProjectConfig', () => {
it('uses a stable managed runtime marker instead of a random daemon port', () => {
expect(
managedLocalEmbeddingProjectConfig({
model: 'all-MiniLM-L6-v2',
dimensions: 384,
}),
).toEqual({
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: {
base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
pathPrefix: '',
},
});
});
});
describe('managedLocalEmbeddingHealthConfig', () => {
it('uses the active managed daemon URL for the immediate health check', () => {
expect(
managedLocalEmbeddingHealthConfig({
baseUrl: 'http://127.0.0.1:61234',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
}),
).toEqual({
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' },
});
});
});
describe('ensureManagedLocalEmbeddingsDaemon', () => {
it('ensures the local-embeddings feature and starts the managed daemon', async () => {
const io = makeIo();
const ensureRuntime = vi.fn(async () => runtime());
const startDaemon = vi.fn(async () => daemonResult('started'));
await expect(
ensureManagedLocalEmbeddingsDaemon({
cliVersion: '0.2.0',
installPolicy: 'auto',
io: io.io,
ensureRuntime,
startDaemon,
}),
).resolves.toEqual({
baseUrl: 'http://127.0.0.1:61234',
env: {
[MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: 'http://127.0.0.1:61234',
},
});
expect(ensureRuntime).toHaveBeenCalledWith({
cliVersion: '0.2.0',
installPolicy: 'auto',
io: io.io,
feature: 'local-embeddings',
});
expect(startDaemon).toHaveBeenCalledWith({
cliVersion: '0.2.0',
features: ['local-embeddings'],
force: false,
});
expect(io.stderr()).toContain('Started KTX local embeddings daemon: http://127.0.0.1:61234');
});
it('reuses an already running daemon without reporting a new start', async () => {
const io = makeIo();
await ensureManagedLocalEmbeddingsDaemon({
cliVersion: '0.2.0',
installPolicy: 'prompt',
io: io.io,
ensureRuntime: vi.fn(async () => runtime()),
startDaemon: vi.fn(async () => daemonResult('reused')),
});
expect(io.stderr()).toContain('Using KTX local embeddings daemon: http://127.0.0.1:61234');
});
});

95
packages/cli/src/managed-local-embeddings.ts Normal file
View file

@ -0,0 +1,95 @@
import {
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV,
} from '@ktx/context';
import type { KtxProjectEmbeddingConfig } from '@ktx/context/project';
import type { KtxEmbeddingConfig } from '@ktx/llm';
import type { KtxCliIo } from './cli-runtime.js';
import {
ensureManagedPythonCommandRuntime,
type KtxManagedPythonInstallPolicy,
type ManagedPythonCommandRuntime,
} from './managed-python-command.js';
import { startManagedPythonDaemon, type ManagedPythonDaemonStartResult } from './managed-python-daemon.js';
export interface ManagedLocalEmbeddingsDaemon {
baseUrl: string;
env: Record<typeof MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, string>;
}
export interface ManagedLocalEmbeddingsOptions {
cliVersion: string;
installPolicy: KtxManagedPythonInstallPolicy;
io: KtxCliIo;
ensureRuntime?: (options: {
cliVersion: string;
installPolicy: KtxManagedPythonInstallPolicy;
io: KtxCliIo;
feature: 'local-embeddings';
}) => Promise<ManagedPythonCommandRuntime>;
startDaemon?: (options: {
cliVersion: string;
features: ['local-embeddings'];
force: boolean;
}) => Promise<ManagedPythonDaemonStartResult>;
}
export function managedLocalEmbeddingProjectConfig(input: {
model: string;
dimensions: number;
}): KtxProjectEmbeddingConfig {
return {
backend: 'sentence-transformers',
model: input.model,
dimensions: input.dimensions,
sentenceTransformers: {
base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
pathPrefix: '',
},
};
}
export function managedLocalEmbeddingHealthConfig(input: {
baseUrl: string;
model: string;
dimensions: number;
}): KtxEmbeddingConfig {
return {
backend: 'sentence-transformers',
model: input.model,
dimensions: input.dimensions,
sentenceTransformers: {
baseURL: input.baseUrl,
pathPrefix: '',
},
};
}
export async function ensureManagedLocalEmbeddingsDaemon(
options: ManagedLocalEmbeddingsOptions,
): Promise<ManagedLocalEmbeddingsDaemon> {
const ensureRuntime = options.ensureRuntime ?? ensureManagedPythonCommandRuntime;
const startDaemon = options.startDaemon ?? startManagedPythonDaemon;
await ensureRuntime({
cliVersion: options.cliVersion,
installPolicy: options.installPolicy,
io: options.io,
feature: 'local-embeddings',
});
const daemon = await startDaemon({
cliVersion: options.cliVersion,
features: ['local-embeddings'],
force: false,
});
const verb = daemon.status === 'started' ? 'Started' : 'Using';
options.io.stderr.write(`${verb} KTX local embeddings daemon: ${daemon.baseUrl}\n`);
return {
baseUrl: daemon.baseUrl,
env: {
[MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: daemon.baseUrl,
},
};
}

224
packages/cli/src/managed-python-command.test.ts Normal file
View file

@ -0,0 +1,224 @@
import { describe, expect, it, vi } from 'vitest';
import {
createManagedPythonSemanticLayerComputePort,
managedRuntimeInstallCommand,
runtimeInstallPolicyFromFlags,
} from './managed-python-command.js';
import type {
InstalledKtxRuntimeManifest,
KtxRuntimeFeature,
ManagedPythonRuntimeInstallResult,
ManagedPythonRuntimeLayout,
ManagedPythonRuntimeStatus,
} from './managed-python-runtime.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
function layout(): ManagedPythonRuntimeLayout {
return {
cliVersion: '0.2.0',
runtimeRoot: '/runtime',
versionDir: '/runtime/0.2.0',
venvDir: '/runtime/0.2.0/.venv',
manifestPath: '/runtime/0.2.0/manifest.json',
installLogPath: '/runtime/0.2.0/install.log',
assetDir: '/assets/python',
assetManifestPath: '/assets/python/manifest.json',
pythonPath: '/runtime/0.2.0/.venv/bin/python',
daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon',
daemonStatePath: '/runtime/0.2.0/daemon.json',
daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log',
daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log',
};
}
function manifest(features: KtxRuntimeFeature[] = ['core']): InstalledKtxRuntimeManifest {
return {
schemaVersion: 1,
cliVersion: '0.2.0',
installedAt: '2026-05-11T00:00:00.000Z',
asset: {
schemaVersion: 1,
distributionName: 'kaelio-ktx',
normalizedName: 'kaelio_ktx',
version: '0.2.0',
wheel: {
file: 'kaelio_ktx-0.2.0-py3-none-any.whl',
sha256: 'a'.repeat(64),
bytes: 123,
},
},
features,
python: {
executable: '/runtime/0.2.0/.venv/bin/python',
daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon',
},
installLog: '/runtime/0.2.0/install.log',
};
}
function readyStatus(features: KtxRuntimeFeature[] = ['core']): ManagedPythonRuntimeStatus {
return {
kind: 'ready',
detail: 'Runtime ready at /runtime/0.2.0',
layout: layout(),
manifest: manifest(features),
};
}
function missingStatus(): ManagedPythonRuntimeStatus {
return {
kind: 'missing',
detail: 'No runtime manifest at /runtime/0.2.0/manifest.json',
layout: layout(),
};
}
function installResult(features: KtxRuntimeFeature[] = ['core']): ManagedPythonRuntimeInstallResult {
const installedManifest = manifest(features);
return {
status: 'installed',
layout: layout(),
asset: {
manifest: installedManifest.asset,
wheelPath: '/assets/python/kaelio_ktx-0.2.0-py3-none-any.whl',
},
manifest: installedManifest,
};
}
describe('managedRuntimeInstallCommand', () => {
it('prints the exact command for each managed runtime feature', () => {
expect(managedRuntimeInstallCommand('core')).toBe('ktx runtime install --yes');
expect(managedRuntimeInstallCommand('local-embeddings')).toBe(
'ktx runtime install --feature local-embeddings --yes',
);
});
});
describe('runtimeInstallPolicyFromFlags', () => {
it('maps command flags to managed runtime install policies', () => {
expect(runtimeInstallPolicyFromFlags({})).toBe('prompt');
expect(runtimeInstallPolicyFromFlags({ yes: false })).toBe('prompt');
expect(runtimeInstallPolicyFromFlags({ yes: true })).toBe('auto');
expect(runtimeInstallPolicyFromFlags({ input: false })).toBe('never');
});
it('rejects conflicting runtime install flags', () => {
expect(() => runtimeInstallPolicyFromFlags({ yes: true, input: false })).toThrow(
'Choose only one runtime install mode: --yes or --no-input',
);
});
});
describe('createManagedPythonSemanticLayerComputePort', () => {
it('uses the managed ktx-daemon executable when the runtime is ready', async () => {
const io = makeIo();
const compute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };
const createPythonCompute = vi.fn(() => compute);
await expect(
createManagedPythonSemanticLayerComputePort({
cliVersion: '0.2.0',
installPolicy: 'never',
io: io.io,
readStatus: vi.fn(async () => readyStatus()),
installRuntime: vi.fn(),
createPythonCompute,
}),
).resolves.toBe(compute);
expect(createPythonCompute).toHaveBeenCalledWith({
command: '/runtime/0.2.0/.venv/bin/ktx-daemon',
args: [],
});
expect(io.stderr()).toBe('');
});
it('fails with a preparation command when input is disabled and the runtime is missing', async () => {
const io = makeIo();
const installRuntime = vi.fn();
await expect(
createManagedPythonSemanticLayerComputePort({
cliVersion: '0.2.0',
installPolicy: 'never',
io: io.io,
readStatus: vi.fn(async () => missingStatus()),
installRuntime,
}),
).rejects.toThrow('KTX Python runtime is required for this command. Run: ktx runtime install --yes');
expect(installRuntime).not.toHaveBeenCalled();
});
it('installs the core runtime without prompting when policy is auto', async () => {
const io = makeIo();
const compute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };
const createPythonCompute = vi.fn(() => compute);
const installRuntime = vi.fn(async () => installResult());
await expect(
createManagedPythonSemanticLayerComputePort({
cliVersion: '0.2.0',
installPolicy: 'auto',
io: io.io,
readStatus: vi.fn(async () => missingStatus()),
installRuntime,
createPythonCompute,
}),
).resolves.toBe(compute);
expect(installRuntime).toHaveBeenCalledWith({
cliVersion: '0.2.0',
features: ['core'],
force: false,
});
expect(io.stderr()).toContain('Installing KTX Python runtime (core) with uv');
expect(io.stderr()).toContain('KTX Python runtime ready: /runtime/0.2.0');
});
it('prompts before installing when policy is prompt', async () => {
const io = makeIo();
const confirmInstall = vi.fn(async () => true);
const installRuntime = vi.fn(async () => installResult());
await createManagedPythonSemanticLayerComputePort({
cliVersion: '0.2.0',
installPolicy: 'prompt',
io: io.io,
readStatus: vi.fn(async () => missingStatus()),
installRuntime,
createPythonCompute: vi.fn(() => ({ query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() })),
confirmInstall,
});
expect(confirmInstall).toHaveBeenCalledWith(
'KTX needs to install the core Python runtime. This downloads Python dependencies with uv. Continue?',
);
expect(installRuntime).toHaveBeenCalledWith({
cliVersion: '0.2.0',
features: ['core'],
force: false,
});
});
});

135
packages/cli/src/managed-python-command.ts Normal file
View file

@ -0,0 +1,135 @@
import { cancel, confirm, isCancel } from '@clack/prompts';
import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon';
import type { KtxCliIo } from './cli-runtime.js';
import {
installManagedPythonRuntime,
readManagedPythonRuntimeStatus,
type InstalledKtxRuntimeManifest,
type KtxRuntimeFeature,
type ManagedPythonRuntimeInstallOptions,
type ManagedPythonRuntimeInstallResult,
type ManagedPythonRuntimeLayout,
type ManagedPythonRuntimeLayoutOptions,
type ManagedPythonRuntimeStatus,
} from './managed-python-runtime.js';
export type KtxManagedPythonInstallPolicy = 'prompt' | 'auto' | 'never';
export function runtimeInstallPolicyFromFlags(options: {
yes?: boolean;
input?: boolean;
}): KtxManagedPythonInstallPolicy {
if (options.yes === true && options.input === false) {
throw new Error('Choose only one runtime install mode: --yes or --no-input');
}
if (options.yes === true) {
return 'auto';
}
return options.input === false ? 'never' : 'prompt';
}
export interface ManagedPythonCommandRuntime {
layout: ManagedPythonRuntimeLayout;
manifest: InstalledKtxRuntimeManifest;
}
export interface ManagedPythonCommandDeps {
readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise<ManagedPythonRuntimeStatus>;
installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise<ManagedPythonRuntimeInstallResult>;
confirmInstall?: (message: string) => Promise<boolean>;
}
export interface ManagedPythonCommandOptions extends ManagedPythonCommandDeps {
cliVersion: string;
installPolicy: KtxManagedPythonInstallPolicy;
io: KtxCliIo;
feature?: KtxRuntimeFeature;
}
export interface ManagedPythonSemanticLayerComputeOptions extends ManagedPythonCommandOptions {
createPythonCompute?: typeof createPythonSemanticLayerComputePort;
}
export function managedRuntimeInstallCommand(feature: KtxRuntimeFeature): string {
return feature === 'local-embeddings'
? 'ktx runtime install --feature local-embeddings --yes'
: 'ktx runtime install --yes';
}
function installPrompt(feature: KtxRuntimeFeature): string {
const label = feature === 'local-embeddings' ? 'local embeddings Python runtime' : 'core Python runtime';
return `KTX needs to install the ${label}. This downloads Python dependencies with uv. Continue?`;
}
function runtimeRequiredMessage(feature: KtxRuntimeFeature): string {
return `KTX Python runtime is required for this command. Run: ${managedRuntimeInstallCommand(feature)}`;
}
function hasFeature(manifest: InstalledKtxRuntimeManifest, feature: KtxRuntimeFeature): boolean {
return manifest.features.includes(feature);
}
async function defaultConfirmInstall(message: string): Promise<boolean> {
if (process.stdin.isTTY !== true || process.stdout.isTTY !== true) {
return false;
}
const response = await confirm({ message, initialValue: true });
if (isCancel(response)) {
cancel('Runtime installation cancelled.');
return false;
}
return response === true;
}
export async function ensureManagedPythonCommandRuntime(
options: ManagedPythonCommandOptions,
): Promise<ManagedPythonCommandRuntime> {
const feature = options.feature ?? 'core';
const readStatus = options.readStatus ?? readManagedPythonRuntimeStatus;
const installRuntime = options.installRuntime ?? installManagedPythonRuntime;
const status = await readStatus({ cliVersion: options.cliVersion });
if (status.kind === 'ready' && status.manifest && hasFeature(status.manifest, feature)) {
return { layout: status.layout, manifest: status.manifest };
}
if (options.installPolicy === 'never') {
throw new Error(runtimeRequiredMessage(feature));
}
if (options.installPolicy === 'prompt') {
const confirmInstall = options.confirmInstall ?? defaultConfirmInstall;
const confirmed = await confirmInstall(installPrompt(feature));
if (!confirmed) {
throw new Error(`KTX Python runtime installation was cancelled. Run: ${managedRuntimeInstallCommand(feature)}`);
}
}
options.io.stderr.write(`Installing KTX Python runtime (${feature}) with uv...\n`);
const installed = await installRuntime({
cliVersion: options.cliVersion,
features: [feature],
force: false,
});
options.io.stderr.write(`KTX Python runtime ready: ${installed.layout.versionDir}\n`);
return { layout: installed.layout, manifest: installed.manifest };
}
export async function createManagedPythonSemanticLayerComputePort(
options: ManagedPythonSemanticLayerComputeOptions,
): Promise<KtxSemanticLayerComputePort> {
const runtime = await ensureManagedPythonCommandRuntime({
cliVersion: options.cliVersion,
installPolicy: options.installPolicy,
io: options.io,
feature: 'core',
...(options.readStatus ? { readStatus: options.readStatus } : {}),
...(options.installRuntime ? { installRuntime: options.installRuntime } : {}),
...(options.confirmInstall ? { confirmInstall: options.confirmInstall } : {}),
});
const createPythonCompute = options.createPythonCompute ?? createPythonSemanticLayerComputePort;
return createPythonCompute({
command: runtime.manifest.python.daemonExecutable,
args: [],
});
}

239
packages/cli/src/managed-python-daemon.test.ts Normal file
View file

@ -0,0 +1,239 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
readManagedPythonDaemonStatus,
startManagedPythonDaemon,
stopManagedPythonDaemon,
type ManagedPythonDaemonChild,
type ManagedPythonDaemonFetch,
type ManagedPythonDaemonSpawn,
type ManagedPythonDaemonState,
} from './managed-python-daemon.js';
import type {
InstalledKtxRuntimeManifest,
ManagedPythonRuntimeInstallResult,
ManagedPythonRuntimeLayout,
} from './managed-python-runtime.js';
function layout(root: string): ManagedPythonRuntimeLayout {
return {
cliVersion: '0.2.0',
runtimeRoot: join(root, 'runtime'),
versionDir: join(root, 'runtime', '0.2.0'),
venvDir: join(root, 'runtime', '0.2.0', '.venv'),
manifestPath: join(root, 'runtime', '0.2.0', 'manifest.json'),
installLogPath: join(root, 'runtime', '0.2.0', 'install.log'),
assetDir: join(root, 'assets', 'python'),
assetManifestPath: join(root, 'assets', 'python', 'manifest.json'),
pythonPath: join(root, 'runtime', '0.2.0', '.venv', 'bin', 'python'),
daemonPath: join(root, 'runtime', '0.2.0', '.venv', 'bin', 'ktx-daemon'),
daemonStatePath: join(root, 'runtime', '0.2.0', 'daemon.json'),
daemonStdoutPath: join(root, 'runtime', '0.2.0', 'daemon.stdout.log'),
daemonStderrPath: join(root, 'runtime', '0.2.0', 'daemon.stderr.log'),
};
}
function manifest(root: string, features: Array<'core' | 'local-embeddings'> = ['core']): InstalledKtxRuntimeManifest {
const runtimeLayout = layout(root);
return {
schemaVersion: 1,
cliVersion: '0.2.0',
installedAt: '2026-05-11T00:00:00.000Z',
asset: {
schemaVersion: 1,
distributionName: 'kaelio-ktx',
normalizedName: 'kaelio_ktx',
version: '0.2.0',
wheel: {
file: 'kaelio_ktx-0.2.0-py3-none-any.whl',
sha256: 'a'.repeat(64),
bytes: 123,
},
},
features,
python: {
executable: runtimeLayout.pythonPath,
daemonExecutable: runtimeLayout.daemonPath,
},
installLog: runtimeLayout.installLogPath,
};
}
function installResult(root: string, features: Array<'core' | 'local-embeddings'> = ['core']): ManagedPythonRuntimeInstallResult {
return {
status: 'ready',
layout: layout(root),
asset: {
manifest: manifest(root, features).asset,
wheelPath: join(root, 'assets', 'python', 'kaelio_ktx-0.2.0-py3-none-any.whl'),
},
manifest: manifest(root, features),
};
}
function makeFetch(version = '0.2.0'): ManagedPythonDaemonFetch {
return vi.fn(async () => ({
ok: true,
status: 200,
json: async () => ({ status: 'healthy', version }),
text: async () => '',
}));
}
function makeSpawn(pid = 4242): ManagedPythonDaemonSpawn {
return vi.fn((_command, _args, _options): ManagedPythonDaemonChild => ({
pid,
unref: vi.fn(),
}));
}
function runningState(root: string, overrides: Partial<ManagedPythonDaemonState> = {}): ManagedPythonDaemonState {
const runtimeLayout = layout(root);
return {
schemaVersion: 1,
pid: 4242,
host: '127.0.0.1',
port: 58731,
version: '0.2.0',
features: ['core'],
startedAt: '2026-05-11T00:00:00.000Z',
stdoutLog: runtimeLayout.daemonStdoutPath,
stderrLog: runtimeLayout.daemonStderrPath,
...overrides,
};
}
describe('managed Python daemon lifecycle', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-managed-daemon-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('reports stopped when no daemon state exists', async () => {
const status = await readManagedPythonDaemonStatus({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
processAlive: vi.fn(() => false),
fetch: makeFetch(),
});
expect(status.kind).toBe('stopped');
expect(status.detail).toContain('No daemon state');
});
it('starts ktx-daemon serve-http, waits for health, and writes state', async () => {
const spawnDaemon = makeSpawn(5555);
const installRuntime = vi.fn(async () => installResult(tempDir));
const result = await startManagedPythonDaemon({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
features: ['core'],
installRuntime,
spawnDaemon,
fetch: makeFetch(),
allocatePort: vi.fn(async () => 61234),
now: () => new Date('2026-05-11T00:00:00.000Z'),
pollIntervalMs: 1,
});
expect(result.status).toBe('started');
expect(result.baseUrl).toBe('http://127.0.0.1:61234');
expect(installRuntime).toHaveBeenCalledWith({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
features: ['core'],
force: false,
});
expect(spawnDaemon).toHaveBeenCalledWith(
layout(tempDir).daemonPath,
['serve-http', '--host', '127.0.0.1', '--port', '61234'],
expect.objectContaining({
detached: true,
env: expect.objectContaining({ KTX_DAEMON_VERSION: '0.2.0' }),
}),
);
expect(JSON.parse(await readFile(layout(tempDir).daemonStatePath, 'utf8'))).toMatchObject({
pid: 5555,
port: 61234,
version: '0.2.0',
features: ['core'],
stdoutLog: layout(tempDir).daemonStdoutPath,
stderrLog: layout(tempDir).daemonStderrPath,
});
});
it('reuses a healthy daemon with the requested feature set', async () => {
await mkdir(layout(tempDir).versionDir, { recursive: true });
await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`);
const spawnDaemon = makeSpawn(9999);
const result = await startManagedPythonDaemon({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
features: ['core'],
installRuntime: vi.fn(async () => installResult(tempDir)),
spawnDaemon,
fetch: makeFetch(),
processAlive: vi.fn(() => true),
pollIntervalMs: 1,
});
expect(result.status).toBe('reused');
expect(result.baseUrl).toBe('http://127.0.0.1:58731');
expect(spawnDaemon).not.toHaveBeenCalled();
});
it('starts a fresh daemon when the previous state is stale', async () => {
await mkdir(layout(tempDir).versionDir, { recursive: true });
await writeFile(
layout(tempDir).daemonStatePath,
`${JSON.stringify(runningState(tempDir, { version: '0.1.0' }), null, 2)}\n`,
);
const result = await startManagedPythonDaemon({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
features: ['core'],
installRuntime: vi.fn(async () => installResult(tempDir)),
spawnDaemon: makeSpawn(6666),
fetch: makeFetch(),
processAlive: vi.fn(() => true),
killProcess: vi.fn(),
allocatePort: vi.fn(async () => 61235),
now: () => new Date('2026-05-11T00:00:00.000Z'),
pollIntervalMs: 1,
});
expect(result.status).toBe('started');
expect(JSON.parse(await readFile(layout(tempDir).daemonStatePath, 'utf8'))).toMatchObject({
pid: 6666,
port: 61235,
version: '0.2.0',
});
});
it('stops a recorded daemon and removes the state file', async () => {
await mkdir(layout(tempDir).versionDir, { recursive: true });
await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`);
const killProcess = vi.fn();
const result = await stopManagedPythonDaemon({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
processAlive: vi.fn(() => true),
killProcess,
});
expect(result.status).toBe('stopped');
expect(killProcess).toHaveBeenCalledWith(4242);
await expect(readFile(layout(tempDir).daemonStatePath, 'utf8')).rejects.toThrow();
});
});

397
packages/cli/src/managed-python-daemon.ts Normal file
View file

@ -0,0 +1,397 @@
import { spawn } from 'node:child_process';
import { mkdir, open, readFile, rm, writeFile } from 'node:fs/promises';
import { createServer } from 'node:net';
import { setTimeout as delay } from 'node:timers/promises';
import { z } from 'zod';
import {
installManagedPythonRuntime,
managedPythonRuntimeLayout,
runtimeFeatureSchema,
type KtxRuntimeFeature,
type ManagedPythonRuntimeInstallOptions,
type ManagedPythonRuntimeInstallResult,
type ManagedPythonRuntimeLayout,
type ManagedPythonRuntimeLayoutOptions,
} from './managed-python-runtime.js';
export interface ManagedPythonDaemonState {
schemaVersion: 1;
pid: number;
host: '127.0.0.1';
port: number;
version: string;
features: KtxRuntimeFeature[];
startedAt: string;
stdoutLog: string;
stderrLog: string;
}
export type ManagedPythonDaemonStatus =
| { kind: 'stopped'; detail: string; layout: ManagedPythonRuntimeLayout }
| { kind: 'running'; detail: string; layout: ManagedPythonRuntimeLayout; state: ManagedPythonDaemonState; baseUrl: string }
| { kind: 'stale'; detail: string; layout: ManagedPythonRuntimeLayout; state?: ManagedPythonDaemonState };
export interface ManagedPythonDaemonStartResult {
status: 'started' | 'reused';
layout: ManagedPythonRuntimeLayout;
state: ManagedPythonDaemonState;
baseUrl: string;
}
export interface ManagedPythonDaemonStopResult {
status: 'stopped' | 'already-stopped';
layout: ManagedPythonRuntimeLayout;
state?: ManagedPythonDaemonState;
}
export interface ManagedPythonDaemonChild {
pid?: number;
unref(): void;
}
export type ManagedPythonDaemonSpawn = (
command: string,
args: string[],
options: {
detached: boolean;
stdio: ['ignore', number, number];
env: NodeJS.ProcessEnv;
},
) => ManagedPythonDaemonChild;
export type ManagedPythonDaemonFetch = (
url: string,
) => Promise<{
ok: boolean;
status: number;
json(): Promise<unknown>;
text(): Promise<string>;
}>;
export interface ManagedPythonDaemonStartOptions extends ManagedPythonRuntimeLayoutOptions {
features: KtxRuntimeFeature[];
force?: boolean;
installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise<ManagedPythonRuntimeInstallResult>;
spawnDaemon?: ManagedPythonDaemonSpawn;
fetch?: ManagedPythonDaemonFetch;
allocatePort?: () => Promise<number>;
processAlive?: (pid: number) => boolean;
killProcess?: (pid: number) => void;
now?: () => Date;
startupTimeoutMs?: number;
pollIntervalMs?: number;
}
export interface ManagedPythonDaemonStatusOptions extends ManagedPythonRuntimeLayoutOptions {
fetch?: ManagedPythonDaemonFetch;
processAlive?: (pid: number) => boolean;
}
export interface ManagedPythonDaemonStopOptions extends ManagedPythonRuntimeLayoutOptions {
processAlive?: (pid: number) => boolean;
killProcess?: (pid: number) => void;
}
const daemonStateSchema = z.object({
schemaVersion: z.literal(1),
pid: z.number().int().positive(),
host: z.literal('127.0.0.1'),
port: z.number().int().min(1).max(65535),
version: z.string().min(1),
features: z.array(runtimeFeatureSchema).min(1),
startedAt: z.string().min(1),
stdoutLog: z.string().min(1),
stderrLog: z.string().min(1),
});
function normalizeFeatures(features: KtxRuntimeFeature[]): KtxRuntimeFeature[] {
const requested = new Set<KtxRuntimeFeature>(['core', ...features]);
return runtimeFeatureSchema.options.filter((feature) => requested.has(feature));
}
function hasFeatures(state: ManagedPythonDaemonState, features: KtxRuntimeFeature[]): boolean {
return normalizeFeatures(features).every((feature) => state.features.includes(feature));
}
function defaultFetch(url: string): ReturnType<ManagedPythonDaemonFetch> {
return fetch(url) as ReturnType<ManagedPythonDaemonFetch>;
}
function defaultProcessAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
function defaultKillProcess(pid: number): void {
try {
process.kill(pid, 'SIGTERM');
} catch (error) {
const code = (error as { code?: unknown }).code;
if (code !== 'ESRCH') {
throw error;
}
}
}
function defaultSpawnDaemon(
command: string,
args: string[],
options: Parameters<ManagedPythonDaemonSpawn>[2],
): ManagedPythonDaemonChild {
return spawn(command, args, options);
}
function baseUrl(state: Pick<ManagedPythonDaemonState, 'host' | 'port'>): string {
return `http://${state.host}:${state.port}`;
}
async function readState(path: string): Promise<ManagedPythonDaemonState | undefined> {
try {
return daemonStateSchema.parse(JSON.parse(await readFile(path, 'utf8')) as unknown);
} catch (error) {
const code = (error as { code?: unknown }).code;
if (code === 'ENOENT') {
return undefined;
}
throw error;
}
}
async function writeState(path: string, state: ManagedPythonDaemonState): Promise<void> {
await writeFile(path, `${JSON.stringify(state, null, 2)}\n`);
}
async function healthOk(input: {
state: ManagedPythonDaemonState;
cliVersion: string;
fetch: ManagedPythonDaemonFetch;
}): Promise<{ ok: true } | { ok: false; detail: string }> {
try {
const response = await input.fetch(`${baseUrl(input.state)}/health`);
if (!response.ok) {
return { ok: false, detail: `Health check returned HTTP ${response.status}: ${await response.text()}` };
}
const body = (await response.json()) as unknown;
if (!body || typeof body !== 'object' || Array.isArray(body)) {
return { ok: false, detail: 'Health check returned non-object JSON' };
}
const record = body as Record<string, unknown>;
if (record.status !== 'healthy') {
return { ok: false, detail: `Health check returned status ${String(record.status)}` };
}
if (record.version !== input.cliVersion) {
return {
ok: false,
detail: `Daemon version ${String(record.version)} does not match CLI ${input.cliVersion}`,
};
}
return { ok: true };
} catch (error) {
return { ok: false, detail: error instanceof Error ? error.message : String(error) };
}
}
export async function readManagedPythonDaemonStatus(
options: ManagedPythonDaemonStatusOptions,
): Promise<ManagedPythonDaemonStatus> {
const layout = managedPythonRuntimeLayout(options);
let state: ManagedPythonDaemonState | undefined;
try {
state = await readState(layout.daemonStatePath);
} catch (error) {
return {
kind: 'stale',
detail: `Daemon state is invalid: ${error instanceof Error ? error.message : String(error)}`,
layout,
};
}
if (!state) {
return { kind: 'stopped', detail: `No daemon state at ${layout.daemonStatePath}`, layout };
}
if (state.version !== options.cliVersion) {
return {
kind: 'stale',
detail: `Daemon is for CLI ${state.version}, current CLI is ${options.cliVersion}`,
layout,
state,
};
}
const processAlive = options.processAlive ?? defaultProcessAlive;
if (!processAlive(state.pid)) {
return { kind: 'stale', detail: `Daemon process ${state.pid} is not running`, layout, state };
}
const health = await healthOk({
state,
cliVersion: options.cliVersion,
fetch: options.fetch ?? defaultFetch,
});
if (!health.ok) {
return { kind: 'stale', detail: health.detail, layout, state };
}
return { kind: 'running', detail: `Daemon running at ${baseUrl(state)}`, layout, state, baseUrl: baseUrl(state) };
}
export async function allocateDaemonPort(): Promise<number> {
return await new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
server.close(() => {
if (address && typeof address === 'object') {
resolve(address.port);
return;
}
reject(new Error('Failed to allocate a daemon port'));
});
});
});
}
async function waitForHealth(input: {
state: ManagedPythonDaemonState;
cliVersion: string;
fetch: ManagedPythonDaemonFetch;
timeoutMs: number;
pollIntervalMs: number;
}): Promise<void> {
const deadline = Date.now() + input.timeoutMs;
let lastDetail = 'daemon did not answer health checks';
while (Date.now() <= deadline) {
const health = await healthOk({
state: input.state,
cliVersion: input.cliVersion,
fetch: input.fetch,
});
if (health.ok) {
return;
}
lastDetail = health.detail;
await delay(input.pollIntervalMs);
}
throw new Error(`KTX Python daemon failed to start: ${lastDetail}. stderr: ${input.state.stderrLog}`);
}
async function removeState(layout: ManagedPythonRuntimeLayout): Promise<void> {
await rm(layout.daemonStatePath, { force: true });
}
async function stopRecordedDaemon(input: {
layout: ManagedPythonRuntimeLayout;
state: ManagedPythonDaemonState;
processAlive: (pid: number) => boolean;
killProcess: (pid: number) => void;
}): Promise<void> {
if (input.processAlive(input.state.pid)) {
input.killProcess(input.state.pid);
}
await removeState(input.layout);
}
export async function startManagedPythonDaemon(
options: ManagedPythonDaemonStartOptions,
): Promise<ManagedPythonDaemonStartResult> {
const features = normalizeFeatures(options.features);
const installRuntime = options.installRuntime ?? installManagedPythonRuntime;
const layoutOverrides = {
...(options.runtimeRoot !== undefined ? { runtimeRoot: options.runtimeRoot } : {}),
...(options.assetDir !== undefined ? { assetDir: options.assetDir } : {}),
...(options.platform !== undefined ? { platform: options.platform } : {}),
...(options.env !== undefined ? { env: options.env } : {}),
...(options.homeDir !== undefined ? { homeDir: options.homeDir } : {}),
};
const layout = managedPythonRuntimeLayout({ cliVersion: options.cliVersion, ...layoutOverrides });
const processAlive = options.processAlive ?? defaultProcessAlive;
const killProcess = options.killProcess ?? defaultKillProcess;
const fetchImpl = options.fetch ?? defaultFetch;
const status = await readManagedPythonDaemonStatus({
cliVersion: options.cliVersion,
...layoutOverrides,
fetch: fetchImpl,
processAlive,
});
if (options.force !== true && status.kind === 'running' && hasFeatures(status.state, features)) {
return { status: 'reused', layout, state: status.state, baseUrl: status.baseUrl };
}
if ('state' in status && status.state) {
await stopRecordedDaemon({ layout, state: status.state, processAlive, killProcess });
} else {
await removeState(layout);
}
const installed = await installRuntime({
cliVersion: options.cliVersion,
...layoutOverrides,
features,
force: false,
});
await mkdir(layout.versionDir, { recursive: true });
const stdout = await open(layout.daemonStdoutPath, 'a');
const stderr = await open(layout.daemonStderrPath, 'a');
try {
const port = await (options.allocatePort ?? allocateDaemonPort)();
const spawnDaemon = options.spawnDaemon ?? defaultSpawnDaemon;
const child = spawnDaemon(
installed.manifest.python.daemonExecutable,
['serve-http', '--host', '127.0.0.1', '--port', String(port)],
{
detached: true,
stdio: ['ignore', stdout.fd, stderr.fd],
env: {
...process.env,
KTX_DAEMON_VERSION: options.cliVersion,
},
},
);
child.unref();
if (!child.pid) {
throw new Error(`KTX Python daemon did not report a pid. stderr: ${layout.daemonStderrPath}`);
}
const state: ManagedPythonDaemonState = {
schemaVersion: 1,
pid: child.pid,
host: '127.0.0.1',
port,
version: options.cliVersion,
features: installed.manifest.features,
startedAt: (options.now ?? (() => new Date()))().toISOString(),
stdoutLog: layout.daemonStdoutPath,
stderrLog: layout.daemonStderrPath,
};
await waitForHealth({
state,
cliVersion: options.cliVersion,
fetch: fetchImpl,
timeoutMs: options.startupTimeoutMs ?? 10_000,
pollIntervalMs: options.pollIntervalMs ?? 100,
});
await writeState(layout.daemonStatePath, state);
return { status: 'started', layout, state, baseUrl: baseUrl(state) };
} finally {
await stdout.close();
await stderr.close();
}
}
export async function stopManagedPythonDaemon(
options: ManagedPythonDaemonStopOptions,
): Promise<ManagedPythonDaemonStopResult> {
const layout = managedPythonRuntimeLayout(options);
const state = await readState(layout.daemonStatePath);
if (!state) {
return { status: 'already-stopped', layout };
}
await stopRecordedDaemon({
layout,
state,
processAlive: options.processAlive ?? defaultProcessAlive,
killProcess: options.killProcess ?? defaultKillProcess,
});
return { status: 'stopped', layout, state };
}

171
packages/cli/src/managed-python-http.test.ts Normal file
View file

@ -0,0 +1,171 @@
import { describe, expect, it, vi } from 'vitest';
import {
createManagedDaemonHttpJsonRunner,
createManagedDaemonLookerTableIdentifierParser,
createManagedDaemonSqlAnalysisPort,
createManagedPythonDaemonBaseUrlResolver,
managedDaemonDatabaseIntrospectionOptions,
} from './managed-python-http.js';
function io() {
let stderr = '';
return {
io: {
stdout: { write: vi.fn() },
stderr: { write: (chunk: string) => (stderr += chunk) },
},
stderr: () => stderr,
};
}
describe('createManagedPythonDaemonBaseUrlResolver', () => {
it('ensures the core runtime, starts the daemon, reports the URL, and caches the result', async () => {
const testIo = io();
const ensureRuntime = vi.fn(async () => ({
layout: {} as never,
manifest: {} as never,
}));
const startDaemon = vi.fn(async () => ({
status: 'started' as const,
layout: {} as never,
state: { pid: 1234 } as never,
baseUrl: 'http://127.0.0.1:61234',
}));
const resolveBaseUrl = createManagedPythonDaemonBaseUrlResolver({
cliVersion: '0.2.0',
installPolicy: 'auto',
io: testIo.io,
ensureRuntime,
startDaemon,
});
await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234');
await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234');
expect(ensureRuntime).toHaveBeenCalledTimes(1);
expect(ensureRuntime).toHaveBeenCalledWith({
cliVersion: '0.2.0',
installPolicy: 'auto',
io: testIo.io,
feature: 'core',
});
expect(startDaemon).toHaveBeenCalledTimes(1);
expect(startDaemon).toHaveBeenCalledWith({
cliVersion: '0.2.0',
features: ['core'],
force: false,
});
expect(testIo.stderr()).toContain('Started KTX Python daemon: http://127.0.0.1:61234');
});
it('reports daemon reuse without reinstalling after the first resolved URL', async () => {
const testIo = io();
const ensureRuntime = vi.fn(async () => ({
layout: {} as never,
manifest: {} as never,
}));
const startDaemon = vi.fn(async () => ({
status: 'reused' as const,
layout: {} as never,
state: { pid: 1234 } as never,
baseUrl: 'http://127.0.0.1:61234',
}));
const resolveBaseUrl = createManagedPythonDaemonBaseUrlResolver({
cliVersion: '0.2.0',
installPolicy: 'never',
io: testIo.io,
ensureRuntime,
startDaemon,
});
await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234');
await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234');
expect(ensureRuntime).toHaveBeenCalledTimes(1);
expect(startDaemon).toHaveBeenCalledTimes(1);
expect(testIo.stderr()).toContain('Using existing KTX Python daemon: http://127.0.0.1:61234');
});
});
describe('createManagedDaemonHttpJsonRunner', () => {
it('resolves the managed base URL lazily for each HTTP JSON request', async () => {
const postJson = vi.fn(async () => ({ ok: true }));
const runner = createManagedDaemonHttpJsonRunner({
resolveBaseUrl: async () => 'http://127.0.0.1:61234',
postJson,
});
await expect(runner('/sql/parse-table-identifier', { items: [] })).resolves.toEqual({ ok: true });
expect(postJson).toHaveBeenCalledWith('http://127.0.0.1:61234', '/sql/parse-table-identifier', { items: [] });
});
});
describe('managed daemon ingest ports', () => {
it('creates a Looker table parser backed by the managed daemon runner', async () => {
const requestJson = vi.fn(async () => ({
results: {
'model.explore': {
ok: true,
catalog: 'warehouse',
schema: 'public',
name: 'orders',
canonical_table: 'public.orders',
},
},
}));
const parser = createManagedDaemonLookerTableIdentifierParser({ requestJson });
await expect(
parser.parse([{ key: 'model.explore', sql_table_name: 'public.orders', dialect: 'postgres' }]),
).resolves.toEqual({
'model.explore': {
ok: true,
catalog: 'warehouse',
schema: 'public',
name: 'orders',
canonical_table: 'public.orders',
},
});
expect(requestJson).toHaveBeenCalledWith('/sql/parse-table-identifier', {
items: [{ key: 'model.explore', sql_table_name: 'public.orders', dialect: 'postgres' }],
});
});
it('creates a SQL analysis port backed by the managed daemon runner', async () => {
const requestJson = vi.fn(async () => ({
fingerprint: 'select-orders',
normalized_sql: 'SELECT * FROM public.orders WHERE id = ?',
tables_touched: ['public.orders'],
literal_slots: [{ position: 1, type: 'number', example_value: '42' }],
}));
const sqlAnalysis = createManagedDaemonSqlAnalysisPort({ requestJson });
await expect(sqlAnalysis.analyzeForFingerprint('SELECT * FROM public.orders WHERE id = 42', 'postgres')).resolves
.toEqual({
fingerprint: 'select-orders',
normalizedSql: 'SELECT * FROM public.orders WHERE id = ?',
tablesTouched: ['public.orders'],
literalSlots: [{ position: 1, type: 'number', exampleValue: '42' }],
});
expect(requestJson).toHaveBeenCalledWith('/api/sql/analyze-for-fingerprint', {
sql: 'SELECT * FROM public.orders WHERE id = 42',
dialect: 'postgres',
});
});
it('returns live-database daemon request options backed by the managed runner', async () => {
const requestJson = vi.fn(async () => ({
connection_id: 'warehouse',
tables: [],
}));
const options = managedDaemonDatabaseIntrospectionOptions({ requestJson });
expect(options.requestJson).toBeDefined();
await expect(options.requestJson?.('/database/introspect', { connection_id: 'warehouse' })).resolves.toEqual({
connection_id: 'warehouse',
tables: [],
});
expect(requestJson).toHaveBeenCalledWith('/database/introspect', { connection_id: 'warehouse' });
});
});

194
packages/cli/src/managed-python-http.ts Normal file
View file

@ -0,0 +1,194 @@
import { request as httpRequest } from 'node:http';
import { request as httpsRequest } from 'node:https';
import { URL } from 'node:url';
import {
createDaemonLookerTableIdentifierParser,
type DaemonLiveDatabaseIntrospectionOptions,
type KtxDaemonDatabaseHttpJsonRunner,
type KtxDaemonTableIdentifierHttpJsonRunner,
type LookerTableIdentifierParser,
} from '@ktx/context/ingest';
import {
createHttpSqlAnalysisPort,
type KtxSqlAnalysisHttpJsonRunner,
type SqlAnalysisPort,
} from '@ktx/context/sql-analysis';
import type { KtxCliIo } from './cli-runtime.js';
import {
ensureManagedPythonCommandRuntime,
type KtxManagedPythonInstallPolicy,
type ManagedPythonCommandRuntime,
} from './managed-python-command.js';
import { startManagedPythonDaemon, type ManagedPythonDaemonStartResult } from './managed-python-daemon.js';
export type ManagedPythonHttpJsonRunner = (
path: string,
payload: Record<string, unknown>,
) => Promise<Record<string, unknown>>;
export type ManagedPythonHttpPostJson = (
baseUrl: string,
path: string,
payload: Record<string, unknown>,
) => Promise<Record<string, unknown>>;
export interface ManagedPythonCoreDaemonOptions {
cliVersion: string;
installPolicy: KtxManagedPythonInstallPolicy;
io: KtxCliIo;
ensureRuntime?: (options: {
cliVersion: string;
installPolicy: KtxManagedPythonInstallPolicy;
io: KtxCliIo;
feature: 'core';
}) => Promise<ManagedPythonCommandRuntime>;
startDaemon?: (options: {
cliVersion: string;
features: ['core'];
force: false;
}) => Promise<ManagedPythonDaemonStartResult>;
}
export type ManagedPythonDaemonHttpOptions =
| {
requestJson: ManagedPythonHttpJsonRunner;
}
| {
resolveBaseUrl: () => Promise<string>;
postJson?: ManagedPythonHttpPostJson;
}
| (ManagedPythonCoreDaemonOptions & {
postJson?: ManagedPythonHttpPostJson;
});
function normalizedBaseUrl(baseUrl: string): string {
return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
}
function parseJsonObject(raw: string, path: string): Record<string, unknown> {
const parsed = JSON.parse(raw) as unknown;
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new Error(`KTX managed daemon HTTP ${path} returned non-object JSON`);
}
return parsed as Record<string, unknown>;
}
export async function postManagedDaemonJson(
baseUrl: string,
path: string,
payload: Record<string, unknown>,
): Promise<Record<string, unknown>> {
return await new Promise((resolve, reject) => {
const target = new URL(path.replace(/^\//, ''), normalizedBaseUrl(baseUrl));
const body = JSON.stringify(payload);
const client = target.protocol === 'https:' ? httpsRequest : httpRequest;
const request = client(
target,
{
method: 'POST',
headers: {
accept: 'application/json',
'content-type': 'application/json',
'content-length': Buffer.byteLength(body),
},
},
(response) => {
const chunks: Buffer[] = [];
response.on('data', (chunk: Buffer) => chunks.push(chunk));
response.on('end', () => {
const text = Buffer.concat(chunks).toString('utf8');
const statusCode = response.statusCode ?? 0;
if (statusCode < 200 || statusCode >= 300) {
reject(new Error(`KTX managed daemon HTTP ${path} failed with ${statusCode}: ${text}`));
return;
}
try {
resolve(parseJsonObject(text, path));
} catch (error) {
reject(error);
}
});
},
);
request.on('error', reject);
request.end(body);
});
}
export function createManagedPythonDaemonBaseUrlResolver(
options: ManagedPythonCoreDaemonOptions,
): () => Promise<string> {
let cachedBaseUrl: string | undefined;
return async () => {
if (cachedBaseUrl) {
return cachedBaseUrl;
}
const ensureRuntime = options.ensureRuntime ?? ensureManagedPythonCommandRuntime;
const startDaemon = options.startDaemon ?? startManagedPythonDaemon;
await ensureRuntime({
cliVersion: options.cliVersion,
installPolicy: options.installPolicy,
io: options.io,
feature: 'core',
});
const daemon = await startDaemon({
cliVersion: options.cliVersion,
features: ['core'],
force: false,
});
const verb = daemon.status === 'started' ? 'Started' : 'Using existing';
options.io.stderr.write(`${verb} KTX Python daemon: ${daemon.baseUrl}\n`);
cachedBaseUrl = daemon.baseUrl;
return cachedBaseUrl;
};
}
function isRequestJsonOnly(options: ManagedPythonDaemonHttpOptions): options is { requestJson: ManagedPythonHttpJsonRunner } {
return 'requestJson' in options;
}
function isResolveBaseUrlOnly(
options: ManagedPythonDaemonHttpOptions,
): options is { resolveBaseUrl: () => Promise<string>; postJson?: ManagedPythonHttpPostJson } {
return 'resolveBaseUrl' in options;
}
export function createManagedDaemonHttpJsonRunner(
options: ManagedPythonDaemonHttpOptions,
): ManagedPythonHttpJsonRunner {
if (isRequestJsonOnly(options)) {
return options.requestJson;
}
const resolveBaseUrl = isResolveBaseUrlOnly(options)
? options.resolveBaseUrl
: createManagedPythonDaemonBaseUrlResolver(options);
const postJson = options.postJson ?? postManagedDaemonJson;
return async (path, payload) => postJson(await resolveBaseUrl(), path, payload);
}
export function createManagedDaemonLookerTableIdentifierParser(
options: ManagedPythonDaemonHttpOptions,
): LookerTableIdentifierParser {
return createDaemonLookerTableIdentifierParser({
baseUrl: 'http://127.0.0.1:0',
requestJson: createManagedDaemonHttpJsonRunner(options) as KtxDaemonTableIdentifierHttpJsonRunner,
});
}
export function createManagedDaemonSqlAnalysisPort(options: ManagedPythonDaemonHttpOptions): SqlAnalysisPort {
return createHttpSqlAnalysisPort({
baseUrl: 'http://127.0.0.1:0',
requestJson: createManagedDaemonHttpJsonRunner(options) as KtxSqlAnalysisHttpJsonRunner,
});
}
export function managedDaemonDatabaseIntrospectionOptions(
options: ManagedPythonDaemonHttpOptions,
): Pick<DaemonLiveDatabaseIntrospectionOptions, 'requestJson'> {
return {
requestJson: createManagedDaemonHttpJsonRunner(options) as KtxDaemonDatabaseHttpJsonRunner,
};
}

479
packages/cli/src/managed-python-runtime.test.ts Normal file
View file

@ -0,0 +1,479 @@
import { createHash } from 'node:crypto';
import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
MISSING_UV_RUNTIME_INSTALL_MESSAGE,
doctorManagedPythonRuntime,
installManagedPythonRuntime,
managedPythonRuntimeLayout,
pruneManagedPythonRuntimes,
readManagedPythonRuntimeStatus,
verifyRuntimeAsset,
type ManagedPythonRuntimeExec,
} from './managed-python-runtime.js';
async function writeAsset(root: string, contents = 'wheel-bytes') {
const assetDir = join(root, 'assets', 'python');
await mkdir(assetDir, { recursive: true });
const wheelPath = join(assetDir, 'kaelio_ktx-0.1.0-py3-none-any.whl');
await writeFile(wheelPath, contents);
await writeFile(
join(assetDir, 'manifest.json'),
`${JSON.stringify(
{
schemaVersion: 1,
distributionName: 'kaelio-ktx',
normalizedName: 'kaelio_ktx',
version: '0.1.0',
wheel: {
file: 'kaelio_ktx-0.1.0-py3-none-any.whl',
sha256: createHash('sha256').update(contents).digest('hex'),
bytes: Buffer.byteLength(contents),
},
},
null,
2,
)}\n`,
);
return { assetDir, wheelPath };
}
describe('managedPythonRuntimeLayout', () => {
it('uses the macOS application-support runtime root', () => {
const layout = managedPythonRuntimeLayout({
cliVersion: '0.2.0',
platform: 'darwin',
env: {},
homeDir: '/Users/alex',
assetDir: '/repo/packages/cli/assets/python',
});
expect(layout.runtimeRoot).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime');
expect(layout.versionDir).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0');
expect(layout.venvDir).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv');
expect(layout.pythonPath).toBe(
'/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv/bin/python',
);
expect(layout.daemonPath).toBe(
'/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv/bin/ktx-daemon',
);
expect(layout.daemonStatePath).toBe(
'/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.json',
);
expect(layout.daemonStdoutPath).toBe(
'/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.stdout.log',
);
expect(layout.daemonStderrPath).toBe(
'/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.stderr.log',
);
expect(layout.assetManifestPath).toBe('/repo/packages/cli/assets/python/manifest.json');
});
it('honors KTX_RUNTIME_ROOT before platform defaults', () => {
const layout = managedPythonRuntimeLayout({
cliVersion: '0.2.0',
platform: 'darwin',
env: { KTX_RUNTIME_ROOT: '/tmp/ktx-runtime' },
homeDir: '/Users/alex',
assetDir: '/repo/packages/cli/assets/python',
});
expect(layout.runtimeRoot).toBe('/tmp/ktx-runtime');
expect(layout.versionDir).toBe('/tmp/ktx-runtime/0.2.0');
});
it('honors XDG_DATA_HOME on Linux', () => {
const layout = managedPythonRuntimeLayout({
cliVersion: '0.2.0',
platform: 'linux',
env: { XDG_DATA_HOME: '/var/xdg' },
homeDir: '/home/alex',
assetDir: '/repo/packages/cli/assets/python',
});
expect(layout.runtimeRoot).toBe('/var/xdg/kaelio/ktx/runtime');
expect(layout.versionDir).toBe('/var/xdg/kaelio/ktx/runtime/0.2.0');
});
it('uses LocalAppData on Windows', () => {
const layout = managedPythonRuntimeLayout({
cliVersion: '0.2.0',
platform: 'win32',
env: { LOCALAPPDATA: 'C:\\Users\\Alex\\AppData\\Local' },
homeDir: 'C:\\Users\\Alex',
assetDir: 'C:\\repo\\packages\\cli\\assets\\python',
});
expect(layout.runtimeRoot).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime');
expect(layout.pythonPath).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime/0.2.0/.venv/Scripts/python.exe');
expect(layout.daemonPath).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime/0.2.0/.venv/Scripts/ktx-daemon.exe');
});
});
describe('verifyRuntimeAsset', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-asset-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('reads the manifest and verifies the wheel checksum', async () => {
const { assetDir, wheelPath } = await writeAsset(tempDir, 'valid-wheel');
const asset = await verifyRuntimeAsset({ assetDir });
expect(asset.manifest.distributionName).toBe('kaelio-ktx');
expect(asset.manifest.normalizedName).toBe('kaelio_ktx');
expect(asset.wheelPath).toBe(wheelPath);
});
it('rejects a wheel whose checksum does not match the manifest', async () => {
const { assetDir, wheelPath } = await writeAsset(tempDir, 'original');
await writeFile(wheelPath, 'tampered');
await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow(
/Bundled Python runtime wheel checksum mismatch/,
);
});
it('rejects an unsafe wheel filename in the manifest', async () => {
const { assetDir } = await writeAsset(tempDir, 'valid-wheel');
await writeFile(
join(assetDir, 'manifest.json'),
`${JSON.stringify({
schemaVersion: 1,
distributionName: 'kaelio-ktx',
normalizedName: 'kaelio_ktx',
version: '0.1.0',
wheel: {
file: '../kaelio_ktx-0.1.0-py3-none-any.whl',
sha256: 'a'.repeat(64),
bytes: 1,
},
})}\n`,
);
await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow(/Unsafe runtime wheel filename/);
});
});
describe('installManagedPythonRuntime', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-install-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('creates a venv, installs the core wheel, and writes a manifest', async () => {
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
const commands: Array<{ command: string; args: string[] }> = [];
const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => {
commands.push({ command, args });
return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' };
});
const result = await installManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
features: ['core'],
exec,
});
expect(result.status).toBe('installed');
expect(commands).toEqual([
{ command: 'uv', args: ['--version'] },
{ command: 'uv', args: ['venv', result.layout.venvDir] },
{
command: 'uv',
args: ['pip', 'install', '--python', result.layout.pythonPath, result.asset.wheelPath],
},
]);
const manifest = JSON.parse(await readFile(result.layout.manifestPath, 'utf8')) as {
cliVersion: string;
features: string[];
python: { executable: string; daemonExecutable: string };
};
expect(manifest.cliVersion).toBe('0.2.0');
expect(manifest.features).toEqual(['core']);
expect(manifest.python.executable).toBe(result.layout.pythonPath);
expect(manifest.python.daemonExecutable).toBe(result.layout.daemonPath);
});
it('installs the local-embeddings extra when requested', async () => {
const { assetDir } = await writeAsset(tempDir, 'embedding-wheel');
const commands: Array<{ command: string; args: string[] }> = [];
const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => {
commands.push({ command, args });
return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' };
});
const result = await installManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
features: ['local-embeddings'],
exec,
});
expect(commands.at(-1)).toEqual({
command: 'uv',
args: ['pip', 'install', '--python', result.layout.pythonPath, `${result.asset.wheelPath}[local-embeddings]`],
});
const manifest = JSON.parse(await readFile(result.layout.manifestPath, 'utf8')) as { features: string[] };
expect(manifest.features).toEqual(['core', 'local-embeddings']);
});
it('fails with the hard-prerequisite message when uv is missing', async () => {
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
const commands: Array<{ command: string; args: string[] }> = [];
const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => {
commands.push({ command, args });
throw new Error('spawn uv ENOENT');
});
await expect(
installManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
features: ['core'],
exec,
}),
).rejects.toThrow(MISSING_UV_RUNTIME_INSTALL_MESSAGE);
expect(commands).toEqual([{ command: 'uv', args: ['--version'] }]);
});
it('reuses an existing compatible runtime when force is false', async () => {
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({
stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '',
stderr: '',
}));
const first = await installManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
features: ['core'],
exec,
});
await mkdir(join(first.layout.venvDir, 'bin'), { recursive: true });
await writeFile(first.layout.pythonPath, '#!/usr/bin/env python\n');
await writeFile(first.layout.daemonPath, '#!/usr/bin/env python\n');
const second = await installManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
features: ['core'],
exec,
});
expect(second.status).toBe('ready');
expect(exec).toHaveBeenCalledTimes(3);
});
it('keeps failed install logs in the versioned runtime directory', async () => {
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => {
if (command === 'uv' && args[0] === 'venv') {
throw Object.assign(new Error('uv venv failed'), { stdout: 'creating\n', stderr: 'bad python\n' });
}
return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' };
});
await expect(
installManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
features: ['core'],
exec,
}),
).rejects.toThrow(/Python runtime install failed/);
const log = await readFile(join(tempDir, 'runtime', '0.2.0', 'install.log'), 'utf8');
expect(log).toContain('$ uv venv');
expect(log).toContain('bad python');
});
});
describe('readManagedPythonRuntimeStatus', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-status-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('reports missing before install', async () => {
const status = await readManagedPythonRuntimeStatus({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir: join(tempDir, 'assets', 'python'),
});
expect(status.kind).toBe('missing');
expect(status.detail).toContain('No runtime manifest');
});
it('reports ready when manifest and executables exist', async () => {
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({
stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '',
stderr: '',
}));
const install = await installManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
features: ['core'],
exec,
});
await mkdir(join(install.layout.venvDir, 'bin'), { recursive: true });
await writeFile(install.layout.pythonPath, '#!/usr/bin/env python\n');
await writeFile(install.layout.daemonPath, '#!/usr/bin/env python\n');
const status = await readManagedPythonRuntimeStatus({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
});
expect(status.kind).toBe('ready');
expect(status.manifest?.features).toEqual(['core']);
});
it('reports broken when an executable is missing', async () => {
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({
stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '',
stderr: '',
}));
await installManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
features: ['core'],
exec,
});
const status = await readManagedPythonRuntimeStatus({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
});
expect(status.kind).toBe('broken');
expect(status.detail).toContain('Missing Python executable');
});
});
describe('doctorManagedPythonRuntime', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-doctor-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('checks uv, bundled assets, and installed runtime status', async () => {
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({
stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '',
stderr: '',
}));
const checks = await doctorManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
exec,
});
expect(checks.map((check) => [check.id, check.status])).toEqual([
['uv', 'pass'],
['asset', 'pass'],
['runtime', 'fail'],
]);
expect(checks[2]?.fix).toBe('Run: ktx runtime install --yes');
});
it('reports uv as a hard prerequisite when uv is missing', async () => {
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
const exec: ManagedPythonRuntimeExec = vi.fn(async () => {
throw new Error('spawn uv ENOENT');
});
const checks = await doctorManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
exec,
});
expect(checks[0]).toEqual({
id: 'uv',
label: 'uv',
status: 'fail',
detail: MISSING_UV_RUNTIME_INSTALL_MESSAGE,
fix: 'Install uv, make sure it is on PATH, and run: ktx runtime install --yes',
});
});
});
describe('pruneManagedPythonRuntimes', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-prune-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('removes stale version directories and keeps the current version', async () => {
const runtimeRoot = join(tempDir, 'runtime');
await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true });
await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true });
await writeFile(join(runtimeRoot, 'README.txt'), 'not a runtime directory\n');
const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot });
expect(result.removed).toEqual([join(runtimeRoot, '0.1.0')]);
expect(result.kept).toEqual([join(runtimeRoot, '0.2.0')]);
await expect(stat(join(runtimeRoot, '0.1.0'))).rejects.toThrow();
expect(await readdir(runtimeRoot)).toEqual(['0.2.0', 'README.txt']);
});
it('supports dry-run without deleting stale directories', async () => {
const runtimeRoot = join(tempDir, 'runtime');
await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true });
await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true });
const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot, dryRun: true });
expect(result.removed).toEqual([]);
expect(result.stale).toEqual([join(runtimeRoot, '0.1.0')]);
expect(await readdir(runtimeRoot)).toEqual(['0.1.0', '0.2.0']);
});
});

444
packages/cli/src/managed-python-runtime.ts Normal file
View file

@ -0,0 +1,444 @@
import { execFile } from 'node:child_process';
import { createHash } from 'node:crypto';
import { access, appendFile, mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
import { homedir } from 'node:os';
import { basename, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import { z } from 'zod';
const execFileAsync = promisify(execFile);
export const runtimeFeatureSchema = z.enum(['core', 'local-embeddings']);
export type KtxRuntimeFeature = z.infer<typeof runtimeFeatureSchema>;
const runtimeAssetManifestSchema = z.object({
schemaVersion: z.literal(1),
distributionName: z.literal('kaelio-ktx'),
normalizedName: z.literal('kaelio_ktx'),
version: z.string().min(1),
wheel: z.object({
file: z.string().min(1),
sha256: z.string().regex(/^[a-f0-9]{64}$/),
bytes: z.number().int().nonnegative(),
}),
});
export type KtxRuntimeAssetManifest = z.infer<typeof runtimeAssetManifestSchema>;
const installedRuntimeManifestSchema = z.object({
schemaVersion: z.literal(1),
cliVersion: z.string().min(1),
installedAt: z.string().min(1),
asset: runtimeAssetManifestSchema,
features: z.array(runtimeFeatureSchema).min(1),
python: z.object({
executable: z.string().min(1),
daemonExecutable: z.string().min(1),
}),
installLog: z.string().min(1),
});
export type InstalledKtxRuntimeManifest = z.infer<typeof installedRuntimeManifestSchema>;
export interface ManagedPythonRuntimeLayoutOptions {
cliVersion: string;
platform?: NodeJS.Platform;
env?: NodeJS.ProcessEnv;
homeDir?: string;
runtimeRoot?: string;
assetDir?: string;
}
export interface ManagedPythonRuntimeLayout {
cliVersion: string;
runtimeRoot: string;
versionDir: string;
venvDir: string;
manifestPath: string;
installLogPath: string;
assetDir: string;
assetManifestPath: string;
pythonPath: string;
daemonPath: string;
daemonStatePath: string;
daemonStdoutPath: string;
daemonStderrPath: string;
}
export interface ManagedRuntimeAsset {
manifest: KtxRuntimeAssetManifest;
wheelPath: string;
}
export type ManagedPythonRuntimeExec = (
command: string,
args: string[],
options?: { cwd?: string; env?: NodeJS.ProcessEnv },
) => Promise<{ stdout: string; stderr: string }>;
export interface ManagedPythonRuntimeInstallOptions extends ManagedPythonRuntimeLayoutOptions {
features: KtxRuntimeFeature[];
force?: boolean;
exec?: ManagedPythonRuntimeExec;
}
export interface ManagedPythonRuntimeInstallResult {
status: 'ready' | 'installed';
layout: ManagedPythonRuntimeLayout;
asset: ManagedRuntimeAsset;
manifest: InstalledKtxRuntimeManifest;
}
export type ManagedPythonRuntimeStatusKind = 'missing' | 'ready' | 'mismatched' | 'broken';
export interface ManagedPythonRuntimeStatus {
kind: ManagedPythonRuntimeStatusKind;
detail: string;
layout: ManagedPythonRuntimeLayout;
manifest?: InstalledKtxRuntimeManifest;
}
export interface ManagedPythonRuntimeDoctorCheck {
id: 'uv' | 'asset' | 'runtime';
label: string;
status: 'pass' | 'fail';
detail: string;
fix?: string;
}
export interface ManagedPythonRuntimePruneResult {
runtimeRoot: string;
stale: string[];
kept: string[];
removed: string[];
}
export const MISSING_UV_RUNTIME_INSTALL_MESSAGE =
'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx runtime install --yes';
function defaultAssetDir(): string {
return fileURLToPath(new URL('../assets/python/', import.meta.url));
}
function runtimeRootFor(input: Required<Pick<ManagedPythonRuntimeLayoutOptions, 'platform' | 'env' | 'homeDir'>>): string {
if (input.env.KTX_RUNTIME_ROOT) {
return input.env.KTX_RUNTIME_ROOT;
}
if (input.platform === 'darwin') {
return join(input.homeDir, 'Library', 'Application Support', 'kaelio', 'ktx', 'runtime');
}
if (input.platform === 'win32') {
return join(input.env.LOCALAPPDATA ?? join(input.homeDir, 'AppData', 'Local'), 'Kaelio', 'KTX', 'runtime');
}
return join(input.env.XDG_DATA_HOME ?? join(input.homeDir, '.local', 'share'), 'kaelio', 'ktx', 'runtime');
}
function executablePath(venvDir: string, platform: NodeJS.Platform, name: string): string {
if (platform === 'win32') {
return join(venvDir, 'Scripts', `${name}.exe`);
}
return join(venvDir, 'bin', name);
}
export function managedPythonRuntimeLayout(options: ManagedPythonRuntimeLayoutOptions): ManagedPythonRuntimeLayout {
const platform = options.platform ?? process.platform;
const env = options.env ?? process.env;
const homeDir = options.homeDir ?? homedir();
const runtimeRoot = options.runtimeRoot ?? runtimeRootFor({ platform, env, homeDir });
const versionDir = join(runtimeRoot, options.cliVersion);
const venvDir = join(versionDir, '.venv');
const assetDir = options.assetDir ?? defaultAssetDir();
return {
cliVersion: options.cliVersion,
runtimeRoot,
versionDir,
venvDir,
manifestPath: join(versionDir, 'manifest.json'),
installLogPath: join(versionDir, 'install.log'),
assetDir,
assetManifestPath: join(assetDir, 'manifest.json'),
pythonPath: executablePath(venvDir, platform, 'python'),
daemonPath: executablePath(venvDir, platform, 'ktx-daemon'),
daemonStatePath: join(versionDir, 'daemon.json'),
daemonStdoutPath: join(versionDir, 'daemon.stdout.log'),
daemonStderrPath: join(versionDir, 'daemon.stderr.log'),
};
}
async function pathExists(path: string): Promise<boolean> {
try {
await access(path);
return true;
} catch {
return false;
}
}
function assertSafeWheelFilename(file: string): void {
if (file !== basename(file) || file.includes('/') || file.includes('\\')) {
throw new Error(`Unsafe runtime wheel filename in bundled manifest: ${file}`);
}
}
async function readJsonFile(path: string): Promise<unknown> {
return JSON.parse(await readFile(path, 'utf8')) as unknown;
}
export async function verifyRuntimeAsset(input: { assetDir: string }): Promise<ManagedRuntimeAsset> {
const manifestPath = join(input.assetDir, 'manifest.json');
const manifest = runtimeAssetManifestSchema.parse(await readJsonFile(manifestPath));
assertSafeWheelFilename(manifest.wheel.file);
const wheelPath = join(input.assetDir, manifest.wheel.file);
const wheel = await readFile(wheelPath);
const sha256 = createHash('sha256').update(wheel).digest('hex');
if (sha256 !== manifest.wheel.sha256 || wheel.byteLength !== manifest.wheel.bytes) {
throw new Error(`Bundled Python runtime wheel checksum mismatch: ${wheelPath}`);
}
return { manifest, wheelPath };
}
function normalizeFeatures(features: KtxRuntimeFeature[]): KtxRuntimeFeature[] {
const requested = new Set<KtxRuntimeFeature>(['core', ...features]);
return runtimeFeatureSchema.options.filter((feature) => requested.has(feature));
}
async function readInstalledManifest(path: string): Promise<InstalledKtxRuntimeManifest | undefined> {
if (!(await pathExists(path))) {
return undefined;
}
return installedRuntimeManifestSchema.parse(await readJsonFile(path));
}
function hasFeatures(manifest: InstalledKtxRuntimeManifest, features: KtxRuntimeFeature[]): boolean {
return normalizeFeatures(features).every((feature) => manifest.features.includes(feature));
}
async function defaultExec(
command: string,
args: string[],
options: { cwd?: string; env?: NodeJS.ProcessEnv } = {},
): Promise<{ stdout: string; stderr: string }> {
const result = await execFileAsync(command, args, {
cwd: options.cwd,
env: options.env,
encoding: 'utf8',
maxBuffer: 1024 * 1024 * 20,
});
return { stdout: result.stdout, stderr: result.stderr };
}
function errorOutput(error: unknown): { stdout: string; stderr: string } {
const value = error as { stdout?: unknown; stderr?: unknown };
return {
stdout: typeof value.stdout === 'string' ? value.stdout : '',
stderr: typeof value.stderr === 'string' ? value.stderr : '',
};
}
async function runLogged(input: {
exec: ManagedPythonRuntimeExec;
logPath: string;
command: string;
args: string[];
cwd?: string;
}): Promise<{ stdout: string; stderr: string }> {
await appendFile(input.logPath, `$ ${input.command} ${input.args.join(' ')}\n`);
try {
const result = await input.exec(input.command, input.args, { cwd: input.cwd });
if (result.stdout) {
await appendFile(input.logPath, result.stdout.endsWith('\n') ? result.stdout : `${result.stdout}\n`);
}
if (result.stderr) {
await appendFile(input.logPath, result.stderr.endsWith('\n') ? result.stderr : `${result.stderr}\n`);
}
return result;
} catch (error) {
const output = errorOutput(error);
if (output.stdout) {
await appendFile(input.logPath, output.stdout.endsWith('\n') ? output.stdout : `${output.stdout}\n`);
}
if (output.stderr) {
await appendFile(input.logPath, output.stderr.endsWith('\n') ? output.stderr : `${output.stderr}\n`);
}
throw new Error(`Python runtime install failed. Install log: ${input.logPath}`);
}
}
async function ensureUv(exec: ManagedPythonRuntimeExec): Promise<string> {
try {
const result = await exec('uv', ['--version']);
return result.stdout.trim() || 'uv available';
} catch {
throw new Error(MISSING_UV_RUNTIME_INSTALL_MESSAGE);
}
}
export async function installManagedPythonRuntime(
options: ManagedPythonRuntimeInstallOptions,
): Promise<ManagedPythonRuntimeInstallResult> {
const layout = managedPythonRuntimeLayout(options);
const exec = options.exec ?? defaultExec;
const features = normalizeFeatures(options.features);
const asset = await verifyRuntimeAsset({ assetDir: layout.assetDir });
const existing = await readInstalledManifest(layout.manifestPath);
if (
options.force !== true &&
existing &&
existing.cliVersion === options.cliVersion &&
existing.asset.wheel.sha256 === asset.manifest.wheel.sha256 &&
hasFeatures(existing, features) &&
(await pathExists(existing.python.executable)) &&
(await pathExists(existing.python.daemonExecutable))
) {
return { status: 'ready', layout, asset, manifest: existing };
}
await rm(layout.versionDir, { recursive: true, force: true });
await mkdir(layout.versionDir, { recursive: true });
await writeFile(layout.installLogPath, '');
await ensureUv(exec);
await runLogged({ exec, logPath: layout.installLogPath, command: 'uv', args: ['venv', layout.venvDir] });
const wheelSpec = features.includes('local-embeddings') ? `${asset.wheelPath}[local-embeddings]` : asset.wheelPath;
await runLogged({
exec,
logPath: layout.installLogPath,
command: 'uv',
args: ['pip', 'install', '--python', layout.pythonPath, wheelSpec],
});
const manifest: InstalledKtxRuntimeManifest = {
schemaVersion: 1,
cliVersion: options.cliVersion,
installedAt: new Date().toISOString(),
asset: asset.manifest,
features,
python: {
executable: layout.pythonPath,
daemonExecutable: layout.daemonPath,
},
installLog: layout.installLogPath,
};
await writeFile(layout.manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
return { status: 'installed', layout, asset, manifest };
}
export async function readManagedPythonRuntimeStatus(
options: ManagedPythonRuntimeLayoutOptions,
): Promise<ManagedPythonRuntimeStatus> {
const layout = managedPythonRuntimeLayout(options);
let manifest: InstalledKtxRuntimeManifest | undefined;
try {
manifest = await readInstalledManifest(layout.manifestPath);
} catch (error) {
return {
kind: 'broken',
detail: `Runtime manifest is invalid: ${error instanceof Error ? error.message : String(error)}`,
layout,
};
}
if (!manifest) {
return { kind: 'missing', detail: `No runtime manifest at ${layout.manifestPath}`, layout };
}
if (manifest.cliVersion !== options.cliVersion) {
return {
kind: 'mismatched',
detail: `Runtime is for CLI ${manifest.cliVersion}, current CLI is ${options.cliVersion}`,
layout,
manifest,
};
}
if (!(await pathExists(manifest.python.executable))) {
return { kind: 'broken', detail: `Missing Python executable: ${manifest.python.executable}`, layout, manifest };
}
if (!(await pathExists(manifest.python.daemonExecutable))) {
return { kind: 'broken', detail: `Missing ktx-daemon executable: ${manifest.python.daemonExecutable}`, layout, manifest };
}
return { kind: 'ready', detail: `Runtime ready at ${layout.versionDir}`, layout, manifest };
}
function check(
status: ManagedPythonRuntimeDoctorCheck['status'],
input: Omit<ManagedPythonRuntimeDoctorCheck, 'status'>,
): ManagedPythonRuntimeDoctorCheck {
return { status, ...input };
}
export async function doctorManagedPythonRuntime(
options: ManagedPythonRuntimeLayoutOptions & { exec?: ManagedPythonRuntimeExec },
): Promise<ManagedPythonRuntimeDoctorCheck[]> {
const exec = options.exec ?? defaultExec;
const checks: ManagedPythonRuntimeDoctorCheck[] = [];
try {
const version = await ensureUv(exec);
checks.push(check('pass', { id: 'uv', label: 'uv', detail: version }));
} catch (error) {
checks.push(
check('fail', {
id: 'uv',
label: 'uv',
detail: error instanceof Error ? error.message : String(error),
fix: 'Install uv, make sure it is on PATH, and run: ktx runtime install --yes',
}),
);
}
try {
const asset = await verifyRuntimeAsset({ assetDir: managedPythonRuntimeLayout(options).assetDir });
checks.push(check('pass', { id: 'asset', label: 'Bundled Python wheel', detail: asset.wheelPath }));
} catch (error) {
checks.push(
check('fail', {
id: 'asset',
label: 'Bundled Python wheel',
detail: error instanceof Error ? error.message : String(error),
fix: 'Run: pnpm run artifacts:check',
}),
);
}
const status = await readManagedPythonRuntimeStatus(options);
checks.push(
check(status.kind === 'ready' ? 'pass' : 'fail', {
id: 'runtime',
label: 'Managed Python runtime',
detail: status.detail,
...(status.kind === 'ready' ? {} : { fix: 'Run: ktx runtime install --yes' }),
}),
);
return checks;
}
export async function pruneManagedPythonRuntimes(options: {
cliVersion: string;
runtimeRoot: string;
dryRun?: boolean;
}): Promise<ManagedPythonRuntimePruneResult> {
if (!(await pathExists(options.runtimeRoot))) {
return { runtimeRoot: options.runtimeRoot, stale: [], kept: [], removed: [] };
}
const entries = await readdir(options.runtimeRoot);
const stale: string[] = [];
const kept: string[] = [];
for (const entry of entries) {
const path = join(options.runtimeRoot, entry);
const info = await stat(path);
if (!info.isDirectory()) {
continue;
}
if (entry === options.cliVersion) {
kept.push(path);
} else {
stale.push(path);
}
}
const removed: string[] = [];
if (options.dryRun !== true) {
for (const path of stale) {
await rm(path, { recursive: true, force: true });
removed.push(path);
}
}
return { runtimeRoot: options.runtimeRoot, stale, kept, removed };
}

315
packages/cli/src/runtime.test.ts Normal file
View file

@ -0,0 +1,315 @@
import { describe, expect, it, vi } from 'vitest';
import type {
ManagedPythonDaemonStartResult,
ManagedPythonDaemonStopResult,
} from './managed-python-daemon.js';
import type {
ManagedPythonRuntimeDoctorCheck,
ManagedPythonRuntimeInstallResult,
ManagedPythonRuntimeStatus,
} from './managed-python-runtime.js';
import { runKtxRuntime, type KtxRuntimeDeps } from './runtime.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('runKtxRuntime', () => {
it('installs the requested runtime feature and prints the manifest path', async () => {
const io = makeIo();
const deps: KtxRuntimeDeps = {
installRuntime: vi.fn(async (): Promise<ManagedPythonRuntimeInstallResult> => ({
status: 'installed',
layout: {
cliVersion: '0.2.0',
runtimeRoot: '/runtime',
versionDir: '/runtime/0.2.0',
venvDir: '/runtime/0.2.0/.venv',
manifestPath: '/runtime/0.2.0/manifest.json',
installLogPath: '/runtime/0.2.0/install.log',
assetDir: '/assets/python',
assetManifestPath: '/assets/python/manifest.json',
pythonPath: '/runtime/0.2.0/.venv/bin/python',
daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon',
daemonStatePath: '/runtime/0.2.0/daemon.json',
daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log',
daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log',
},
asset: {
wheelPath: '/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl',
manifest: {
schemaVersion: 1,
distributionName: 'kaelio-ktx',
normalizedName: 'kaelio_ktx',
version: '0.1.0',
wheel: {
file: 'kaelio_ktx-0.1.0-py3-none-any.whl',
sha256: 'a'.repeat(64),
bytes: 10,
},
},
},
manifest: {
schemaVersion: 1,
cliVersion: '0.2.0',
installedAt: '2026-05-11T00:00:00.000Z',
asset: {
schemaVersion: 1,
distributionName: 'kaelio-ktx',
normalizedName: 'kaelio_ktx',
version: '0.1.0',
wheel: {
file: 'kaelio_ktx-0.1.0-py3-none-any.whl',
sha256: 'a'.repeat(64),
bytes: 10,
},
},
features: ['core', 'local-embeddings'],
python: {
executable: '/runtime/0.2.0/.venv/bin/python',
daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon',
},
installLog: '/runtime/0.2.0/install.log',
},
})),
};
await expect(
runKtxRuntime(
{ command: 'install', cliVersion: '0.2.0', feature: 'local-embeddings', force: true },
io.io,
deps,
),
).resolves.toBe(0);
expect(deps.installRuntime).toHaveBeenCalledWith({
cliVersion: '0.2.0',
features: ['local-embeddings'],
force: true,
});
expect(io.stdout()).toContain('Installed KTX Python runtime');
expect(io.stdout()).toContain('features: core, local-embeddings');
expect(io.stdout()).toContain('manifest: /runtime/0.2.0/manifest.json');
expect(io.stderr()).toBe('');
});
it('starts the managed Python daemon and prints the base URL', async () => {
const io = makeIo();
const deps: KtxRuntimeDeps = {
startDaemon: vi.fn(async (): Promise<ManagedPythonDaemonStartResult> => ({
status: 'started',
baseUrl: 'http://127.0.0.1:61234',
layout: {
cliVersion: '0.2.0',
runtimeRoot: '/runtime',
versionDir: '/runtime/0.2.0',
venvDir: '/runtime/0.2.0/.venv',
manifestPath: '/runtime/0.2.0/manifest.json',
installLogPath: '/runtime/0.2.0/install.log',
assetDir: '/assets/python',
assetManifestPath: '/assets/python/manifest.json',
pythonPath: '/runtime/0.2.0/.venv/bin/python',
daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon',
daemonStatePath: '/runtime/0.2.0/daemon.json',
daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log',
daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log',
},
state: {
schemaVersion: 1,
pid: 4242,
host: '127.0.0.1',
port: 61234,
version: '0.2.0',
features: ['core', 'local-embeddings'],
startedAt: '2026-05-11T00:00:00.000Z',
stdoutLog: '/runtime/0.2.0/daemon.stdout.log',
stderrLog: '/runtime/0.2.0/daemon.stderr.log',
},
})),
};
await expect(
runKtxRuntime(
{ command: 'start', cliVersion: '0.2.0', feature: 'local-embeddings', force: true },
io.io,
deps,
),
).resolves.toBe(0);
expect(deps.startDaemon).toHaveBeenCalledWith({
cliVersion: '0.2.0',
features: ['local-embeddings'],
force: true,
});
expect(io.stdout()).toContain('Started KTX Python daemon');
expect(io.stdout()).toContain('url: http://127.0.0.1:61234');
expect(io.stdout()).toContain('pid: 4242');
expect(io.stdout()).toContain('features: core, local-embeddings');
expect(io.stdout()).toContain('stderr: /runtime/0.2.0/daemon.stderr.log');
});
it('stops the managed Python daemon', async () => {
const io = makeIo();
const deps: KtxRuntimeDeps = {
stopDaemon: vi.fn(async (): Promise<ManagedPythonDaemonStopResult> => ({
status: 'stopped',
layout: {
cliVersion: '0.2.0',
runtimeRoot: '/runtime',
versionDir: '/runtime/0.2.0',
venvDir: '/runtime/0.2.0/.venv',
manifestPath: '/runtime/0.2.0/manifest.json',
installLogPath: '/runtime/0.2.0/install.log',
assetDir: '/assets/python',
assetManifestPath: '/assets/python/manifest.json',
pythonPath: '/runtime/0.2.0/.venv/bin/python',
daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon',
daemonStatePath: '/runtime/0.2.0/daemon.json',
daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log',
daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log',
},
state: {
schemaVersion: 1,
pid: 4242,
host: '127.0.0.1',
port: 61234,
version: '0.2.0',
features: ['core'],
startedAt: '2026-05-11T00:00:00.000Z',
stdoutLog: '/runtime/0.2.0/daemon.stdout.log',
stderrLog: '/runtime/0.2.0/daemon.stderr.log',
},
})),
};
await expect(runKtxRuntime({ command: 'stop', cliVersion: '0.2.0' }, io.io, deps)).resolves.toBe(0);
expect(deps.stopDaemon).toHaveBeenCalledWith({ cliVersion: '0.2.0' });
expect(io.stdout()).toContain('Stopped KTX Python daemon');
expect(io.stdout()).toContain('pid: 4242');
});
it('prints runtime status as JSON', async () => {
const io = makeIo();
const deps: KtxRuntimeDeps = {
readStatus: vi.fn(async (): Promise<ManagedPythonRuntimeStatus> => ({
kind: 'missing',
detail: 'No runtime manifest at /runtime/0.2.0/manifest.json',
layout: {
cliVersion: '0.2.0',
runtimeRoot: '/runtime',
versionDir: '/runtime/0.2.0',
venvDir: '/runtime/0.2.0/.venv',
manifestPath: '/runtime/0.2.0/manifest.json',
installLogPath: '/runtime/0.2.0/install.log',
assetDir: '/assets/python',
assetManifestPath: '/assets/python/manifest.json',
pythonPath: '/runtime/0.2.0/.venv/bin/python',
daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon',
daemonStatePath: '/runtime/0.2.0/daemon.json',
daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log',
daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log',
},
})),
};
await expect(runKtxRuntime({ command: 'status', cliVersion: '0.2.0', json: true }, io.io, deps)).resolves.toBe(0);
expect(JSON.parse(io.stdout())).toMatchObject({
kind: 'missing',
detail: 'No runtime manifest at /runtime/0.2.0/manifest.json',
layout: { runtimeRoot: '/runtime' },
});
});
it('returns failure for doctor when any check fails', async () => {
const io = makeIo();
const deps: KtxRuntimeDeps = {
doctorRuntime: vi.fn(async (): Promise<ManagedPythonRuntimeDoctorCheck[]> => [
{ id: 'uv', label: 'uv', status: 'pass', detail: 'uv 0.9.5' },
{
id: 'runtime',
label: 'Managed Python runtime',
status: 'fail',
detail: 'No runtime manifest',
fix: 'Run: ktx runtime install --yes',
},
]),
};
await expect(runKtxRuntime({ command: 'doctor', cliVersion: '0.2.0', json: false }, io.io, deps)).resolves.toBe(1);
expect(io.stdout()).toContain('PASS uv: uv 0.9.5');
expect(io.stdout()).toContain('FAIL Managed Python runtime: No runtime manifest');
expect(io.stdout()).toContain('Fix: Run: ktx runtime install --yes');
});
it('requires --yes before pruning stale runtime directories', async () => {
const io = makeIo();
const deps: KtxRuntimeDeps = {
pruneRuntime: vi.fn(async () => {
throw new Error('should not prune without --yes');
}),
};
await expect(runKtxRuntime({ command: 'prune', cliVersion: '0.2.0', dryRun: false, yes: false }, io.io, deps))
.resolves.toBe(1);
expect(io.stderr()).toContain('Refusing to prune without --yes');
expect(deps.pruneRuntime).not.toHaveBeenCalled();
});
it('prints stale directories during prune dry-run', async () => {
const io = makeIo();
const deps: KtxRuntimeDeps = {
readStatus: vi.fn(async (): Promise<ManagedPythonRuntimeStatus> => ({
kind: 'missing',
detail: 'No runtime manifest at /runtime/0.2.0/manifest.json',
layout: {
cliVersion: '0.2.0',
runtimeRoot: '/runtime',
versionDir: '/runtime/0.2.0',
venvDir: '/runtime/0.2.0/.venv',
manifestPath: '/runtime/0.2.0/manifest.json',
installLogPath: '/runtime/0.2.0/install.log',
assetDir: '/assets/python',
assetManifestPath: '/assets/python/manifest.json',
pythonPath: '/runtime/0.2.0/.venv/bin/python',
daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon',
daemonStatePath: '/runtime/0.2.0/daemon.json',
daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log',
daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log',
},
})),
pruneRuntime: vi.fn(async () => ({
runtimeRoot: '/runtime',
stale: ['/runtime/0.1.0'],
kept: ['/runtime/0.2.0'],
removed: [],
})),
};
await expect(runKtxRuntime({ command: 'prune', cliVersion: '0.2.0', dryRun: true, yes: false }, io.io, deps))
.resolves.toBe(0);
expect(io.stdout()).toContain('Stale KTX Python runtimes');
expect(io.stdout()).toContain('/runtime/0.1.0');
});
});

187
packages/cli/src/runtime.ts Normal file
View file

@ -0,0 +1,187 @@
import type { KtxCliIo } from './cli-runtime.js';
import {
startManagedPythonDaemon,
stopManagedPythonDaemon,
type ManagedPythonDaemonStartResult,
type ManagedPythonDaemonStopResult,
} from './managed-python-daemon.js';
import {
doctorManagedPythonRuntime,
installManagedPythonRuntime,
pruneManagedPythonRuntimes,
readManagedPythonRuntimeStatus,
type KtxRuntimeFeature,
type ManagedPythonRuntimeDoctorCheck,
type ManagedPythonRuntimeInstallOptions,
type ManagedPythonRuntimeInstallResult,
type ManagedPythonRuntimeLayoutOptions,
type ManagedPythonRuntimePruneResult,
type ManagedPythonRuntimeStatus,
} from './managed-python-runtime.js';
export type KtxRuntimeArgs =
| { command: 'install'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean }
| { command: 'start'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean }
| { command: 'stop'; cliVersion: string }
| { command: 'status'; cliVersion: string; json: boolean }
| { command: 'doctor'; cliVersion: string; json: boolean }
| { command: 'prune'; cliVersion: string; dryRun: boolean; yes: boolean };
export interface KtxRuntimeDeps {
installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise<ManagedPythonRuntimeInstallResult>;
startDaemon?: (options: {
cliVersion: string;
features: KtxRuntimeFeature[];
force?: boolean;
}) => Promise<ManagedPythonDaemonStartResult>;
stopDaemon?: (options: { cliVersion: string }) => Promise<ManagedPythonDaemonStopResult>;
readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise<ManagedPythonRuntimeStatus>;
doctorRuntime?: (options: ManagedPythonRuntimeLayoutOptions) => Promise<ManagedPythonRuntimeDoctorCheck[]>;
pruneRuntime?: (options: {
cliVersion: string;
runtimeRoot: string;
dryRun?: boolean;
}) => Promise<ManagedPythonRuntimePruneResult>;
}
function writeJson(io: KtxCliIo, value: unknown): void {
io.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
}
function writeInstallResult(io: KtxCliIo, result: ManagedPythonRuntimeInstallResult): void {
const verb = result.status === 'ready' ? 'Using existing' : 'Installed';
io.stdout.write(`${verb} KTX Python runtime\n`);
io.stdout.write(`version: ${result.manifest.cliVersion}\n`);
io.stdout.write(`features: ${result.manifest.features.join(', ')}\n`);
io.stdout.write(`python: ${result.manifest.python.executable}\n`);
io.stdout.write(`daemon: ${result.manifest.python.daemonExecutable}\n`);
io.stdout.write(`manifest: ${result.layout.manifestPath}\n`);
io.stdout.write(`install log: ${result.layout.installLogPath}\n`);
}
function writeDaemonStart(io: KtxCliIo, result: ManagedPythonDaemonStartResult): void {
const verb = result.status === 'reused' ? 'Using existing' : 'Started';
io.stdout.write(`${verb} KTX Python daemon\n`);
io.stdout.write(`url: ${result.baseUrl}\n`);
io.stdout.write(`pid: ${result.state.pid}\n`);
io.stdout.write(`version: ${result.state.version}\n`);
io.stdout.write(`features: ${result.state.features.join(', ')}\n`);
io.stdout.write(`state: ${result.layout.daemonStatePath}\n`);
io.stdout.write(`stdout: ${result.state.stdoutLog}\n`);
io.stdout.write(`stderr: ${result.state.stderrLog}\n`);
}
function writeDaemonStop(io: KtxCliIo, result: ManagedPythonDaemonStopResult): void {
if (result.status === 'already-stopped') {
io.stdout.write('KTX Python daemon already stopped\n');
return;
}
io.stdout.write('Stopped KTX Python daemon\n');
io.stdout.write(`pid: ${result.state?.pid ?? 'unknown'}\n`);
io.stdout.write(`state: ${result.layout.daemonStatePath}\n`);
}
function writeStatus(io: KtxCliIo, status: ManagedPythonRuntimeStatus): void {
io.stdout.write('KTX Python runtime\n');
io.stdout.write(`status: ${status.kind}\n`);
io.stdout.write(`detail: ${status.detail}\n`);
io.stdout.write(`runtime root: ${status.layout.runtimeRoot}\n`);
io.stdout.write(`version dir: ${status.layout.versionDir}\n`);
if (status.manifest) {
io.stdout.write(`features: ${status.manifest.features.join(', ')}\n`);
io.stdout.write(`python: ${status.manifest.python.executable}\n`);
io.stdout.write(`daemon: ${status.manifest.python.daemonExecutable}\n`);
}
}
function writeDoctor(io: KtxCliIo, checks: ManagedPythonRuntimeDoctorCheck[]): void {
io.stdout.write('KTX Python runtime doctor\n');
for (const check of checks) {
io.stdout.write(`${check.status.toUpperCase()} ${check.label}: ${check.detail}\n`);
if (check.fix) {
io.stdout.write(` Fix: ${check.fix}\n`);
}
}
}
function writePrune(io: KtxCliIo, result: ManagedPythonRuntimePruneResult, dryRun: boolean): void {
if (result.stale.length === 0) {
io.stdout.write(`No stale KTX Python runtimes found under ${result.runtimeRoot}\n`);
return;
}
io.stdout.write(dryRun ? 'Stale KTX Python runtimes\n' : 'Removed stale KTX Python runtimes\n');
for (const path of dryRun ? result.stale : result.removed) {
io.stdout.write(`${path}\n`);
}
}
export async function runKtxRuntime(
args: KtxRuntimeArgs,
io: KtxCliIo = process,
deps: KtxRuntimeDeps = {},
): Promise<number> {
try {
if (args.command === 'install') {
const installRuntime = deps.installRuntime ?? installManagedPythonRuntime;
const result = await installRuntime({
cliVersion: args.cliVersion,
features: [args.feature],
force: args.force,
});
writeInstallResult(io, result);
return 0;
}
if (args.command === 'start') {
const startDaemon = deps.startDaemon ?? startManagedPythonDaemon;
const result = await startDaemon({
cliVersion: args.cliVersion,
features: [args.feature],
force: args.force,
});
writeDaemonStart(io, result);
return 0;
}
if (args.command === 'stop') {
const stopDaemon = deps.stopDaemon ?? stopManagedPythonDaemon;
const result = await stopDaemon({ cliVersion: args.cliVersion });
writeDaemonStop(io, result);
return 0;
}
if (args.command === 'status') {
const readStatus = deps.readStatus ?? readManagedPythonRuntimeStatus;
const status = await readStatus({ cliVersion: args.cliVersion });
if (args.json) {
writeJson(io, status);
} else {
writeStatus(io, status);
}
return 0;
}
if (args.command === 'doctor') {
const doctorRuntime = deps.doctorRuntime ?? doctorManagedPythonRuntime;
const checks = await doctorRuntime({ cliVersion: args.cliVersion });
if (args.json) {
writeJson(io, { checks });
} else {
writeDoctor(io, checks);
}
return checks.some((check) => check.status === 'fail') ? 1 : 0;
}
if (!args.dryRun && !args.yes) {
io.stderr.write('Refusing to prune without --yes. Preview with: ktx runtime prune --dry-run\n');
return 1;
}
const status = await (deps.readStatus ?? readManagedPythonRuntimeStatus)({ cliVersion: args.cliVersion });
const pruneRuntime = deps.pruneRuntime ?? pruneManagedPythonRuntimes;
const result = await pruneRuntime({
cliVersion: args.cliVersion,
runtimeRoot: status.layout.runtimeRoot,
dryRun: args.dryRun,
});
writePrune(io, result, args.dryRun);
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

43
packages/cli/src/scan.test.ts
View file

@ -356,6 +356,49 @@ describe('runKtxScan', () => {
expect(io.stdout()).not.toContain('/~');
});
it('passes managed daemon options to local ingest adapters when no explicit daemon URL is set', async () => {
await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
const createLocalIngestAdapters = vi.fn(() => []);
const runLocalScan = vi.fn(
async (_input: RunLocalScanOptions): Promise<LocalScanRunResult> => ({
runId: 'scan-run-1',
status: 'done',
done: true,
connectionId: 'warehouse',
mode: 'structural',
dryRun: false,
syncId: 'sync-1',
report,
}),
);
const io = makeIo();
await expect(
runKtxScan(
{
command: 'run',
projectDir: tempDir,
connectionId: 'warehouse',
mode: 'structural',
detectRelationships: false,
dryRun: false,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
},
io.io,
{ runLocalScan, createLocalIngestAdapters },
),
).resolves.toBe(0);
expect(createLocalIngestAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir }), {
managedDaemon: {
cliVersion: '0.2.0',
installPolicy: 'auto',
io: io.io,
},
});
});
it('explains warnings, capability gaps, and relationships in human scan summaries', async () => {
await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
const runLocalScan = vi.fn(

18
packages/cli/src/scan.ts
View file

@ -33,6 +33,7 @@ import {
import type { KtxCliIo } from './index.js';
import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
import { profileMark } from './startup-profile.js';
profileMark('module:scan');
@ -46,6 +47,8 @@ export type KtxScanArgs =
detectRelationships: boolean;
dryRun: boolean;
databaseIntrospectionUrl?: string;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
}
| { command: 'status'; projectDir: string; runId: string }
| { command: 'report'; projectDir: string; runId: string; json: boolean }
@ -220,6 +223,17 @@ function warningLine(warning: KtxScanWarning): string {
return `${warning.code}: ${location}${warning.message}`;
}
function managedDaemonOptionsForScanRun(args: Extract<KtxScanArgs, { command: 'run' }>, io: KtxCliIo) {
if (args.databaseIntrospectionUrl || !args.cliVersion || !args.runtimeInstallPolicy) {
return undefined;
}
return {
cliVersion: args.cliVersion,
installPolicy: args.runtimeInstallPolicy,
io,
};
}
function writeNeedsAttention(report: KtxScanReport, io: KtxCliIo): void {
io.stdout.write('\nNeeds attention\n');
if (report.warnings.length === 0 && report.capabilityGaps.length === 0) {
@ -704,6 +718,7 @@ export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps
return 0;
}
const managedDaemon = managedDaemonOptionsForScanRun(args, io);
const connector =
args.mode !== 'structural' || args.detectRelationships
? await createKtxCliScanConnector(project, args.connectionId)
@ -720,7 +735,8 @@ export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps
databaseIntrospectionUrl: args.databaseIntrospectionUrl,
connector,
adapters: (deps.createLocalIngestAdapters ?? createKtxCliLocalIngestAdapters)(project, {
databaseIntrospectionUrl: args.databaseIntrospectionUrl,
...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}),
...(managedDaemon ? { managedDaemon } : {}),
}),
progress,
});

120
packages/cli/src/serve.test.ts
View file

@ -6,6 +6,19 @@ import { initKtxProject } from '@ktx/context/project';
import { describe, expect, it, vi } from 'vitest';
import { runKtxServeStdio } from './serve.js';
function makeManagedRuntimeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: { write: (chunk: string) => (stdout += chunk) },
stderr: { write: (chunk: string) => (stderr += chunk) },
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('runKtxServeStdio', () => {
it('loads the project, creates local ports, and connects the server to stdio', async () => {
const connect = vi.fn().mockResolvedValue(undefined);
@ -149,6 +162,9 @@ describe('runKtxServeStdio', () => {
expect.objectContaining({
localIngest: expect.objectContaining({
adapters: expect.any(Array),
pullConfigOptions: {
databaseIntrospectionUrl: 'http://127.0.0.1:8765',
},
}),
localScan: expect.objectContaining({
adapters: createdAdapters,
@ -161,6 +177,63 @@ describe('runKtxServeStdio', () => {
});
});
it('passes managed daemon options to MCP local ingest adapters and pull-config options', async () => {
const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never;
const adapters: SourceAdapter[] = [
{ source: 'looker', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) },
];
const createIngestAdapters = vi.fn(() => adapters);
const createContextTools = vi.fn(() => ({ connections: { list: async () => [] } }));
const managedRuntimeIo = makeManagedRuntimeIo();
await expect(
runKtxServeStdio(
{
mcp: 'stdio',
projectDir: '/tmp/ktx-project',
userId: 'agent',
semanticCompute: false,
semanticComputeUrl: undefined,
databaseIntrospectionUrl: undefined,
executeQueries: false,
memoryCapture: false,
memoryModel: undefined,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
},
{
loadProject: async () => project,
createContextTools,
createIngestAdapters,
managedRuntimeIo: managedRuntimeIo.io,
createServer: vi.fn(() => ({ connect: vi.fn(async () => undefined) }) as never),
createTransport: vi.fn(() => ({}) as never),
stderr: { write: vi.fn() },
},
),
).resolves.toBe(0);
const expectedManagedDaemon = {
cliVersion: '0.2.0',
installPolicy: 'auto',
io: managedRuntimeIo.io,
};
expect(createIngestAdapters).toHaveBeenCalledWith(project, {
managedDaemon: expectedManagedDaemon,
});
expect(createContextTools).toHaveBeenCalledWith(
project,
expect.objectContaining({
localIngest: expect.objectContaining({
adapters,
pullConfigOptions: {
managedDaemon: expectedManagedDaemon,
},
}),
}),
);
});
it('uses CLI-native local ingest adapters for standalone scan tools', async () => {
const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never;
const createContextTools = vi.fn(() => ({}) as never);
@ -241,6 +314,53 @@ describe('runKtxServeStdio', () => {
}
});
it('uses managed semantic compute when MCP semantic compute has no explicit HTTP URL', async () => {
const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never;
const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };
const createManagedSemanticLayerCompute = vi.fn(async () => semanticLayerCompute);
const createContextTools = vi.fn(() => ({ connections: { list: async () => [] } }));
const managedRuntimeIo = makeManagedRuntimeIo();
await expect(
runKtxServeStdio(
{
mcp: 'stdio',
projectDir: '/tmp/ktx-project',
userId: 'agent',
semanticCompute: true,
semanticComputeUrl: undefined,
databaseIntrospectionUrl: undefined,
executeQueries: false,
memoryCapture: false,
memoryModel: undefined,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
},
{
loadProject: async () => project,
createContextTools,
createManagedSemanticLayerCompute,
managedRuntimeIo: managedRuntimeIo.io,
createServer: vi.fn(() => ({ connect: vi.fn(async () => undefined) }) as never),
createTransport: vi.fn(() => ({}) as never),
stderr: { write: vi.fn() },
},
),
).resolves.toBe(0);
expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({
cliVersion: '0.2.0',
installPolicy: 'auto',
io: managedRuntimeIo.io,
});
expect(createContextTools).toHaveBeenCalledWith(
project,
expect.objectContaining({
semanticLayerCompute,
}),
);
});
it('uses the HTTP semantic compute port when a daemon URL is provided', async () => {
const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never;
const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };

76
packages/cli/src/serve.ts
View file

@ -2,7 +2,6 @@ import { createLocalKtxLlmProviderFromConfig } from '@ktx/context';
import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from '@ktx/context/connections';
import {
createHttpSemanticLayerComputePort,
createPythonSemanticLayerComputePort,
type KtxSemanticLayerComputePort,
} from '@ktx/context/daemon';
import { createDefaultLocalIngestAdapters, type LocalIngestMcpOptions } from '@ktx/context/ingest';
@ -15,8 +14,14 @@ import { createLocalProjectMemoryCapture, type MemoryCaptureService } from '@ktx
import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project';
import type { LocalScanMcpOptions } from '@ktx/context/scan';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import type { KtxCliIo } from './cli-runtime.js';
import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
import {
createManagedPythonSemanticLayerComputePort,
type KtxManagedPythonInstallPolicy,
} from './managed-python-command.js';
import type { ManagedPythonCoreDaemonOptions } from './managed-python-http.js';
import { profileMark } from './startup-profile.js';
profileMark('module:serve');
@ -31,6 +36,8 @@ export interface KtxServeArgs {
executeQueries: boolean;
memoryCapture: boolean;
memoryModel?: string;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
}
interface KtxServeIo {
@ -48,8 +55,10 @@ interface KtxServeDeps {
loadProject?: typeof loadKtxProject;
createContextTools?: (project: KtxLocalProject, options?: LocalProjectContextToolOptions) => KtxMcpContextPorts;
createSemanticLayerCompute?: () => KtxSemanticLayerComputePort;
createManagedSemanticLayerCompute?: typeof createManagedPythonSemanticLayerComputePort;
managedRuntimeIo?: KtxCliIo;
createHttpSemanticLayerCompute?: (baseUrl: string) => KtxSemanticLayerComputePort;
createIngestAdapters?: typeof createDefaultLocalIngestAdapters;
createIngestAdapters?: typeof createKtxCliLocalIngestAdapters;
createQueryExecutor?: () => KtxSqlQueryExecutorPort;
createMemoryCapture?: typeof createLocalProjectMemoryCapture;
createServer?: typeof createDefaultKtxMcpServer;
@ -57,6 +66,51 @@ interface KtxServeDeps {
stderr?: KtxServeIo['stderr'];
}
function requiredManagedRuntimeCliVersion(args: KtxServeArgs): string {
if (!args.cliVersion) {
throw new Error('Managed Python semantic compute requires a CLI version.');
}
return args.cliVersion;
}
function managedDaemonOptionsForServe(
args: KtxServeArgs,
deps: KtxServeDeps,
): ManagedPythonCoreDaemonOptions | undefined {
if (args.databaseIntrospectionUrl || !args.cliVersion) {
return undefined;
}
return {
cliVersion: args.cliVersion,
installPolicy: args.runtimeInstallPolicy ?? 'prompt',
io: deps.managedRuntimeIo ?? process,
};
}
async function createServeSemanticLayerCompute(
args: KtxServeArgs,
deps: KtxServeDeps,
): Promise<KtxSemanticLayerComputePort | undefined> {
if (!args.semanticCompute) {
return undefined;
}
if (args.semanticComputeUrl) {
return (deps.createHttpSemanticLayerCompute ?? ((baseUrl) => createHttpSemanticLayerComputePort({ baseUrl })))(
args.semanticComputeUrl,
);
}
if (deps.createSemanticLayerCompute) {
return deps.createSemanticLayerCompute();
}
const createManagedSemanticLayerCompute =
deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort;
return createManagedSemanticLayerCompute({
cliVersion: requiredManagedRuntimeCliVersion(args),
installPolicy: args.runtimeInstallPolicy ?? 'prompt',
io: deps.managedRuntimeIo ?? process,
});
}
export async function runKtxServeStdio(args: KtxServeArgs, deps: KtxServeDeps = {}): Promise<number> {
const loadProjectFn = deps.loadProject ?? loadKtxProject;
const createContextToolsFn = deps.createContextTools ?? createLocalProjectMcpContextPorts;
@ -65,20 +119,17 @@ export async function runKtxServeStdio(args: KtxServeArgs, deps: KtxServeDeps =
const stderr = deps.stderr ?? process.stderr;
const project = await loadProjectFn({ projectDir: args.projectDir });
const semanticLayerCompute = args.semanticCompute
? args.semanticComputeUrl
? (deps.createHttpSemanticLayerCompute ?? ((baseUrl) => createHttpSemanticLayerComputePort({ baseUrl })))(
args.semanticComputeUrl,
)
: (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)()
: undefined;
const semanticLayerCompute = await createServeSemanticLayerCompute(args, deps);
const queryExecutor = args.executeQueries
? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)()
: undefined;
const createIngestAdapters = deps.createIngestAdapters ?? createKtxCliLocalIngestAdapters;
const localAdapters = createIngestAdapters(project, {
databaseIntrospectionUrl: args.databaseIntrospectionUrl,
});
const managedDaemon = managedDaemonOptionsForServe(args, deps);
const localAdapterOptions = {
...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}),
...(managedDaemon ? { managedDaemon } : {}),
};
const localAdapters = createIngestAdapters(project, localAdapterOptions);
const llmProvider = args.memoryCapture
? (createLocalKtxLlmProviderFromConfig(project.config.llm) ?? undefined)
: undefined;
@ -90,6 +141,7 @@ export async function runKtxServeStdio(args: KtxServeArgs, deps: KtxServeDeps =
: undefined;
const localIngest: LocalIngestMcpOptions = {
adapters: localAdapters,
pullConfigOptions: localAdapterOptions,
...(semanticLayerCompute ? { semanticLayerCompute } : {}),
...(queryExecutor ? { queryExecutor } : {}),
};

134
packages/cli/src/setup-embeddings.test.ts
View file

@ -46,6 +46,15 @@ function makePromptAdapter(options: {
};
}
function managedDaemon(baseUrl = 'http://127.0.0.1:61234') {
return {
baseUrl,
env: {
KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL: baseUrl,
},
};
}
describe('setup embeddings step', () => {
let tempDir: string;
@ -67,6 +76,8 @@ describe('setup embeddings step', () => {
{
projectDir: tempDir,
inputMode: 'auto',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
io.io,
@ -94,10 +105,12 @@ describe('setup embeddings step', () => {
{
projectDir: tempDir,
inputMode: 'auto',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
io.io,
{ prompts, env: {}, healthCheck },
{ prompts, env: {}, healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()) },
);
expect(result.status).toBe('ready');
@ -106,7 +119,7 @@ describe('setup embeddings step', () => {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' },
sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' },
});
expect(vi.mocked(prompts.select).mock.calls.map((call) => call[0].message)).toEqual([
EMBEDDING_OPTION_PROMPT_MESSAGE,
@ -119,30 +132,38 @@ describe('setup embeddings step', () => {
const io = makeIo();
const healthCheck = vi.fn(async () => ({ ok: true as const }));
const prompts = makePromptAdapter({ selectValues: ['sentence-transformers'] });
const ensureLocalEmbeddings = vi.fn(async () => managedDaemon());
const result = await runKtxSetupEmbeddingsStep(
{
projectDir: tempDir,
inputMode: 'auto',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
io.io,
{ prompts, env: {}, healthCheck },
{ prompts, env: {}, healthCheck, ensureLocalEmbeddings },
);
expect(result.status).toBe('ready');
expect(ensureLocalEmbeddings).toHaveBeenCalledWith({
cliVersion: '0.2.0',
installPolicy: 'auto',
io: io.io,
});
expect(healthCheck).toHaveBeenCalledWith({
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' },
sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' },
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.ingest.embeddings).toMatchObject({
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { base_url: 'http://127.0.0.1:8765', pathPrefix: '' },
sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' },
});
expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings);
expect(config.setup?.completed_steps).toContain('embeddings');
@ -167,10 +188,12 @@ describe('setup embeddings step', () => {
{
projectDir: tempDir,
inputMode: 'auto',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
io.io,
{ prompts, env: {}, healthCheck },
{ prompts, env: {}, healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()) },
);
await vi.waitFor(() => {
@ -192,10 +215,12 @@ describe('setup embeddings step', () => {
{
projectDir: tempDir,
inputMode: 'disabled',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
io.io,
{ env: {}, healthCheck },
{ env: {}, healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()) },
);
expect(result.status).toBe('ready');
@ -203,30 +228,59 @@ describe('setup embeddings step', () => {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' },
sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' },
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.ingest.embeddings).toMatchObject({
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { base_url: 'http://127.0.0.1:8765', pathPrefix: '' },
sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' },
});
expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings);
expect(config.setup?.completed_steps).toContain('embeddings');
});
it('fails non-interactive local setup when the managed local embeddings runtime is missing', async () => {
const io = makeIo();
const ensureLocalEmbeddings = vi.fn(async () => {
throw new Error(
'KTX Python runtime is required for this command. Run: ktx runtime install --feature local-embeddings --yes',
);
});
const result = await runKtxSetupEmbeddingsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'never',
skipEmbeddings: false,
},
io.io,
{ env: {}, ensureLocalEmbeddings },
);
expect(result.status).toBe('failed');
expect(io.stderr()).toContain(
'KTX Python runtime is required for this command. Run: ktx runtime install --feature local-embeddings --yes',
);
});
it('does not persist embedding completion when the health check fails', async () => {
const io = makeIo();
const result = await runKtxSetupEmbeddingsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
io.io,
{
env: {},
ensureLocalEmbeddings: vi.fn(async () => managedDaemon()),
healthCheck: vi.fn(async () => ({ ok: false as const, message: '401 invalid api key [redacted]' })),
},
);
@ -236,7 +290,7 @@ describe('setup embeddings step', () => {
expect(config.setup?.completed_steps ?? []).not.toContain('embeddings');
expect(config.ingest.embeddings.backend).toBe('deterministic');
expect(io.stderr()).toContain('Local embedding health check failed: 401 invalid api key [redacted]');
expect(io.stderr()).toContain('ktx-daemon serve-http --host 127.0.0.1 --port 8765');
expect(io.stderr()).toContain('Prepare the runtime with: ktx runtime start --feature local-embeddings');
expect(io.stderr()).not.toContain('skip for now');
});
@ -250,6 +304,8 @@ describe('setup embeddings step', () => {
inputMode: 'disabled',
embeddingBackend: 'openai',
embeddingApiKeyEnv: 'OPENAI_API_KEY',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
io.io,
@ -285,9 +341,20 @@ describe('setup embeddings step', () => {
.mockResolvedValueOnce({ ok: true as const });
const result = await runKtxSetupEmbeddingsStep(
{ projectDir: tempDir, inputMode: 'auto', skipEmbeddings: false },
{
projectDir: tempDir,
inputMode: 'auto',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
io.io,
{ prompts, env: { OPENAI_API_KEY: 'sk-openai-test' }, healthCheck },
{
prompts,
env: { OPENAI_API_KEY: 'sk-openai-test' },
healthCheck,
ensureLocalEmbeddings: vi.fn(async () => managedDaemon()),
},
);
expect(result.status).toBe('ready');
@ -295,7 +362,7 @@ describe('setup embeddings step', () => {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' },
sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' },
});
expect(healthCheck).toHaveBeenNthCalledWith(2, {
backend: 'openai',
@ -320,7 +387,13 @@ describe('setup embeddings step', () => {
it('leaves setup incomplete when skipped', async () => {
const result = await runKtxSetupEmbeddingsStep(
{ projectDir: tempDir, inputMode: 'disabled', skipEmbeddings: true },
{
projectDir: tempDir,
inputMode: 'disabled',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: true,
},
makeIo().io,
);
@ -333,9 +406,20 @@ describe('setup embeddings step', () => {
it('returns back without writing config when the local health check fails and Back is selected', async () => {
const prompts = makePromptAdapter({ selectValues: ['sentence-transformers', 'back'] });
const result = await runKtxSetupEmbeddingsStep(
{ projectDir: tempDir, inputMode: 'auto', skipEmbeddings: false },
{
projectDir: tempDir,
inputMode: 'auto',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
makeIo().io,
{ prompts, env: {}, healthCheck: vi.fn(async () => ({ ok: false as const, message: 'daemon unavailable' })) },
{
prompts,
env: {},
ensureLocalEmbeddings: vi.fn(async () => managedDaemon()),
healthCheck: vi.fn(async () => ({ ok: false as const, message: 'daemon unavailable' })),
},
);
expect(result.status).toBe('back');
@ -371,10 +455,20 @@ describe('setup embeddings step', () => {
const healthCheck = vi.fn(async () => ({ ok: true as const }));
await expect(
runKtxSetupEmbeddingsStep({ projectDir: tempDir, inputMode: 'disabled', skipEmbeddings: false }, makeIo().io, {
env: { OPENAI_API_KEY: 'sk-openai-test' },
healthCheck,
}),
runKtxSetupEmbeddingsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
makeIo().io,
{
env: { OPENAI_API_KEY: 'sk-openai-test' },
healthCheck,
},
),
).resolves.toMatchObject({ status: 'ready' });
expect(healthCheck).not.toHaveBeenCalled();
});

73
packages/cli/src/setup-embeddings.ts
View file

@ -10,6 +10,13 @@ import {
} from '@ktx/context/project';
import { type KtxEmbeddingConfig, type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from '@ktx/llm';
import type { KtxCliIo } from './cli-runtime.js';
import {
ensureManagedLocalEmbeddingsDaemon,
managedLocalEmbeddingHealthConfig,
managedLocalEmbeddingProjectConfig,
type ManagedLocalEmbeddingsDaemon,
} from './managed-local-embeddings.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
import { withMenuOptionsSpacing, withTextInputNavigation } from './prompt-navigation.js';
import { withSetupInterruptConfirmation } from './setup-interrupt.js';
import { envCredentialReference, writeProjectLocalSecretReference } from './setup-secrets.js';
@ -19,6 +26,8 @@ export type KtxSetupEmbeddingBackend = 'openai' | 'sentence-transformers';
export interface KtxSetupEmbeddingsArgs {
projectDir: string;
inputMode: 'auto' | 'disabled';
cliVersion: string;
runtimeInstallPolicy: KtxManagedPythonInstallPolicy;
embeddingBackend?: KtxSetupEmbeddingBackend;
embeddingApiKeyEnv?: string;
embeddingApiKeyFile?: string;
@ -44,6 +53,11 @@ export interface KtxSetupEmbeddingsDeps {
env?: NodeJS.ProcessEnv;
prompts?: KtxSetupEmbeddingsPromptAdapter;
healthCheck?: (config: KtxEmbeddingConfig) => Promise<KtxEmbeddingHealthCheckResult>;
ensureLocalEmbeddings?: (options: {
cliVersion: string;
installPolicy: KtxManagedPythonInstallPolicy;
io: KtxCliIo;
}) => Promise<ManagedLocalEmbeddingsDaemon>;
}
type BackendChoice = KtxSetupEmbeddingBackend | 'back';
@ -62,9 +76,6 @@ const DEFAULTS: Record<
};
const LOCAL_EMBEDDING_BACKEND: KtxSetupEmbeddingBackend = 'sentence-transformers';
const LOCAL_EMBEDDING_DAEMON_COMMAND = 'ktx-daemon serve-http --host 127.0.0.1 --port 8765';
const LOCAL_EMBEDDING_DAEMON_DEV_COMMAND =
'cd ktx && source .venv/bin/activate && uv run ktx-daemon serve-http --host 127.0.0.1 --port 8765';
const EMBEDDING_OPTION_PROMPT_CONTEXT =
'KTX uses embeddings for semantic search over semantic-layer sources, wiki context, schema metadata, ' +
'and relationship evidence.';
@ -302,10 +313,10 @@ async function chooseEmbeddingBackend(
function localEmbeddingSetupMessage(message: string): string {
return [
`Local embedding health check failed: ${message}`,
'Local embeddings use the KTX Python daemon. KTX can call ktx-daemon automatically when it is on PATH.',
`For repeated inference, start the HTTP daemon in another terminal with: ${LOCAL_EMBEDDING_DAEMON_COMMAND}`,
`From the KTX repo, use: ${LOCAL_EMBEDDING_DAEMON_DEV_COMMAND}`,
'The first run may download the all-MiniLM-L6-v2 model, so it can take a minute.',
'Local embeddings use the KTX-managed Python runtime.',
'Prepare the runtime with: ktx runtime start --feature local-embeddings',
'Use --yes with setup to install and start the runtime without prompting.',
'The first run may download Python packages and the all-MiniLM-L6-v2 model.',
].join('\n');
}
@ -432,12 +443,34 @@ export async function runKtxSetupEmbeddingsStep(
credentialValue = credential.value;
}
const healthConfig = buildHealthConfig({
backend: selectedBackend,
model,
dimensions,
credentialValue,
});
let managedLocalEmbeddings: ManagedLocalEmbeddingsDaemon | undefined;
if (selectedBackend === LOCAL_EMBEDDING_BACKEND) {
const ensureLocalEmbeddings = deps.ensureLocalEmbeddings ?? ensureManagedLocalEmbeddingsDaemon;
try {
managedLocalEmbeddings = await ensureLocalEmbeddings({
cliVersion: args.cliVersion,
installPolicy: args.runtimeInstallPolicy,
io,
});
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return { status: 'failed', projectDir: args.projectDir };
}
}
const healthConfig =
selectedBackend === LOCAL_EMBEDDING_BACKEND && managedLocalEmbeddings
? managedLocalEmbeddingHealthConfig({
baseUrl: managedLocalEmbeddings.baseUrl,
model,
dimensions,
})
: buildHealthConfig({
backend: selectedBackend,
model,
dimensions,
credentialValue,
});
const progress = startHealthCheckProgress(io, healthCheckStartText(selectedBackend, model, dimensions));
let health: KtxEmbeddingHealthCheckResult;
try {
@ -450,12 +483,14 @@ export async function runKtxSetupEmbeddingsStep(
progress.succeed(`Embedding test passed (${model}, ${dimensions} dimensions)`);
await persistEmbeddingConfig(
args.projectDir,
buildProjectEmbeddingConfig({
backend: selectedBackend,
model,
dimensions,
credentialRef,
}),
selectedBackend === LOCAL_EMBEDDING_BACKEND
? managedLocalEmbeddingProjectConfig({ model, dimensions })
: buildProjectEmbeddingConfig({
backend: selectedBackend,
model,
dimensions,
credentialRef,
}),
);
io.stdout.write(`Embeddings ready: yes (${model}, ${dimensions} dimensions)\n`);
return { status: 'ready', projectDir: args.projectDir };

70
packages/cli/src/setup.test.ts
View file

@ -318,6 +318,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'disabled',
yes: true,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: true,
databaseSchemas: [],
@ -364,6 +365,7 @@ describe('setup status', () => {
skipAgents: false,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: false,
databaseSchemas: [],
@ -410,6 +412,7 @@ describe('setup status', () => {
skipAgents: false,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: false,
databaseSchemas: [],
@ -434,6 +437,7 @@ describe('setup status', () => {
skipAgents: false,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: false,
databaseSchemas: [],
@ -472,6 +476,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: true,
databaseSchemas: [],
@ -530,6 +535,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: true,
databaseSchemas: [],
@ -597,6 +603,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: true,
databaseSchemas: [],
@ -661,6 +668,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: true,
databaseSchemas: [],
@ -697,6 +705,7 @@ describe('setup status', () => {
skipAgents: false,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: false,
databaseSchemas: [],
@ -733,6 +742,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'disabled',
yes: false,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: true,
databaseSchemas: [],
@ -764,6 +774,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'disabled',
yes: false,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: true,
databaseSchemas: [],
@ -791,6 +802,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'disabled',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: true,
databaseSchemas: [],
@ -819,6 +831,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'disabled',
yes: false,
cliVersion: '0.2.0',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
@ -858,7 +871,8 @@ describe('setup status', () => {
agents: false,
skipAgents: true,
inputMode: 'disabled',
yes: false,
yes: true,
cliVersion: '0.2.0',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
@ -878,6 +892,8 @@ describe('setup status', () => {
expect.objectContaining({
projectDir: tempDir,
inputMode: 'disabled',
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
embeddingBackend: 'openai',
embeddingApiKeyEnv: 'OPENAI_API_KEY',
skipEmbeddings: false,
@ -886,6 +902,43 @@ describe('setup status', () => {
);
});
it('passes no-input runtime policy to the embeddings step', async () => {
const io = makeIo();
const embeddings = vi.fn(async () => ({ status: 'failed' as const, projectDir: tempDir }));
await expect(
runKtxSetup(
{
command: 'run',
projectDir: tempDir,
mode: 'new',
agents: false,
agentScope: 'project',
agentInstallMode: 'cli',
skipAgents: true,
inputMode: 'disabled',
yes: false,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: false,
databaseSchemas: [],
skipDatabases: true,
skipSources: true,
},
io.io,
{ embeddings },
),
).resolves.toBe(1);
expect(embeddings).toHaveBeenCalledWith(
expect.objectContaining({
cliVersion: '0.2.0',
runtimeInstallPolicy: 'never',
}),
io.io,
);
});
it('lets Back from embedding setup return to the model step instead of exiting', async () => {
const testIo = makeIo();
const modelResults = [
@ -905,6 +958,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: false,
databaseSchemas: [],
@ -952,6 +1006,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: false,
databaseSchemas: [],
@ -997,6 +1052,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: true,
databaseSchemas: [],
@ -1037,6 +1093,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'disabled',
yes: false,
cliVersion: '0.2.0',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
@ -1084,6 +1141,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'disabled',
yes: true,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: true,
skipDatabases: true,
@ -1130,6 +1188,7 @@ describe('setup status', () => {
agents: false,
inputMode: 'disabled',
yes: true,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: true,
skipDatabases: true,
@ -1191,6 +1250,7 @@ describe('setup status', () => {
agentInstallMode: 'cli',
inputMode: 'disabled',
yes: true,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: true,
skipDatabases: true,
@ -1244,6 +1304,7 @@ describe('setup status', () => {
agentInstallMode: 'cli',
inputMode: 'disabled',
yes: true,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: true,
skipDatabases: true,
@ -1277,6 +1338,7 @@ describe('setup status', () => {
agents: false,
inputMode: 'disabled',
yes: true,
cliVersion: '0.2.0',
skipLlm: true,
skipEmbeddings: true,
skipDatabases: true,
@ -1352,6 +1414,7 @@ describe('setup status', () => {
agents: false,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: false,
skipDatabases: false,
@ -1416,6 +1479,7 @@ describe('setup status', () => {
agents: false,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: false,
skipDatabases: false,
@ -1509,6 +1573,7 @@ describe('setup status', () => {
agents: false,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: false,
skipDatabases: false,
@ -1604,6 +1669,7 @@ describe('setup status', () => {
agents: false,
inputMode: 'auto',
yes: false,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: false,
skipDatabases: false,
@ -1667,6 +1733,7 @@ describe('setup status', () => {
agentInstallMode: 'both',
inputMode: 'disabled',
yes: true,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: false,
skipDatabases: false,
@ -1715,6 +1782,7 @@ describe('setup status', () => {
skipAgents: true,
inputMode: 'disabled',
yes: false,
cliVersion: '0.2.0',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,

10
packages/cli/src/setup.ts
View file

@ -65,6 +65,7 @@ export type KtxSetupArgs =
skipAgents?: boolean;
inputMode: 'auto' | 'disabled';
yes: boolean;
cliVersion: string;
anthropicApiKeyEnv?: string;
anthropicApiKeyFile?: string;
anthropicModel?: string;
@ -406,6 +407,13 @@ function writeContextNotReadyForAgents(projectDir: string, io: KtxCliIo): void {
io.stderr.write(`Then install agent integration:\n ktx setup --agents --project-dir ${resolve(projectDir)}\n`);
}
function setupRuntimeInstallPolicy(args: Extract<KtxSetupArgs, { command: 'run' }>): 'prompt' | 'auto' | 'never' {
if (args.yes) {
return 'auto';
}
return args.inputMode === 'disabled' ? 'never' : 'prompt';
}
export async function runKtxSetup(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetupDeps = {}): Promise<number> {
try {
return await runKtxSetupInner(args, io, deps);
@ -609,6 +617,8 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
{
projectDir: projectResult.projectDir,
inputMode: args.inputMode,
cliVersion: args.cliVersion,
runtimeInstallPolicy: setupRuntimeInstallPolicy(args),
...(args.embeddingBackend ? { embeddingBackend: args.embeddingBackend } : {}),
...(args.embeddingApiKeyEnv ? { embeddingApiKeyEnv: args.embeddingApiKeyEnv } : {}),
...(args.embeddingApiKeyFile ? { embeddingApiKeyFile: args.embeddingApiKeyFile } : {}),

67
packages/cli/src/sl.test.ts
View file

@ -129,6 +129,8 @@ joins: []
query: { measures: ['orders.order_count'], dimensions: [] },
format: 'sql',
execute: false,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
},
{ stdout, stderr },
{ loadProject, createSemanticLayerCompute },
@ -139,6 +141,67 @@ joins: []
expect(stderr.write).not.toHaveBeenCalled();
});
it('creates default sl query compute through the managed runtime helper', async () => {
const projectDir = join(tempDir, 'project');
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
`name: orders
table: public.orders
grain: [id]
columns:
- name: id
type: number
measures:
- name: order_count
expr: count(*)
joins: []
`,
'ktx',
'ktx@example.com',
'Add orders source',
);
const stdout = { write: vi.fn() };
const stderr = { write: vi.fn() };
const compute = {
query: vi.fn(async () => ({
sql: 'select count(*) as order_count from public.orders',
dialect: 'postgres',
columns: [{ name: 'orders.order_count' }],
plan: {},
})),
validateSources: vi.fn(),
generateSources: vi.fn(),
};
const createManagedSemanticLayerCompute = vi.fn(async () => compute);
await expect(
runKtxSl(
{
command: 'query',
projectDir,
connectionId: 'warehouse',
query: { measures: ['orders.order_count'], dimensions: [] },
format: 'sql',
execute: false,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
},
{ stdout, stderr },
{ createManagedSemanticLayerCompute },
),
).resolves.toBe(0);
expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({
cliVersion: '0.2.0',
installPolicy: 'auto',
io: { stdout, stderr },
});
expect(stdout.write).toHaveBeenCalledWith('select count(*) as order_count from public.orders\n');
});
it('executes sl query through the injected query executor', async () => {
const projectDir = join(tempDir, 'project');
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
@ -194,6 +257,8 @@ joins: []
format: 'json',
execute: true,
maxRows: 20,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
},
{ stdout, stderr },
{
@ -292,6 +357,8 @@ joins: []
format: 'json',
execute: true,
maxRows: 20,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
},
{ stdout, stderr },
{ createSemanticLayerCompute },

21
packages/cli/src/sl.ts
View file

@ -1,5 +1,5 @@
import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from '@ktx/context/connections';
import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon';
import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon';
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
import {
compileLocalSlQuery,
@ -9,6 +9,10 @@ import {
writeLocalSlSource,
type SemanticLayerQueryInput,
} from '@ktx/context/sl';
import {
createManagedPythonSemanticLayerComputePort,
type KtxManagedPythonInstallPolicy,
} from './managed-python-command.js';
import { profileMark } from './startup-profile.js';
profileMark('module:sl');
@ -28,6 +32,8 @@ export type KtxSlArgs =
format: SlQueryFormat;
execute: boolean;
maxRows?: number;
cliVersion: string;
runtimeInstallPolicy: KtxManagedPythonInstallPolicy;
};
interface KtxSlIo {
@ -38,6 +44,11 @@ interface KtxSlIo {
interface KtxSlDeps {
loadProject?: typeof loadKtxProject;
createSemanticLayerCompute?: () => KtxSemanticLayerComputePort;
createManagedSemanticLayerCompute?: (options: {
cliVersion: string;
installPolicy: KtxManagedPythonInstallPolicy;
io: KtxSlIo;
}) => Promise<KtxSemanticLayerComputePort>;
createQueryExecutor?: () => KtxSqlQueryExecutorPort;
}
@ -97,7 +108,13 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx
return 0;
}
if (args.command === 'query') {
const compute = (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)();
const compute = deps.createSemanticLayerCompute
? deps.createSemanticLayerCompute()
: await (deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort)({
cliVersion: args.cliVersion,
installPolicy: args.runtimeInstallPolicy,
io,
});
const queryExecutor = args.execute ? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)() : undefined;
const result = await compileLocalSlQuery(project as KtxLocalProject, {
connectionId: args.connectionId,

11
packages/context/src/ingest/local-ingest.ts
View file

@ -41,10 +41,17 @@ export interface RunLocalIngestOptions {
export interface LocalIngestMcpOptions
extends Pick<
RunLocalIngestOptions,
'agentRunner' | 'llmProvider' | 'memoryModel' | 'semanticLayerCompute' | 'queryExecutor' | 'logger'
> {
| 'agentRunner'
| 'llmProvider'
| 'memoryModel'
| 'semanticLayerCompute'
| 'queryExecutor'
| 'logger'
| 'pullConfigOptions'
> {
adapters?: SourceAdapter[];
jobIdFactory?: () => string;
runLocalIngest?: (options: RunLocalIngestOptions) => Promise<LocalIngestResult>;
runLocalMetabaseIngest?: (options: RunLocalMetabaseIngestOptions) => Promise<LocalMetabaseFanoutResult>;
}

2
packages/context/src/llm/index.ts
View file

@ -11,6 +11,8 @@ export {
summarizeKtxLlmDebugRequest,
} from './debug-request-recorder.js';
export {
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV,
createLocalKtxEmbeddingProviderFromConfig,
createLocalKtxLlmProviderFromConfig,
resolveLocalKtxEmbeddingConfig,

41
packages/context/src/llm/local-config.test.ts
View file

@ -5,6 +5,8 @@ import {
type KtxProjectLlmConfig,
} from '../project/config.js';
import {
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV,
createLocalKtxEmbeddingProviderFromConfig,
createLocalKtxLlmProviderFromConfig,
resolveLocalKtxEmbeddingConfig,
@ -104,6 +106,45 @@ describe('local KTX embedding config', () => {
});
});
it('resolves managed sentence-transformers config from the CLI-provided daemon URL', () => {
const config: KtxProjectEmbeddingConfig = {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: {
base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
pathPrefix: '',
},
batchSize: 32,
};
expect(
resolveLocalKtxEmbeddingConfig(config, {
[MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: 'http://127.0.0.1:61234',
}),
).toEqual({
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' },
batchSize: 32,
});
});
it('returns null for managed sentence-transformers when no daemon URL is available', () => {
const config: KtxProjectEmbeddingConfig = {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: {
base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
pathPrefix: '',
},
};
expect(resolveLocalKtxEmbeddingConfig(config, {})).toBeNull();
});
it('constructs deterministic embeddings from the default project config', () => {
const createKtxEmbeddingProvider = vi.fn(() => ({}) as never);
const provider = createLocalKtxEmbeddingProviderFromConfig(

32
packages/context/src/llm/local-config.ts
View file

@ -16,6 +16,9 @@ interface LocalConfigDeps {
createKtxEmbeddingProvider?: typeof createKtxEmbeddingProvider;
}
export const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL = 'managed:local-embeddings';
export const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV = 'KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL';
function resolveOptional(value: string | undefined, env: NodeJS.ProcessEnv): string | undefined {
return resolveKtxConfigReference(value, env) || undefined;
}
@ -89,6 +92,19 @@ export function createLocalKtxLlmProviderFromConfig(
return resolved ? (deps.createKtxLlmProvider ?? createKtxLlmProvider)(resolved) : null;
}
function resolveSentenceTransformersBaseUrl(
value: string | undefined,
env: NodeJS.ProcessEnv,
): string | undefined {
if (!value) {
return undefined;
}
if (value === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL) {
return resolveOptional(`env:${MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV}`, env);
}
return value;
}
export function resolveLocalKtxEmbeddingConfig(
config: KtxProjectEmbeddingConfig,
env: NodeJS.ProcessEnv,
@ -96,6 +112,22 @@ export function resolveLocalKtxEmbeddingConfig(
if (config.backend === 'none') {
return null;
}
if (config.backend === 'sentence-transformers') {
const baseURL = resolveSentenceTransformersBaseUrl(config.sentenceTransformers?.base_url, env);
if (!baseURL) {
return null;
}
return {
backend: config.backend,
model: config.model ?? 'all-MiniLM-L6-v2',
dimensions: config.dimensions,
sentenceTransformers: {
baseURL,
pathPrefix: config.sentenceTransformers?.pathPrefix,
},
batchSize: config.batchSize,
};
}
return {
backend: config.backend,
model: config.model ?? 'deterministic',

59
packages/context/src/mcp/local-project-ports.test.ts
View file

@ -845,6 +845,65 @@ describe('createLocalProjectMcpContextPorts', () => {
expect(agentRunner.runLoop).toHaveBeenCalledTimes(1);
});
it('passes local ingest pull-config options into runLocalIngest', async () => {
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres' };
project.config.ingest.adapters = ['looker'];
const runLocalIngest = vi.fn(async () => ({
result: { ok: true },
report: {
id: 'report-1',
runId: 'run-1',
jobId: 'job-1',
sourceKey: 'looker',
connectionId: 'warehouse',
body: {
syncId: 'sync-1',
workUnits: [],
failedWorkUnits: [],
diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 0 },
provenanceRows: [],
},
},
}) as never);
const ports = createLocalProjectMcpContextPorts(project, {
localIngest: {
adapters: [
{ source: 'looker', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) },
],
pullConfigOptions: {
looker: {
daemonBaseUrl: 'http://127.0.0.1:61234',
},
},
runLocalIngest,
},
});
await expect(
ports.ingest?.trigger({
adapter: 'looker',
connectionId: 'warehouse',
trigger: 'manual_resync',
config: {},
}),
).resolves.toMatchObject({
runId: 'run-1',
jobId: 'job-1',
reportId: 'report-1',
});
expect(runLocalIngest).toHaveBeenCalledWith(
expect.objectContaining({
pullConfigOptions: {
looker: {
daemonBaseUrl: 'http://127.0.0.1:61234',
},
},
}),
);
});
it('triggers fetch-capable local ingest without sourceDir config', async () => {
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
project.config.connections.warehouse = {

5
packages/context/src/mcp/local-project-ports.ts
View file

@ -586,6 +586,7 @@ export function createLocalProjectMcpContextPorts(
metabaseConnectionId: input.connectionId,
trigger: input.trigger,
jobIdFactory: options.localIngest?.jobIdFactory,
pullConfigOptions: options.localIngest?.pullConfigOptions,
agentRunner: options.localIngest?.agentRunner,
llmProvider: options.localIngest?.llmProvider,
memoryModel: options.localIngest?.memoryModel,
@ -610,12 +611,14 @@ export function createLocalProjectMcpContextPorts(
};
}
const result = await runLocalIngest({
const executeLocalIngest = options.localIngest?.runLocalIngest ?? runLocalIngest;
const result = await executeLocalIngest({
project,
adapters: options.localIngest?.adapters ?? createDefaultLocalIngestAdapters(project),
adapter: input.adapter,
connectionId: input.connectionId,
sourceDir,
pullConfigOptions: options.localIngest?.pullConfigOptions,
trigger: input.trigger,
jobId: options.localIngest?.jobIdFactory?.(),
agentRunner: options.localIngest?.agentRunner,

4
packages/context/src/package-exports.test.ts
View file

@ -132,6 +132,10 @@ describe('@ktx/context package exports', () => {
expect(root.assertSearchBackendConformanceCase).toBeTypeOf('function');
expect(root.assertSearchBackendCapabilities).toBeTypeOf('function');
expect(root.createLocalKtxEmbeddingProviderFromConfig).toBeTypeOf('function');
expect(root.MANAGED_SENTENCE_TRANSFORMERS_BASE_URL).toBe('managed:local-embeddings');
expect(root.MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV).toBe(
'KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL',
);
expect(agent).toBeDefined();
expect(agent.AgentRunnerService).toBeTypeOf('function');
expect(root.AgentRunnerService).toBeTypeOf('function');

8
python/ktx-daemon/pyproject.toml
View file

@ -15,15 +15,19 @@ dependencies = [
"psycopg[binary]>=3.2.0",
"pydantic>=2.9.0",
"requests>=2.32.0",
"sentence-transformers>=5.1.1",
"sqlglot>=26",
"torch>=2.2.0",
"uvicorn[standard]>=0.32.0",
]
[project.scripts]
ktx-daemon = "ktx_daemon.__main__:main"
[project.optional-dependencies]
local-embeddings = [
"sentence-transformers>=5.1.1",
"torch>=2.2.0",
]
[project.urls]
Homepage = "https://github.com/kaelio/ktx"
Repository = "https://github.com/kaelio/ktx"

7
python/ktx-daemon/src/ktx_daemon/app.py
View file

@ -3,6 +3,7 @@
from __future__ import annotations
import logging
import os
from collections.abc import Callable
from typing import Any
@ -80,7 +81,11 @@ def create_app(
@app.get("/health")
async def health() -> dict[str, str]:
return {"status": "healthy"}
response = {"status": "healthy"}
version = os.environ.get("KTX_DAEMON_VERSION")
if version:
response["version"] = version
return response
@app.post("/database/introspect", response_model=DatabaseIntrospectionResponse)
async def database_introspect(

10
python/ktx-daemon/tests/test_app.py
View file

@ -69,6 +69,16 @@ def test_health_endpoint_returns_healthy() -> None:
assert response.json() == {"status": "healthy"}
def test_health_endpoint_returns_managed_runtime_version(monkeypatch) -> None:
monkeypatch.setenv("KTX_DAEMON_VERSION", "0.2.0")
client = TestClient(create_app())
response = client.get("/health")
assert response.status_code == 200
assert response.json() == {"status": "healthy", "version": "0.2.0"}
def test_database_introspect_endpoint_returns_snapshot() -> None:
calls = []

38
release-policy.json
View file

@ -1,37 +1,27 @@
{
"schemaVersion": 1,
"releaseMode": "ci-artifact-only",
"releaseMode": "npm-public-release-ready",
"npm": {
"publish": false,
"publish": true,
"registry": null,
"packages": [
"@ktx/cli",
"@ktx/connector-bigquery",
"@ktx/connector-clickhouse",
"@ktx/connector-mysql",
"@ktx/connector-postgres",
"@ktx/connector-snowflake",
"@ktx/connector-sqlite",
"@ktx/connector-sqlserver",
"@ktx/context",
"@ktx/llm"
]
"access": "public",
"tag": "next",
"packages": ["@kaelio/ktx"]
},
"python": {
"publish": false,
"repository": null,
"packages": ["ktx-sl", "ktx-daemon"]
"packages": ["kaelio-ktx"]
},
"publishedPackageSmoke": {
"packageName": null,
"version": "latest",
"packageName": "@kaelio/ktx",
"version": "0.1.0-rc.0",
"registry": null
},
"requiredBeforePublishing": [
"Choose npm registry and package visibility.",
"Choose Python package repository.",
"Choose public release versions.",
"Configure registry credentials outside source control.",
"Choose release tag and provenance policy."
]
"runtimeInstaller": {
"uvStrategy": "path-prerequisite",
"bootstrapUv": false,
"missingUvBehavior": "focused-error"
},
"requiredBeforePublishing": []
}

263
scripts/build-public-npm-package.mjs Normal file
View file

@ -0,0 +1,263 @@
#!/usr/bin/env node
import { execFile } from 'node:child_process';
import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { promisify } from 'node:util';
const execFileAsync = promisify(execFile);
export const PUBLIC_NPM_PACKAGE_NAME = '@kaelio/ktx';
export const PUBLIC_NPM_PACKAGE_VERSION = '0.1.0-rc.0';
export function publicNpmPackageTarballName(version = PUBLIC_NPM_PACKAGE_VERSION) {
return `kaelio-ktx-${version}.tgz`;
}
export const PUBLIC_BUNDLED_WORKSPACE_PACKAGES = [
'@ktx/llm',
'@ktx/context',
'@ktx/connector-bigquery',
'@ktx/connector-clickhouse',
'@ktx/connector-mysql',
'@ktx/connector-postgres',
'@ktx/connector-snowflake',
'@ktx/connector-sqlite',
'@ktx/connector-sqlserver',
];
export const PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS = {
'@ktx/llm': 'packages/llm',
'@ktx/context': 'packages/context',
'@ktx/connector-bigquery': 'packages/connector-bigquery',
'@ktx/connector-clickhouse': 'packages/connector-clickhouse',
'@ktx/connector-mysql': 'packages/connector-mysql',
'@ktx/connector-postgres': 'packages/connector-postgres',
'@ktx/connector-snowflake': 'packages/connector-snowflake',
'@ktx/connector-sqlite': 'packages/connector-sqlite',
'@ktx/connector-sqlserver': 'packages/connector-sqlserver',
};
function scriptRootDir() {
return resolve(dirname(fileURLToPath(import.meta.url)), '..');
}
export function publicNpmPackageLayout(rootDir = scriptRootDir(), version = PUBLIC_NPM_PACKAGE_VERSION) {
return {
rootDir,
packageVersion: version,
cliPackageRoot: join(rootDir, 'packages', 'cli'),
packRoot: join(rootDir, 'dist', 'public-npm-package'),
npmDir: join(rootDir, 'dist', 'artifacts', 'npm'),
tarballPath: join(rootDir, 'dist', 'artifacts', 'npm', publicNpmPackageTarballName(version)),
};
}
async function readJson(path) {
return JSON.parse(await readFile(path, 'utf8'));
}
async function writeJson(path, value) {
await writeFile(path, `${JSON.stringify(value, null, 2)}\n`);
}
function sortedObject(entries) {
return Object.fromEntries([...entries].sort(([left], [right]) => left.localeCompare(right)));
}
function isWorkspacePackageName(name) {
return name.startsWith('@ktx/');
}
function parseCaretVersion(value) {
const match = /^\^(\d+)\.(\d+)\.(\d+)$/.exec(value);
if (!match) {
return null;
}
return {
major: Number(match[1]),
minor: Number(match[2]),
patch: Number(match[3]),
};
}
function compareParsedVersions(left, right) {
return left.major - right.major || left.minor - right.minor || left.patch - right.patch;
}
function mergeDependencyVersion(name, previous, next) {
if (previous === next) {
return previous;
}
const previousCaret = parseCaretVersion(previous);
const nextCaret = parseCaretVersion(next);
if (previousCaret && nextCaret && previousCaret.major === nextCaret.major) {
return compareParsedVersions(previousCaret, nextCaret) >= 0 ? previous : next;
}
throw new Error(`Incompatible dependency versions for ${name}: ${previous} and ${next}`);
}
export function collectPublicDependencies(packageJsons) {
const dependencies = new Map();
for (const packageJson of packageJsons) {
for (const [name, version] of Object.entries(packageJson.dependencies ?? {})) {
if (isWorkspacePackageName(name)) {
continue;
}
const previous = dependencies.get(name);
dependencies.set(name, previous ? mergeDependencyVersion(name, previous, version) : version);
}
}
return sortedObject(dependencies);
}
export function publicNpmPackageJson(cliPackageJson, dependencies, version = PUBLIC_NPM_PACKAGE_VERSION) {
return {
name: PUBLIC_NPM_PACKAGE_NAME,
version,
description: 'Standalone KTX context layer for database agents',
private: false,
type: 'module',
engines: cliPackageJson.engines ?? { node: '>=22.0.0' },
bin: { ktx: './dist/bin.js' },
main: cliPackageJson.main ?? 'dist/index.js',
types: cliPackageJson.types ?? 'dist/index.d.ts',
exports: cliPackageJson.exports ?? {
'.': {
types: './dist/index.d.ts',
import: './dist/index.js',
default: './dist/index.js',
},
'./package.json': './package.json',
},
files: ['dist', 'assets'],
dependencies,
bundledDependencies: PUBLIC_BUNDLED_WORKSPACE_PACKAGES,
license: cliPackageJson.license ?? 'Apache-2.0',
repository: {
type: 'git',
url: 'git+https://github.com/kaelio/ktx.git',
},
bugs: {
url: 'https://github.com/kaelio/ktx/issues',
},
homepage: 'https://github.com/kaelio/ktx#readme',
};
}
function bundledWorkspacePackageJson(packageJson) {
const dependencies = Object.fromEntries(
Object.entries(packageJson.dependencies ?? {}).filter(([name]) => !isWorkspacePackageName(name)),
);
return {
name: packageJson.name,
version: packageJson.version ?? PUBLIC_NPM_PACKAGE_VERSION,
private: true,
type: packageJson.type ?? 'module',
main: packageJson.main,
types: packageJson.types,
exports: packageJson.exports,
files: packageJson.files,
dependencies: sortedObject(Object.entries(dependencies)),
license: packageJson.license ?? 'Apache-2.0',
};
}
async function copyPackageFileEntries(sourceRoot, targetRoot, packageJson) {
for (const entry of packageJson.files ?? ['dist']) {
await cp(join(sourceRoot, entry), join(targetRoot, entry), {
recursive: true,
force: true,
});
}
}
async function copyCliPackage(layout, cliPackageJson, dependencies) {
await copyPackageFileEntries(layout.cliPackageRoot, layout.packRoot, cliPackageJson);
await writeJson(
join(layout.packRoot, 'package.json'),
publicNpmPackageJson(cliPackageJson, dependencies, layout.packageVersion),
);
}
async function copyBundledWorkspacePackage(rootDir, packageName, packageJson) {
const packageRoot = PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS[packageName];
if (!packageRoot) {
throw new Error(`Missing bundled workspace package root for ${packageName}`);
}
const sourceRoot = join(rootDir, packageRoot);
const targetRoot = join(rootDir, 'dist', 'public-npm-package', 'node_modules', ...packageName.split('/'));
await mkdir(targetRoot, { recursive: true });
await copyPackageFileEntries(sourceRoot, targetRoot, packageJson);
await writeJson(join(targetRoot, 'package.json'), bundledWorkspacePackageJson(packageJson));
}
export async function createPublicNpmPackageTree(layout = publicNpmPackageLayout()) {
const cliPackageJson = await readJson(join(layout.cliPackageRoot, 'package.json'));
const bundledPackageJsons = await Promise.all(
PUBLIC_BUNDLED_WORKSPACE_PACKAGES.map(async (packageName) => {
const packageRoot = PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS[packageName];
const packageJson = await readJson(join(layout.rootDir, packageRoot, 'package.json'));
if (packageJson.name !== packageName) {
throw new Error(`Unexpected package name in ${packageRoot}/package.json: ${packageJson.name}`);
}
return packageJson;
}),
);
const dependencies = collectPublicDependencies([cliPackageJson, ...bundledPackageJsons]);
await rm(layout.packRoot, { recursive: true, force: true });
await mkdir(layout.packRoot, { recursive: true });
await mkdir(layout.npmDir, { recursive: true });
await copyCliPackage(layout, cliPackageJson, dependencies);
for (const packageJson of bundledPackageJsons) {
await copyBundledWorkspacePackage(layout.rootDir, packageJson.name, packageJson);
}
return {
layout,
packageJson: publicNpmPackageJson(cliPackageJson, dependencies, layout.packageVersion),
bundledPackages: PUBLIC_BUNDLED_WORKSPACE_PACKAGES,
};
}
export function publicNpmPackCommand(layout = publicNpmPackageLayout()) {
return {
command: 'pnpm',
args: ['--config.node-linker=hoisted', 'pack', '--out', layout.tarballPath],
cwd: layout.packRoot,
};
}
export async function buildPublicNpmPackage(layout = publicNpmPackageLayout()) {
await createPublicNpmPackageTree(layout);
const pack = publicNpmPackCommand(layout);
await execFileAsync(pack.command, pack.args, {
cwd: pack.cwd,
encoding: 'utf8',
maxBuffer: 10 * 1024 * 1024,
});
return layout.tarballPath;
}
async function main() {
const tarball = await buildPublicNpmPackage();
process.stdout.write(`Built ${PUBLIC_NPM_PACKAGE_NAME} package: ${tarball}\n`);
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
try {
await main();
} catch (error) {
process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`);
process.exitCode = 1;
}
}

275
scripts/build-public-npm-package.test.mjs Normal file
View file

@ -0,0 +1,275 @@
import assert from 'node:assert/strict';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, it } from 'node:test';
import {
PUBLIC_BUNDLED_WORKSPACE_PACKAGES,
PUBLIC_NPM_PACKAGE_NAME,
PUBLIC_NPM_PACKAGE_VERSION,
collectPublicDependencies,
createPublicNpmPackageTree,
publicNpmPackageJson,
publicNpmPackageLayout,
publicNpmPackageTarballName,
publicNpmPackCommand,
} from './build-public-npm-package.mjs';
async function writeJson(path, value) {
await writeFile(path, `${JSON.stringify(value, null, 2)}\n`);
}
async function writePackage(root, packageRoot, packageJson, files = {}) {
const absoluteRoot = join(root, packageRoot);
await mkdir(absoluteRoot, { recursive: true });
await writeJson(join(absoluteRoot, 'package.json'), packageJson);
for (const [relativePath, contents] of Object.entries(files)) {
const target = join(absoluteRoot, relativePath);
await mkdir(join(target, '..'), { recursive: true });
await writeFile(target, contents);
}
}
async function writeWorkspaceFixture(root) {
await writePackage(
root,
'packages/cli',
{
name: '@ktx/cli',
version: '0.0.0-private',
description: 'CLI wrapper for KTX',
type: 'module',
engines: { node: '>=22.0.0' },
bin: { ktx: './dist/bin.js' },
main: 'dist/index.js',
types: 'dist/index.d.ts',
exports: {
'.': {
types: './dist/index.d.ts',
import: './dist/index.js',
default: './dist/index.js',
},
'./package.json': './package.json',
},
files: ['dist', 'assets'],
dependencies: {
'@clack/prompts': '1.3.0',
'@ktx/context': 'workspace:*',
commander: '14.0.3',
},
license: 'Apache-2.0',
repository: {
type: 'git',
url: 'git+https://github.com/kaelio/ktx.git',
directory: 'packages/cli',
},
},
{
'dist/bin.js': '#!/usr/bin/env node\n',
'dist/index.js': 'export const cli = true;\n',
'dist/index.d.ts': 'export declare const cli: true;\n',
'assets/python/manifest.json': '{"schemaVersion":1}\n',
},
);
await writePackage(
root,
'packages/context',
{
name: '@ktx/context',
version: '0.0.0-private',
type: 'module',
main: 'dist/index.js',
exports: { '.': './dist/index.js' },
files: ['dist', 'prompts', 'skills'],
dependencies: {
'@ktx/llm': 'workspace:*',
yaml: '^2.8.2',
},
},
{
'dist/index.js': 'export const context = true;\n',
'prompts/system.md': 'prompt\n',
'skills/sl/SKILL.md': 'skill\n',
},
);
await writePackage(
root,
'packages/llm',
{
name: '@ktx/llm',
version: '0.0.0-private',
type: 'module',
main: 'dist/index.js',
exports: { '.': './dist/index.js' },
files: ['dist'],
dependencies: {
ai: '^6.0.168',
},
},
{
'dist/index.js': 'export const llm = true;\n',
},
);
for (const packageName of PUBLIC_BUNDLED_WORKSPACE_PACKAGES.filter((name) => name.startsWith('@ktx/connector-'))) {
const directory = packageName.replace('@ktx/', '');
await writePackage(
root,
`packages/${directory}`,
{
name: packageName,
version: '0.0.0-private',
type: 'module',
main: 'dist/index.js',
exports: { '.': './dist/index.js' },
files: ['dist'],
dependencies: {
'@ktx/context': 'workspace:*',
},
},
{
'dist/index.js': `export const name = ${JSON.stringify(packageName)};\n`,
},
);
}
}
describe('publicNpmPackageLayout', () => {
it('uses the first public npm release version for the tarball name', () => {
const layout = publicNpmPackageLayout('/repo/ktx');
assert.equal(PUBLIC_NPM_PACKAGE_VERSION, '0.1.0-rc.0');
assert.equal(publicNpmPackageTarballName(), 'kaelio-ktx-0.1.0-rc.0.tgz');
assert.equal(layout.tarballPath, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz');
});
});
describe('collectPublicDependencies', () => {
it('unions external runtime dependencies and omits workspace packages', () => {
assert.deepEqual(
collectPublicDependencies([
{
name: '@ktx/cli',
dependencies: {
'@ktx/context': 'workspace:*',
commander: '14.0.3',
zod: '^4.4.3',
},
},
{
name: '@ktx/context',
dependencies: {
'@ktx/llm': 'workspace:*',
commander: '14.0.3',
yaml: '^2.8.2',
zod: '^4.1.13',
},
},
]),
{
commander: '14.0.3',
yaml: '^2.8.2',
zod: '^4.4.3',
},
);
});
it('fails on incompatible external dependency ranges', () => {
assert.throws(
() =>
collectPublicDependencies([
{ name: '@ktx/cli', dependencies: { zod: '^4.4.3' } },
{ name: '@ktx/context', dependencies: { zod: '^3.25.0' } },
]),
/Incompatible dependency versions for zod/,
);
});
});
describe('publicNpmPackageJson', () => {
it('does not bundle the removed PostHog connector package', () => {
assert.equal(PUBLIC_BUNDLED_WORKSPACE_PACKAGES.includes('@ktx/connector-posthog'), false);
});
it('describes the public @kaelio/ktx binary package', () => {
const packageJson = publicNpmPackageJson(
{
name: '@ktx/cli',
version: '0.0.0-private',
engines: { node: '>=22.0.0' },
bin: { ktx: './dist/bin.js' },
main: 'dist/index.js',
types: 'dist/index.d.ts',
exports: { '.': './dist/index.js', './package.json': './package.json' },
license: 'Apache-2.0',
},
{ commander: '14.0.3' },
);
assert.equal(packageJson.name, PUBLIC_NPM_PACKAGE_NAME);
assert.equal(packageJson.version, '0.1.0-rc.0');
assert.equal(packageJson.private, false);
assert.deepEqual(packageJson.bin, { ktx: './dist/bin.js' });
assert.deepEqual(packageJson.dependencies, { commander: '14.0.3' });
assert.deepEqual(packageJson.bundledDependencies, PUBLIC_BUNDLED_WORKSPACE_PACKAGES);
assert.deepEqual(packageJson.files, ['dist', 'assets']);
});
});
describe('createPublicNpmPackageTree', () => {
it('copies CLI files, assets, and bundled internal workspace packages', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-public-npm-test-'));
try {
await writeWorkspaceFixture(root);
const layout = publicNpmPackageLayout(root);
const result = await createPublicNpmPackageTree(layout);
assert.equal(result.packageJson.name, '@kaelio/ktx');
assert.equal(result.packageJson.dependencies.commander, '14.0.3');
assert.equal(result.packageJson.dependencies.yaml, '^2.8.2');
assert.equal(result.packageJson.dependencies.ai, '^6.0.168');
assert.equal(
await readFile(join(layout.packRoot, 'assets', 'python', 'manifest.json'), 'utf8'),
'{"schemaVersion":1}\n',
);
assert.equal(
await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'dist', 'index.js'), 'utf8'),
'export const context = true;\n',
);
assert.equal(
await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'prompts', 'system.md'), 'utf8'),
'prompt\n',
);
const bundledContextJson = JSON.parse(
await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'package.json'), 'utf8'),
);
assert.equal(bundledContextJson.private, true);
assert.deepEqual(bundledContextJson.dependencies, { yaml: '^2.8.2' });
} finally {
await rm(root, { recursive: true, force: true });
}
});
});
describe('publicNpmPackCommand', () => {
it('packs the assembled public package with pnpm', () => {
const layout = publicNpmPackageLayout('/repo/ktx');
assert.deepEqual(publicNpmPackCommand(layout), {
command: 'pnpm',
args: [
'--config.node-linker=hoisted',
'pack',
'--out',
'/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz',
],
cwd: '/repo/ktx/dist/public-npm-package',
});
});
});

144
scripts/build-python-runtime-wheel.mjs Normal file
View file

@ -0,0 +1,144 @@
#!/usr/bin/env node
import { execFile } from 'node:child_process';
import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { promisify } from 'node:util';
const execFileAsync = promisify(execFile);
export const RUNTIME_WHEEL_DISTRIBUTION_NAME = 'kaelio-ktx';
export const RUNTIME_WHEEL_NORMALIZED_NAME = 'kaelio_ktx';
export const RUNTIME_WHEEL_PACKAGE_VERSION = '0.1.0';
function scriptRootDir() {
return resolve(dirname(fileURLToPath(import.meta.url)), '..');
}
export function runtimeWheelLayout(rootDir = scriptRootDir()) {
return {
rootDir,
semanticLayerSourceDir: join(rootDir, 'python', 'ktx-sl', 'semantic_layer'),
daemonSourceDir: join(rootDir, 'python', 'ktx-daemon', 'src', 'ktx_daemon'),
buildRoot: join(rootDir, 'dist', 'runtime-wheel-src'),
outputDir: join(rootDir, 'dist', 'artifacts', 'python'),
};
}
export function runtimeWheelPyproject() {
return `[project]
name = "${RUNTIME_WHEEL_DISTRIBUTION_NAME}"
version = "${RUNTIME_WHEEL_PACKAGE_VERSION}"
description = "Bundled Python runtime payload for the KTX npm package"
readme = "README.md"
requires-python = ">=3.13"
license = "Apache-2.0"
dependencies = [
"fastapi>=0.115.0",
"lkml>=1.3.7",
"numpy>=2.2.6",
"orjson>=3.11.4",
"pandas>=2.2.3",
"psycopg[binary]>=3.2.0",
"pydantic>=2.9.0",
"pyyaml>=6",
"requests>=2.32.0",
"sqlglot>=26",
"uvicorn[standard]>=0.32.0",
]
[project.optional-dependencies]
local-embeddings = [
"sentence-transformers>=5.1.1",
"torch>=2.2.0",
]
[project.scripts]
ktx-daemon = "ktx_daemon.__main__:main"
[project.urls]
Homepage = "https://github.com/kaelio/ktx"
Repository = "https://github.com/kaelio/ktx"
Issues = "https://github.com/kaelio/ktx/issues"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["semantic_layer", "ktx_daemon"]
`;
}
export function runtimeWheelReadme() {
return `# kaelio-ktx Python runtime
Bundled Python runtime wheel for KTX.
This wheel is built from the repository's \`semantic_layer\` and
\`ktx_daemon\` source trees for inclusion in the npm package. It is not a
separate public PyPI release artifact.
`;
}
export async function createRuntimeWheelBuildTree(layout = runtimeWheelLayout()) {
await rm(layout.buildRoot, { recursive: true, force: true });
await mkdir(layout.buildRoot, { recursive: true });
await cp(layout.semanticLayerSourceDir, join(layout.buildRoot, 'semantic_layer'), {
recursive: true,
});
await cp(layout.daemonSourceDir, join(layout.buildRoot, 'ktx_daemon'), {
recursive: true,
});
await writeFile(join(layout.buildRoot, 'pyproject.toml'), runtimeWheelPyproject());
await writeFile(join(layout.buildRoot, 'README.md'), runtimeWheelReadme());
}
export function runtimeWheelBuildCommand(layout = runtimeWheelLayout()) {
return {
command: 'uv',
args: ['build', '--wheel', '--out-dir', layout.outputDir, layout.buildRoot],
cwd: layout.rootDir,
};
}
async function runCommand(command, args, options) {
const result = await execFileAsync(command, args, {
cwd: options.cwd,
encoding: 'utf8',
maxBuffer: 1024 * 1024 * 20,
});
if (result.stdout) {
process.stdout.write(result.stdout);
}
if (result.stderr) {
process.stderr.write(result.stderr);
}
}
export async function buildRuntimeWheel(layout = runtimeWheelLayout()) {
await mkdir(layout.outputDir, { recursive: true });
await createRuntimeWheelBuildTree(layout);
const command = runtimeWheelBuildCommand(layout);
await runCommand(command.command, command.args, { cwd: command.cwd });
const pyproject = await readFile(join(layout.buildRoot, 'pyproject.toml'), 'utf8');
return {
buildRoot: layout.buildRoot,
outputDir: layout.outputDir,
pyproject,
};
}
async function main() {
await buildRuntimeWheel(runtimeWheelLayout());
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
try {
await main();
} catch (error) {
process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`);
process.exitCode = 1;
}
}

115
scripts/build-python-runtime-wheel.test.mjs Normal file
View file

@ -0,0 +1,115 @@
import assert from 'node:assert/strict';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, it } from 'node:test';
import {
RUNTIME_WHEEL_DISTRIBUTION_NAME,
RUNTIME_WHEEL_PACKAGE_VERSION,
createRuntimeWheelBuildTree,
runtimeWheelBuildCommand,
runtimeWheelLayout,
runtimeWheelPyproject,
} from './build-python-runtime-wheel.mjs';
async function writeRuntimeSourceFixture(root) {
await mkdir(join(root, 'python', 'ktx-sl', 'semantic_layer'), {
recursive: true,
});
await mkdir(join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon'), {
recursive: true,
});
await writeFile(
join(root, 'python', 'ktx-sl', 'semantic_layer', '__init__.py'),
'SEMANTIC_LAYER_FIXTURE = True\n',
);
await writeFile(
join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon', '__init__.py'),
'KTX_DAEMON_FIXTURE = True\n',
);
await writeFile(
join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon', '__main__.py'),
'def main():\n return 0\n',
);
}
describe('runtimeWheelLayout', () => {
it('uses stable source, build, and output paths', () => {
const layout = runtimeWheelLayout('/repo/ktx');
assert.equal(layout.rootDir, '/repo/ktx');
assert.equal(layout.semanticLayerSourceDir, '/repo/ktx/python/ktx-sl/semantic_layer');
assert.equal(layout.daemonSourceDir, '/repo/ktx/python/ktx-daemon/src/ktx_daemon');
assert.equal(layout.buildRoot, '/repo/ktx/dist/runtime-wheel-src');
assert.equal(layout.outputDir, '/repo/ktx/dist/artifacts/python');
});
});
describe('runtimeWheelPyproject', () => {
it('describes one kaelio-ktx wheel with lazy local embeddings', () => {
const pyproject = runtimeWheelPyproject();
assert.match(pyproject, /name = "kaelio-ktx"/);
assert.match(pyproject, /version = "0\.1\.0"/);
assert.match(pyproject, /ktx-daemon = "ktx_daemon\.__main__:main"/);
assert.match(pyproject, /packages = \["semantic_layer", "ktx_daemon"\]/);
assert.match(pyproject, /\[project\.optional-dependencies\]/);
assert.match(pyproject, /local-embeddings = \[/);
assert.match(pyproject, /"sentence-transformers>=5\.1\.1"/);
assert.match(pyproject, /"torch>=2\.2\.0"/);
assert.doesNotMatch(
pyproject.match(/dependencies = \[[\s\S]*?\]/)?.[0] ?? '',
/sentence-transformers|torch/,
);
});
});
describe('createRuntimeWheelBuildTree', () => {
it('copies KTX-owned Python packages into the build tree', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-wheel-test-'));
try {
await writeRuntimeSourceFixture(root);
const layout = runtimeWheelLayout(root);
await createRuntimeWheelBuildTree(layout);
assert.equal(
await readFile(join(layout.buildRoot, 'semantic_layer', '__init__.py'), 'utf8'),
'SEMANTIC_LAYER_FIXTURE = True\n',
);
assert.equal(
await readFile(join(layout.buildRoot, 'ktx_daemon', '__main__.py'), 'utf8'),
'def main():\n return 0\n',
);
const pyproject = await readFile(join(layout.buildRoot, 'pyproject.toml'), 'utf8');
assert.match(pyproject, /name = "kaelio-ktx"/);
assert.match(pyproject, /local-embeddings = \[/);
const readme = await readFile(join(layout.buildRoot, 'README.md'), 'utf8');
assert.match(readme, /Bundled Python runtime wheel for KTX/);
} finally {
await rm(root, { recursive: true, force: true });
}
});
});
describe('runtimeWheelBuildCommand', () => {
it('runs uv build against the generated build tree', () => {
const layout = runtimeWheelLayout('/repo/ktx');
assert.deepEqual(runtimeWheelBuildCommand(layout), {
command: 'uv',
args: [
'build',
'--wheel',
'--out-dir',
'/repo/ktx/dist/artifacts/python',
'/repo/ktx/dist/runtime-wheel-src',
],
cwd: '/repo/ktx',
});
assert.equal(RUNTIME_WHEEL_DISTRIBUTION_NAME, 'kaelio-ktx');
assert.equal(RUNTIME_WHEEL_PACKAGE_VERSION, '0.1.0');
});
});

14
scripts/check-boundaries.mjs
View file

@ -7,6 +7,10 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
const codeExtensions = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.py']);
const runtimeAssetPatterns = [/^packages\/[^/]+\/prompts\/.+\.md$/, /^packages\/[^/]+\/skills\/.+\.md$/];
const identifierSkipPrefixes = ['docs/', 'examples/', 'python/ktx-sl/plans/', 'python/ktx-sl/openspec/'];
const identifierAllowPatterns = [
/^packages\/cli\/src\/(?:index|managed-local-embeddings|managed-python-command|managed-python-daemon|managed-python-runtime|runtime)(?:\.test)?\.ts$/,
/^scripts\/(?:build-public-npm-package|build-python-runtime-wheel|local-embeddings-runtime-smoke|package-artifacts|publish-public-npm-package|published-package-smoke|release-readiness)(?:\.test)?\.mjs$/,
];
const forbiddenIdentifierTerms = ['kae' + 'lio', 'Kae' + 'lio', 'KAE' + 'LIO_'];
const appImportPatterns = [
@ -98,6 +102,10 @@ function skipsIdentifierScan(relativePath) {
return identifierSkipPrefixes.some((prefix) => relativePath.startsWith(prefix));
}
function allowsForbiddenIdentifier(relativePath) {
return identifierAllowPatterns.some((pattern) => pattern.test(relativePath));
}
export function scanFileContent(relativePath, content) {
const normalizedPath = normalizePath(relativePath);
const violations = [];
@ -138,7 +146,11 @@ export function scanFileContent(relativePath, content) {
}
}
if (scansForForbiddenIdentifiers(normalizedPath) && !skipsIdentifierScan(normalizedPath)) {
if (
scansForForbiddenIdentifiers(normalizedPath) &&
!skipsIdentifierScan(normalizedPath) &&
!allowsForbiddenIdentifier(normalizedPath)
) {
for (const term of forbiddenIdentifierTerms) {
if (content.includes(term)) {
violations.push({

9
scripts/check-boundaries.test.mjs
View file

@ -65,6 +65,15 @@ describe('scanFileContent', () => {
assert.equal(scanFileContent('python/ktx-sl/openspec/specs/semantic-layer/spec.md', name).length, 0);
});
it('allows public package identifiers in release packaging and managed runtime source', () => {
const name = lowerProductName();
assert.equal(scanFileContent('scripts/local-embeddings-runtime-smoke.mjs', `@${name}/ktx`).length, 0);
assert.equal(scanFileContent('scripts/package-artifacts.test.mjs', `${name}-ktx`).length, 0);
assert.equal(scanFileContent('scripts/publish-public-npm-package.test.mjs', `@${name}/ktx`).length, 0);
assert.equal(scanFileContent('packages/cli/src/managed-python-runtime.ts', `${name}_ktx`).length, 0);
});
it('allows clean source files and clean runtime prompt assets', () => {
assert.deepEqual(
scanFileContent('packages/context/src/index.ts', "export const packageName = '@ktx/context';"),

84
scripts/examples-docs.test.mjs
View file

@ -6,6 +6,26 @@ async function readText(relativePath) {
return readFile(new URL(`../${relativePath}`, import.meta.url), 'utf8');
}
function publicNpmPackageName() {
return `@${['kae', 'lio'].join('')}/ktx`;
}
function runtimeWheelPackageName() {
return `${['kae', 'lio'].join('')}-ktx`;
}
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
function publicPackagePattern(text) {
return new RegExp(text.replaceAll('{package}', escapeRegExp(publicNpmPackageName())));
}
function runtimeWheelPackagePattern(text) {
return new RegExp(text.replaceAll('{package}', escapeRegExp(runtimeWheelPackageName())));
}
describe('standalone example docs', () => {
it('documents the local warehouse example from the examples index', async () => {
const examples = await readText('examples/README.md');
@ -63,6 +83,16 @@ describe('standalone example docs', () => {
assert.match(workload, /app_user/);
assert.match(workload, /etl_user/);
assert.match(smoke, /pg_stat_statements_reset/);
assert.match(smoke, /KTX_RUNTIME_ROOT/);
assert.match(smoke, /managedDaemon/);
assert.match(smoke, /installPolicy: 'auto'/);
assert.match(smoke, /getKtxCliPackageInfo/);
assert.doesNotMatch(smoke, /python-service/);
assert.doesNotMatch(smoke, /PYTHON_SERVICE/);
assert.doesNotMatch(smoke, /uvicorn app\.main:app/);
assert.doesNotMatch(smoke, /export KTX_SQL_ANALYSIS_URL/);
assert.doesNotMatch(readme, /python-service/);
assert.doesNotMatch(readme, /KTX_SQL_ANALYSIS_URL/);
assert.match(smoke, /assert_manifest "\$FIRST_MANIFEST" true/);
assert.match(smoke, /assert_manifest "\$SECOND_MANIFEST" false/);
assert.match(smoke, /assert_manifest "\$RESET_MANIFEST" true/);
@ -113,6 +143,60 @@ describe('standalone example docs', () => {
assert.match(rootReadme, /Tables: 1/);
});
it('documents public npm and managed runtime usage in the README', async () => {
const rootReadme = await readText('README.md');
assert.match(rootReadme, publicPackagePattern('npx {package} setup demo --no-input'));
assert.match(rootReadme, publicPackagePattern('npx {package} sl query'));
assert.match(rootReadme, publicPackagePattern('npm install {package}'));
assert.match(rootReadme, publicPackagePattern('npm install -g {package}'));
assert.match(rootReadme, /ktx runtime install/);
assert.match(rootReadme, /ktx runtime status/);
assert.match(rootReadme, /ktx runtime doctor/);
assert.match(rootReadme, /ktx runtime start/);
assert.match(rootReadme, /ktx runtime stop/);
assert.match(rootReadme, /ktx runtime prune --dry-run/);
assert.match(rootReadme, /ktx runtime prune --yes/);
assert.match(rootReadme, /KTX requires `uv` on `PATH`/);
assert.match(rootReadme, /KTX doesn't download `uv` automatically/);
assert.match(
rootReadme,
runtimeWheelPackagePattern(
'release\\s+artifact manifest contains the public npm tarball and the\\s+bundled `{package}`\\s+runtime wheel',
),
);
assert.match(rootReadme, /source packages for\s+development, not public release artifacts/);
assert.match(rootReadme, /ktx serve --mcp stdio/);
assert.doesNotMatch(rootReadme, /uv run ktx-daemon serve-http/);
assert.doesNotMatch(rootReadme, /--semantic-compute-url http:\/\/127\.0\.0\.1:8765/);
});
it('documents the public package artifact smoke shape', async () => {
const readme = await readText('examples/package-artifacts/README.md');
assert.match(readme, publicPackagePattern('{package}'));
assert.match(readme, /managed Python runtime/);
assert.match(
readme,
new RegExp(
`public \`${escapeRegExp(publicNpmPackageName())}\` npm tarball and the\\s+bundled \`${escapeRegExp(
runtimeWheelPackageName(),
)}\`\\s+runtime wheel`,
),
);
assert.match(readme, /does not install standalone\s+Python packages directly/);
assert.doesNotMatch(readme, /standalone Python distributions/);
assert.doesNotMatch(readme, /installs the Python artifacts directly/);
assert.match(readme, /requires `uv` on `PATH`/);
assert.match(readme, /ktx runtime status/);
assert.match(readme, /ktx runtime doctor/);
assert.match(readme, /ktx runtime prune --dry-run/);
assert.match(readme, /ktx runtime prune --yes/);
assert.doesNotMatch(readme, /@ktx\/context/);
assert.doesNotMatch(readme, /@ktx\/cli/);
assert.doesNotMatch(readme, /python -m ktx_daemon semantic-validate/);
});
it('replaces the fake-ingest smoke with a ktx scan walkthrough in the README', async () => {
const rootReadme = await readText('README.md');

144
scripts/installed-live-database-smoke.mjs
View file

@ -1,19 +1,15 @@
#!/usr/bin/env node
import { execFile, spawn } from 'node:child_process';
import { execFile } from 'node:child_process';
import { once } from 'node:events';
import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { request as httpRequest } from 'node:http';
import { createServer } from 'node:net';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { pathToFileURL } from 'node:url';
import {
findPythonArtifacts,
npmSmokePackageJson,
npmSmokePythonEnv,
packageArtifactLayout,
pythonArtifactInstallArgs,
} from './package-artifacts.mjs';
const POSTGRES_IMAGE = process.env.KTX_ARTIFACT_POSTGRES_IMAGE ?? 'postgres:16-alpine';
@ -238,93 +234,37 @@ async function seedPostgres(containerName) {
requireSuccess('seed postgres catalog', result);
}
function httpGetOk(url) {
return new Promise((resolve, reject) => {
const request = httpRequest(url, { method: 'GET' }, (response) => {
response.resume();
response.on('end', () => resolve((response.statusCode ?? 0) >= 200 && (response.statusCode ?? 0) < 300));
});
request.on('error', reject);
request.end();
});
}
function spawnLogged(command, args, options = {}) {
const stdout = [];
const stderr = [];
let spawnError;
const child = spawn(command, args, {
cwd: options.cwd,
env: options.env ?? process.env,
stdio: ['ignore', 'pipe', 'pipe'],
});
child.stdout.on('data', (chunk) => stdout.push(chunk));
child.stderr.on('data', (chunk) => stderr.push(chunk));
child.on('error', (error) => {
spawnError = error;
});
function managedRuntimeEnv(cleanInstallDir) {
return {
child,
error() {
return spawnError;
},
output() {
return {
stdout: Buffer.concat(stdout).toString('utf8'),
stderr: Buffer.concat(stderr).toString('utf8'),
};
},
...process.env,
KTX_RUNTIME_ROOT: join(cleanInstallDir, 'managed-runtime'),
};
}
async function waitForHttpHealth(url, daemon) {
const deadline = Date.now() + 15_000;
while (Date.now() < deadline) {
if (daemon.error()) {
const output = daemon.output();
throw new Error(
`Failed to start ktx-daemon: ${daemon.error().message}\nstdout:\n${output.stdout}\nstderr:\n${output.stderr}`,
);
}
if (daemon.child.exitCode !== null || daemon.child.signalCode !== null) {
const output = daemon.output();
throw new Error(`ktx-daemon exited before health check passed\nstdout:\n${output.stdout}\nstderr:\n${output.stderr}`);
}
try {
if (await httpGetOk(url)) {
return;
}
} catch {
await new Promise((resolve) => setTimeout(resolve, 100));
continue;
}
await new Promise((resolve) => setTimeout(resolve, 100));
function parseDaemonBaseUrl(stdout) {
const match = stdout.match(/^url: (http:\/\/127\.0\.0\.1:\d+)$/m);
if (!match) {
throw new Error(`Daemon URL was not printed by runtime start:\n${stdout}`);
}
const output = daemon.output();
throw new Error(`Timed out waiting for ${url}\nstdout:\n${output.stdout}\nstderr:\n${output.stderr}`);
return match[1];
}
async function startDaemon(port, cleanInstallDir) {
const daemon = spawnLogged(
'ktx-daemon',
['serve-http', '--host', '127.0.0.1', '--port', String(port), '--log-level', 'warning'],
{ cwd: cleanInstallDir, env: npmSmokePythonEnv(cleanInstallDir) },
);
await waitForHttpHealth(`http://127.0.0.1:${port}/health`, daemon);
return daemon;
async function startDaemon(cleanInstallDir) {
const result = await run('pnpm', ['exec', 'ktx', 'runtime', 'start'], {
cwd: cleanInstallDir,
env: managedRuntimeEnv(cleanInstallDir),
timeout: 120_000,
});
requireSuccess('ktx runtime start', result);
return parseDaemonBaseUrl(result.stdout);
}
async function stopDaemon(daemon) {
if (daemon.child.exitCode !== null || daemon.child.signalCode !== null) {
return;
}
daemon.child.kill('SIGTERM');
const closed = once(daemon.child, 'close').then(() => true);
const timedOut = new Promise((resolve) => setTimeout(() => resolve(false), 5_000));
if (!(await Promise.race([closed, timedOut]))) {
daemon.child.kill('SIGKILL');
await once(daemon.child, 'close');
}
async function stopDaemon(cleanInstallDir) {
await run('pnpm', ['exec', 'ktx', 'runtime', 'stop'], {
cwd: cleanInstallDir,
env: managedRuntimeEnv(cleanInstallDir),
timeout: 30_000,
});
}
async function assertPathExists(path, label) {
@ -336,7 +276,6 @@ async function assertPathExists(path, label) {
}
async function prepareCleanInstall(layout, cleanInstallDir) {
const pythonArtifacts = await findPythonArtifacts(layout.pythonDir);
await assertPathExists(layout.contextTarball, '@ktx/context tarball');
await assertPathExists(layout.cliTarball, '@ktx/cli tarball');
await mkdir(cleanInstallDir, { recursive: true });
@ -344,34 +283,24 @@ async function prepareCleanInstall(layout, cleanInstallDir) {
await run('pnpm', ['install'], { cwd: cleanInstallDir, timeout: 120_000 }).then((result) =>
requireSuccess('pnpm install clean artifact project', result),
);
await run('uv', ['venv', '.venv'], { cwd: cleanInstallDir, timeout: 120_000 }).then((result) =>
requireSuccess('uv venv clean artifact project', result),
);
await run(
'uv',
pythonArtifactInstallArgs(
join(cleanInstallDir, '.venv', process.platform === 'win32' ? 'Scripts/python.exe' : 'bin/python'),
pythonArtifacts,
),
{
cwd: cleanInstallDir,
timeout: 120_000,
},
).then((result) => requireSuccess('install Python artifacts', result));
await run('pnpm', ['exec', 'ktx', 'runtime', 'install', '--yes'], {
cwd: cleanInstallDir,
env: managedRuntimeEnv(cleanInstallDir),
timeout: 120_000,
}).then((result) => requireSuccess('install managed runtime', result));
}
async function main() {
const layout = packageArtifactLayout();
const root = await mkdtemp(join(tmpdir(), 'ktx-live-db-artifact-smoke-'));
const containerName = smokeContainerName();
let daemon;
let cleanInstallDir;
let daemonStarted = false;
try {
const postgresPort = await getAvailablePort();
const daemonPort = await getAvailablePort();
const postgresUrl = buildPostgresUrl(postgresPort);
const cleanInstallDir = join(root, 'npm-clean-install');
cleanInstallDir = join(root, 'npm-clean-install');
const projectDir = join(root, 'project');
const databaseIntrospectionUrl = `http://127.0.0.1:${daemonPort}`;
await startPostgresContainer(containerName, postgresPort);
await waitForPostgres(containerName);
@ -386,11 +315,12 @@ async function main() {
requireSuccess('ktx init', init);
await writeFile(join(projectDir, 'ktx.yaml'), buildKtxYaml(postgresUrl), 'utf8');
daemon = await startDaemon(daemonPort, cleanInstallDir);
const databaseIntrospectionUrl = await startDaemon(cleanInstallDir);
daemonStarted = true;
const ingestRun = await run('pnpm', buildLiveDatabaseIngestArgs(projectDir, databaseIntrospectionUrl), {
cwd: cleanInstallDir,
env: npmSmokePythonEnv(cleanInstallDir),
env: managedRuntimeEnv(cleanInstallDir),
timeout: 120_000,
});
requireSuccess('ktx dev ingest run live-database', ingestRun);
@ -403,7 +333,7 @@ async function main() {
const runId = getRunId(ingestRun.stdout);
const ingestStatus = await run('pnpm', buildLiveDatabaseStatusArgs(projectDir, runId), {
cwd: cleanInstallDir,
env: npmSmokePythonEnv(cleanInstallDir),
env: managedRuntimeEnv(cleanInstallDir),
timeout: 30_000,
});
requireSuccess('ktx ingest status live-database', ingestStatus);
@ -414,8 +344,8 @@ async function main() {
await assertPathExists(join(projectDir, '.ktx', 'db.sqlite'), 'SQLite local ingest state');
process.stdout.write(`Installed live-database artifact smoke passed: ${runId}\n`);
} finally {
if (daemon) {
await stopDaemon(daemon);
if (daemonStarted && cleanInstallDir) {
await stopDaemon(cleanInstallDir);
}
await stopPostgresContainer(containerName);
await rm(root, { recursive: true, force: true });

397
scripts/local-embeddings-runtime-smoke.mjs Normal file
View file

@ -0,0 +1,397 @@
import { execFile } from 'node:child_process';
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import {
PUBLIC_NPM_PACKAGE_NAME,
PUBLIC_NPM_PACKAGE_VERSION,
} from './build-public-npm-package.mjs';
const execFileAsync = promisify(execFile);
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
const DEFAULT_ROOT_DIR = resolve(SCRIPT_DIR, '..');
const PUBLIC_NPM_ARTIFACT_DIR = join('dist', 'artifacts', 'npm');
const OPT_IN_MESSAGE =
'Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke.';
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
export function expectedPublicKtxVersionPattern() {
return new RegExp(
`${escapeRegExp(PUBLIC_NPM_PACKAGE_NAME)} ${escapeRegExp(PUBLIC_NPM_PACKAGE_VERSION)}`,
);
}
export function localEmbeddingsSmokeOptIn(env = process.env, args = process.argv.slice(2)) {
if (env.KTX_RUN_LOCAL_EMBEDDINGS_SMOKE === '1' || args.includes('--force')) {
return { run: true };
}
return { run: false, message: OPT_IN_MESSAGE };
}
export function publicKtxTarballName(files) {
const matches = files.filter((file) => /^kaelio-ktx-.+\.tgz$/.test(file)).sort();
if (matches.length !== 1) {
throw new Error(
`Expected exactly one @kaelio/ktx tarball in ${PUBLIC_NPM_ARTIFACT_DIR}, found ${matches.length}: ${
matches.join(', ') || 'none'
}. Run pnpm run artifacts:build first.`,
);
}
return matches[0];
}
export async function selectPublicKtxTarball(rootDir = DEFAULT_ROOT_DIR) {
const npmArtifactDir = join(rootDir, PUBLIC_NPM_ARTIFACT_DIR);
const files = await readdir(npmArtifactDir);
return join(npmArtifactDir, publicKtxTarballName(files));
}
export function buildLocalEmbeddingsSmokeEnv(root, baseEnv = process.env) {
return {
...baseEnv,
KTX_RUN_LOCAL_EMBEDDINGS_SMOKE: '1',
KTX_RUNTIME_ROOT: join(root, 'managed-runtime'),
HF_HOME: join(root, 'hf-home'),
TRANSFORMERS_CACHE: join(root, 'transformers-cache'),
SENTENCE_TRANSFORMERS_HOME: join(root, 'sentence-transformers-home'),
TORCH_HOME: join(root, 'torch-home'),
};
}
export function localEmbeddingsSmokeCommands(input) {
return [
{
label: 'ktx public package version',
command: 'pnpm',
args: ['exec', 'ktx', '--version'],
timeoutMs: 60_000,
},
{
label: 'ktx runtime status missing',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'status', '--json'],
timeoutMs: 60_000,
},
{
label: 'ktx runtime install local embeddings',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'],
timeoutMs: 1_200_000,
},
{
label: 'ktx runtime status local embeddings ready',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'status', '--json'],
timeoutMs: 60_000,
},
{
label: 'ktx runtime start local embeddings',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'start', '--feature', 'local-embeddings'],
timeoutMs: 300_000,
},
{
label: 'ktx setup local embeddings',
command: 'pnpm',
args: [
'exec',
'ktx',
'setup',
'--project-dir',
input.projectDir,
'--new',
'--no-input',
'--yes',
'--skip-llm',
'--embedding-backend',
'sentence-transformers',
'--skip-databases',
'--skip-sources',
'--skip-agents',
],
timeoutMs: 900_000,
},
{
label: 'ktx runtime stop local embeddings',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'stop'],
timeoutMs: 60_000,
},
];
}
export function parseDaemonBaseUrl(stdout) {
const match = stdout.match(/^url: (http:\/\/127\.0\.0\.1:\d+)$/m);
if (!match) {
throw new Error(`Daemon URL was not printed by runtime start:\n${stdout}`);
}
return match[1];
}
export function validateEmbeddingResponse(raw, expectedDimensions) {
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
throw new Error('Embedding response must be a JSON object');
}
const embedding = raw.embedding;
if (!Array.isArray(embedding)) {
throw new Error('Embedding response must include an embedding array');
}
if (embedding.length !== expectedDimensions) {
throw new Error(`Expected embedding dimension ${expectedDimensions}, got ${embedding.length}`);
}
for (const [index, value] of embedding.entries()) {
if (typeof value !== 'number' || !Number.isFinite(value)) {
throw new Error(`Embedding value at index ${index} is not a finite number`);
}
}
}
async function run(command, args, options = {}) {
process.stdout.write(`$ ${command} ${args.join(' ')}\n`);
try {
const result = await execFileAsync(command, args, {
cwd: options.cwd,
env: { ...process.env, ...options.env },
encoding: 'utf8',
maxBuffer: 1024 * 1024 * 20,
timeout: options.timeoutMs ?? 120_000,
});
if (result.stdout) {
process.stdout.write(result.stdout);
}
if (result.stderr) {
process.stderr.write(result.stderr);
}
return { code: 0, stdout: result.stdout, stderr: result.stderr };
} catch (error) {
const stdout = typeof error.stdout === 'string' ? error.stdout : '';
const stderr = typeof error.stderr === 'string' ? error.stderr : error.message;
if (stdout) {
process.stdout.write(stdout);
}
if (stderr) {
process.stderr.write(stderr);
}
return {
code: typeof error.code === 'number' ? error.code : 1,
stdout,
stderr,
};
}
}
function requireSuccess(label, result, options = {}) {
if (result.code !== 0) {
throw new Error(`${label} failed with code ${result.code}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`);
}
if (options.stderrPattern && !options.stderrPattern.test(result.stderr)) {
throw new Error(`${label} stderr did not match ${options.stderrPattern}\nstderr:\n${result.stderr}`);
}
}
function parseJsonStdout(label, result) {
requireSuccess(label, result);
try {
return JSON.parse(result.stdout);
} catch (error) {
throw new Error(`${label} did not write JSON stdout: ${error.message}\nstdout:\n${result.stdout}`);
}
}
function requireOutput(label, result, pattern) {
if (!pattern.test(result.stdout)) {
throw new Error(`${label} stdout did not match ${pattern}\nstdout:\n${result.stdout}`);
}
}
async function postJson(baseUrl, path, payload, timeoutMs) {
const response = await fetch(new URL(path, baseUrl), {
method: 'POST',
headers: {
accept: 'application/json',
'content-type': 'application/json',
},
body: JSON.stringify(payload),
signal: AbortSignal.timeout(timeoutMs),
});
const text = await response.text();
if (!response.ok) {
throw new Error(`POST ${path} failed with ${response.status}: ${text}`);
}
try {
return JSON.parse(text);
} catch (error) {
throw new Error(`POST ${path} returned non-JSON response: ${error.message}\n${text}`);
}
}
async function writeSmokePackage(projectDir, tarballPath) {
await mkdir(projectDir, { recursive: true });
await writeFile(
join(projectDir, 'package.json'),
`${JSON.stringify(
{
name: 'ktx-local-embeddings-runtime-smoke',
version: '0.0.0',
private: true,
type: 'module',
dependencies: {
'@kaelio/ktx': `file:${tarballPath}`,
},
},
null,
2,
)}\n`,
);
}
export async function runLocalEmbeddingsRuntimeSmoke(options = {}) {
const rootDir = options.rootDir ?? DEFAULT_ROOT_DIR;
const tarballPath = options.tarballPath ?? (await selectPublicKtxTarball(rootDir));
const root = await mkdtemp(join(tmpdir(), 'ktx-local-embeddings-smoke-'));
const keepTemp = options.keepTemp ?? process.env.KTX_KEEP_LOCAL_EMBEDDINGS_SMOKE === '1';
const installDir = join(root, 'installed-package');
const projectDir = join(root, 'project');
const smokeEnv = buildLocalEmbeddingsSmokeEnv(root);
const commands = localEmbeddingsSmokeCommands({ projectDir });
let daemonStarted = false;
try {
await writeSmokePackage(installDir, tarballPath);
requireSuccess(
'pnpm install public package',
await run('pnpm', ['install', '--ignore-scripts=false'], {
cwd: installDir,
env: smokeEnv,
timeoutMs: 300_000,
}),
);
const version = await run(commands[0].command, commands[0].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[0].timeoutMs,
});
requireSuccess(commands[0].label, version);
requireOutput(commands[0].label, version, expectedPublicKtxVersionPattern());
const missingStatus = parseJsonStdout(
commands[1].label,
await run(commands[1].command, commands[1].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[1].timeoutMs,
}),
);
if (missingStatus.kind !== 'missing') {
throw new Error(`Expected missing runtime before install, got ${JSON.stringify(missingStatus)}`);
}
const install = await run(commands[2].command, commands[2].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[2].timeoutMs,
});
requireSuccess(commands[2].label, install);
requireOutput(commands[2].label, install, /Installed KTX Python runtime/);
requireOutput(commands[2].label, install, /features: core, local-embeddings/);
const readyStatus = parseJsonStdout(
commands[3].label,
await run(commands[3].command, commands[3].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[3].timeoutMs,
}),
);
if (readyStatus.kind !== 'ready') {
throw new Error(`Expected ready runtime after install, got ${JSON.stringify(readyStatus)}`);
}
if (!readyStatus.manifest?.features?.includes('local-embeddings')) {
throw new Error(`Runtime manifest did not include local-embeddings: ${JSON.stringify(readyStatus.manifest)}`);
}
const start = await run(commands[4].command, commands[4].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[4].timeoutMs,
});
requireSuccess(commands[4].label, start);
daemonStarted = true;
const baseUrl = parseDaemonBaseUrl(start.stdout);
const embeddingResponse = await postJson(
baseUrl,
'/embeddings/compute',
{ text: 'KTX local embeddings release smoke' },
900_000,
);
validateEmbeddingResponse(embeddingResponse, 384);
process.stdout.write('KTX local embeddings daemon computed a 384-dimensional embedding\n');
const setup = await run(commands[5].command, commands[5].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[5].timeoutMs,
});
requireSuccess(commands[5].label, setup);
requireOutput(commands[5].label, setup, /Embeddings ready: yes \(all-MiniLM-L6-v2\)/);
const config = await readFile(join(projectDir, 'ktx.yaml'), 'utf8');
if (!config.includes('base_url: managed:local-embeddings')) {
throw new Error(`ktx.yaml did not contain managed local embeddings marker:\n${config}`);
}
process.stdout.write('KTX setup persisted managed local embeddings marker\n');
const stop = await run(commands[6].command, commands[6].args, {
cwd: installDir,
env: smokeEnv,
timeoutMs: commands[6].timeoutMs,
});
requireSuccess(commands[6].label, stop);
daemonStarted = false;
requireOutput(commands[6].label, stop, /Stopped KTX Python daemon/);
process.stdout.write('KTX local embeddings runtime smoke verified\n');
} finally {
if (daemonStarted) {
await run('pnpm', ['exec', 'ktx', 'runtime', 'stop'], {
cwd: installDir,
env: smokeEnv,
timeoutMs: 60_000,
});
}
if (!keepTemp) {
await rm(root, { recursive: true, force: true });
} else {
process.stdout.write(`Kept local embeddings smoke root: ${root}\n`);
}
}
}
async function main() {
const args = process.argv.slice(2);
const optIn = localEmbeddingsSmokeOptIn(process.env, args);
if (!optIn.run) {
process.stdout.write(`Skipping KTX local embeddings runtime smoke. ${optIn.message}\n`);
if (args.includes('--require-opt-in')) {
process.exitCode = 1;
}
return;
}
await runLocalEmbeddingsRuntimeSmoke();
}
if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`);
process.exitCode = 1;
});
}

172
scripts/local-embeddings-runtime-smoke.test.mjs Normal file
View file

@ -0,0 +1,172 @@
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import { describe, it } from 'node:test';
import {
buildLocalEmbeddingsSmokeEnv,
expectedPublicKtxVersionPattern,
localEmbeddingsSmokeCommands,
localEmbeddingsSmokeOptIn,
parseDaemonBaseUrl,
publicKtxTarballName,
validateEmbeddingResponse,
} from './local-embeddings-runtime-smoke.mjs';
describe('localEmbeddingsSmokeOptIn', () => {
it('skips unless the smoke is explicitly enabled', () => {
assert.deepEqual(localEmbeddingsSmokeOptIn({}, []), {
run: false,
message: 'Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke.',
});
});
it('runs when the environment opt-in is set', () => {
assert.deepEqual(localEmbeddingsSmokeOptIn({ KTX_RUN_LOCAL_EMBEDDINGS_SMOKE: '1' }, []), {
run: true,
});
});
it('runs when --force is present', () => {
assert.deepEqual(localEmbeddingsSmokeOptIn({}, ['--force']), {
run: true,
});
});
});
describe('publicKtxTarballName', () => {
it('selects the public @kaelio/ktx tarball name', () => {
assert.equal(
publicKtxTarballName(['kaelio-ktx-0.1.0-rc.0.tgz', 'ignore-me.tgz']),
'kaelio-ktx-0.1.0-rc.0.tgz',
);
});
it('fails when the public package tarball is missing', () => {
assert.throws(
() => publicKtxTarballName(['ktx-cli-0.0.0-private.tgz']),
/Expected exactly one @kaelio\/ktx tarball/,
);
});
it('fails when multiple public package tarballs are present', () => {
assert.throws(
() => publicKtxTarballName(['kaelio-ktx-0.1.0-rc.0.tgz', 'kaelio-ktx-0.2.0.tgz']),
/Expected exactly one @kaelio\/ktx tarball/,
);
});
});
describe('expectedPublicKtxVersionPattern', () => {
it('matches the public package version and rejects the private workspace version', () => {
const pattern = expectedPublicKtxVersionPattern();
assert.match('@kaelio/ktx 0.1.0-rc.0\n', pattern);
assert.doesNotMatch('@kaelio/ktx 0.0.0-private\n', pattern);
});
});
describe('buildLocalEmbeddingsSmokeEnv', () => {
it('isolates the runtime root and model caches inside the smoke root', () => {
const env = buildLocalEmbeddingsSmokeEnv('/tmp/ktx-local-embedding-smoke', {
PATH: '/usr/bin',
});
assert.equal(env.PATH, '/usr/bin');
assert.equal(env.KTX_RUN_LOCAL_EMBEDDINGS_SMOKE, '1');
assert.equal(env.KTX_RUNTIME_ROOT, '/tmp/ktx-local-embedding-smoke/managed-runtime');
assert.equal(env.HF_HOME, '/tmp/ktx-local-embedding-smoke/hf-home');
assert.equal(env.TRANSFORMERS_CACHE, '/tmp/ktx-local-embedding-smoke/transformers-cache');
assert.equal(env.SENTENCE_TRANSFORMERS_HOME, '/tmp/ktx-local-embedding-smoke/sentence-transformers-home');
assert.equal(env.TORCH_HOME, '/tmp/ktx-local-embedding-smoke/torch-home');
});
});
describe('localEmbeddingsSmokeCommands', () => {
it('describes the installed-package commands needed for the smoke', () => {
const commands = localEmbeddingsSmokeCommands({
projectDir: '/tmp/ktx-local-embedding-smoke/project',
});
assert.deepEqual(commands.map((command) => command.label), [
'ktx public package version',
'ktx runtime status missing',
'ktx runtime install local embeddings',
'ktx runtime status local embeddings ready',
'ktx runtime start local embeddings',
'ktx setup local embeddings',
'ktx runtime stop local embeddings',
]);
assert.deepEqual(commands[2], {
label: 'ktx runtime install local embeddings',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'],
timeoutMs: 1_200_000,
});
assert.deepEqual(commands[4], {
label: 'ktx runtime start local embeddings',
command: 'pnpm',
args: ['exec', 'ktx', 'runtime', 'start', '--feature', 'local-embeddings'],
timeoutMs: 300_000,
});
assert.deepEqual(commands[5].args, [
'exec',
'ktx',
'setup',
'--project-dir',
'/tmp/ktx-local-embedding-smoke/project',
'--new',
'--no-input',
'--yes',
'--skip-llm',
'--embedding-backend',
'sentence-transformers',
'--skip-databases',
'--skip-sources',
'--skip-agents',
]);
});
});
describe('parseDaemonBaseUrl', () => {
it('extracts the daemon URL from runtime start output', () => {
assert.equal(
parseDaemonBaseUrl('Started KTX Python daemon\nurl: http://127.0.0.1:61234\nfeatures: local-embeddings\n'),
'http://127.0.0.1:61234',
);
});
it('rejects output without a daemon URL', () => {
assert.throws(() => parseDaemonBaseUrl('Started KTX Python daemon\n'), /Daemon URL was not printed/);
});
});
describe('validateEmbeddingResponse', () => {
it('accepts a finite embedding vector with the expected dimensions', () => {
validateEmbeddingResponse({ embedding: [0.1, -0.2, 0.3] }, 3);
});
it('rejects a vector with the wrong dimensions', () => {
assert.throws(
() => validateEmbeddingResponse({ embedding: [0.1, 0.2] }, 3),
/Expected embedding dimension 3, got 2/,
);
});
it('rejects non-finite embedding values', () => {
assert.throws(
() => validateEmbeddingResponse({ embedding: [0.1, Number.NaN, 0.3] }, 3),
/Embedding value at index 1 is not a finite number/,
);
});
});
describe('package script', () => {
it('registers the opt-in local embeddings smoke command', async () => {
const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'));
assert.equal(
packageJson.scripts['release:local-embeddings-smoke'],
'node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in',
);
});
});

968
scripts/package-artifacts.mjs
View file

File diff suppressed because it is too large Load diff

486
scripts/package-artifacts.test.mjs
View file

@ -6,19 +6,22 @@ import { join } from 'node:path';
import { describe, it } from 'node:test';
import {
CLI_PYTHON_ASSET_MANIFEST,
INTERNAL_NPM_WORKSPACE_PACKAGES,
RUNTIME_WHEEL_DISTRIBUTION_NAME,
RUNTIME_WHEEL_NORMALIZED_NAME,
RUNTIME_WHEEL_PACKAGE_VERSION,
artifactManifestPath,
buildArtifactCommands,
copyRuntimeWheelAssets,
findPythonArtifacts,
NPM_ARTIFACT_PACKAGES,
npmDemoSmokeSource,
npmRuntimeSmokeSource,
npmSmokePackageJson,
npmSmokePythonEnv,
npmVerifySource,
packageArtifactLayout,
packageReleaseMetadata,
pythonArtifactInstallArgs,
pythonVerifySource,
verifyArtifactManifest,
writeArtifactManifest,
} from './package-artifacts.mjs';
@ -29,47 +32,21 @@ async function writeJson(path, value) {
await writeFile(path, `${JSON.stringify(value, null, 2)}\n`);
}
const CONNECTOR_PACKAGE_NAMES = [
'@ktx/connector-bigquery',
'@ktx/connector-clickhouse',
'@ktx/connector-mysql',
'@ktx/connector-postgres',
'@ktx/connector-snowflake',
'@ktx/connector-sqlite',
'@ktx/connector-sqlserver',
];
function packageRootForName(packageName) {
return `packages/${packageName.replace('@ktx/', '')}`;
}
function expectedNpmArtifactPath(packageName) {
return `npm/${packageName.replace('@ktx/', 'ktx-')}-0.0.0-private.tgz`;
}
const INTERNAL_BUILD_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => packageInfo.name);
const CONNECTOR_PACKAGE_NAMES = INTERNAL_BUILD_PACKAGE_NAMES.filter((packageName) =>
packageName.startsWith('@ktx/connector-'),
);
const NPM_BUILD_PACKAGE_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli'];
async function writeReleaseMetadataInputs(root) {
const npmPackages = ['@ktx/context', '@ktx/llm', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli'];
for (const packageName of npmPackages) {
const packageRoot = packageName === '@ktx/context' ? 'packages/context' : packageRootForName(packageName);
await mkdir(join(root, packageRoot), { recursive: true });
await writeJson(join(root, packageRoot, 'package.json'), {
name: packageName,
for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) {
await mkdir(join(root, packageInfo.packageRoot), { recursive: true });
await writeJson(join(root, packageInfo.packageRoot, 'package.json'), {
name: packageInfo.name,
version: '0.0.0-private',
private: true,
});
}
await mkdir(join(root, 'python', 'ktx-sl'), { recursive: true });
await mkdir(join(root, 'python', 'ktx-daemon'), { recursive: true });
await writeFile(
join(root, 'python', 'ktx-sl', 'pyproject.toml'),
['[project]', 'name = "ktx-sl"', 'version = "0.1.0"', ''].join('\n'),
);
await writeFile(
join(root, 'python', 'ktx-daemon', 'pyproject.toml'),
['[project]', 'name = "ktx-daemon"', 'version = "0.1.0"', ''].join('\n'),
);
}
async function writeUploadableArtifactFixtures(layout) {
@ -81,10 +58,10 @@ async function writeUploadableArtifactFixtures(layout) {
layout.npmTarballs[packageInfo.name],
`${packageInfo.name}-tarball`,
]),
[join(layout.pythonDir, 'ktx_sl-0.1.0-py3-none-any.whl'), 'ktx-sl-wheel'],
[join(layout.pythonDir, 'ktx_sl-0.1.0.tar.gz'), 'ktx-sl-sdist'],
[join(layout.pythonDir, 'ktx_daemon-0.1.0-py3-none-any.whl'), 'ktx-daemon-wheel'],
[join(layout.pythonDir, 'ktx_daemon-0.1.0.tar.gz'), 'ktx-daemon-sdist'],
[
join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
'kaelio-ktx-runtime-wheel',
],
]);
for (const [path, contents] of fileContents) {
@ -99,47 +76,30 @@ describe('packageArtifactLayout', () => {
assert.equal(layout.artifactDir, '/repo/ktx/dist/artifacts');
assert.equal(layout.npmDir, '/repo/ktx/dist/artifacts/npm');
assert.equal(layout.pythonDir, '/repo/ktx/dist/artifacts/python');
assert.equal(layout.contextTarball, '/repo/ktx/dist/artifacts/npm/ktx-context-0.0.0-private.tgz');
assert.equal(layout.cliTarball, '/repo/ktx/dist/artifacts/npm/ktx-cli-0.0.0-private.tgz');
assert.equal(
layout.connectorTarballs['@ktx/connector-sqlite'],
'/repo/ktx/dist/artifacts/npm/ktx-connector-sqlite-0.0.0-private.tgz',
);
assert.equal(
layout.connectorTarballs['@ktx/connector-postgres'],
'/repo/ktx/dist/artifacts/npm/ktx-connector-postgres-0.0.0-private.tgz',
);
assert.deepEqual(
Object.keys(layout.npmTarballs),
NPM_ARTIFACT_PACKAGES.map((packageInfo) => packageInfo.name),
);
assert.equal(layout.cliTarball, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz');
assert.deepEqual(Object.keys(layout.npmTarballs), ['@kaelio/ktx']);
});
});
describe('buildArtifactCommands', () => {
it('builds all TypeScript packages before packing npm artifacts and builds both Python packages', () => {
it('builds TypeScript packages and the runtime wheel before packing npm artifacts', () => {
const layout = packageArtifactLayout('/repo/ktx');
const commands = buildArtifactCommands(layout);
assert.deepEqual(
commands.slice(0, NPM_ARTIFACT_PACKAGES.length).map((command) => [command.command, command.args]),
NPM_ARTIFACT_PACKAGES.map((packageInfo) => ['pnpm', ['--filter', packageInfo.name, 'run', 'build']]),
commands.slice(0, NPM_BUILD_PACKAGE_ORDER.length).map((command) => [command.command, command.args]),
NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]),
);
assert.deepEqual(
commands
.slice(NPM_ARTIFACT_PACKAGES.length, NPM_ARTIFACT_PACKAGES.length * 2)
.map((command) => [command.command, command.args]),
NPM_ARTIFACT_PACKAGES.map((packageInfo) => [
'pnpm',
['--filter', packageInfo.name, 'pack', '--out', layout.npmTarballs[packageInfo.name]],
commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [
command.command,
command.args,
]),
[[process.execPath, ['scripts/build-python-runtime-wheel.mjs']]],
);
assert.deepEqual(
commands.slice(NPM_ARTIFACT_PACKAGES.length * 2).map((command) => [command.command, command.args]),
[
['uv', ['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python']],
['uv', ['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python']],
],
commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [command.command, command.args]),
[[process.execPath, ['scripts/build-public-npm-package.mjs']]],
);
});
});
@ -151,26 +111,18 @@ describe('packageReleaseMetadata', () => {
await writeReleaseMetadataInputs(root);
assert.deepEqual(await packageReleaseMetadata(root), [
...NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({
ecosystem: 'npm',
packageName: packageInfo.name,
packageRoot: packageInfo.packageRoot,
packageVersion: '0.0.0-private',
private: true,
releaseMode: 'ci-artifact-only',
})),
{
ecosystem: 'python',
packageName: 'ktx-sl',
packageRoot: 'python/ktx-sl',
packageVersion: '0.1.0',
ecosystem: 'npm',
packageName: '@kaelio/ktx',
packageRoot: 'packages/cli',
packageVersion: '0.1.0-rc.0',
private: false,
releaseMode: 'ci-artifact-only',
},
{
ecosystem: 'python',
packageName: 'ktx-daemon',
packageRoot: 'python/ktx-daemon',
packageName: 'kaelio-ktx',
packageRoot: 'python/runtime-wheel',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
@ -183,19 +135,13 @@ describe('packageReleaseMetadata', () => {
});
describe('findPythonArtifacts', () => {
it('finds one wheel and one source distribution for each Python package', async () => {
it('finds the bundled runtime wheel only', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-'));
try {
await writeFile(join(root, 'ktx_sl-0.1.0-py3-none-any.whl'), '');
await writeFile(join(root, 'ktx_sl-0.1.0.tar.gz'), '');
await writeFile(join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'), '');
await writeFile(join(root, 'ktx_daemon-0.1.0.tar.gz'), '');
await writeFile(join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), '');
assert.deepEqual(await findPythonArtifacts(root), {
ktxSlWheel: join(root, 'ktx_sl-0.1.0-py3-none-any.whl'),
ktxSlSdist: join(root, 'ktx_sl-0.1.0.tar.gz'),
ktxDaemonWheel: join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'),
ktxDaemonSdist: join(root, 'ktx_daemon-0.1.0.tar.gz'),
runtimeWheel: join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
});
} finally {
await rm(root, { recursive: true, force: true });
@ -205,7 +151,7 @@ describe('findPythonArtifacts', () => {
it('throws when a required Python artifact is missing', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-'));
try {
await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: ktx-sl wheel/);
await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: kaelio-ktx runtime wheel/);
} finally {
await rm(root, { recursive: true, force: true });
}
@ -230,30 +176,24 @@ describe('artifact manifest', () => {
assert.equal(manifest.sourceRevision, 'abc123');
assert.deepEqual(
manifest.packages.filter((entry) => entry.ecosystem === 'npm'),
NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({
ecosystem: 'npm',
packageName: packageInfo.name,
packageRoot: packageInfo.packageRoot,
packageVersion: '0.0.0-private',
private: true,
releaseMode: 'ci-artifact-only',
})),
[
{
ecosystem: 'npm',
packageName: '@kaelio/ktx',
packageRoot: 'packages/cli',
packageVersion: '0.1.0-rc.0',
private: false,
releaseMode: 'ci-artifact-only',
},
],
);
assert.deepEqual(
manifest.packages.filter((entry) => entry.ecosystem === 'python'),
[
{
ecosystem: 'python',
packageName: 'ktx-sl',
packageRoot: 'python/ktx-sl',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
},
{
ecosystem: 'python',
packageName: 'ktx-daemon',
packageRoot: 'python/ktx-daemon',
packageName: 'kaelio-ktx',
packageRoot: 'python/runtime-wheel',
packageVersion: '0.1.0',
private: false,
releaseMode: 'ci-artifact-only',
@ -271,13 +211,15 @@ describe('artifact manifest', () => {
path: file.path,
}))
.sort((left, right) => left.packageName.localeCompare(right.packageName)),
NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({
artifactKind: 'tarball',
ecosystem: 'npm',
packageName: packageInfo.name,
packageVersion: '0.0.0-private',
path: expectedNpmArtifactPath(packageInfo.name),
})).sort((left, right) => left.packageName.localeCompare(right.packageName)),
[
{
artifactKind: 'tarball',
ecosystem: 'npm',
packageName: '@kaelio/ktx',
packageVersion: '0.1.0-rc.0',
path: 'npm/kaelio-ktx-0.1.0-rc.0.tgz',
},
],
);
assert.deepEqual(
manifest.files
@ -293,38 +235,17 @@ describe('artifact manifest', () => {
{
artifactKind: 'wheel',
ecosystem: 'python',
packageName: 'ktx-daemon',
packageName: 'kaelio-ktx',
packageVersion: '0.1.0',
path: 'python/ktx_daemon-0.1.0-py3-none-any.whl',
},
{
artifactKind: 'sdist',
ecosystem: 'python',
packageName: 'ktx-daemon',
packageVersion: '0.1.0',
path: 'python/ktx_daemon-0.1.0.tar.gz',
},
{
artifactKind: 'wheel',
ecosystem: 'python',
packageName: 'ktx-sl',
packageVersion: '0.1.0',
path: 'python/ktx_sl-0.1.0-py3-none-any.whl',
},
{
artifactKind: 'sdist',
ecosystem: 'python',
packageName: 'ktx-sl',
packageVersion: '0.1.0',
path: 'python/ktx_sl-0.1.0.tar.gz',
path: 'python/kaelio_ktx-0.1.0-py3-none-any.whl',
},
],
);
const sqliteEntry = manifest.files.find((file) => file.path === 'npm/ktx-connector-sqlite-0.0.0-private.tgz');
assert.ok(sqliteEntry);
assert.equal(sqliteEntry.bytes, Buffer.byteLength('@ktx/connector-sqlite-tarball'));
assert.equal(sqliteEntry.sha256, createHash('sha256').update('@ktx/connector-sqlite-tarball').digest('hex'));
const npmEntry = manifest.files.find((file) => file.path === 'npm/kaelio-ktx-0.1.0-rc.0.tgz');
assert.ok(npmEntry);
assert.equal(npmEntry.bytes, Buffer.byteLength('@kaelio/ktx-tarball'));
assert.equal(npmEntry.sha256, createHash('sha256').update('@kaelio/ktx-tarball').digest('hex'));
const writtenManifest = JSON.parse(await readFile(artifactManifestPath(layout), 'utf-8'));
assert.deepEqual(writtenManifest, manifest);
@ -351,7 +272,7 @@ describe('verifyArtifactManifest', () => {
assert.equal(manifest.schemaVersion, 2);
assert.equal(manifest.sourceRevision, 'abc123');
assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 4);
assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 1);
} finally {
await rm(root, { recursive: true, force: true });
}
@ -418,48 +339,89 @@ describe('verifyArtifactManifest', () => {
});
});
describe('pythonArtifactInstallArgs', () => {
it('installs the built Python wheels by artifact path', () => {
const args = pythonArtifactInstallArgs('/tmp/smoke/.venv/bin/python', {
ktxSlWheel: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0-py3-none-any.whl',
ktxSlSdist: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0.tar.gz',
ktxDaemonWheel: '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0-py3-none-any.whl',
ktxDaemonSdist: '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0.tar.gz',
});
describe('copyRuntimeWheelAssets', () => {
it('copies the runtime wheel and checksum manifest into CLI assets', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-assets-test-'));
const layout = packageArtifactLayout(root);
try {
await mkdir(layout.pythonDir, { recursive: true });
await writeFile(
join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
'kaelio-ktx-runtime-wheel',
);
assert.deepEqual(args, [
'pip',
'install',
'--python',
'/tmp/smoke/.venv/bin/python',
'/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0-py3-none-any.whl',
'/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0-py3-none-any.whl',
]);
assert.equal(args.includes('ktx-daemon'), false);
assert.equal(args.includes('--find-links'), false);
const assets = await copyRuntimeWheelAssets(layout, {
runtimeWheel: join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'),
});
assert.equal(
assets.wheelPath,
join(root, 'packages', 'cli', 'assets', 'python', 'kaelio_ktx-0.1.0-py3-none-any.whl'),
);
assert.equal(
assets.manifestPath,
join(root, 'packages', 'cli', 'assets', 'python', CLI_PYTHON_ASSET_MANIFEST),
);
const manifest = JSON.parse(await readFile(assets.manifestPath, 'utf8'));
assert.deepEqual(manifest, {
schemaVersion: 1,
distributionName: RUNTIME_WHEEL_DISTRIBUTION_NAME,
normalizedName: RUNTIME_WHEEL_NORMALIZED_NAME,
version: RUNTIME_WHEEL_PACKAGE_VERSION,
wheel: {
file: 'kaelio_ktx-0.1.0-py3-none-any.whl',
sha256: createHash('sha256')
.update('kaelio-ktx-runtime-wheel')
.digest('hex'),
bytes: Buffer.byteLength('kaelio-ktx-runtime-wheel'),
},
});
} finally {
await rm(root, { recursive: true, force: true });
}
});
});
describe('npmSmokePythonEnv', () => {
it('prepends the npm smoke virtualenv bin directory to PATH', () => {
const env = npmSmokePythonEnv('/tmp/ktx-npm-smoke', { PATH: '/usr/bin' });
describe('verifyNpmArtifacts', () => {
it('does not prepare an external Python environment for the npm smoke', async () => {
const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8');
const start = source.indexOf('async function verifyNpmArtifacts');
const end = source.indexOf('async function verifyNpmDemoArtifacts');
assert.ok(start > 0, 'verifyNpmArtifacts function must exist');
assert.ok(end > start, 'verifyNpmDemoArtifacts must follow verifyNpmArtifacts');
assert.match(env.PATH, /^\/tmp\/ktx-npm-smoke\/\.venv\/(bin|Scripts)/);
assert.match(env.PATH, /\/usr\/bin$/);
const body = source.slice(start, end);
assert.doesNotMatch(body, /uv', \['venv', '\.venv'\]/);
assert.doesNotMatch(body, /pythonArtifactInstallArgs/);
assert.doesNotMatch(body, /npmSmokePythonEnv/);
});
});
describe('standalone Python artifact cleanup', () => {
it('does not build or verify standalone Python package artifacts', async () => {
const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8');
assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-sl'/);
assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-daemon'/);
assert.doesNotMatch(source, /async function verifyPythonArtifacts/);
assert.doesNotMatch(source, /pythonArtifactInstallArgs/);
assert.doesNotMatch(source, /pythonVerifySource/);
assert.doesNotMatch(source, /ktx_sl-0\.1\.0/);
assert.doesNotMatch(source, /ktx_daemon-0\.1\.0/);
});
});
describe('verification snippets', () => {
it('pins smoke dependencies and connector packages to clean-install-safe artifacts', () => {
it('pins the smoke project to the public package artifact', () => {
const layout = packageArtifactLayout('/repo/ktx');
const packageJson = npmSmokePackageJson(layout);
for (const packageInfo of NPM_ARTIFACT_PACKAGES) {
assert.equal(packageJson.dependencies[packageInfo.name], `file:${layout.npmTarballs[packageInfo.name]}`);
assert.equal(packageJson.pnpm.overrides[packageInfo.name], `file:${layout.npmTarballs[packageInfo.name]}`);
}
assert.equal(packageJson.dependencies['@modelcontextprotocol/sdk'], '^1.27.1');
assert.deepEqual(packageJson.pnpm.onlyBuiltDependencies, ['better-sqlite3']);
const packageJson = npmSmokePackageJson(layout);
assert.deepEqual(packageJson.dependencies, {
'@kaelio/ktx': `file:${layout.cliTarball}`,
});
assert.deepEqual(packageJson.devDependencies, {
'better-sqlite3': '^12.6.2',
});
});
it('exposes manifest verification as a package artifact command', async () => {
@ -472,115 +434,64 @@ describe('verification snippets', () => {
assert.equal(packageJson.scripts['artifacts:verify-manifest'], 'node scripts/package-artifacts.mjs verify-manifest');
});
it('verifies installed dbt extraction exports from @ktx/context/ingest', () => {
const source = npmVerifySource();
assert.match(source, /const ingest = await import\('@ktx\/context\/ingest'\);/);
assert.match(source, /const dbtExtractionExports = \[/);
assert.match(source, /throw new Error\('Missing dbt extraction export: ' \+ exportName\);/);
for (const exportName of [
'parseMetricflowFiles',
'parseMetricflowPullConfig',
'importMetricflowSemanticModels',
'parseDbtSchemaFiles',
'toDescriptionUpdates',
'toRelationshipUpdates',
'mergeSemanticModelTables',
'loadProjectInfo',
'loadDbtSchemaFiles',
]) {
assert.match(source, new RegExp(`\\['${exportName}', ingest\\.${exportName}\\]`));
}
});
it('asserts the public npm and connector entry points that clean installs must expose', () => {
const source = npmVerifySource();
assert.match(source, /@ktx\/context/);
assert.match(source, /@ktx\/context\/project/);
assert.match(source, /@ktx\/context\/mcp/);
assert.match(source, /@ktx\/context\/memory/);
assert.match(source, /@ktx\/context\/daemon/);
assert.match(source, /@ktx\/cli/);
assert.match(source, /@ktx\/llm/);
assert.match(source, /createKtxLlmProvider/);
assert.match(source, /KtxMessageBuilder/);
assert.match(source, /createKtxEmbeddingProvider/);
assert.doesNotMatch(source, /createGatewayLlmProvider/);
assert.match(source, /createLocalProjectMemoryCapture/);
for (const packageName of CONNECTOR_PACKAGE_NAMES) {
assert.match(source, new RegExp(packageName.replace('/', '\\/')));
}
assert.match(source, /KtxSqliteScanConnector/);
assert.match(source, /KtxPostgresScanConnector/);
assert.match(source, /KtxBigQueryScanConnector/);
assert.match(source, /KtxSnowflakeScanConnector/);
});
it('asserts installed hybrid search exports and CLI smoke coverage', () => {
it('asserts the public npm entry point that clean installs must expose', () => {
const verifySource = npmVerifySource();
const runtimeSource = npmRuntimeSmokeSource();
const demoSource = npmDemoSmokeSource();
assert.match(verifySource, /const search = await import\('@ktx\/context\/search'\);/);
assert.match(verifySource, /HybridSearchCore/);
assert.match(verifySource, /assertSearchBackendConformanceCase/);
assert.match(verifySource, /assertSearchBackendCapabilities/);
assert.match(runtimeSource, /ktx agent wiki search hybrid metadata verified/);
assert.match(runtimeSource, /ktx agent sl list hybrid metadata verified/);
assert.match(runtimeSource, /agent_sl_search_missing_project/);
assert.match(runtimeSource, /agent_sl_search_no_connections/);
assert.match(runtimeSource, /agent_sl_search_no_indexed_sources/);
assert.match(demoSource, /ktx seeded demo agent wiki search verified/);
assert.match(demoSource, /ktx seeded demo agent sl search verified/);
assert.match(verifySource, /const cli = await import\('@kaelio\/ktx'\);/);
assert.match(verifySource, /getKtxCliPackageInfo/);
assert.match(verifySource, /runKtxCli/);
assert.doesNotMatch(verifySource, /@ktx\/context/);
assert.doesNotMatch(verifySource, /@ktx\/llm/);
assert.doesNotMatch(verifySource, /@ktx\/connector-/);
});
it('runs installed CLI commands and MCP through an installed daemon HTTP server', () => {
it('runs installed CLI commands through the public package runtime', () => {
const source = npmRuntimeSmokeSource();
assert.match(source, /@modelcontextprotocol\/sdk\/client\/index\.js/);
assert.match(source, /@modelcontextprotocol\/sdk\/client\/stdio\.js/);
assert.match(source, /spawn\(command, args/);
assert.match(source, /createServer/);
assert.match(source, /request as httpRequest/);
assert.match(source, /getAvailablePort/);
assert.match(source, /startSemanticDaemon/);
assert.match(source, /waitForHttpHealth/);
assert.match(source, /stopSemanticDaemon/);
assert.match(source, /'ktx-daemon'/);
assert.match(source, /'serve-http'/);
assert.match(source, /'--host'/);
assert.match(source, /'127\.0\.0\.1'/);
assert.match(source, /'--port'/);
assert.match(source, /\/health/);
assert.match(source, /--semantic-compute-url/);
assert.match(source, /createDaemonLookerTableIdentifierParser/);
assert.match(source, /LocalLookerRuntimeStore/);
assert.match(source, /Looker daemon table identifier parser verified/);
assert.match(source, /Looker local runtime store verified/);
assert.match(source, /semanticComputeUrl/);
assert.match(source, /ktx public package version/);
assert.match(source, /@kaelio\\\/ktx 0\\\.1\\\.0/);
assert.match(source, /'ktx', 'sl', 'query'/);
assert.doesNotMatch(source, /@ktx\/context/);
assert.doesNotMatch(source, /@modelcontextprotocol/);
assert.doesNotMatch(source, /startSemanticDaemon/);
assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'setup'/);
assert.match(source, /knowledge', 'global', 'revenue\.md'/);
assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'agent',\s*'wiki',\s*'search'/);
assert.match(source, /semantic-layer', 'warehouse', 'orders\.yaml'/);
assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'agent',\s*'sl',\s*'list'/);
assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'agent',\s*'sl',\s*'query'/);
assert.match(source, /orders\.order_count/);
assert.match(source, /sqlite3/);
assert.match(source, /driver: sqlite/);
assert.match(source, /path: warehouse\.db/);
assert.match(source, /live-database/);
assert.match(source, /'--execute'/);
assert.match(source, /'--execute-queries'/);
assert.match(source, /slValidateResult\.success, true/);
assert.match(source, /slQueryResult\.dialect, 'sqlite'/);
assert.match(source, /slQueryResult\.plan\.execution\.driver, 'sqlite'/);
assert.match(source, /"mode": "compile_only"/);
assert.match(source, /"mode": "executed"/);
assert.match(source, /ktx agent sl query sqlite execute/);
assert.match(source, /ktx sl query sqlite execute/);
assert.match(source, /import Database from 'better-sqlite3'/);
assert.doesNotMatch(source, /run\('python'/);
assert.match(source, /KTX_RUNTIME_ROOT/);
assert.match(source, /managed-runtime/);
assert.match(source, /ktx runtime status missing/);
assert.match(source, /runtimeStatusBefore\.kind, 'missing'/);
assert.ok(source.includes(String.raw`Installing KTX Python runtime \(core\) with uv`));
assert.match(source, /KTX Python runtime ready:/);
assert.match(source, /ktx runtime status ready/);
assert.match(source, /runtimeStatusAfter\.kind, 'ready'/);
assert.match(source, /runtimeStatusAfter\.manifest\.features/);
assert.match(source, /ktx runtime doctor/);
assert.match(source, /PASS Managed Python runtime/);
assert.match(source, /ktx runtime start/);
assert.match(source, /ktx runtime start reuse/);
assert.match(source, /Using existing KTX Python daemon/);
assert.match(source, /ktx runtime stop/);
assert.match(source, /ktx runtime prune dry run/);
assert.match(source, /0\.0\.0/);
assert.match(source, /ktx runtime prune needs confirmation/);
assert.match(source, /Refusing to prune without --yes/);
assert.match(source, /ktx runtime prune confirmed/);
assert.match(source, /Removed stale KTX Python runtimes/);
assert.match(source, /assert\.rejects\(\(\) => access\(staleRuntimeDir\)\)/);
assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'dev',\s*'scan',\s*'warehouse'/);
assert.match(source, /'--mode',\s*'enriched'/);
assert.doesNotMatch(source, /'--enrich'/);
@ -590,28 +501,7 @@ describe('verification snippets', () => {
assert.match(source, /scanReportJson\.artifactPaths\.enrichmentArtifacts/);
assert.match(source, /enrichment:/);
assert.match(source, /mode: deterministic/);
assert.match(source, /backend: gateway/);
assert.match(source, /models:/);
assert.match(source, /default: smoke\/provider/);
assert.match(source, /api_key: env:AI_GATEWAY_API_KEY/);
assert.match(source, /run\('pnpm', \['exec', 'ktx', 'dev', 'ingest', 'run'/);
assert.match(source, /'serve', '--mcp', 'stdio'/);
assert.doesNotMatch(source, /'--semantic-compute',\n\s*'--execute-queries'/);
assert.match(source, /'--memory-capture', '--memory-model', 'smoke\/provider'/);
assert.match(source, /mcpServerStderr/);
assert.match(source, /ktx serve stderr/);
assert.match(source, /sl_validate/);
assert.match(source, /sl_query/);
assert.match(source, /memory_capture/);
assert.match(source, /memory_capture_status/);
assert.match(source, /connection_test/);
assert.match(source, /scan_trigger/);
assert.match(source, /scan_status/);
assert.match(source, /scan_report/);
assert.match(source, /scan_list_artifacts/);
assert.match(source, /scan_read_artifact/);
assert.match(source, /mcpScanArtifacts\.artifacts\.find/);
assert.match(source, /AI_GATEWAY_API_KEY/);
assert.match(source, /access\(join\(projectDir, '\.ktx', 'db\.sqlite'\)\)/);
assert.match(source, /SQLite knowledge index/);
assert.match(source, /ktx dev ingest run requires llm\\.provider\\.backend: anthropic, vertex, or gateway/);
@ -632,22 +522,8 @@ describe('verification snippets', () => {
assert.match(source, /'dev', 'doctor', 'setup', '--no-input'/);
assert.match(source, /'--plain'/);
assert.match(source, /ktx setup demo seeded wrote unexpected stderr/);
assert.match(source, /Object\.keys\(packageJson\.dependencies\)/);
assert.match(source, /'@kaelio\/ktx'/);
});
});
it('checks packaged ingest runtime assets in the installed npm smoke', () => {
const source = npmRuntimeSmokeSource();
assert.match(source, /notion_synthesize\/SKILL\.md/);
assert.match(source, /skills\/page_triage_classifier\.md/);
assert.match(source, /skills\/light_extraction\.md/);
});
it('asserts the Python modules that clean installs must expose', () => {
const source = pythonVerifySource();
assert.match(source, /semantic_layer/);
assert.match(source, /ktx_daemon/);
assert.match(source, /importlib.metadata/);
});
});

87
scripts/publish-public-npm-package.mjs Normal file
View file

@ -0,0 +1,87 @@
#!/usr/bin/env node
import { execFile } from 'node:child_process';
import { access } from 'node:fs/promises';
import { pathToFileURL } from 'node:url';
import { promisify } from 'node:util';
import { packageArtifactLayout } from './package-artifacts.mjs';
import { releaseReadinessReport } from './release-readiness.mjs';
const execFileAsync = promisify(execFile);
export function resolvePublishMode(args = process.argv.slice(2)) {
return { live: args.includes('--publish') };
}
export function requireNpmPublicReleaseReady(report) {
if (report.releaseMode !== 'npm-public-release-ready' || report.npmPublishEnabled !== true || !report.npmPublish) {
throw new Error('release-policy.json must use npm-public-release-ready before publishing');
}
return report.npmPublish;
}
export function buildNpmPublishCommand(tarballPath, publish, mode) {
return {
command: 'pnpm',
args: [
'publish',
tarballPath,
'--access',
publish.access,
'--tag',
publish.tag,
...(mode.live ? [] : ['--dry-run', '--no-git-checks']),
],
env: publish.registry ? { npm_config_registry: publish.registry } : {},
};
}
async function assertFileExists(path) {
try {
await access(path);
} catch {
throw new Error(`Missing npm tarball: ${path}. Run pnpm run artifacts:check first.`);
}
}
async function runPublishCommand(command) {
process.stdout.write(`$ ${command.command} ${command.args.join(' ')}\n`);
await execFileAsync(command.command, command.args, {
env: { ...process.env, ...command.env },
encoding: 'utf8',
maxBuffer: 1024 * 1024 * 20,
});
}
export async function publishPublicNpmPackage(options = {}) {
const rootDir = options.rootDir;
const mode = options.mode ?? resolvePublishMode(options.args);
const report = await releaseReadinessReport(rootDir);
const publish = requireNpmPublicReleaseReady(report);
const layout = packageArtifactLayout(rootDir);
const tarballPath = layout.cliTarball;
await assertFileExists(tarballPath);
const command = buildNpmPublishCommand(tarballPath, publish, mode);
await runPublishCommand(command);
process.stdout.write(
mode.live
? `Published ${publish.packageName}@${publish.version} with tag ${publish.tag}\n`
: `Dry-run verified ${publish.packageName}@${publish.version} with tag ${publish.tag}\n`,
);
}
async function main() {
await publishPublicNpmPackage({ args: process.argv.slice(2) });
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
try {
await main();
} catch (error) {
process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`);
process.exitCode = 1;
}
}

109
scripts/publish-public-npm-package.test.mjs Normal file
View file

@ -0,0 +1,109 @@
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import { describe, it } from 'node:test';
import {
buildNpmPublishCommand,
requireNpmPublicReleaseReady,
resolvePublishMode,
} from './publish-public-npm-package.mjs';
const readyReport = {
releaseMode: 'npm-public-release-ready',
npmPublishEnabled: true,
npmPublish: {
packageName: '@kaelio/ktx',
version: '0.1.0-rc.0',
access: 'public',
tag: 'next',
registry: null,
},
};
describe('resolvePublishMode', () => {
it('dry-runs by default', () => {
assert.deepEqual(resolvePublishMode([]), { live: false });
});
it('requires an explicit flag for live publish', () => {
assert.deepEqual(resolvePublishMode(['--publish']), { live: true });
});
});
describe('requireNpmPublicReleaseReady', () => {
it('accepts the npm public release ready report', () => {
assert.equal(requireNpmPublicReleaseReady(readyReport), readyReport.npmPublish);
});
it('rejects artifact-only reports', () => {
assert.throws(
() =>
requireNpmPublicReleaseReady({
releaseMode: 'ci-artifact-only',
npmPublishEnabled: false,
npmPublish: null,
}),
/release-policy.json must use npm-public-release-ready before publishing/,
);
});
});
describe('buildNpmPublishCommand', () => {
it('builds a dry-run pnpm publish command by default', () => {
assert.deepEqual(
buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz', readyReport.npmPublish, {
live: false,
}),
{
command: 'pnpm',
args: [
'publish',
'/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz',
'--access',
'public',
'--tag',
'next',
'--dry-run',
'--no-git-checks',
],
env: {},
},
);
});
it('omits dry-run only for explicit live publish', () => {
assert.deepEqual(
buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz', readyReport.npmPublish, {
live: true,
}).args,
[
'publish',
'/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz',
'--access',
'public',
'--tag',
'next',
],
);
});
it('uses npm_config_registry when a registry is configured', () => {
const publish = {
...readyReport.npmPublish,
registry: 'https://registry.npmjs.org/',
};
assert.deepEqual(
buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz', publish, { live: false }).env,
{ npm_config_registry: 'https://registry.npmjs.org/' },
);
});
});
describe('package script', () => {
it('registers release:npm-publish', async () => {
const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'));
assert.equal(packageJson.scripts['release:npm-publish'], 'node scripts/publish-public-npm-package.mjs');
});
});

102
scripts/published-package-smoke-config.mjs
View file

@ -1,3 +1,4 @@
import { dirname, join } from 'node:path';
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
@ -28,6 +29,30 @@ function assertHttpRegistry(registry, label) {
}
}
function registryEnv(config) {
return config.registry ? { npm_config_registry: config.registry } : {};
}
function runtimeCommandEnv(config, runtimeRoot) {
return { ...registryEnv(config), KTX_RUNTIME_ROOT: runtimeRoot };
}
function semanticQueryArgs(projectDir) {
return [
'sl',
'query',
'--project-dir',
projectDir,
'--connection-id',
'orbit_demo',
'--measure',
'contracts.contract_count',
'--format',
'sql',
'--yes',
];
}
function normalizePolicyConfig(policyConfig = {}) {
if (policyConfig === null || policyConfig === undefined) {
return { packageName: null, version: DEFAULT_VERSION_TAG, registry: null };
@ -114,39 +139,70 @@ export function publishedPackageSpec(config) {
return `${config.packageName}@${config.packageVersion}`;
}
export function buildPublishedPackageNpxCommand(config, args, label = 'published package command') {
const env = config.registry ? { npm_config_registry: config.registry } : {};
export function buildPublishedPackageNpxCommand(config, args, label = 'published package command', extraEnv = {}) {
return {
label,
command: 'npx',
args: ['--yes', publishedPackageSpec(config), ...args],
env,
env: { ...registryEnv(config), ...extraEnv },
};
}
export function buildPublishedPackageSmokeCommands(config, projectDir, emptyProjectDir) {
export function buildPublishedPackageSmokeCommands(
config,
projectDir,
runtimeRoot = join(dirname(projectDir), 'managed-runtime'),
) {
const runtimeEnv = runtimeCommandEnv(config, runtimeRoot);
const packageEnv = registryEnv(config);
const queryArgs = semanticQueryArgs(projectDir);
return [
buildPublishedPackageNpxCommand(config, ['--version'], 'published package version'),
buildPublishedPackageNpxCommand(config, ['--version'], 'published package npx version'),
buildPublishedPackageNpxCommand(
config,
['demo', '--project-dir', projectDir, '--no-input', '--plain'],
'published package demo',
),
buildPublishedPackageNpxCommand(
config,
['agent', 'wiki', 'search', 'ARR contract', '--json', '--limit', '5', '--project-dir', projectDir],
'published package wiki hybrid search',
),
buildPublishedPackageNpxCommand(
config,
['agent', 'sl', 'list', '--json', '--query', 'ARR', '--project-dir', projectDir],
'published package semantic-layer hybrid search',
),
buildPublishedPackageNpxCommand(
config,
['agent', 'sl', 'list', '--json', '--query', 'revenue', '--project-dir', emptyProjectDir],
'published package missing-project readiness',
['setup', 'demo', '--project-dir', projectDir, '--no-input', '--plain'],
'published package setup demo',
{ KTX_RUNTIME_ROOT: runtimeRoot },
),
buildPublishedPackageNpxCommand(config, queryArgs, 'published package npx sl query', {
KTX_RUNTIME_ROOT: runtimeRoot,
}),
{
label: 'published package local install',
command: 'pnpm',
args: ['add', publishedPackageSpec(config)],
env: packageEnv,
},
{
label: 'published package local version',
command: 'npx',
args: ['ktx', '--version'],
env: packageEnv,
},
{
label: 'published package local sl query',
command: 'npx',
args: ['ktx', ...queryArgs],
env: runtimeEnv,
},
{
label: 'published package global install',
command: 'pnpm',
args: ['add', '--global', publishedPackageSpec(config)],
env: packageEnv,
},
{
label: 'published package global version',
command: 'ktx',
args: ['--version'],
env: packageEnv,
},
{
label: 'published package global sl query',
command: 'ktx',
args: queryArgs,
env: runtimeEnv,
},
];
}

102
scripts/published-package-smoke.mjs
View file

@ -2,7 +2,7 @@
import assert from 'node:assert/strict';
import { execFile } from 'node:child_process';
import { mkdir, mkdtemp, rm } from 'node:fs/promises';
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
@ -23,6 +23,26 @@ export {
const execFileAsync = promisify(execFile);
const SMOKE_TIMEOUT_MS = 180_000;
const VERSION_LABELS = new Set([
'published package npx version',
'published package local version',
'published package global version',
]);
const SEMANTIC_QUERY_LABELS = new Set([
'published package npx sl query',
'published package local sl query',
'published package global sl query',
]);
export function isPublishedPackageVersionLabel(label) {
return VERSION_LABELS.has(label);
}
export function isPublishedPackageSemanticQueryLabel(label) {
return SEMANTIC_QUERY_LABELS.has(label);
}
function scriptRootDir() {
return resolve(dirname(fileURLToPath(import.meta.url)), '..');
}
@ -59,78 +79,34 @@ function requireSuccess(label, result) {
);
}
function parseJson(label, text) {
try {
return JSON.parse(text);
} catch (error) {
throw new Error(`${label} did not produce JSON: ${error instanceof Error ? error.message : String(error)}\n${text}`);
}
}
function assertHybridWikiSearch(result) {
const payload = parseJson('published package wiki search', result.stdout);
assert.ok(payload.totalFound > 0, 'published package wiki search should return results');
assert.ok(
payload.results.some((entry) => Array.isArray(entry.matchReasons) && entry.matchReasons.length > 0),
'published package wiki search should expose match reasons',
);
}
function assertHybridSlSearch(result) {
const payload = parseJson('published package semantic-layer search', result.stdout);
assert.ok(payload.totalSources > 0, 'published package semantic-layer search should return sources');
assert.ok(
payload.sources.some((entry) => Array.isArray(entry.matchReasons) && entry.matchReasons.length > 0),
'published package semantic-layer search should expose match reasons',
);
}
function assertMissingProjectReadiness(result, emptyProjectDir) {
assert.equal(result.code, 1, 'missing-project semantic-layer search should exit 1');
assert.equal(result.stdout, '', 'missing-project semantic-layer search should not write JSON errors to stdout');
const payload = parseJson('published package missing-project semantic-layer search', result.stderr);
assert.deepEqual(payload, {
ok: false,
error: {
code: 'agent_sl_search_missing_project',
message: `Semantic-layer search needs an initialized KTX project at ${emptyProjectDir}.`,
nextSteps: [
'ktx demo',
`ktx setup --project-dir ${emptyProjectDir}`,
'ktx ingest <connection>',
`ktx agent sl list --json --query "revenue" --project-dir ${emptyProjectDir}`,
],
},
});
}
export async function runPublishedPackageSmoke(config) {
const root = await mkdtemp(join(tmpdir(), 'ktx-published-package-smoke-'));
try {
const projectDir = join(root, 'demo-project');
const emptyProjectDir = join(root, 'empty-project');
await mkdir(emptyProjectDir, { recursive: true });
const commands = buildPublishedPackageSmokeCommands(config, projectDir, emptyProjectDir);
for (const command of commands.slice(0, 4)) {
const result = await runCommand(command.command, command.args, { env: command.env });
const commands = buildPublishedPackageSmokeCommands(config, projectDir);
const pnpmHome = join(root, 'pnpm-home');
const globalEnv = {
PNPM_HOME: pnpmHome,
PATH: `${pnpmHome}${process.platform === 'win32' ? ';' : ':'}${process.env.PATH ?? ''}`,
};
for (const command of commands) {
const isGlobalCommand = command.label.includes('global');
const result = await runCommand(command.command, command.args, {
cwd: command.label.includes('local') || isGlobalCommand ? root : undefined,
env: isGlobalCommand ? { ...globalEnv, ...command.env } : command.env,
});
requireSuccess(command.label, result);
if (command.label === 'published package wiki hybrid search') {
assertHybridWikiSearch(result);
if (isPublishedPackageVersionLabel(command.label)) {
assert.match(result.stdout, /@kaelio\/ktx /);
}
if (command.label === 'published package semantic-layer hybrid search') {
assertHybridSlSearch(result);
if (isPublishedPackageSemanticQueryLabel(command.label)) {
assert.match(result.stdout, /SELECT/i);
assert.match(result.stdout, /contracts/i);
}
}
const missingProjectCommand = commands[4];
const missingProject = await runCommand(missingProjectCommand.command, missingProjectCommand.args, {
env: missingProjectCommand.env,
});
assertMissingProjectReadiness(missingProject, emptyProjectDir);
process.stdout.write('published package hybrid search smoke verified\n');
process.stdout.write('published package invocation smoke verified\n');
} finally {
await rm(root, { recursive: true, force: true });
}

239
scripts/published-package-smoke.test.mjs
View file

@ -5,6 +5,8 @@ import { describe, it } from 'node:test';
import {
buildPublishedPackageNpxCommand,
buildPublishedPackageSmokeCommands,
isPublishedPackageSemanticQueryLabel,
isPublishedPackageVersionLabel,
publishedPackageSpec,
readPublishedPackageSmokeConfig,
} from './published-package-smoke.mjs';
@ -32,7 +34,7 @@ describe('published package smoke config', () => {
assert.deepEqual(
readPublishedPackageSmokeConfig(
{
KTX_PUBLISHED_KTX_PACKAGE: '@ktx/cli-public',
KTX_PUBLISHED_KTX_PACKAGE: '@kaelio/ktx',
KTX_PUBLISHED_KTX_VERSION: 'latest',
KTX_PUBLISHED_KTX_REGISTRY: 'https://registry.npmjs.org/',
},
@ -42,7 +44,7 @@ describe('published package smoke config', () => {
enabled: true,
requireConfig: false,
configSource: 'environment',
packageName: '@ktx/cli-public',
packageName: '@kaelio/ktx',
packageVersion: 'latest',
registry: 'https://registry.npmjs.org/',
},
@ -55,7 +57,7 @@ describe('published package smoke config', () => {
{},
[],
{
packageName: '@ktx/cli-public',
packageName: '@kaelio/ktx',
version: '2026.5.8',
registry: 'https://registry.npmjs.org/',
},
@ -64,7 +66,7 @@ describe('published package smoke config', () => {
enabled: true,
requireConfig: false,
configSource: 'release-policy',
packageName: '@ktx/cli-public',
packageName: '@kaelio/ktx',
packageVersion: '2026.5.8',
registry: 'https://registry.npmjs.org/',
},
@ -75,12 +77,12 @@ describe('published package smoke config', () => {
assert.deepEqual(
readPublishedPackageSmokeConfig(
{
KTX_PUBLISHED_KTX_PACKAGE: '@ktx/cli-from-env',
KTX_PUBLISHED_KTX_PACKAGE: '@kaelio/ktx',
KTX_PUBLISHED_KTX_VERSION: 'latest',
},
[],
{
packageName: '@ktx/cli-from-policy',
packageName: '@kaelio/ktx',
version: '2026.5.8',
registry: 'https://registry.npmjs.org/',
},
@ -89,7 +91,7 @@ describe('published package smoke config', () => {
enabled: true,
requireConfig: false,
configSource: 'environment',
packageName: '@ktx/cli-from-env',
packageName: '@kaelio/ktx',
packageVersion: 'latest',
registry: 'https://registry.npmjs.org/',
},
@ -125,7 +127,7 @@ describe('published package smoke config', () => {
() =>
readPublishedPackageSmokeConfig(
{
KTX_PUBLISHED_KTX_PACKAGE: '@ktx/cli-public',
KTX_PUBLISHED_KTX_PACKAGE: '@kaelio/ktx',
KTX_PUBLISHED_KTX_VERSION: '--tag latest',
},
[],
@ -136,7 +138,7 @@ describe('published package smoke config', () => {
() =>
readPublishedPackageSmokeConfig(
{
KTX_PUBLISHED_KTX_PACKAGE: '@ktx/cli-public',
KTX_PUBLISHED_KTX_PACKAGE: '@kaelio/ktx',
KTX_PUBLISHED_KTX_REGISTRY: 'file:///tmp/npm',
},
[],
@ -146,103 +148,166 @@ describe('published package smoke config', () => {
});
});
describe('published package smoke output validation labels', () => {
it('classifies version and semantic query commands', () => {
assert.equal(isPublishedPackageVersionLabel('published package npx version'), true);
assert.equal(isPublishedPackageVersionLabel('published package local version'), true);
assert.equal(isPublishedPackageVersionLabel('published package global version'), true);
assert.equal(isPublishedPackageVersionLabel('published package setup demo'), false);
assert.equal(isPublishedPackageSemanticQueryLabel('published package npx sl query'), true);
assert.equal(isPublishedPackageSemanticQueryLabel('published package local sl query'), true);
assert.equal(isPublishedPackageSemanticQueryLabel('published package global sl query'), true);
assert.equal(isPublishedPackageSemanticQueryLabel('published package local install'), false);
});
});
describe('published package smoke command construction', () => {
const config = {
enabled: true,
requireConfig: false,
packageName: '@ktx/cli-public',
packageName: '@kaelio/ktx',
packageVersion: 'latest',
registry: 'https://registry.npmjs.org/',
};
it('builds the npx package spec from package name and version tag', () => {
assert.equal(publishedPackageSpec(config), '@ktx/cli-public@latest');
assert.equal(publishedPackageSpec(config), '@kaelio/ktx@latest');
});
it('builds npx commands with a registry env patch instead of shell interpolation', () => {
assert.deepEqual(buildPublishedPackageNpxCommand(config, ['--version']), {
label: 'published package command',
command: 'npx',
args: ['--yes', '@ktx/cli-public@latest', '--version'],
args: ['--yes', '@kaelio/ktx@latest', '--version'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
});
});
it('builds the full hybrid-search smoke command list', () => {
assert.deepEqual(buildPublishedPackageSmokeCommands(config, '/tmp/ktx-smoke/demo', '/tmp/ktx-smoke/empty'), [
{
label: 'published package version',
command: 'npx',
args: ['--yes', '@ktx/cli-public@latest', '--version'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package demo',
command: 'npx',
args: [
'--yes',
'@ktx/cli-public@latest',
'demo',
'--project-dir',
'/tmp/ktx-smoke/demo',
'--no-input',
'--plain',
],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package wiki hybrid search',
command: 'npx',
args: [
'--yes',
'@ktx/cli-public@latest',
'agent',
'wiki',
'search',
'ARR contract',
'--json',
'--limit',
'5',
'--project-dir',
'/tmp/ktx-smoke/demo',
],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package semantic-layer hybrid search',
command: 'npx',
args: [
'--yes',
'@ktx/cli-public@latest',
'agent',
'sl',
'list',
'--json',
'--query',
'ARR',
'--project-dir',
'/tmp/ktx-smoke/demo',
],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package missing-project readiness',
command: 'npx',
args: [
'--yes',
'@ktx/cli-public@latest',
'agent',
'sl',
'list',
'--json',
'--query',
'revenue',
'--project-dir',
'/tmp/ktx-smoke/empty',
],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
]);
it('builds the full public package smoke command list', () => {
assert.deepEqual(
buildPublishedPackageSmokeCommands(
config,
'/tmp/ktx-smoke/demo',
'/tmp/ktx-smoke/managed-runtime',
),
[
{
label: 'published package npx version',
command: 'npx',
args: ['--yes', '@kaelio/ktx@latest', '--version'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package setup demo',
command: 'npx',
args: [
'--yes',
'@kaelio/ktx@latest',
'setup',
'demo',
'--project-dir',
'/tmp/ktx-smoke/demo',
'--no-input',
'--plain',
],
env: {
npm_config_registry: 'https://registry.npmjs.org/',
KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime',
},
},
{
label: 'published package npx sl query',
command: 'npx',
args: [
'--yes',
'@kaelio/ktx@latest',
'sl',
'query',
'--project-dir',
'/tmp/ktx-smoke/demo',
'--connection-id',
'orbit_demo',
'--measure',
'contracts.contract_count',
'--format',
'sql',
'--yes',
],
env: {
npm_config_registry: 'https://registry.npmjs.org/',
KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime',
},
},
{
label: 'published package local install',
command: 'pnpm',
args: ['add', '@kaelio/ktx@latest'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package local version',
command: 'npx',
args: ['ktx', '--version'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package local sl query',
command: 'npx',
args: [
'ktx',
'sl',
'query',
'--project-dir',
'/tmp/ktx-smoke/demo',
'--connection-id',
'orbit_demo',
'--measure',
'contracts.contract_count',
'--format',
'sql',
'--yes',
],
env: {
npm_config_registry: 'https://registry.npmjs.org/',
KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime',
},
},
{
label: 'published package global install',
command: 'pnpm',
args: ['add', '--global', '@kaelio/ktx@latest'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package global version',
command: 'ktx',
args: ['--version'],
env: { npm_config_registry: 'https://registry.npmjs.org/' },
},
{
label: 'published package global sl query',
command: 'ktx',
args: [
'sl',
'query',
'--project-dir',
'/tmp/ktx-smoke/demo',
'--connection-id',
'orbit_demo',
'--measure',
'contracts.contract_count',
'--format',
'sql',
'--yes',
],
env: {
npm_config_registry: 'https://registry.npmjs.org/',
KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime',
},
},
],
);
});
it('exposes the smoke through the package release script', async () => {

135
scripts/release-readiness.mjs
View file

@ -5,6 +5,7 @@ import { dirname, join, resolve } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { packageArtifactLayout, packageReleaseMetadata, verifyArtifactManifest } from './package-artifacts.mjs';
import { PUBLIC_NPM_PACKAGE_VERSION } from './build-public-npm-package.mjs';
import { readPublishedPackageSmokeConfig } from './published-package-smoke-config.mjs';
function scriptRootDir() {
@ -21,9 +22,11 @@ async function readJson(path) {
const CI_ARTIFACT_ONLY_RELEASE_MODE = 'ci-artifact-only';
const PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE = 'published-package-smoke-required';
const NPM_PUBLIC_RELEASE_READY_MODE = 'npm-public-release-ready';
const SUPPORTED_RELEASE_MODES = new Set([
CI_ARTIFACT_ONLY_RELEASE_MODE,
PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE,
NPM_PUBLIC_RELEASE_READY_MODE,
]);
export async function readReleasePolicy(rootDir = scriptRootDir()) {
@ -64,6 +67,19 @@ function assertStringArray(value, label) {
}
}
function assertNpmAccess(value) {
if (value !== 'public') {
throw new Error('Release policy npm.access must be public');
}
}
function assertNpmTag(value) {
assertString(value, 'Release policy npm.tag');
if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(value)) {
throw new Error(`Invalid Release policy npm.tag: ${value}`);
}
}
function assertSupportedReleaseMode(releaseMode) {
assertString(releaseMode, 'Release policy releaseMode');
if (!SUPPORTED_RELEASE_MODES.has(releaseMode)) {
@ -79,10 +95,31 @@ function assertRequiredBeforePublishing(policy) {
}
if (
policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE &&
(policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE ||
policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) &&
policy.requiredBeforePublishing.length > 0
) {
throw new Error('published-package-smoke-required release mode requires requiredBeforePublishing to be empty');
throw new Error(`${policy.releaseMode} release mode requires requiredBeforePublishing to be empty`);
}
}
function assertRuntimeInstallerPolicy(policy) {
assertPlainObject(policy.runtimeInstaller, 'Release policy runtimeInstaller');
assertString(policy.runtimeInstaller.uvStrategy, 'Release policy runtimeInstaller.uvStrategy');
assertBoolean(policy.runtimeInstaller.bootstrapUv, 'Release policy runtimeInstaller.bootstrapUv');
assertString(
policy.runtimeInstaller.missingUvBehavior,
'Release policy runtimeInstaller.missingUvBehavior',
);
if (policy.runtimeInstaller.uvStrategy !== 'path-prerequisite') {
throw new Error('Release policy runtimeInstaller.uvStrategy must be path-prerequisite');
}
if (policy.runtimeInstaller.bootstrapUv !== false) {
throw new Error('Release policy runtimeInstaller.bootstrapUv must be false');
}
if (policy.runtimeInstaller.missingUvBehavior !== 'focused-error') {
throw new Error('Release policy runtimeInstaller.missingUvBehavior must be focused-error');
}
}
@ -107,6 +144,8 @@ export function validateReleasePolicy(policy) {
assertBoolean(policy.npm.publish, 'Release policy npm.publish');
assertNullableString(policy.npm.registry, 'Release policy npm.registry');
assertNpmAccess(policy.npm.access);
assertNpmTag(policy.npm.tag);
assertStringArray(policy.npm.packages, 'Release policy npm.packages');
assertBoolean(policy.python.publish, 'Release policy python.publish');
@ -117,6 +156,7 @@ export function validateReleasePolicy(policy) {
assertNullableString(policy.publishedPackageSmoke.registry, 'Release policy publishedPackageSmoke.registry');
readPublishedPackageSmokeConfig({}, [], policy.publishedPackageSmoke);
assertRequiredBeforePublishing(policy);
assertRuntimeInstallerPolicy(policy);
return policy;
}
@ -128,10 +168,12 @@ function metadataNames(metadata, ecosystem) {
function publishedPackageSmokeGate(policy) {
const config = readPublishedPackageSmokeConfig({}, [], policy.publishedPackageSmoke);
if (policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE && !config.enabled) {
throw new Error(
'published-package-smoke-required release mode requires release-policy.json publishedPackageSmoke.packageName',
);
if (
(policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE ||
policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) &&
!config.enabled
) {
throw new Error(`${policy.releaseMode} release mode requires release-policy.json publishedPackageSmoke.packageName`);
}
const base =
@ -140,6 +182,11 @@ function publishedPackageSmokeGate(policy) {
status: 'not_required',
reason: 'Published package smoke remains pending until release-policy.json enables npm registry publishing.',
}
: policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE
? {
status: 'required',
reason: 'Run the published package smoke after the npm package is published.',
}
: {
status: 'required',
reason: 'Run the published package smoke before accepting the hybrid-search release.',
@ -180,23 +227,68 @@ function assertNonPublishingArtifactPolicy(policy, metadata) {
throw new Error(`Package ${entry.packageName} releaseMode must remain ci-artifact-only`);
}
if (entry.ecosystem === 'npm') {
if (entry.private !== true) {
const isPublicKtxPackage = entry.packageName === '@kaelio/ktx';
if (isPublicKtxPackage) {
if (entry.private !== false) {
throw new Error(`${policyLabel} npm package @kaelio/ktx must be publishable when npm.publish is false`);
}
if (entry.packageVersion !== PUBLIC_NPM_PACKAGE_VERSION) {
throw new Error(`${policyLabel} npm package @kaelio/ktx must use public version ${PUBLIC_NPM_PACKAGE_VERSION}`);
}
} else if (entry.private !== true) {
throw new Error(`${policyLabel} npm package ${entry.packageName} must remain private`);
}
if (!entry.packageVersion.endsWith('-private')) {
} else if (!entry.packageVersion.endsWith('-private')) {
throw new Error(`${policyLabel} npm package ${entry.packageName} must use a private version suffix`);
}
}
}
}
function assertNpmPublicReleaseReadyPolicy(policy, metadata) {
if (policy.npm.publish !== true) {
throw new Error('npm-public-release-ready policy requires npm.publish true');
}
if (policy.python.publish !== false) {
throw new Error('npm-public-release-ready policy keeps python.publish false');
}
if (policy.python.repository !== null) {
throw new Error('npm-public-release-ready policy keeps python.repository null');
}
assertSameMembers(policy.npm.packages, ['@kaelio/ktx'], 'Release policy npm.packages');
assertSameMembers(policy.python.packages, metadataNames(metadata, 'python'), 'Release policy python.packages');
const npmMetadata = metadata.find((entry) => entry.ecosystem === 'npm' && entry.packageName === '@kaelio/ktx');
if (!npmMetadata) {
throw new Error('npm-public-release-ready policy requires @kaelio/ktx artifact metadata');
}
if (npmMetadata.private !== false) {
throw new Error('npm-public-release-ready policy requires @kaelio/ktx to be publishable');
}
if (npmMetadata.packageVersion !== PUBLIC_NPM_PACKAGE_VERSION) {
throw new Error(
`npm-public-release-ready policy expected @kaelio/ktx ${PUBLIC_NPM_PACKAGE_VERSION}, got ${npmMetadata.packageVersion}`,
);
}
if (policy.publishedPackageSmoke.packageName !== '@kaelio/ktx') {
throw new Error('npm-public-release-ready policy requires publishedPackageSmoke.packageName @kaelio/ktx');
}
if (policy.publishedPackageSmoke.version !== PUBLIC_NPM_PACKAGE_VERSION) {
throw new Error(`npm-public-release-ready policy requires publishedPackageSmoke.version ${PUBLIC_NPM_PACKAGE_VERSION}`);
}
}
export async function releaseReadinessReport(rootDir = scriptRootDir()) {
const policy = validateReleasePolicy(await readReleasePolicy(rootDir));
const layout = packageArtifactLayout(rootDir);
const manifest = await verifyArtifactManifest(layout);
const metadata = await packageReleaseMetadata(rootDir);
assertNonPublishingArtifactPolicy(policy, metadata);
if (policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) {
assertNpmPublicReleaseReadyPolicy(policy, metadata);
} else {
assertNonPublishingArtifactPolicy(policy, metadata);
}
return {
schemaVersion: 1,
@ -206,6 +298,17 @@ export async function releaseReadinessReport(rootDir = scriptRootDir()) {
pythonPublishEnabled: policy.python.publish,
packageNames: metadata.map((entry) => entry.packageName),
publishedPackageSmokeGate: publishedPackageSmokeGate(policy),
runtimeInstaller: policy.runtimeInstaller,
npmPublish:
policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE
? {
packageName: '@kaelio/ktx',
version: PUBLIC_NPM_PACKAGE_VERSION,
access: policy.npm.access,
tag: policy.npm.tag,
registry: policy.npm.registry,
}
: null,
blockedPublishingDecisions: policy.requiredBeforePublishing,
};
}
@ -229,7 +332,17 @@ async function main() {
process.stdout.write(
`Published package smoke registry: ${report.publishedPackageSmokeGate.registry ?? 'default npm registry'}\n`,
);
process.stdout.write('Registry publishing remains disabled by release-policy.json.\n');
process.stdout.write(`Runtime uv strategy: ${report.runtimeInstaller.uvStrategy}\n`);
process.stdout.write(
`Runtime uv bootstrap: ${report.runtimeInstaller.bootstrapUv ? 'enabled' : 'disabled'}\n`,
);
if (report.npmPublish) {
process.stdout.write(
`NPM publish target: ${report.npmPublish.packageName}@${report.npmPublish.version} (${report.npmPublish.tag})\n`,
);
} else {
process.stdout.write('Registry publishing remains disabled by release-policy.json.\n');
}
process.stdout.write('Required decisions before publishing:\n');
for (const decision of report.blockedPublishingDecisions) {
process.stdout.write(`- ${decision}\n`);

296
scripts/release-readiness.test.mjs
View file

@ -4,39 +4,28 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, it } from 'node:test';
import { NPM_ARTIFACT_PACKAGES, packageArtifactLayout, writeArtifactManifest } from './package-artifacts.mjs';
import {
INTERNAL_NPM_WORKSPACE_PACKAGES,
NPM_ARTIFACT_PACKAGES,
packageArtifactLayout,
writeArtifactManifest,
} from './package-artifacts.mjs';
import { PUBLIC_NPM_PACKAGE_VERSION } from './build-public-npm-package.mjs';
import { readReleasePolicy, releasePolicyPath, releaseReadinessReport } from './release-readiness.mjs';
async function writeJson(path, value) {
await writeFile(path, `${JSON.stringify(value, null, 2)}\n`);
}
async function writeReleaseMetadataInputs(root, options = {}) {
for (const packageInfo of NPM_ARTIFACT_PACKAGES) {
async function writeReleaseMetadataInputs(root) {
for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) {
await mkdir(join(root, packageInfo.packageRoot), { recursive: true });
await writeJson(join(root, packageInfo.packageRoot, 'package.json'), {
name: packageInfo.name,
version: '0.0.0-private',
private:
packageInfo.name === '@ktx/context'
? (options.contextPrivate ?? true)
: packageInfo.name === '@ktx/cli'
? (options.cliPrivate ?? true)
: true,
private: true,
});
}
await mkdir(join(root, 'python', 'ktx-sl'), { recursive: true });
await mkdir(join(root, 'python', 'ktx-daemon'), { recursive: true });
await writeFile(
join(root, 'python', 'ktx-sl', 'pyproject.toml'),
['[project]', 'name = "ktx-sl"', 'version = "0.1.0"', ''].join('\n'),
);
await writeFile(
join(root, 'python', 'ktx-daemon', 'pyproject.toml'),
['[project]', 'name = "ktx-daemon"', 'version = "0.1.0"', ''].join('\n'),
);
}
async function writeUploadableArtifactFixtures(layout) {
@ -48,10 +37,7 @@ async function writeUploadableArtifactFixtures(layout) {
layout.npmTarballs[packageInfo.name],
`${packageInfo.name}-tarball`,
]),
[join(layout.pythonDir, 'ktx_sl-0.1.0-py3-none-any.whl'), 'ktx-sl-wheel'],
[join(layout.pythonDir, 'ktx_sl-0.1.0.tar.gz'), 'ktx-sl-sdist'],
[join(layout.pythonDir, 'ktx_daemon-0.1.0-py3-none-any.whl'), 'ktx-daemon-wheel'],
[join(layout.pythonDir, 'ktx_daemon-0.1.0.tar.gz'), 'ktx-daemon-sdist'],
[join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), 'kaelio-ktx-runtime-wheel'],
]);
for (const [path, contents] of fileContents) {
@ -68,24 +54,29 @@ function releasePolicy(overrides = {}) {
npm: {
publish: false,
registry: null,
packages: NPM_ARTIFACT_PACKAGES.map((packageInfo) => packageInfo.name),
access: 'public',
tag: 'latest',
packages: ['@kaelio/ktx'],
...npmOverrides,
},
python: {
publish: false,
repository: null,
packages: ['ktx-sl', 'ktx-daemon'],
packages: ['kaelio-ktx'],
...pythonOverrides,
},
publishedPackageSmoke: {
packageName: null,
packageName: '@kaelio/ktx',
version: 'latest',
registry: null,
},
runtimeInstaller: {
uvStrategy: 'path-prerequisite',
bootstrapUv: false,
missingUvBehavior: 'focused-error',
},
requiredBeforePublishing: [
'Choose npm registry and package visibility.',
'Choose Python package repository.',
'Choose public release versions.',
'Choose public release version.',
'Configure registry credentials outside source control.',
'Choose release tag and provenance policy.',
],
@ -98,7 +89,7 @@ async function writePolicy(root, policy = releasePolicy()) {
}
async function writeReadyFixture(root, options = {}) {
await writeReleaseMetadataInputs(root, options);
await writeReleaseMetadataInputs(root);
await writePolicy(root, options.policy ?? releasePolicy());
const layout = packageArtifactLayout(root);
await writeUploadableArtifactFixtures(layout);
@ -135,20 +126,24 @@ describe('release readiness policy', () => {
sourceRevision: 'abc123',
npmPublishEnabled: false,
pythonPublishEnabled: false,
packageNames: [...NPM_ARTIFACT_PACKAGES.map((packageInfo) => packageInfo.name), 'ktx-sl', 'ktx-daemon'],
packageNames: ['@kaelio/ktx', 'kaelio-ktx'],
publishedPackageSmokeGate: {
status: 'not_required',
script: 'pnpm run release:published-smoke',
reason: 'Published package smoke remains pending until release-policy.json enables npm registry publishing.',
configSource: null,
packageName: null,
configSource: 'release-policy',
packageName: '@kaelio/ktx',
version: 'latest',
registry: null,
},
runtimeInstaller: {
uvStrategy: 'path-prerequisite',
bootstrapUv: false,
missingUvBehavior: 'focused-error',
},
npmPublish: null,
blockedPublishingDecisions: [
'Choose npm registry and package visibility.',
'Choose Python package repository.',
'Choose public release versions.',
'Choose public release version.',
'Configure registry credentials outside source control.',
'Choose release tag and provenance policy.',
],
@ -164,7 +159,7 @@ describe('release readiness policy', () => {
await writeReadyFixture(root, {
policy: releasePolicy({
publishedPackageSmoke: {
packageName: '@ktx/cli-public',
packageName: '@kaelio/ktx',
version: '2026.5.8',
registry: 'https://registry.npmjs.org/',
},
@ -178,7 +173,7 @@ describe('release readiness policy', () => {
script: 'pnpm run release:published-smoke',
reason: 'Published package smoke remains pending until release-policy.json enables npm registry publishing.',
configSource: 'release-policy',
packageName: '@ktx/cli-public',
packageName: '@kaelio/ktx',
version: '2026.5.8',
registry: 'https://registry.npmjs.org/',
});
@ -194,7 +189,7 @@ describe('release readiness policy', () => {
policy: releasePolicy({
releaseMode: 'published-package-smoke-required',
publishedPackageSmoke: {
packageName: '@ktx/cli-public',
packageName: '@kaelio/ktx',
version: '2026.5.8',
registry: 'https://registry.npmjs.org/',
},
@ -210,16 +205,22 @@ describe('release readiness policy', () => {
sourceRevision: 'abc123',
npmPublishEnabled: false,
pythonPublishEnabled: false,
packageNames: [...NPM_ARTIFACT_PACKAGES.map((packageInfo) => packageInfo.name), 'ktx-sl', 'ktx-daemon'],
packageNames: ['@kaelio/ktx', 'kaelio-ktx'],
publishedPackageSmokeGate: {
status: 'required',
script: 'pnpm run release:published-smoke',
reason: 'Run the published package smoke before accepting the hybrid-search release.',
configSource: 'release-policy',
packageName: '@ktx/cli-public',
packageName: '@kaelio/ktx',
version: '2026.5.8',
registry: 'https://registry.npmjs.org/',
},
runtimeInstaller: {
uvStrategy: 'path-prerequisite',
bootstrapUv: false,
missingUvBehavior: 'focused-error',
},
npmPublish: null,
blockedPublishingDecisions: [],
});
} finally {
@ -227,12 +228,205 @@ describe('release readiness policy', () => {
}
});
it('accepts the npm public release ready policy', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-test-'));
try {
await writeReadyFixture(root, {
policy: releasePolicy({
releaseMode: 'npm-public-release-ready',
npm: {
publish: true,
registry: null,
access: 'public',
tag: 'latest',
},
publishedPackageSmoke: {
packageName: '@kaelio/ktx',
version: PUBLIC_NPM_PACKAGE_VERSION,
registry: null,
},
requiredBeforePublishing: [],
}),
});
const report = await releaseReadinessReport(root);
assert.deepEqual(report, {
schemaVersion: 1,
releaseMode: 'npm-public-release-ready',
sourceRevision: 'abc123',
npmPublishEnabled: true,
pythonPublishEnabled: false,
packageNames: ['@kaelio/ktx', 'kaelio-ktx'],
publishedPackageSmokeGate: {
status: 'required',
script: 'pnpm run release:published-smoke',
reason: 'Run the published package smoke after the npm package is published.',
configSource: 'release-policy',
packageName: '@kaelio/ktx',
version: PUBLIC_NPM_PACKAGE_VERSION,
registry: null,
},
runtimeInstaller: {
uvStrategy: 'path-prerequisite',
bootstrapUv: false,
missingUvBehavior: 'focused-error',
},
npmPublish: {
packageName: '@kaelio/ktx',
version: PUBLIC_NPM_PACKAGE_VERSION,
access: 'public',
tag: 'latest',
registry: null,
},
blockedPublishingDecisions: [],
});
} finally {
await rm(root, { recursive: true, force: true });
}
});
it('rejects npm public release ready mode without a runtime installer policy', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-policy-missing-test-'));
try {
await writeReadyFixture(root, {
policy: releasePolicy({
releaseMode: 'npm-public-release-ready',
npm: {
publish: true,
registry: null,
access: 'public',
tag: 'latest',
},
publishedPackageSmoke: {
packageName: '@kaelio/ktx',
version: PUBLIC_NPM_PACKAGE_VERSION,
registry: null,
},
runtimeInstaller: undefined,
requiredBeforePublishing: [],
}),
});
await assert.rejects(
() => releaseReadinessReport(root),
/Release policy runtimeInstaller must be a JSON object/,
);
} finally {
await rm(root, { recursive: true, force: true });
}
});
it('rejects uv bootstrap download policy for the first public npm release', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-policy-bootstrap-test-'));
try {
await writeReadyFixture(root, {
policy: releasePolicy({
releaseMode: 'npm-public-release-ready',
npm: {
publish: true,
registry: null,
access: 'public',
tag: 'latest',
},
publishedPackageSmoke: {
packageName: '@kaelio/ktx',
version: PUBLIC_NPM_PACKAGE_VERSION,
registry: null,
},
runtimeInstaller: {
uvStrategy: 'bootstrap-download',
bootstrapUv: true,
missingUvBehavior: 'download',
},
requiredBeforePublishing: [],
}),
});
await assert.rejects(
() => releaseReadinessReport(root),
/Release policy runtimeInstaller\.uvStrategy must be path-prerequisite/,
);
} finally {
await rm(root, { recursive: true, force: true });
}
});
it('rejects npm public release ready mode when npm publish is disabled', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-disabled-test-'));
try {
await writeReadyFixture(root, {
policy: releasePolicy({
releaseMode: 'npm-public-release-ready',
npm: {
publish: false,
registry: null,
access: 'public',
tag: 'latest',
},
publishedPackageSmoke: {
packageName: '@kaelio/ktx',
version: PUBLIC_NPM_PACKAGE_VERSION,
registry: null,
},
requiredBeforePublishing: [],
}),
});
await assert.rejects(
() => releaseReadinessReport(root),
/npm-public-release-ready policy requires npm.publish true/,
);
} finally {
await rm(root, { recursive: true, force: true });
}
});
it('rejects npm public release ready mode when Python publishing is enabled', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-python-test-'));
try {
await writeReadyFixture(root, {
policy: releasePolicy({
releaseMode: 'npm-public-release-ready',
npm: {
publish: true,
registry: null,
access: 'public',
tag: 'latest',
},
python: {
publish: true,
repository: 'pypi',
},
publishedPackageSmoke: {
packageName: '@kaelio/ktx',
version: PUBLIC_NPM_PACKAGE_VERSION,
registry: null,
},
requiredBeforePublishing: [],
}),
});
await assert.rejects(
() => releaseReadinessReport(root),
/npm-public-release-ready policy keeps python.publish false/,
);
} finally {
await rm(root, { recursive: true, force: true });
}
});
it('rejects required published smoke mode without a package name', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-release-smoke-required-missing-config-test-'));
try {
await writeReadyFixture(root, {
policy: releasePolicy({
releaseMode: 'published-package-smoke-required',
publishedPackageSmoke: {
packageName: null,
version: 'latest',
registry: null,
},
requiredBeforePublishing: [],
}),
});
@ -253,7 +447,7 @@ describe('release readiness policy', () => {
policy: releasePolicy({
releaseMode: 'published-package-smoke-required',
publishedPackageSmoke: {
packageName: '@ktx/cli-public',
packageName: '@kaelio/ktx',
version: 'latest',
registry: null,
},
@ -345,14 +539,20 @@ describe('release readiness policy', () => {
}
});
it('rejects a public npm package while releaseMode is ci-artifact-only', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-release-public-npm-test-'));
it('rejects release policy that still lists internal npm packages', async () => {
const root = await mkdtemp(join(tmpdir(), 'ktx-release-stale-internal-npm-policy-test-'));
try {
await writeReadyFixture(root, { contextPrivate: false });
await writeReadyFixture(root, {
policy: releasePolicy({
npm: {
packages: ['@kaelio/ktx', '@ktx/context'],
},
}),
});
await assert.rejects(
() => releaseReadinessReport(root),
/ci-artifact-only policy npm package @ktx\/context must remain private/,
/Release policy npm\.packages mismatch/,
);
} finally {
await rm(root, { recursive: true, force: true });

21
scripts/release-workflow.test.mjs Normal file
View file

@ -0,0 +1,21 @@
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import { describe, it } from 'node:test';
describe('release workflow', () => {
it('publishes only from manual dispatch with an explicit live input', async () => {
const workflow = await readFile(new URL('../.github/workflows/release.yml', import.meta.url), 'utf8');
assert.match(workflow, /^name: KTX Release$/m);
assert.match(workflow, /^ workflow_dispatch:$/m);
assert.match(workflow, /publish_live:/);
assert.match(workflow, /default: false/);
assert.match(workflow, /pnpm run artifacts:check/);
assert.match(workflow, /pnpm run release:readiness/);
assert.match(workflow, /pnpm run release:npm-publish$/m);
assert.match(workflow, /pnpm run release:npm-publish -- --publish/);
assert.match(workflow, /NODE_AUTH_TOKEN: \$\{\{ secrets.NPM_TOKEN \}\}/);
assert.doesNotMatch(workflow, /^ push:/m);
assert.doesNotMatch(workflow, /^ pull_request:/m);
});
});

13
uv.lock generated
View file

@ -452,11 +452,15 @@ dependencies = [
{ name = "psycopg", extra = ["binary"] },
{ name = "pydantic" },
{ name = "requests" },
{ name = "sentence-transformers" },
{ name = "sqlglot" },
{ name = "uvicorn", extra = ["standard"] },
]
[package.optional-dependencies]
local-embeddings = [
{ name = "sentence-transformers" },
{ name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" },
{ name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" },
{ name = "uvicorn", extra = ["standard"] },
]
[package.dev-dependencies]
@ -476,11 +480,12 @@ requires-dist = [
{ name = "psycopg", extras = ["binary"], specifier = ">=3.2.0" },
{ name = "pydantic", specifier = ">=2.9.0" },
{ name = "requests", specifier = ">=2.32.0" },
{ name = "sentence-transformers", specifier = ">=5.1.1" },
{ name = "sentence-transformers", marker = "extra == 'local-embeddings'", specifier = ">=5.1.1" },
{ name = "sqlglot", specifier = ">=26" },
{ name = "torch", specifier = ">=2.2.0", index = "https://download.pytorch.org/whl/cpu" },
{ name = "torch", marker = "extra == 'local-embeddings'", specifier = ">=2.2.0", index = "https://download.pytorch.org/whl/cpu" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" },
]
provides-extras = ["local-embeddings"]
[package.metadata.requires-dev]
dev = [