From 9dad936ac7e5ee7ff25c0122d140835610377e04 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Mon, 11 May 2026 15:50:34 +0200 Subject: [PATCH] feat: npm-managed Python runtime for @kaelio/ktx (#7) * docs: add npm managed python runtime design * build: add bundled python runtime wheel builder * build: make local embedding dependencies optional * build: bundle python runtime wheel in cli artifacts * build: track bundled python runtime release artifact * test: verify bundled python runtime wheel * docs: add plan for bundled python runtime wheel * test: cover managed python runtime lifecycle * feat: add managed python runtime installer * feat: add runtime command runner * feat: expose runtime management commands * test: verify managed python runtime commands * docs: add plan for managed python runtime installer * feat: add managed python command helper * feat: use managed runtime for sl query compute * feat: route sl query managed runtime policy * docs: add plan for managed runtime sl query integration * feat: add managed runtime daemon metadata * feat: manage python daemon lifecycle * feat: add runtime daemon start stop commands * fix: verify managed runtime daemon lifecycle * docs: add plan for managed runtime daemon lifecycle * feat: add managed local embeddings config marker * feat: add managed local embeddings daemon helper * feat: use managed runtime for local embedding setup * feat: pass managed runtime policy through setup * docs: add plan for managed local embeddings runtime * feat: read CLI package metadata dynamically * feat: assemble public kaelio ktx npm package * feat: release one public kaelio ktx npm artifact * test: cover public kaelio ktx package invocations * chore: verify public kaelio ktx package artifacts * docs: add plan for public kaelio ktx npm package * test: verify managed runtime in public package smoke * test: finalize managed runtime release smoke * docs: add plan for managed runtime release smoke * test: specify local embeddings release smoke * feat: add local embeddings runtime smoke * chore: register local embeddings smoke * fix: verify local embeddings smoke * fix: restore artifact smoke python env helper * docs: add plan for managed local embeddings release smoke * refactor: share managed runtime install policy parsing * feat: use managed runtime for agent semantic queries * feat: use managed runtime for MCP semantic compute * docs: add plan for managed agent and MCP semantic runtime * feat(cli): add managed daemon HTTP helpers * feat(cli): route local adapters through managed daemon * feat(cli): use managed daemon for ingest helpers * feat(cli): pass managed daemon options to scan * feat(context): pass MCP ingest pull config options * feat(cli): pass managed daemon options to serve ingest * test: verify managed local ingest daemon runtime * docs: add plan for managed local ingest daemon runtime * docs: align managed runtime examples * docs: add plan for managed runtime docs cleanup * test: cover published package runtime smoke commands * test: validate published package smoke outputs * docs: add plan for published package runtime smoke * build: stamp public npm package version * release: add npm public release policy * release: add guarded npm publish script * release: document public npm release handoff * docs: add plan for public npm release handoff * test: cover managed runtime prune in package smoke * docs: document managed runtime prune * docs: add plan for managed runtime prune smoke and docs * chore: encode uv runtime prerequisite policy * fix: clarify missing uv runtime error * docs: document uv runtime prerequisite * docs: add plan for uv runtime prerequisite contract * refactor: limit release artifacts to public package runtime * chore: align release policy with bundled runtime wheel * docs: describe single public runtime artifact surface * test: verify single public runtime artifact contract * docs: add plan for single public runtime artifact cleanup * fix: align local embeddings smoke with public version * docs: add plan for local embeddings smoke public version * release: soft-launch as @kaelio/ktx@0.1.0-rc.0 on next tag Publish target moves to the pre-release version 0.1.0-rc.0 under the next dist-tag so npm install @kaelio/ktx (which resolves to latest) does not pick up the soft-launch build. Users opt in via @kaelio/ktx@next. * Fix release script boundary checks * Remove PostHog from public package bundle --- .github/workflows/release.yml | 69 + .gitignore | 1 + README.md | 120 +- ...2026-05-11-bundled-python-runtime-wheel.md | 1144 ++++++++++ ...5-11-managed-agent-mcp-semantic-runtime.md | 1109 ++++++++++ ...-managed-local-embeddings-release-smoke.md | 856 ++++++++ ...-05-11-managed-local-embeddings-runtime.md | 1122 ++++++++++ ...d-local-embeddings-smoke-public-version.md | 239 +++ ...-11-managed-local-ingest-daemon-runtime.md | 1650 ++++++++++++++ ...aged-python-runtime-command-integration.md | 935 ++++++++ ...managed-python-runtime-daemon-lifecycle.md | 1546 +++++++++++++ ...-05-11-managed-python-runtime-installer.md | 1750 +++++++++++++++ ...11-managed-python-runtime-release-smoke.md | 585 +++++ ...runtime-docs-and-postgres-smoke-cleanup.md | 657 ++++++ ...11-managed-runtime-prune-smoke-and-docs.md | 377 ++++ ...anaged-runtime-uv-prerequisite-contract.md | 647 ++++++ ...026-05-11-public-kaelio-ktx-npm-package.md | 1904 +++++++++++++++++ .../2026-05-11-public-npm-release-handoff.md | 1332 ++++++++++++ ...published-package-managed-runtime-smoke.md | 602 ++++++ ...-single-public-runtime-artifact-cleanup.md | 978 +++++++++ ...05-11-npm-managed-python-runtime-design.md | 234 ++ examples/package-artifacts/README.md | 21 +- examples/postgres-historic/README.md | 17 +- examples/postgres-historic/scripts/smoke.sh | 27 +- package.json | 2 + packages/cli/src/agent-runtime.test.ts | 44 + packages/cli/src/agent-runtime.ts | 36 +- packages/cli/src/agent.test.ts | 35 + packages/cli/src/agent.ts | 37 +- packages/cli/src/cli-program.ts | 7 + packages/cli/src/cli-runtime.ts | 29 +- packages/cli/src/command-schemas.ts | 2 + packages/cli/src/commands/agent-commands.ts | 14 +- packages/cli/src/commands/ingest-commands.ts | 4 + packages/cli/src/commands/runtime-commands.ts | 100 + packages/cli/src/commands/scan-commands.ts | 5 + packages/cli/src/commands/serve-commands.ts | 5 + packages/cli/src/commands/setup-commands.ts | 1 + packages/cli/src/commands/sl-commands.ts | 5 + packages/cli/src/dev.test.ts | 6 + packages/cli/src/index.test.ts | 414 +++- packages/cli/src/index.ts | 21 + packages/cli/src/ingest.test.ts | 55 +- packages/cli/src/ingest.ts | 20 + packages/cli/src/local-adapters.ts | 65 +- .../cli/src/managed-local-embeddings.test.ts | 180 ++ packages/cli/src/managed-local-embeddings.ts | 95 + .../cli/src/managed-python-command.test.ts | 224 ++ packages/cli/src/managed-python-command.ts | 135 ++ .../cli/src/managed-python-daemon.test.ts | 239 +++ packages/cli/src/managed-python-daemon.ts | 397 ++++ packages/cli/src/managed-python-http.test.ts | 171 ++ packages/cli/src/managed-python-http.ts | 194 ++ .../cli/src/managed-python-runtime.test.ts | 479 +++++ packages/cli/src/managed-python-runtime.ts | 444 ++++ packages/cli/src/runtime.test.ts | 315 +++ packages/cli/src/runtime.ts | 187 ++ packages/cli/src/scan.test.ts | 43 + packages/cli/src/scan.ts | 18 +- packages/cli/src/serve.test.ts | 120 ++ packages/cli/src/serve.ts | 76 +- packages/cli/src/setup-embeddings.test.ts | 134 +- packages/cli/src/setup-embeddings.ts | 73 +- packages/cli/src/setup.test.ts | 70 +- packages/cli/src/setup.ts | 10 + packages/cli/src/sl.test.ts | 67 + packages/cli/src/sl.ts | 21 +- packages/context/src/ingest/local-ingest.ts | 11 +- packages/context/src/llm/index.ts | 2 + packages/context/src/llm/local-config.test.ts | 41 + packages/context/src/llm/local-config.ts | 32 + .../src/mcp/local-project-ports.test.ts | 59 + .../context/src/mcp/local-project-ports.ts | 5 +- packages/context/src/package-exports.test.ts | 4 + python/ktx-daemon/pyproject.toml | 8 +- python/ktx-daemon/src/ktx_daemon/app.py | 7 +- python/ktx-daemon/tests/test_app.py | 10 + release-policy.json | 38 +- scripts/build-public-npm-package.mjs | 263 +++ scripts/build-public-npm-package.test.mjs | 275 +++ scripts/build-python-runtime-wheel.mjs | 144 ++ scripts/build-python-runtime-wheel.test.mjs | 115 + scripts/check-boundaries.mjs | 14 +- scripts/check-boundaries.test.mjs | 9 + scripts/examples-docs.test.mjs | 84 + scripts/installed-live-database-smoke.mjs | 144 +- scripts/local-embeddings-runtime-smoke.mjs | 397 ++++ .../local-embeddings-runtime-smoke.test.mjs | 172 ++ scripts/package-artifacts.mjs | 968 +++------ scripts/package-artifacts.test.mjs | 486 ++--- scripts/publish-public-npm-package.mjs | 87 + scripts/publish-public-npm-package.test.mjs | 109 + scripts/published-package-smoke-config.mjs | 102 +- scripts/published-package-smoke.mjs | 102 +- scripts/published-package-smoke.test.mjs | 239 ++- scripts/release-readiness.mjs | 135 +- scripts/release-readiness.test.mjs | 296 ++- scripts/release-workflow.test.mjs | 21 + uv.lock | 13 +- 99 files changed, 25375 insertions(+), 1538 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-local-embeddings-smoke-public-version.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md create mode 100644 docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md create mode 100644 docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md create mode 100644 docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md create mode 100644 docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md create mode 100644 docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md create mode 100644 docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md create mode 100644 packages/cli/src/commands/runtime-commands.ts create mode 100644 packages/cli/src/managed-local-embeddings.test.ts create mode 100644 packages/cli/src/managed-local-embeddings.ts create mode 100644 packages/cli/src/managed-python-command.test.ts create mode 100644 packages/cli/src/managed-python-command.ts create mode 100644 packages/cli/src/managed-python-daemon.test.ts create mode 100644 packages/cli/src/managed-python-daemon.ts create mode 100644 packages/cli/src/managed-python-http.test.ts create mode 100644 packages/cli/src/managed-python-http.ts create mode 100644 packages/cli/src/managed-python-runtime.test.ts create mode 100644 packages/cli/src/managed-python-runtime.ts create mode 100644 packages/cli/src/runtime.test.ts create mode 100644 packages/cli/src/runtime.ts create mode 100644 scripts/build-public-npm-package.mjs create mode 100644 scripts/build-public-npm-package.test.mjs create mode 100644 scripts/build-python-runtime-wheel.mjs create mode 100644 scripts/build-python-runtime-wheel.test.mjs create mode 100644 scripts/local-embeddings-runtime-smoke.mjs create mode 100644 scripts/local-embeddings-runtime-smoke.test.mjs create mode 100644 scripts/publish-public-npm-package.mjs create mode 100644 scripts/publish-public-npm-package.test.mjs create mode 100644 scripts/release-workflow.test.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..16c9f1e2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,69 @@ +name: KTX Release + +on: + workflow_dispatch: + inputs: + publish_live: + description: "Publish @kaelio/ktx to npm instead of running a dry-run" + required: true + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: ktx-release-${{ github.ref }} + cancel-in-progress: false + +jobs: + npm-public-release: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + with: + node-version: "24" + cache: "pnpm" + cache-dependency-path: "pnpm-lock.yaml" + + - name: Install TypeScript dependencies + run: pnpm install --frozen-lockfile + + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Setup uv + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install Python dependencies + run: uv sync --all-packages + + - name: Build and verify artifacts + run: pnpm run artifacts:check + + - name: Check release readiness + run: pnpm run release:readiness + + - name: Dry-run npm publish + if: ${{ !inputs.publish_live }} + run: pnpm run release:npm-publish + + - name: Publish npm package + if: ${{ inputs.publish_live }} + run: pnpm run release:npm-publish -- --publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.gitignore b/.gitignore index 3b262d15..b2d82b54 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ venv/ env/ build/ dist/ +packages/cli/assets/python/ *.egg-info/ .pytest_cache/ .coverage diff --git a/README.md b/README.md index c92371a4..8b4d32a2 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,11 @@ artifacts. You can inspect them, commit them, and serve them to any MCP client. ## Quick start -Run the pre-seeded demo from the repository root: +Run the pre-seeded demo through the public npm package: ```bash -pnpm install -pnpm run setup:dev -pnpm run ktx -- setup demo --no-input -pnpm run ktx -- setup demo inspect +npx @kaelio/ktx setup demo --no-input +npx @kaelio/ktx setup demo inspect ``` The default demo uses packaged sample data and prebuilt context. It does not @@ -35,7 +33,7 @@ require API keys, network access, or an LLM provider. To replay the packaged ingest run, use: ```bash -pnpm run ktx -- setup demo --mode replay --no-input +npx @kaelio/ktx setup demo --mode replay --no-input ``` To run the full agentic demo with an LLM provider, set a provider key for the @@ -43,22 +41,29 @@ current process: ```bash ANTHROPIC_API_KEY=$YOUR_ANTHROPIC_API_KEY \ - pnpm run ktx -- setup demo --mode full --no-input + npx @kaelio/ktx setup demo --mode full --no-input ``` Interactive full-demo setup can prompt for a provider key without writing the key to `ktx.yaml`. -## Build a local project - -Create a project from the repository root: +You can also install the CLI in a project or globally: ```bash -uv sync --all-packages -source .venv/bin/activate +npm install @kaelio/ktx +npx ktx --help +npm install -g @kaelio/ktx +ktx --help +``` +## Build a local project + +Create a project from a local workspace: + +```bash +npm install @kaelio/ktx PROJECT_DIR="$(mktemp -d)/ktx-demo" -pnpm run ktx -- init "$PROJECT_DIR" --name ktx-demo +npx ktx init "$PROJECT_DIR" --name ktx-demo ``` Create a SQLite warehouse: @@ -112,7 +117,7 @@ YAML Write and validate a semantic-layer source: ```bash -pnpm run ktx -- sl write accounts --project-dir "$PROJECT_DIR" \ +npx ktx sl write accounts --project-dir "$PROJECT_DIR" \ --connection-id warehouse --yaml 'name: accounts table: accounts description: CRM accounts with segmentation attributes. @@ -133,14 +138,14 @@ measures: joins: [] ' -pnpm run ktx -- sl validate accounts --project-dir "$PROJECT_DIR" \ +npx ktx sl validate accounts --project-dir "$PROJECT_DIR" \ --connection-id warehouse ``` Generate SQL and execute the query: ```bash -pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \ +npx ktx sl query --project-dir "$PROJECT_DIR" \ --connection-id warehouse \ --measure accounts.account_count \ --dimension accounts.segment \ @@ -148,7 +153,7 @@ pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \ --limit 5 \ --format sql -pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \ +npx ktx sl query --project-dir "$PROJECT_DIR" \ --connection-id warehouse \ --measure accounts.account_count \ --dimension accounts.segment \ @@ -161,8 +166,8 @@ pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \ List and test the warehouse connection: ```bash -pnpm run ktx -- connection list --project-dir "$PROJECT_DIR" -pnpm run ktx -- connection test warehouse --project-dir "$PROJECT_DIR" +npx ktx connection list --project-dir "$PROJECT_DIR" +npx ktx connection test warehouse --project-dir "$PROJECT_DIR" ``` The connection test prints the configured driver and discovered table count: @@ -179,34 +184,66 @@ Scan artifacts are written under ```bash -SCAN_OUTPUT="$(pnpm run ktx -- scan warehouse --project-dir "$PROJECT_DIR")" +SCAN_OUTPUT="$(npx ktx scan warehouse --project-dir "$PROJECT_DIR")" printf '%s\n' "$SCAN_OUTPUT" SCAN_RUN_ID="$(printf '%s\n' "$SCAN_OUTPUT" | awk '/^Run: / { print $2 }')" -pnpm run ktx -- scan status --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID" -pnpm run ktx -- scan report --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID" +npx ktx scan status --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID" +npx ktx scan report --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID" ``` For non-SQLite drivers, prefer credential references such as `--url env:NAME` or `--url file:PATH` over literal credential URLs. +## Managed Python runtime + +KTX installs its Python runtime only when a Python-backed command needs it. +The runtime lives outside the npm cache, is versioned by the installed CLI +version, and is managed by `ktx runtime` commands. + +KTX requires `uv` on `PATH` to create the managed runtime. Install `uv` with +your system package manager or the official installer before running Python- +backed KTX commands. KTX doesn't download `uv` automatically; run +`ktx runtime doctor` if runtime installation fails: + +```bash +npx ktx runtime install --yes +npx ktx runtime status +npx ktx runtime doctor +npx ktx runtime start +npx ktx runtime stop +npx ktx runtime prune --dry-run +npx ktx runtime prune --yes +``` + +Use `runtime prune --dry-run` to preview stale runtime directories from older +CLI versions. Add `--yes` to remove those stale directories after daemon +processes are stopped. + +Commands such as `npx @kaelio/ktx sl query ... --yes` can install the core +runtime lazily from the bundled wheel. Local embeddings remain lazy; prepare +them only when you select local `sentence-transformers` embeddings: + +```bash +npx ktx runtime install --feature local-embeddings --yes +npx ktx runtime start --feature local-embeddings +``` + ## Serve MCP -Start the Python compute daemon in one terminal: +Start the stdio MCP server from the project directory: ```bash -source .venv/bin/activate -uv run ktx-daemon serve-http --host 127.0.0.1 --port 8765 -``` - -Start the stdio MCP server in another terminal: - -```bash -pnpm run ktx -- serve --mcp stdio --project-dir "$PROJECT_DIR" \ +npx ktx serve --mcp stdio --project-dir "$PROJECT_DIR" \ --user-id local \ - --semantic-compute-url http://127.0.0.1:8765 \ - --execute-queries + --semantic-compute \ + --execute-queries \ + --yes ``` +The `--semantic-compute` flag uses the managed Python runtime when no explicit +semantic compute URL is provided. KTX starts or reuses the managed runtime as +needed. + The MCP server exposes `connection_list`, `knowledge_search`, `knowledge_read`, `knowledge_write`, `sl_list_sources`, `sl_read_source`, `sl_write_source`, `sl_validate`, `sl_query`, `ingest_trigger`, @@ -251,19 +288,26 @@ packages. ## Release status -This repository is prepared for source publication. Package publishing is still -disabled by `release-policy.json`; registry names, public versions, package -visibility, and provenance policy must be chosen before publishing artifacts to -npm or Python package indexes. +This repository builds one public npm artifact named `@kaelio/ktx`. The release +artifact manifest contains the public npm tarball and the bundled `kaelio-ktx` +runtime wheel. The first public npm handoff is policy-gated through +`release-policy.json`, which keeps Python package publishing disabled because +KTX-owned Python code ships inside the npm package as a bundled wheel. The +`python/ktx-sl` and `python/ktx-daemon` directories remain source packages for +development, not public release artifacts. -Build local package artifacts with: +Build local package artifacts and verify the guarded dry-run publish path with: ```bash source .venv/bin/activate pnpm run artifacts:check pnpm run release:readiness +pnpm run release:npm-publish ``` +Run the live npm publish only from the manual `KTX Release` workflow with the +`publish_live` input enabled after the `NPM_TOKEN` secret is configured. + ## License KTX is licensed under the Apache License, Version 2.0. See `LICENSE`. diff --git a/docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md b/docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md new file mode 100644 index 00000000..5a523605 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md @@ -0,0 +1,1144 @@ +# Bundled Python Runtime Wheel Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build and package one bundled `kaelio-ktx` Python wheel that contains +KTX-owned Python runtime code and keeps local embedding dependencies optional. + +**Architecture:** Add a deterministic Node assembly script that copies the +existing `semantic_layer` and `ktx_daemon` source trees into a temporary wheel +source tree, writes a runtime-only `pyproject.toml`, and builds one wheel with +`uv build`. Wire package artifacts so the CLI npm tarball includes the bundled +wheel plus a checksum manifest under `assets/python/`. + +**Tech Stack:** Node 22 ESM scripts, `node:test`, `uv`, Hatchling, Python 3.13, +pnpm, TypeScript package artifacts. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. +There are no committed plan files under `docs/superpowers/plans/` in this +worktree or in git history for this spec. The spec itself is the only tracked +Superpowers document. + +The following pieces are already implemented: + +- `packages/context/src/daemon/semantic-layer-compute.ts` can invoke + `python -m ktx_daemon` for one-shot semantic-layer operations. +- `python/ktx-daemon` exposes `ktx-daemon` one-shot commands and an HTTP + `serve-http` daemon with `/health`. +- `scripts/package-artifacts.mjs` builds npm package tarballs and separate + `ktx-sl` and `ktx-daemon` Python artifacts. +- `scripts/package-artifacts.mjs` writes a checksummed artifact manifest. + +The following spec requirements are not implemented yet: + +- A single public `@kaelio/ktx` npm surface. +- One KTX-owned bundled Python wheel inside the npm package. +- A managed runtime root, installer, runtime manifest, and runtime command + family. +- Lazy `local-embeddings` installation that keeps `sentence-transformers` and + `torch` out of the default Python dependency set. + +This plan implements the bundled wheel prerequisite. Runtime install commands +must be planned after this lands because they need a real wheel payload and +checksum manifest to install. + +## File structure + +- Create `scripts/build-python-runtime-wheel.mjs`: assembles the temporary + runtime wheel source tree and runs `uv build`. +- Create `scripts/build-python-runtime-wheel.test.mjs`: tests source copying, + generated `pyproject.toml`, and the `uv build` command shape. +- Modify `scripts/package-artifacts.mjs`: builds the runtime wheel before npm + packing, copies it into `packages/cli/assets/python/`, includes it in the + artifact manifest, and installs it in artifact smoke tests. +- Modify `scripts/package-artifacts.test.mjs`: covers runtime wheel metadata, + manifest entries, install arguments, and CLI asset copy behavior. +- Modify `scripts/release-readiness.test.mjs`: expects `kaelio-ktx` in Python + release metadata and policy fixtures. +- Modify `release-policy.json`: lists `kaelio-ktx` as a CI-only Python + artifact. +- Modify `python/ktx-daemon/pyproject.toml`: moves + `sentence-transformers` and `torch` to a `local-embeddings` optional + dependency group. +- Modify `uv.lock`: records the dependency metadata change. +- Modify `.gitignore`: ignores generated `packages/cli/assets/python/` + contents. + +## Plan status + +No earlier plans were found for this spec. This is plan 1 for the spec. + +### Task 1: Add failing tests for the runtime wheel builder + +**Files:** + +- Create: `scripts/build-python-runtime-wheel.test.mjs` +- Test: `scripts/build-python-runtime-wheel.test.mjs` + +- [ ] **Step 1: Write the failing test file** + +Create `scripts/build-python-runtime-wheel.test.mjs` with this content: + +```javascript +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, + createRuntimeWheelBuildTree, + runtimeWheelBuildCommand, + runtimeWheelLayout, + runtimeWheelPyproject, +} from './build-python-runtime-wheel.mjs'; + +async function writeRuntimeSourceFixture(root) { + await mkdir(join(root, 'python', 'ktx-sl', 'semantic_layer'), { + recursive: true, + }); + await mkdir(join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon'), { + recursive: true, + }); + + await writeFile( + join(root, 'python', 'ktx-sl', 'semantic_layer', '__init__.py'), + 'SEMANTIC_LAYER_FIXTURE = True\n', + ); + await writeFile( + join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon', '__init__.py'), + 'KTX_DAEMON_FIXTURE = True\n', + ); + await writeFile( + join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon', '__main__.py'), + 'def main():\n return 0\n', + ); +} + +describe('runtimeWheelLayout', () => { + it('uses stable source, build, and output paths', () => { + const layout = runtimeWheelLayout('/repo/ktx'); + + assert.equal(layout.rootDir, '/repo/ktx'); + assert.equal(layout.semanticLayerSourceDir, '/repo/ktx/python/ktx-sl/semantic_layer'); + assert.equal(layout.daemonSourceDir, '/repo/ktx/python/ktx-daemon/src/ktx_daemon'); + assert.equal(layout.buildRoot, '/repo/ktx/dist/runtime-wheel-src'); + assert.equal(layout.outputDir, '/repo/ktx/dist/artifacts/python'); + }); +}); + +describe('runtimeWheelPyproject', () => { + it('describes one kaelio-ktx wheel with lazy local embeddings', () => { + const pyproject = runtimeWheelPyproject(); + + assert.match(pyproject, /name = "kaelio-ktx"/); + assert.match(pyproject, /version = "0\.1\.0"/); + assert.match(pyproject, /ktx-daemon = "ktx_daemon\.__main__:main"/); + assert.match(pyproject, /packages = \["semantic_layer", "ktx_daemon"\]/); + assert.match(pyproject, /\[project\.optional-dependencies\]/); + assert.match(pyproject, /local-embeddings = \[/); + assert.match(pyproject, /"sentence-transformers>=5\.1\.1"/); + assert.match(pyproject, /"torch>=2\.2\.0"/); + assert.doesNotMatch( + pyproject.match(/dependencies = \[[\s\S]*?\]/)?.[0] ?? '', + /sentence-transformers|torch/, + ); + }); +}); + +describe('createRuntimeWheelBuildTree', () => { + it('copies KTX-owned Python packages into the build tree', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-wheel-test-')); + try { + await writeRuntimeSourceFixture(root); + const layout = runtimeWheelLayout(root); + + await createRuntimeWheelBuildTree(layout); + + assert.equal( + await readFile(join(layout.buildRoot, 'semantic_layer', '__init__.py'), 'utf8'), + 'SEMANTIC_LAYER_FIXTURE = True\n', + ); + assert.equal( + await readFile(join(layout.buildRoot, 'ktx_daemon', '__main__.py'), 'utf8'), + 'def main():\n return 0\n', + ); + const pyproject = await readFile(join(layout.buildRoot, 'pyproject.toml'), 'utf8'); + assert.match(pyproject, /name = "kaelio-ktx"/); + assert.match(pyproject, /local-embeddings = \[/); + const readme = await readFile(join(layout.buildRoot, 'README.md'), 'utf8'); + assert.match(readme, /Bundled Python runtime wheel for KTX/); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); + +describe('runtimeWheelBuildCommand', () => { + it('runs uv build against the generated build tree', () => { + const layout = runtimeWheelLayout('/repo/ktx'); + + assert.deepEqual(runtimeWheelBuildCommand(layout), { + command: 'uv', + args: [ + 'build', + '--wheel', + '--out-dir', + '/repo/ktx/dist/artifacts/python', + '/repo/ktx/dist/runtime-wheel-src', + ], + cwd: '/repo/ktx', + }); + assert.equal(RUNTIME_WHEEL_DISTRIBUTION_NAME, 'kaelio-ktx'); + assert.equal(RUNTIME_WHEEL_PACKAGE_VERSION, '0.1.0'); + }); +}); +``` + +- [ ] **Step 2: Run the failing test** + +Run: + +```bash +node --test scripts/build-python-runtime-wheel.test.mjs +``` + +Expected: FAIL with an import error for +`./build-python-runtime-wheel.mjs`. + +### Task 2: Implement the runtime wheel builder + +**Files:** + +- Create: `scripts/build-python-runtime-wheel.mjs` +- Test: `scripts/build-python-runtime-wheel.test.mjs` + +- [ ] **Step 1: Create the builder script** + +Create `scripts/build-python-runtime-wheel.mjs` with this content: + +```javascript +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +export const RUNTIME_WHEEL_DISTRIBUTION_NAME = 'kaelio-ktx'; +export const RUNTIME_WHEEL_NORMALIZED_NAME = 'kaelio_ktx'; +export const RUNTIME_WHEEL_PACKAGE_VERSION = '0.1.0'; + +function scriptRootDir() { + return resolve(dirname(fileURLToPath(import.meta.url)), '..'); +} + +export function runtimeWheelLayout(rootDir = scriptRootDir()) { + return { + rootDir, + semanticLayerSourceDir: join(rootDir, 'python', 'ktx-sl', 'semantic_layer'), + daemonSourceDir: join(rootDir, 'python', 'ktx-daemon', 'src', 'ktx_daemon'), + buildRoot: join(rootDir, 'dist', 'runtime-wheel-src'), + outputDir: join(rootDir, 'dist', 'artifacts', 'python'), + }; +} + +export function runtimeWheelPyproject() { + return `[project] +name = "${RUNTIME_WHEEL_DISTRIBUTION_NAME}" +version = "${RUNTIME_WHEEL_PACKAGE_VERSION}" +description = "Bundled Python runtime payload for the KTX npm package" +readme = "README.md" +requires-python = ">=3.13" +license = "Apache-2.0" +dependencies = [ + "fastapi>=0.115.0", + "lkml>=1.3.7", + "numpy>=2.2.6", + "orjson>=3.11.4", + "pandas>=2.2.3", + "psycopg[binary]>=3.2.0", + "pydantic>=2.9.0", + "pyyaml>=6", + "requests>=2.32.0", + "sqlglot>=26", + "uvicorn[standard]>=0.32.0", +] + +[project.optional-dependencies] +local-embeddings = [ + "sentence-transformers>=5.1.1", + "torch>=2.2.0", +] + +[project.scripts] +ktx-daemon = "ktx_daemon.__main__:main" + +[project.urls] +Homepage = "https://github.com/kaelio/ktx" +Repository = "https://github.com/kaelio/ktx" +Issues = "https://github.com/kaelio/ktx/issues" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["semantic_layer", "ktx_daemon"] +`; +} + +export function runtimeWheelReadme() { + return `# kaelio-ktx Python runtime + +Bundled Python runtime wheel for KTX. + +This wheel is built from the repository's \`semantic_layer\` and +\`ktx_daemon\` source trees for inclusion in the npm package. It is not a +separate public PyPI release artifact. +`; +} + +export async function createRuntimeWheelBuildTree(layout = runtimeWheelLayout()) { + await rm(layout.buildRoot, { recursive: true, force: true }); + await mkdir(layout.buildRoot, { recursive: true }); + await cp(layout.semanticLayerSourceDir, join(layout.buildRoot, 'semantic_layer'), { + recursive: true, + }); + await cp(layout.daemonSourceDir, join(layout.buildRoot, 'ktx_daemon'), { + recursive: true, + }); + await writeFile(join(layout.buildRoot, 'pyproject.toml'), runtimeWheelPyproject()); + await writeFile(join(layout.buildRoot, 'README.md'), runtimeWheelReadme()); +} + +export function runtimeWheelBuildCommand(layout = runtimeWheelLayout()) { + return { + command: 'uv', + args: ['build', '--wheel', '--out-dir', layout.outputDir, layout.buildRoot], + cwd: layout.rootDir, + }; +} + +async function runCommand(command, args, options) { + const result = await execFileAsync(command, args, { + cwd: options.cwd, + encoding: 'utf8', + maxBuffer: 1024 * 1024 * 20, + }); + if (result.stdout) { + process.stdout.write(result.stdout); + } + if (result.stderr) { + process.stderr.write(result.stderr); + } +} + +export async function buildRuntimeWheel(layout = runtimeWheelLayout()) { + await mkdir(layout.outputDir, { recursive: true }); + await createRuntimeWheelBuildTree(layout); + const command = runtimeWheelBuildCommand(layout); + await runCommand(command.command, command.args, { cwd: command.cwd }); + const pyproject = await readFile(join(layout.buildRoot, 'pyproject.toml'), 'utf8'); + return { + buildRoot: layout.buildRoot, + outputDir: layout.outputDir, + pyproject, + }; +} + +async function main() { + await buildRuntimeWheel(runtimeWheelLayout()); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + try { + await main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} +``` + +- [ ] **Step 2: Run the builder test** + +Run: + +```bash +node --test scripts/build-python-runtime-wheel.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 3: Commit the builder** + +Run: + +```bash +git add scripts/build-python-runtime-wheel.mjs scripts/build-python-runtime-wheel.test.mjs +git commit -m "build: add bundled python runtime wheel builder" +``` + +### Task 3: Move heavy local embedding dependencies behind an extra + +**Files:** + +- Modify: `python/ktx-daemon/pyproject.toml` +- Modify: `uv.lock` +- Test: `python/ktx-daemon/tests/test_embeddings.py` +- Test: `scripts/build-python-runtime-wheel.test.mjs` + +- [ ] **Step 1: Update daemon dependencies** + +In `python/ktx-daemon/pyproject.toml`, remove these two lines from +`[project].dependencies`: + +```toml + "sentence-transformers>=5.1.1", + "torch>=2.2.0", +``` + +Add this block immediately after `[project.scripts]`: + +```toml +[project.optional-dependencies] +local-embeddings = [ + "sentence-transformers>=5.1.1", + "torch>=2.2.0", +] +``` + +The relevant section must read: + +```toml +[project] +name = "ktx-daemon" +version = "0.1.0" +description = "Portable compute package for KTX semantic-layer operations" +readme = "README.md" +requires-python = ">=3.13" +license = "Apache-2.0" +dependencies = [ + "fastapi>=0.115.0", + "ktx-sl", + "lkml>=1.3.7", + "numpy>=2.2.6", + "orjson>=3.11.4", + "pandas>=2.2.3", + "psycopg[binary]>=3.2.0", + "pydantic>=2.9.0", + "requests>=2.32.0", + "sqlglot>=26", + "uvicorn[standard]>=0.32.0", +] + +[project.scripts] +ktx-daemon = "ktx_daemon.__main__:main" + +[project.optional-dependencies] +local-embeddings = [ + "sentence-transformers>=5.1.1", + "torch>=2.2.0", +] +``` + +- [ ] **Step 2: Refresh the uv lockfile** + +Run: + +```bash +uv lock +``` + +Expected: PASS and `uv.lock` records the `ktx-daemon` optional dependency +metadata. If the local `uv` version is older than `tool.uv.required-version`, +record the version mismatch and do not edit `pyproject.toml` to lower the pin. + +- [ ] **Step 3: Run Python tests that cover lazy embedding imports** + +Run: + +```bash +uv run pytest python/ktx-daemon/tests/test_embeddings.py -q +``` + +Expected: PASS. The tests use injected fake providers and do not require +`sentence-transformers` or `torch`. + +- [ ] **Step 4: Run the runtime wheel metadata test** + +Run: + +```bash +node --test scripts/build-python-runtime-wheel.test.mjs +``` + +Expected: PASS and the generated runtime `pyproject.toml` keeps +`sentence-transformers` and `torch` under `local-embeddings`. + +- [ ] **Step 5: Commit the dependency split** + +Run: + +```bash +git add python/ktx-daemon/pyproject.toml uv.lock +git commit -m "build: make local embedding dependencies optional" +``` + +### Task 4: Add artifact tests for the bundled runtime wheel + +**Files:** + +- Modify: `scripts/package-artifacts.test.mjs` +- Test: `scripts/package-artifacts.test.mjs` + +- [ ] **Step 1: Extend imports** + +In `scripts/package-artifacts.test.mjs`, extend the import from +`./package-artifacts.mjs` with these names: + +```javascript + CLI_PYTHON_ASSET_MANIFEST, + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_NORMALIZED_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, + copyRuntimeWheelAssets, +``` + +- [ ] **Step 2: Update Python metadata fixtures** + +In `writeReleaseMetadataInputs`, keep the existing `ktx-sl` and `ktx-daemon` +fixture files and add no new on-disk Python package. The runtime wheel metadata +will come from constants exported by `package-artifacts.mjs`. + +- [ ] **Step 3: Update uploadable artifact fixtures** + +In `writeUploadableArtifactFixtures`, add this runtime wheel entry to +`fileContents`: + +```javascript + [ + join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + 'kaelio-ktx-runtime-wheel', + ], +``` + +- [ ] **Step 4: Update build command expectations** + +Replace the `buildArtifactCommands` expectations with these three assertions: + +```javascript + assert.deepEqual( + commands.slice(0, NPM_ARTIFACT_PACKAGES.length).map((command) => [command.command, command.args]), + NPM_ARTIFACT_PACKAGES.map((packageInfo) => ['pnpm', ['--filter', packageInfo.name, 'run', 'build']]), + ); + assert.deepEqual( + commands + .slice(NPM_ARTIFACT_PACKAGES.length, NPM_ARTIFACT_PACKAGES.length + 3) + .map((command) => [command.command, command.args]), + [ + [ + process.execPath, + ['scripts/build-python-runtime-wheel.mjs'], + ], + [ + 'uv', + ['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python'], + ], + [ + 'uv', + ['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python'], + ], + ], + ); + assert.deepEqual( + commands.slice(NPM_ARTIFACT_PACKAGES.length + 3).map((command) => [command.command, command.args]), + NPM_ARTIFACT_PACKAGES.map((packageInfo) => [ + 'pnpm', + ['--filter', packageInfo.name, 'pack', '--out', layout.npmTarballs[packageInfo.name]], + ]), + ); +``` + +- [ ] **Step 5: Update release metadata expectations** + +In the `packageReleaseMetadata` test, add this Python metadata entry after +`ktx-daemon`: + +```javascript + { + ecosystem: 'python', + packageName: 'kaelio-ktx', + packageRoot: 'python/runtime-wheel', + packageVersion: '0.1.0', + private: false, + releaseMode: 'ci-artifact-only', + }, +``` + +- [ ] **Step 6: Update Python artifact discovery expectations** + +In the `findPythonArtifacts` test, create the runtime wheel fixture: + +```javascript + await writeFile(join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), ''); +``` + +Then update the expected object: + +```javascript + assert.deepEqual(await findPythonArtifacts(root), { + runtimeWheel: join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + ktxSlWheel: join(root, 'ktx_sl-0.1.0-py3-none-any.whl'), + ktxSlSdist: join(root, 'ktx_sl-0.1.0.tar.gz'), + ktxDaemonWheel: join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'), + ktxDaemonSdist: join(root, 'ktx_daemon-0.1.0.tar.gz'), + }); +``` + +- [ ] **Step 7: Update manifest file count expectations** + +In the `verifyArtifactManifest` test, replace: + +```javascript + assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 4); +``` + +with: + +```javascript + assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 5); +``` + +- [ ] **Step 8: Add CLI asset copy test** + +Add this test near the other artifact helper tests: + +```javascript +describe('copyRuntimeWheelAssets', () => { + it('copies the runtime wheel and checksum manifest into CLI assets', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-assets-test-')); + const layout = packageArtifactLayout(root); + try { + await mkdir(layout.pythonDir, { recursive: true }); + await writeFile( + join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + 'kaelio-ktx-runtime-wheel', + ); + + const assets = await copyRuntimeWheelAssets(layout, { + runtimeWheel: join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + }); + + assert.equal( + assets.wheelPath, + join(root, 'packages', 'cli', 'assets', 'python', 'kaelio_ktx-0.1.0-py3-none-any.whl'), + ); + assert.equal( + assets.manifestPath, + join(root, 'packages', 'cli', 'assets', 'python', CLI_PYTHON_ASSET_MANIFEST), + ); + const manifest = JSON.parse(await readFile(assets.manifestPath, 'utf8')); + assert.deepEqual(manifest, { + schemaVersion: 1, + distributionName: RUNTIME_WHEEL_DISTRIBUTION_NAME, + normalizedName: RUNTIME_WHEEL_NORMALIZED_NAME, + version: RUNTIME_WHEEL_PACKAGE_VERSION, + wheel: { + file: 'kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: createHash('sha256') + .update('kaelio-ktx-runtime-wheel') + .digest('hex'), + bytes: Buffer.byteLength('kaelio-ktx-runtime-wheel'), + }, + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); +``` + +- [ ] **Step 9: Update install argument test** + +Replace the `pythonArtifactInstallArgs` expectation with one runtime wheel: + +```javascript + assert.deepEqual(args, [ + 'pip', + 'install', + '--python', + '/tmp/smoke/.venv/bin/python', + '/repo/ktx/dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl', + ]); + assert.equal(args.includes('ktx-daemon'), false); + assert.equal(args.includes('ktx-sl'), false); + assert.equal(args.includes('--find-links'), false); +``` + +- [ ] **Step 10: Run the failing package artifact tests** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs +``` + +Expected: FAIL with missing exports from `scripts/package-artifacts.mjs`. + +### Task 5: Wire the runtime wheel into artifact packaging + +**Files:** + +- Modify: `scripts/package-artifacts.mjs` +- Modify: `scripts/package-artifacts.test.mjs` +- Test: `scripts/package-artifacts.test.mjs` + +- [ ] **Step 1: Import runtime wheel builder constants** + +Add this import near the top of `scripts/package-artifacts.mjs`: + +```javascript +import { + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_NORMALIZED_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, +} from './build-python-runtime-wheel.mjs'; +``` + +Then re-export those constants after the existing constants: + +```javascript +export { + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_NORMALIZED_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, +}; +``` + +- [ ] **Step 2: Add CLI asset manifest constant** + +Add this constant after `PYTHON_PACKAGE_VERSION`: + +```javascript +export const CLI_PYTHON_ASSET_MANIFEST = 'manifest.json'; +``` + +- [ ] **Step 3: Change build command order** + +Replace `buildArtifactCommands(layout)` with this implementation: + +```javascript +export function buildArtifactCommands(layout) { + const npmBuildCommands = NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({ + command: 'pnpm', + args: ['--filter', packageInfo.name, 'run', 'build'], + cwd: layout.rootDir, + })); + const npmPackCommands = NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({ + command: 'pnpm', + args: ['--filter', packageInfo.name, 'pack', '--out', layout.npmTarballs[packageInfo.name]], + cwd: layout.rootDir, + })); + + return [ + ...npmBuildCommands, + { + command: process.execPath, + args: ['scripts/build-python-runtime-wheel.mjs'], + cwd: layout.rootDir, + }, + { + command: 'uv', + args: ['build', '--package', 'ktx-sl', '--out-dir', layout.pythonDir], + cwd: layout.rootDir, + }, + { + command: 'uv', + args: ['build', '--package', 'ktx-daemon', '--out-dir', layout.pythonDir], + cwd: layout.rootDir, + }, + ...npmPackCommands, + ]; +} +``` + +- [ ] **Step 4: Discover the runtime wheel** + +Update `findPythonArtifacts(pythonDir)` to return `runtimeWheel`: + +```javascript +export async function findPythonArtifacts(pythonDir) { + const files = await readdir(pythonDir); + + return { + runtimeWheel: findOne( + files, + RUNTIME_WHEEL_DISTRIBUTION_NAME, + '.whl', + 'kaelio-ktx runtime wheel', + pythonDir, + RUNTIME_WHEEL_PACKAGE_VERSION, + ), + ktxSlWheel: findOne(files, 'ktx-sl', '.whl', 'ktx-sl wheel', pythonDir), + ktxSlSdist: findOne(files, 'ktx-sl', '.tar.gz', 'ktx-sl source distribution', pythonDir), + ktxDaemonWheel: findOne(files, 'ktx-daemon', '.whl', 'ktx-daemon wheel', pythonDir), + ktxDaemonSdist: findOne(files, 'ktx-daemon', '.tar.gz', 'ktx-daemon source distribution', pythonDir), + }; +} +``` + +Change `findOne` to accept an optional version: + +```javascript +function findOne(files, distributionName, suffix, label, pythonDir, version = PYTHON_PACKAGE_VERSION) { + const normalized = normalizePythonDistributionName(distributionName); + const found = files.find((file) => file.startsWith(`${normalized}-${version}`) && file.endsWith(suffix)); + if (!found) { + throw new Error(`Missing Python artifact: ${label}`); + } + return join(pythonDir, found); +} +``` + +- [ ] **Step 5: Add runtime wheel release metadata** + +In `packageReleaseMetadata`, append this entry after `ktxDaemonPackage`: + +```javascript + releaseMetadataEntry({ + ecosystem: 'python', + packageName: RUNTIME_WHEEL_DISTRIBUTION_NAME, + packageRoot: 'python/runtime-wheel', + packageVersion: RUNTIME_WHEEL_PACKAGE_VERSION, + privatePackage: false, + }), +``` + +- [ ] **Step 6: Add runtime wheel to artifact manifest records** + +In `artifactPackageRecords`, add this record after npm records: + +```javascript + { + artifactKind: 'wheel', + artifactPath: pythonArtifacts.runtimeWheel, + metadata: requirePackageMetadata(packagesByName, RUNTIME_WHEEL_DISTRIBUTION_NAME), + }, +``` + +- [ ] **Step 7: Add CLI Python asset copy helper** + +Add this function before `pythonArtifactInstallArgs`: + +```javascript +function runtimeWheelAssetName(runtimeWheelPath) { + return runtimeWheelPath.split(sep).at(-1); +} + +export async function copyRuntimeWheelAssets(layout, pythonArtifacts) { + const assetDir = join(layout.rootDir, 'packages', 'cli', 'assets', 'python'); + const wheelFile = runtimeWheelAssetName(pythonArtifacts.runtimeWheel); + if (!wheelFile) { + throw new Error(`Unable to determine runtime wheel filename: ${pythonArtifacts.runtimeWheel}`); + } + const wheelContents = await readFile(pythonArtifacts.runtimeWheel); + await rm(assetDir, { recursive: true, force: true }); + await mkdir(assetDir, { recursive: true }); + const wheelPath = join(assetDir, wheelFile); + const manifestPath = join(assetDir, CLI_PYTHON_ASSET_MANIFEST); + await writeFile(wheelPath, wheelContents); + await writeFile( + manifestPath, + `${JSON.stringify( + { + schemaVersion: 1, + distributionName: RUNTIME_WHEEL_DISTRIBUTION_NAME, + normalizedName: RUNTIME_WHEEL_NORMALIZED_NAME, + version: RUNTIME_WHEEL_PACKAGE_VERSION, + wheel: { + file: wheelFile, + sha256: createHash('sha256').update(wheelContents).digest('hex'), + bytes: wheelContents.byteLength, + }, + }, + null, + 2, + )}\n`, + ); + return { assetDir, wheelPath, manifestPath }; +} +``` + +- [ ] **Step 8: Install the runtime wheel in artifact smokes** + +Replace `pythonArtifactInstallArgs` with: + +```javascript +export function pythonArtifactInstallArgs(python, pythonArtifacts) { + return ['pip', 'install', '--python', python, pythonArtifacts.runtimeWheel]; +} +``` + +Update `pythonVerifySource()` to assert `kaelio-ktx` metadata and keep module +imports: + +```javascript +export function pythonVerifySource() { + return ` +import importlib.metadata + +import semantic_layer +import ktx_daemon + +assert importlib.metadata.version("kaelio-ktx") == "0.1.0" +assert semantic_layer is not None +assert ktx_daemon.PACKAGE_NAME == "ktx-daemon" +`; +} +``` + +- [ ] **Step 9: Copy runtime assets before npm packing** + +Replace the loop in `buildArtifacts(layout)` with these explicit phases: + +```javascript + const commands = buildArtifactCommands(layout); + const npmBuildCount = NPM_ARTIFACT_PACKAGES.length; + const npmPackStart = commands.length - NPM_ARTIFACT_PACKAGES.length; + + for (const command of commands.slice(0, npmBuildCount)) { + await runCommand(command.command, command.args, { cwd: command.cwd }); + } + for (const command of commands.slice(npmBuildCount, npmPackStart)) { + await runCommand(command.command, command.args, { cwd: command.cwd }); + } + const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); + await copyRuntimeWheelAssets(layout, pythonArtifacts); + for (const command of commands.slice(npmPackStart)) { + await runCommand(command.command, command.args, { cwd: command.cwd }); + } +``` + +- [ ] **Step 10: Run package artifact tests** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 11: Commit artifact wiring** + +Run: + +```bash +git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs +git commit -m "build: bundle python runtime wheel in cli artifacts" +``` + +### Task 6: Update release policy and generated asset ignores + +**Files:** + +- Modify: `release-policy.json` +- Modify: `.gitignore` +- Modify: `scripts/release-readiness.test.mjs` +- Test: `scripts/release-readiness.test.mjs` + +- [ ] **Step 1: Ignore generated CLI Python assets** + +Add this block to `.gitignore` after the `dist/` ignore: + +```gitignore +packages/cli/assets/python/ +``` + +- [ ] **Step 2: Add runtime wheel to release policy** + +Update `release-policy.json` so the Python packages list is: + +```json + "python": { + "publish": false, + "repository": null, + "packages": ["ktx-sl", "ktx-daemon", "kaelio-ktx"] + }, +``` + +- [ ] **Step 3: Update release readiness fixtures** + +In `scripts/release-readiness.test.mjs`, update fixture policy objects that +list Python packages from: + +```javascript +packages: ['ktx-sl', 'ktx-daemon'], +``` + +to: + +```javascript +packages: ['ktx-sl', 'ktx-daemon', 'kaelio-ktx'], +``` + +Update expected package name arrays to include `kaelio-ktx`: + +```javascript +packageNames: [ + ...NPM_ARTIFACT_PACKAGES.map((packageInfo) => packageInfo.name), + 'ktx-sl', + 'ktx-daemon', + 'kaelio-ktx', +], +``` + +- [ ] **Step 4: Run release readiness tests** + +Run: + +```bash +node --test scripts/release-readiness.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 5: Commit policy updates** + +Run: + +```bash +git add .gitignore release-policy.json scripts/release-readiness.test.mjs +git commit -m "build: track bundled python runtime release artifact" +``` + +### Task 7: Verify the built runtime wheel end to end + +**Files:** + +- Build output: `dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl` +- Build output: `packages/cli/assets/python/manifest.json` +- Build output: + `packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl` + +- [ ] **Step 1: Run focused script tests** + +Run: + +```bash +node --test scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 2: Run Python package tests affected by dependency split** + +Run: + +```bash +uv run pytest python/ktx-daemon/tests -q +``` + +Expected: PASS. + +- [ ] **Step 3: Run package artifact check** + +Run: + +```bash +pnpm run artifacts:check +``` + +Expected: PASS. This command builds the runtime wheel, copies it into CLI +assets before npm packing, installs the packed npm packages in a clean smoke +project, installs the bundled runtime wheel with `uv pip install`, and verifies +`semantic_layer` plus `ktx_daemon` imports from the one `kaelio-ktx` wheel. + +- [ ] **Step 4: Inspect the generated CLI asset manifest** + +Run: + +```bash +node -e "const fs=require('node:fs'); const m=JSON.parse(fs.readFileSync('packages/cli/assets/python/manifest.json','utf8')); console.log(m.distributionName, m.version, m.wheel.file, m.wheel.sha256.length)" +``` + +Expected output: + +```text +kaelio-ktx 0.1.0 kaelio_ktx-0.1.0-py3-none-any.whl 64 +``` + +- [ ] **Step 5: Run pre-commit when configured** + +Run this only if `.pre-commit-config.yaml` exists: + +```bash +uv run pre-commit run --files python/ktx-daemon/pyproject.toml uv.lock pyproject.toml scripts/build-python-runtime-wheel.mjs scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs release-policy.json .gitignore +``` + +Expected: PASS. If no pre-commit config exists, record that no pre-commit +configuration exists in this repository and skip this command. + +- [ ] **Step 6: Commit verification-only updates if any** + +If verification required small code or test fixes, commit them: + +```bash +git add scripts/build-python-runtime-wheel.mjs scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs python/ktx-daemon/pyproject.toml uv.lock release-policy.json .gitignore +git commit -m "test: verify bundled python runtime wheel" +``` + +If no files changed after verification, do not create an empty commit. + +## Acceptance criteria + +- `dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl` is built by + `pnpm run artifacts:check`. +- The built CLI npm tarball includes + `assets/python/kaelio_ktx-0.1.0-py3-none-any.whl` and + `assets/python/manifest.json`. +- The asset manifest records the wheel filename, byte count, and SHA-256. +- Installing only the bundled runtime wheel exposes `semantic_layer`, + `ktx_daemon`, and the `ktx-daemon` console script. +- `sentence-transformers` and `torch` are absent from default dependencies and + present under the `local-embeddings` extra. +- Existing separate `ktx-sl` and `ktx-daemon` artifacts can remain CI artifacts + in this plan; the npm runtime payload uses `kaelio-ktx`. + +## Self-review + +Spec coverage: + +- Covers the package-model requirement for one bundled KTX-owned Python wheel. +- Covers the wheel checksum or runtime manifest requirement by adding the npm + asset manifest. +- Covers lazy local embedding dependencies by moving heavy packages into the + `local-embeddings` extra. +- Leaves managed runtime directories, install commands, daemon reuse, and + `@kaelio/ktx` npm renaming for later plans. + +Placeholder scan: + +- The plan contains no placeholder markers and no unspecified implementation + steps. + +Type and name consistency: + +- Runtime distribution name is consistently `kaelio-ktx`. +- Wheel filename prefix is consistently `kaelio_ktx`. +- Runtime version is consistently `0.1.0`. diff --git a/docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md b/docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md new file mode 100644 index 00000000..a73f163f --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md @@ -0,0 +1,1109 @@ +# Managed Agent and MCP Semantic Runtime Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make hidden agent semantic queries and MCP semantic compute use the +KTX-managed core Python runtime instead of relying on a user-provided +`python -m ktx_daemon`. + +**Architecture:** Reuse the existing managed runtime command helper so every +CLI semantic compute surface resolves the same bundled `ktx-daemon` executable. +Keep explicit HTTP daemon URLs working for `ktx serve --semantic-compute-url`, +and add runtime install policy flags where commands can lazily install the core +runtime. + +**Tech Stack:** TypeScript, Commander, Vitest, KTX CLI managed Python runtime, +`@ktx/context/daemon`. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +The following plans are based on that spec and are already implemented in this +worktree: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` +- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` + +Implementation evidence found before writing this plan includes: + +- `scripts/build-python-runtime-wheel.mjs` and + `packages/cli/assets/python/manifest.json`. +- `packages/cli/src/managed-python-runtime.ts`, + `packages/cli/src/runtime.ts`, and + `packages/cli/src/commands/runtime-commands.ts`. +- `packages/cli/src/managed-python-command.ts` and `ktx sl query` runtime + install policy flags. +- `packages/cli/src/managed-python-daemon.ts`, daemon state files, and + `ktx runtime start` / `ktx runtime stop`. +- `packages/cli/src/managed-local-embeddings.ts` and setup embedding wiring. +- `scripts/build-public-npm-package.mjs`, `release-policy.json`, and release + smoke coverage for `@kaelio/ktx`. +- `scripts/package-artifacts.mjs` release smoke coverage for lazy core runtime + install, `ktx sl query`, runtime status, doctor, daemon start, daemon reuse, + and daemon stop. +- `scripts/local-embeddings-runtime-smoke.mjs` opt-in release smoke coverage + for `local-embeddings`. + +The next remaining semantic compute gap is that these CLI paths still create a +raw Python semantic-layer compute port: + +- `packages/cli/src/agent-runtime.ts` +- `packages/cli/src/serve.ts` + +Those paths can call `semantic-query`, `semantic-validate`, and +`semantic-generate-sources` through `@ktx/context/daemon`, so they must resolve +the managed runtime just like `ktx sl query`. + +This plan intentionally does not change live-database introspection or Looker +table-identifier parsing. Those use daemon HTTP endpoints through local ingest +adapters and fit a separate managed-daemon adapter plan. + +## File structure + +- Modify `packages/cli/src/managed-python-command.ts`: export a shared + `runtimeInstallPolicyFromFlags()` helper so CLI commands do not duplicate + `--yes` / `--no-input` behavior. +- Modify `packages/cli/src/managed-python-command.test.ts`: cover the shared + policy helper. +- Modify `packages/cli/src/commands/sl-commands.ts`: replace its private + runtime policy helper with the shared helper. +- Modify `packages/cli/src/agent-runtime.ts`: create managed semantic compute + when agent SL query needs Python and no dependency override is injected. +- Modify `packages/cli/src/agent-runtime.test.ts`: cover the managed agent + runtime path. +- Modify `packages/cli/src/agent.ts`: pass CLI version, install policy, and + CLI IO into default agent runtime creation for `sl-query`. +- Modify `packages/cli/src/agent.test.ts`: cover runtime options passed through + agent SL query execution. +- Modify `packages/cli/src/commands/agent-commands.ts`: add `--yes` and + `--no-input` to hidden `ktx agent sl query`. +- Modify `packages/cli/src/serve.ts`: create managed semantic compute for + `ktx serve --semantic-compute` when no explicit HTTP URL is provided. +- Modify `packages/cli/src/serve.test.ts`: cover the managed MCP semantic + compute path. +- Modify `packages/cli/src/commands/serve-commands.ts`: add `--yes` and + `--no-input` to `ktx serve`. +- Modify `packages/cli/src/index.test.ts`: update CLI argument routing for the + new managed runtime policy fields. + +### Task 1: Share managed runtime install policy parsing + +**Files:** + +- Modify: `packages/cli/src/managed-python-command.test.ts` +- Modify: `packages/cli/src/managed-python-command.ts` +- Modify: `packages/cli/src/commands/sl-commands.ts` +- Test: `packages/cli/src/managed-python-command.test.ts` +- Test: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Write failing policy helper tests** + +In `packages/cli/src/managed-python-command.test.ts`, update the import from +`./managed-python-command.js` to include `runtimeInstallPolicyFromFlags`: + +```typescript +import { + createManagedPythonSemanticLayerComputePort, + managedRuntimeInstallCommand, + runtimeInstallPolicyFromFlags, +} from './managed-python-command.js'; +``` + +Add this block after the existing `describe('managedRuntimeInstallCommand', ...)` +block: + +```typescript +describe('runtimeInstallPolicyFromFlags', () => { + it('maps command flags to managed runtime install policies', () => { + expect(runtimeInstallPolicyFromFlags({})).toBe('prompt'); + expect(runtimeInstallPolicyFromFlags({ yes: false })).toBe('prompt'); + expect(runtimeInstallPolicyFromFlags({ yes: true })).toBe('auto'); + expect(runtimeInstallPolicyFromFlags({ input: false })).toBe('never'); + }); + + it('rejects conflicting runtime install flags', () => { + expect(() => runtimeInstallPolicyFromFlags({ yes: true, input: false })).toThrow( + 'Choose only one runtime install mode: --yes or --no-input', + ); + }); +}); +``` + +- [ ] **Step 2: Run the failing helper tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts +``` + +Expected: FAIL with an import error for `runtimeInstallPolicyFromFlags`. + +- [ ] **Step 3: Export the shared policy helper** + +In `packages/cli/src/managed-python-command.ts`, add this function immediately +after the `KtxManagedPythonInstallPolicy` type: + +```typescript +export function runtimeInstallPolicyFromFlags(options: { + yes?: boolean; + input?: boolean; +}): KtxManagedPythonInstallPolicy { + if (options.yes === true && options.input === false) { + throw new Error('Choose only one runtime install mode: --yes or --no-input'); + } + if (options.yes === true) { + return 'auto'; + } + return options.input === false ? 'never' : 'prompt'; +} +``` + +- [ ] **Step 4: Replace the private SL policy helper** + +In `packages/cli/src/commands/sl-commands.ts`, replace this import: + +```typescript +import type { KtxManagedPythonInstallPolicy } from '../managed-python-command.js'; +``` + +with this import: + +```typescript +import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; +``` + +Delete this private function from `packages/cli/src/commands/sl-commands.ts`: + +```typescript +function runtimeInstallPolicy(options: { yes?: boolean; input?: boolean }): KtxManagedPythonInstallPolicy { + if (options.yes === true && options.input === false) { + throw new Error('Choose only one runtime install mode: --yes or --no-input'); + } + if (options.yes === true) { + return 'auto'; + } + return options.input === false ? 'never' : 'prompt'; +} +``` + +In the `sl.command('query')` action, replace: + +```typescript +runtimeInstallPolicy: runtimeInstallPolicy(options), +``` + +with: + +```typescript +runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), +``` + +- [ ] **Step 5: Run focused helper and routing tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 6: Commit the shared helper** + +```bash +git add packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts packages/cli/src/commands/sl-commands.ts +git commit -m "refactor: share managed runtime install policy parsing" +``` + +### Task 2: Use managed semantic compute for hidden agent SL query + +**Files:** + +- Modify: `packages/cli/src/agent-runtime.test.ts` +- Modify: `packages/cli/src/agent-runtime.ts` +- Modify: `packages/cli/src/agent.test.ts` +- Modify: `packages/cli/src/agent.ts` +- Modify: `packages/cli/src/commands/agent-commands.ts` +- Modify: `packages/cli/src/index.test.ts` +- Test: `packages/cli/src/agent-runtime.test.ts` +- Test: `packages/cli/src/agent.test.ts` +- Test: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Add failing agent runtime tests** + +In `packages/cli/src/agent-runtime.test.ts`, add this test after +`constructs local context ports with semantic compute and query executor`: + +```typescript + it('creates managed semantic compute when no test override is injected', async () => { + const project = { + projectDir: tempDir, + configPath: join(tempDir, 'ktx.yaml'), + config: { project: 'revenue', connections: {} }, + coreConfig: {}, + git: {}, + fileStore: {}, + } as never; + const ports = { semanticLayer: {} } as never; + const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; + const loadProject = vi.fn(async () => project); + const createContextTools = vi.fn(() => ports); + const createManagedSemanticLayerCompute = vi.fn(async () => semanticLayerCompute); + const { io } = makeIo(); + + await expect( + createKtxAgentRuntime( + { + projectDir: tempDir, + enableSemanticCompute: true, + enableQueryExecution: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + io, + }, + { + loadProject, + createContextTools, + createManagedSemanticLayerCompute, + }, + ), + ).resolves.toMatchObject({ project, ports, semanticLayerCompute }); + + expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io, + }); + expect(createContextTools).toHaveBeenCalledWith(project, { + semanticLayerCompute, + }); + }); +``` + +- [ ] **Step 2: Add failing agent command/runtime tests** + +In `packages/cli/src/agent.test.ts`, update the existing +`executes SL queries from a JSON query file` test so the `sl-query` args include +the managed runtime fields: + +```typescript + { + command: 'sl-query', + projectDir: tempDir, + json: true, + connectionId: 'warehouse', + queryFile, + execute: true, + maxRows: 100, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'never', + }, +``` + +Add this test immediately after `executes SL queries from a JSON query file`: + +```typescript + it('passes managed runtime options into default SL query runtime creation', async () => { + const queryFile = join(tempDir, 'sl-query.json'); + const io = makeIo(); + const createRuntime = vi.fn(async () => runtime()); + await writeFile(queryFile, '{"measures":["total_revenue"],"dimensions":[]}', 'utf-8'); + + await expect( + runKtxAgent( + { + command: 'sl-query', + projectDir: tempDir, + json: true, + connectionId: 'warehouse', + queryFile, + execute: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + io.io, + { createRuntime }, + ), + ).resolves.toBe(0); + + expect(createRuntime).toHaveBeenCalledWith({ + projectDir: tempDir, + enableSemanticCompute: true, + enableQueryExecution: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + io: io.io, + }); + }); +``` + +- [ ] **Step 3: Add failing CLI routing tests** + +In `packages/cli/src/index.test.ts`, update the existing +`dispatches full hidden agent commands without exposing agent in root help` +case for `agent sl query` so its expected args include: + +```typescript + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', +``` + +Add this test after that existing full hidden agent command test: + +```typescript + it('routes hidden agent SL query managed runtime policies', async () => { + const autoIo = makeIo(); + const neverIo = makeIo(); + const conflictIo = makeIo(); + const agent = vi.fn(async () => 0); + + await expect( + runKtxCli( + [ + '--project-dir', + tempDir, + 'agent', + 'sl', + 'query', + '--json', + '--connection-id', + 'warehouse', + '--query-file', + '/tmp/query.json', + '--yes', + ], + autoIo.io, + { agent }, + ), + ).resolves.toBe(0); + + await expect( + runKtxCli( + [ + '--project-dir', + tempDir, + 'agent', + 'sl', + 'query', + '--json', + '--connection-id', + 'warehouse', + '--query-file', + '/tmp/query.json', + '--no-input', + ], + neverIo.io, + { agent }, + ), + ).resolves.toBe(0); + + await expect( + runKtxCli( + [ + '--project-dir', + tempDir, + 'agent', + 'sl', + 'query', + '--json', + '--connection-id', + 'warehouse', + '--query-file', + '/tmp/query.json', + '--yes', + '--no-input', + ], + conflictIo.io, + { agent }, + ), + ).resolves.toBe(1); + + expect(agent).toHaveBeenNthCalledWith( + 1, + { + command: 'sl-query', + projectDir: tempDir, + json: true, + connectionId: 'warehouse', + queryFile: '/tmp/query.json', + execute: false, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'auto', + }, + autoIo.io, + ); + expect(agent).toHaveBeenNthCalledWith( + 2, + { + command: 'sl-query', + projectDir: tempDir, + json: true, + connectionId: 'warehouse', + queryFile: '/tmp/query.json', + execute: false, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'never', + }, + neverIo.io, + ); + expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); + }); +``` + +- [ ] **Step 4: Run the failing agent tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/agent-runtime.test.ts src/agent.test.ts src/index.test.ts +``` + +Expected: FAIL with TypeScript or runtime errors for +`createManagedSemanticLayerCompute`, missing `cliVersion`, missing +`runtimeInstallPolicy`, or unsupported hidden agent `--yes` / `--no-input`. + +- [ ] **Step 5: Implement managed agent runtime creation** + +In `packages/cli/src/agent-runtime.ts`, replace the direct +`@ktx/context/daemon` import: + +```typescript +import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon'; +``` + +with: + +```typescript +import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon'; +import { + createManagedPythonSemanticLayerComputePort, + type KtxManagedPythonInstallPolicy, +} from './managed-python-command.js'; +``` + +Update `KtxAgentRuntimeOptions` to: + +```typescript +export interface KtxAgentRuntimeOptions { + projectDir: string; + enableSemanticCompute: boolean; + enableQueryExecution: boolean; + cliVersion?: string; + runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; + io?: KtxCliIo; +} +``` + +Update `KtxAgentRuntimeDeps` to: + +```typescript +export interface KtxAgentRuntimeDeps { + loadProject?: typeof loadKtxProject; + createContextTools?: typeof createLocalProjectMcpContextPorts; + createSemanticLayerCompute?: () => KtxSemanticLayerComputePort; + createManagedSemanticLayerCompute?: typeof createManagedPythonSemanticLayerComputePort; + createQueryExecutor?: () => KtxSqlQueryExecutorPort; +} +``` + +Add this helper before `createKtxAgentRuntime`: + +```typescript +async function createAgentSemanticLayerCompute( + options: KtxAgentRuntimeOptions, + deps: KtxAgentRuntimeDeps, +): Promise { + if (!options.enableSemanticCompute) { + return undefined; + } + if (deps.createSemanticLayerCompute) { + return deps.createSemanticLayerCompute(); + } + if (!options.cliVersion || !options.runtimeInstallPolicy || !options.io) { + throw new Error('Managed Python semantic compute requires cliVersion, runtimeInstallPolicy, and io.'); + } + const createManagedSemanticLayerCompute = + deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort; + return createManagedSemanticLayerCompute({ + cliVersion: options.cliVersion, + installPolicy: options.runtimeInstallPolicy, + io: options.io, + }); +} +``` + +In `createKtxAgentRuntime`, replace: + +```typescript + const semanticLayerCompute = options.enableSemanticCompute + ? (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)() + : undefined; +``` + +with: + +```typescript + const semanticLayerCompute = await createAgentSemanticLayerCompute(options, deps); +``` + +- [ ] **Step 6: Pass runtime options through agent execution** + +In `packages/cli/src/agent.ts`, add this import: + +```typescript +import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; +``` + +Update the `sl-query` variant in `KtxAgentArgs` to: + +```typescript + | { + command: 'sl-query'; + projectDir: string; + json: true; + connectionId: string; + queryFile: string; + execute: boolean; + maxRows?: number; + cliVersion: string; + runtimeInstallPolicy: KtxManagedPythonInstallPolicy; + } +``` + +Update `KtxAgentDeps.createRuntime` to use the shared runtime options type: + +```typescript + createRuntime?: (options: { + projectDir: string; + enableSemanticCompute: boolean; + enableQueryExecution: boolean; + cliVersion?: string; + runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; + io?: KtxCliIo; + }) => Promise; +``` + +Change `runtimeFor` from: + +```typescript +async function runtimeFor(args: KtxAgentArgs, deps: KtxAgentDeps): Promise { + const needsSemanticCompute = args.command === 'sl-query'; + const needsQueryExecution = args.command === 'sql-execute' || (args.command === 'sl-query' && args.execute); + return deps.createRuntime + ? deps.createRuntime({ + projectDir: args.projectDir, + enableSemanticCompute: needsSemanticCompute, + enableQueryExecution: needsQueryExecution, + }) + : createKtxAgentRuntime( + { + projectDir: args.projectDir, + enableSemanticCompute: needsSemanticCompute, + enableQueryExecution: needsQueryExecution, + }, + deps, + ); +} +``` + +to: + +```typescript +async function runtimeFor(args: KtxAgentArgs, deps: KtxAgentDeps, io: KtxCliIo): Promise { + const needsSemanticCompute = args.command === 'sl-query'; + const needsQueryExecution = args.command === 'sql-execute' || (args.command === 'sl-query' && args.execute); + const runtimeOptions = { + projectDir: args.projectDir, + enableSemanticCompute: needsSemanticCompute, + enableQueryExecution: needsQueryExecution, + ...(args.command === 'sl-query' + ? { + cliVersion: args.cliVersion, + runtimeInstallPolicy: args.runtimeInstallPolicy, + io, + } + : {}), + }; + return deps.createRuntime ? deps.createRuntime(runtimeOptions) : createKtxAgentRuntime(runtimeOptions, deps); +} +``` + +In `runKtxAgent`, replace: + +```typescript + const runtime = await runtimeFor(args, deps); +``` + +with: + +```typescript + const runtime = await runtimeFor(args, deps, io); +``` + +- [ ] **Step 7: Add hidden agent runtime policy flags** + +In `packages/cli/src/commands/agent-commands.ts`, add this import: + +```typescript +import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; +``` + +In the `agent sl query` command chain, add these options after +`.option('--execute', ...)`: + +```typescript + .option('--yes', 'Install the managed Python runtime without prompting when required', false) + .option('--no-input', 'Disable interactive managed runtime installation') +``` + +Update the action options type from: + +```typescript + options: { connectionId: string; queryFile: string; execute: boolean; maxRows?: number }, +``` + +to: + +```typescript + options: { + connectionId: string; + queryFile: string; + execute: boolean; + maxRows?: number; + yes?: boolean; + input?: boolean; + }, +``` + +Add these fields to the `runAgent` argument object: + +```typescript + cliVersion: context.packageInfo.version, + runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), +``` + +- [ ] **Step 8: Run focused agent tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/agent-runtime.test.ts src/agent.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 9: Commit the agent integration** + +```bash +git add packages/cli/src/agent-runtime.ts packages/cli/src/agent-runtime.test.ts packages/cli/src/agent.ts packages/cli/src/agent.test.ts packages/cli/src/commands/agent-commands.ts packages/cli/src/index.test.ts +git commit -m "feat: use managed runtime for agent semantic queries" +``` + +### Task 3: Use managed semantic compute for MCP serve + +**Files:** + +- Modify: `packages/cli/src/serve.test.ts` +- Modify: `packages/cli/src/serve.ts` +- Modify: `packages/cli/src/commands/serve-commands.ts` +- Modify: `packages/cli/src/index.test.ts` +- Test: `packages/cli/src/serve.test.ts` +- Test: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Add a failing serve managed runtime test** + +In `packages/cli/src/serve.test.ts`, add this helper after the imports: + +```typescript +function makeManagedRuntimeIo() { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { write: (chunk: string) => (stdout += chunk) }, + stderr: { write: (chunk: string) => (stderr += chunk) }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} +``` + +Add this test before `uses the HTTP semantic compute port when a daemon URL is +provided`: + +```typescript + it('uses managed semantic compute when MCP semantic compute has no explicit HTTP URL', async () => { + const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never; + const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; + const createManagedSemanticLayerCompute = vi.fn(async () => semanticLayerCompute); + const createContextTools = vi.fn(() => ({ connections: { list: async () => [] } })); + const managedRuntimeIo = makeManagedRuntimeIo(); + + await expect( + runKtxServeStdio( + { + mcp: 'stdio', + projectDir: '/tmp/ktx-project', + userId: 'agent', + semanticCompute: true, + semanticComputeUrl: undefined, + databaseIntrospectionUrl: undefined, + executeQueries: false, + memoryCapture: false, + memoryModel: undefined, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + { + loadProject: async () => project, + createContextTools, + createManagedSemanticLayerCompute, + managedRuntimeIo: managedRuntimeIo.io, + createServer: vi.fn(() => ({ connect: vi.fn(async () => undefined) }) as never), + createTransport: vi.fn(() => ({}) as never), + stderr: { write: vi.fn() }, + }, + ), + ).resolves.toBe(0); + + expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: managedRuntimeIo.io, + }); + expect(createContextTools).toHaveBeenCalledWith( + project, + expect.objectContaining({ + semanticLayerCompute, + }), + ); + }); +``` + +- [ ] **Step 2: Add failing serve routing tests** + +In `packages/cli/src/index.test.ts`, update both existing `serveStdio` +expectations so the expected args include: + +```typescript + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', +``` + +Add this test after `dispatches serve public command options through Commander`: + +```typescript + it('routes serve managed runtime install policies', async () => { + const autoIo = makeIo(); + const neverIo = makeIo(); + const conflictIo = makeIo(); + const serveStdio = vi.fn(async () => 0); + + await expect( + runKtxCli(['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--yes'], autoIo.io, { + serveStdio, + }), + ).resolves.toBe(0); + await expect( + runKtxCli(['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--no-input'], neverIo.io, { + serveStdio, + }), + ).resolves.toBe(0); + await expect( + runKtxCli( + ['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--yes', '--no-input'], + conflictIo.io, + { serveStdio }, + ), + ).resolves.toBe(1); + + expect(serveStdio).toHaveBeenNthCalledWith(1, { + mcp: 'stdio', + projectDir: tempDir, + userId: 'local', + semanticCompute: true, + semanticComputeUrl: undefined, + databaseIntrospectionUrl: undefined, + executeQueries: false, + memoryCapture: false, + memoryModel: undefined, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'auto', + }); + expect(serveStdio).toHaveBeenNthCalledWith(2, { + mcp: 'stdio', + projectDir: tempDir, + userId: 'local', + semanticCompute: true, + semanticComputeUrl: undefined, + databaseIntrospectionUrl: undefined, + executeQueries: false, + memoryCapture: false, + memoryModel: undefined, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'never', + }); + expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); + }); +``` + +- [ ] **Step 3: Run the failing serve tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/serve.test.ts src/index.test.ts +``` + +Expected: FAIL with missing `createManagedSemanticLayerCompute` support and +missing `cliVersion` / `runtimeInstallPolicy` fields in command routing. + +- [ ] **Step 4: Implement managed serve semantic compute** + +In `packages/cli/src/serve.ts`, add this import: + +```typescript +import type { KtxCliIo } from './cli-runtime.js'; +import { + createManagedPythonSemanticLayerComputePort, + type KtxManagedPythonInstallPolicy, +} from './managed-python-command.js'; +``` + +Update `KtxServeArgs` to: + +```typescript +export interface KtxServeArgs { + mcp: 'stdio'; + projectDir: string; + userId: string; + semanticCompute: boolean; + semanticComputeUrl?: string; + databaseIntrospectionUrl?: string; + executeQueries: boolean; + memoryCapture: boolean; + memoryModel?: string; + cliVersion?: string; + runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; +} +``` + +Update `KtxServeDeps` to include: + +```typescript + createManagedSemanticLayerCompute?: typeof createManagedPythonSemanticLayerComputePort; + managedRuntimeIo?: KtxCliIo; +``` + +Add these helpers before `runKtxServeStdio`: + +```typescript +function requiredManagedRuntimeCliVersion(args: KtxServeArgs): string { + if (!args.cliVersion) { + throw new Error('Managed Python semantic compute requires a CLI version.'); + } + return args.cliVersion; +} + +async function createServeSemanticLayerCompute( + args: KtxServeArgs, + deps: KtxServeDeps, +): Promise { + if (!args.semanticCompute) { + return undefined; + } + if (args.semanticComputeUrl) { + return (deps.createHttpSemanticLayerCompute ?? ((baseUrl) => createHttpSemanticLayerComputePort({ baseUrl })))( + args.semanticComputeUrl, + ); + } + if (deps.createSemanticLayerCompute) { + return deps.createSemanticLayerCompute(); + } + const createManagedSemanticLayerCompute = + deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort; + return createManagedSemanticLayerCompute({ + cliVersion: requiredManagedRuntimeCliVersion(args), + installPolicy: args.runtimeInstallPolicy ?? 'prompt', + io: deps.managedRuntimeIo ?? process, + }); +} +``` + +In `runKtxServeStdio`, replace: + +```typescript + const semanticLayerCompute = args.semanticCompute + ? args.semanticComputeUrl + ? (deps.createHttpSemanticLayerCompute ?? ((baseUrl) => createHttpSemanticLayerComputePort({ baseUrl })))( + args.semanticComputeUrl, + ) + : (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)() + : undefined; +``` + +with: + +```typescript + const semanticLayerCompute = await createServeSemanticLayerCompute(args, deps); +``` + +Remove `createPythonSemanticLayerComputePort` from the +`@ktx/context/daemon` import list. + +- [ ] **Step 5: Add serve runtime policy flags** + +In `packages/cli/src/commands/serve-commands.ts`, add this import: + +```typescript +import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; +``` + +Add these command options after `.option('--semantic-compute-url ', ...)`: + +```typescript + .option('--yes', 'Install the managed Python runtime without prompting when required', false) + .option('--no-input', 'Disable interactive managed runtime installation') +``` + +Add these fields to the `KtxServeArgs` object: + +```typescript + cliVersion: context.packageInfo.version, + runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), +``` + +- [ ] **Step 6: Run focused serve tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/serve.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 7: Commit the serve integration** + +```bash +git add packages/cli/src/serve.ts packages/cli/src/serve.test.ts packages/cli/src/commands/serve-commands.ts packages/cli/src/index.test.ts +git commit -m "feat: use managed runtime for MCP semantic compute" +``` + +### Task 4: Verify managed semantic runtime integration + +**Files:** + +- Verify: `packages/cli/src/managed-python-command.ts` +- Verify: `packages/cli/src/agent-runtime.ts` +- Verify: `packages/cli/src/agent.ts` +- Verify: `packages/cli/src/commands/agent-commands.ts` +- Verify: `packages/cli/src/serve.ts` +- Verify: `packages/cli/src/commands/serve-commands.ts` +- Verify: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Run all focused CLI tests touched by this plan** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts src/agent-runtime.test.ts src/agent.test.ts src/serve.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 2: Run CLI type-check** + +Run: + +```bash +pnpm --filter @ktx/cli run type-check +``` + +Expected: PASS. + +- [ ] **Step 3: Run CLI tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test +``` + +Expected: PASS. + +- [ ] **Step 4: Run package build** + +Run: + +```bash +pnpm --filter @ktx/cli run build +``` + +Expected: PASS. + +- [ ] **Step 5: Commit any verification fixes** + +If verification required code edits, run: + +```bash +git add packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts packages/cli/src/commands/sl-commands.ts packages/cli/src/agent-runtime.ts packages/cli/src/agent-runtime.test.ts packages/cli/src/agent.ts packages/cli/src/agent.test.ts packages/cli/src/commands/agent-commands.ts packages/cli/src/serve.ts packages/cli/src/serve.test.ts packages/cli/src/commands/serve-commands.ts packages/cli/src/index.test.ts +git commit -m "fix: verify managed semantic runtime surfaces" +``` + +If no files changed after Step 1 through Step 4, do not create an empty commit. + +## Acceptance criteria + +- `ktx agent sl query` has `--yes` and `--no-input` managed runtime policy + flags. +- `ktx agent sl query --yes` passes `runtimeInstallPolicy: 'auto'` and the + current CLI package version into default runtime creation. +- `ktx agent sl query --no-input` passes `runtimeInstallPolicy: 'never'`. +- `ktx agent sl query --yes --no-input` exits with + `Choose only one runtime install mode: --yes or --no-input`. +- Default agent SL query runtime creation uses + `createManagedPythonSemanticLayerComputePort()` and therefore invokes the + bundled managed `ktx-daemon` executable. +- `ktx serve --mcp stdio --semantic-compute` has `--yes` and `--no-input` + managed runtime policy flags. +- `ktx serve --mcp stdio --semantic-compute --yes` passes + `runtimeInstallPolicy: 'auto'` and the current CLI package version into + serve runtime creation. +- `ktx serve --mcp stdio --semantic-compute --no-input` passes + `runtimeInstallPolicy: 'never'`. +- `ktx serve --mcp stdio --semantic-compute-url ` continues to use the + explicit HTTP semantic compute port and does not install or start a managed + runtime. +- Focused CLI tests, full CLI tests, CLI type-check, and CLI build pass. + +## Self-review + +- Spec coverage: this plan extends managed Python one-shot semantic compute to + hidden agent SL query and MCP `serve --semantic-compute`, covering additional + semantic query, validation, and source-generation paths that use + `@ktx/context/daemon`. +- Remaining intentional gap: local ingest daemon-backed database introspection + and Looker SQL table-identifier parsing still need a managed daemon adapter + plan because they use HTTP daemon endpoints rather than the one-shot semantic + compute port. +- Placeholder scan: all steps contain concrete edits, commands, and expected + results. +- Type consistency: runtime policy values stay `prompt`, `auto`, and `never`; + runtime feature values stay `core` and `local-embeddings`; package version + fields are named `cliVersion`. diff --git a/docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md b/docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md new file mode 100644 index 00000000..907c3ba9 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md @@ -0,0 +1,856 @@ +# Managed Local Embeddings Release Smoke Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add an opt-in release smoke that proves the public `@kaelio/ktx` +package can install `local-embeddings`, start the managed daemon, compute a real +local embedding, and persist the managed embedding marker through setup. + +**Architecture:** Keep the default `artifacts:verify` path lightweight. Add a +separate Node smoke script with an explicit opt-in gate, source-level tests, and +a package script that a release job can run only when large Python and model +downloads are acceptable. + +**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, uv, KTX managed Python +runtime assets, FastAPI embedding endpoint, sentence-transformers. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +The following plans are based on that spec and are already implemented in this +worktree: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` +- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` + +Implementation evidence found before writing this plan includes: + +- `scripts/build-python-runtime-wheel.mjs` and matching tests. +- `packages/cli/src/managed-python-runtime.ts`, `runtime.ts`, and + `commands/runtime-commands.ts`. +- `packages/cli/src/managed-python-command.ts` and `ktx sl query` runtime + install policy flags. +- `packages/cli/src/managed-python-daemon.ts` and `ktx runtime start` / + `ktx runtime stop`. +- `packages/cli/src/managed-local-embeddings.ts`, + `packages/context/src/llm/local-config.ts`, and setup embedding wiring. +- `scripts/build-public-npm-package.mjs`, `release-policy.json` listing + `@kaelio/ktx`, and public-package smoke command construction. +- `scripts/package-artifacts.mjs` installed CLI smoke that isolates + `KTX_RUNTIME_ROOT`, lazily installs the core runtime, runs `ktx sl query`, + checks runtime status and doctor output, and starts, reuses, and stops the + core daemon. + +The remaining spec gap is the release-check item that permits local embeddings +coverage in a separate job or opt-in check. The default release artifact smoke +must not download `sentence-transformers`, `torch`, or the +`all-MiniLM-L6-v2` model. + +## File structure + +- Create `scripts/local-embeddings-runtime-smoke.mjs`: an opt-in smoke script + that consumes the built public npm tarball, installs it in a temporary pnpm + project, isolates all runtime and model caches, installs the + `local-embeddings` feature, starts the managed daemon, computes one real + embedding, runs setup with local embeddings, verifies the managed config + marker, and stops the daemon. +- Create `scripts/local-embeddings-runtime-smoke.test.mjs`: fast source-level + tests for opt-in gating, public tarball selection, cache isolation, command + construction, daemon URL parsing, embedding response validation, and package + script registration. +- Modify `package.json`: add `release:local-embeddings-smoke` without adding + it to default `check`, `test`, `artifacts:verify`, or release readiness. + +### Task 1: Add failing local embeddings smoke tests + +**Files:** + +- Create: `scripts/local-embeddings-runtime-smoke.test.mjs` +- Test: `scripts/local-embeddings-runtime-smoke.test.mjs` + +- [ ] **Step 1: Write the failing test file** + +Create `scripts/local-embeddings-runtime-smoke.test.mjs` with this content: + +```javascript +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { describe, it } from 'node:test'; + +import { + buildLocalEmbeddingsSmokeEnv, + localEmbeddingsSmokeCommands, + localEmbeddingsSmokeOptIn, + parseDaemonBaseUrl, + publicKtxTarballName, + validateEmbeddingResponse, +} from './local-embeddings-runtime-smoke.mjs'; + +describe('localEmbeddingsSmokeOptIn', () => { + it('skips unless the smoke is explicitly enabled', () => { + assert.deepEqual(localEmbeddingsSmokeOptIn({}, []), { + run: false, + message: 'Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke.', + }); + }); + + it('runs when the environment opt-in is set', () => { + assert.deepEqual(localEmbeddingsSmokeOptIn({ KTX_RUN_LOCAL_EMBEDDINGS_SMOKE: '1' }, []), { + run: true, + }); + }); + + it('runs when --force is present', () => { + assert.deepEqual(localEmbeddingsSmokeOptIn({}, ['--force']), { + run: true, + }); + }); +}); + +describe('publicKtxTarballName', () => { + it('selects the public @kaelio/ktx tarball name', () => { + assert.equal( + publicKtxTarballName(['kaelio-ktx-0.0.0-private.tgz', 'ignore-me.tgz']), + 'kaelio-ktx-0.0.0-private.tgz', + ); + }); + + it('fails when the public package tarball is missing', () => { + assert.throws( + () => publicKtxTarballName(['ktx-cli-0.0.0-private.tgz']), + /Expected exactly one @kaelio\/ktx tarball/, + ); + }); + + it('fails when multiple public package tarballs are present', () => { + assert.throws( + () => publicKtxTarballName(['kaelio-ktx-0.1.0.tgz', 'kaelio-ktx-0.2.0.tgz']), + /Expected exactly one @kaelio\/ktx tarball/, + ); + }); +}); + +describe('buildLocalEmbeddingsSmokeEnv', () => { + it('isolates the runtime root and model caches inside the smoke root', () => { + const env = buildLocalEmbeddingsSmokeEnv('/tmp/ktx-local-embedding-smoke', { + PATH: '/usr/bin', + }); + + assert.equal(env.PATH, '/usr/bin'); + assert.equal(env.KTX_RUN_LOCAL_EMBEDDINGS_SMOKE, '1'); + assert.equal(env.KTX_RUNTIME_ROOT, '/tmp/ktx-local-embedding-smoke/managed-runtime'); + assert.equal(env.HF_HOME, '/tmp/ktx-local-embedding-smoke/hf-home'); + assert.equal(env.TRANSFORMERS_CACHE, '/tmp/ktx-local-embedding-smoke/transformers-cache'); + assert.equal(env.SENTENCE_TRANSFORMERS_HOME, '/tmp/ktx-local-embedding-smoke/sentence-transformers-home'); + assert.equal(env.TORCH_HOME, '/tmp/ktx-local-embedding-smoke/torch-home'); + }); +}); + +describe('localEmbeddingsSmokeCommands', () => { + it('describes the installed-package commands needed for the smoke', () => { + const commands = localEmbeddingsSmokeCommands({ + projectDir: '/tmp/ktx-local-embedding-smoke/project', + }); + + assert.deepEqual(commands.map((command) => command.label), [ + 'ktx public package version', + 'ktx runtime status missing', + 'ktx runtime install local embeddings', + 'ktx runtime status local embeddings ready', + 'ktx runtime start local embeddings', + 'ktx setup local embeddings', + 'ktx runtime stop local embeddings', + ]); + assert.deepEqual(commands[2], { + label: 'ktx runtime install local embeddings', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], + timeoutMs: 1_200_000, + }); + assert.deepEqual(commands[4], { + label: 'ktx runtime start local embeddings', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'start', '--feature', 'local-embeddings'], + timeoutMs: 300_000, + }); + assert.deepEqual(commands[5].args, [ + 'exec', + 'ktx', + 'setup', + '--project-dir', + '/tmp/ktx-local-embedding-smoke/project', + '--new', + '--no-input', + '--yes', + '--skip-llm', + '--embedding-backend', + 'sentence-transformers', + '--skip-databases', + '--skip-sources', + '--skip-agents', + ]); + }); +}); + +describe('parseDaemonBaseUrl', () => { + it('extracts the daemon URL from runtime start output', () => { + assert.equal( + parseDaemonBaseUrl('Started KTX Python daemon\nurl: http://127.0.0.1:61234\nfeatures: local-embeddings\n'), + 'http://127.0.0.1:61234', + ); + }); + + it('rejects output without a daemon URL', () => { + assert.throws(() => parseDaemonBaseUrl('Started KTX Python daemon\n'), /Daemon URL was not printed/); + }); +}); + +describe('validateEmbeddingResponse', () => { + it('accepts a finite embedding vector with the expected dimensions', () => { + validateEmbeddingResponse({ embedding: [0.1, -0.2, 0.3] }, 3); + }); + + it('rejects a vector with the wrong dimensions', () => { + assert.throws( + () => validateEmbeddingResponse({ embedding: [0.1, 0.2] }, 3), + /Expected embedding dimension 3, got 2/, + ); + }); + + it('rejects non-finite embedding values', () => { + assert.throws( + () => validateEmbeddingResponse({ embedding: [0.1, Number.NaN, 0.3] }, 3), + /Embedding value at index 1 is not a finite number/, + ); + }); +}); + +describe('package script', () => { + it('registers the opt-in local embeddings smoke command', async () => { + const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); + + assert.equal( + packageJson.scripts['release:local-embeddings-smoke'], + 'node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in', + ); + }); +}); +``` + +- [ ] **Step 2: Run the failing test** + +Run: + +```bash +node --test scripts/local-embeddings-runtime-smoke.test.mjs +``` + +Expected: FAIL with an import error for +`./local-embeddings-runtime-smoke.mjs`. + +- [ ] **Step 3: Commit the failing tests** + +Run: + +```bash +git add scripts/local-embeddings-runtime-smoke.test.mjs +git commit -m "test: specify local embeddings release smoke" +``` + +### Task 2: Implement the opt-in smoke script + +**Files:** + +- Create: `scripts/local-embeddings-runtime-smoke.mjs` +- Test: `scripts/local-embeddings-runtime-smoke.test.mjs` + +- [ ] **Step 1: Create the smoke script** + +Create `scripts/local-embeddings-runtime-smoke.mjs` with this content: + +```javascript +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); +const DEFAULT_ROOT_DIR = resolve(SCRIPT_DIR, '..'); +const PUBLIC_NPM_ARTIFACT_DIR = join('dist', 'artifacts', 'npm'); +const OPT_IN_MESSAGE = + 'Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke.'; + +export function localEmbeddingsSmokeOptIn(env = process.env, args = process.argv.slice(2)) { + if (env.KTX_RUN_LOCAL_EMBEDDINGS_SMOKE === '1' || args.includes('--force')) { + return { run: true }; + } + return { run: false, message: OPT_IN_MESSAGE }; +} + +export function publicKtxTarballName(files) { + const matches = files.filter((file) => /^kaelio-ktx-.+\.tgz$/.test(file)).sort(); + if (matches.length !== 1) { + throw new Error( + `Expected exactly one @kaelio/ktx tarball in ${PUBLIC_NPM_ARTIFACT_DIR}, found ${matches.length}: ${ + matches.join(', ') || 'none' + }. Run pnpm run artifacts:build first.`, + ); + } + return matches[0]; +} + +export async function selectPublicKtxTarball(rootDir = DEFAULT_ROOT_DIR) { + const npmArtifactDir = join(rootDir, PUBLIC_NPM_ARTIFACT_DIR); + const files = await readdir(npmArtifactDir); + return join(npmArtifactDir, publicKtxTarballName(files)); +} + +export function buildLocalEmbeddingsSmokeEnv(root, baseEnv = process.env) { + return { + ...baseEnv, + KTX_RUN_LOCAL_EMBEDDINGS_SMOKE: '1', + KTX_RUNTIME_ROOT: join(root, 'managed-runtime'), + HF_HOME: join(root, 'hf-home'), + TRANSFORMERS_CACHE: join(root, 'transformers-cache'), + SENTENCE_TRANSFORMERS_HOME: join(root, 'sentence-transformers-home'), + TORCH_HOME: join(root, 'torch-home'), + }; +} + +export function localEmbeddingsSmokeCommands(input) { + return [ + { + label: 'ktx public package version', + command: 'pnpm', + args: ['exec', 'ktx', '--version'], + timeoutMs: 60_000, + }, + { + label: 'ktx runtime status missing', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'status', '--json'], + timeoutMs: 60_000, + }, + { + label: 'ktx runtime install local embeddings', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], + timeoutMs: 1_200_000, + }, + { + label: 'ktx runtime status local embeddings ready', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'status', '--json'], + timeoutMs: 60_000, + }, + { + label: 'ktx runtime start local embeddings', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'start', '--feature', 'local-embeddings'], + timeoutMs: 300_000, + }, + { + label: 'ktx setup local embeddings', + command: 'pnpm', + args: [ + 'exec', + 'ktx', + 'setup', + '--project-dir', + input.projectDir, + '--new', + '--no-input', + '--yes', + '--skip-llm', + '--embedding-backend', + 'sentence-transformers', + '--skip-databases', + '--skip-sources', + '--skip-agents', + ], + timeoutMs: 900_000, + }, + { + label: 'ktx runtime stop local embeddings', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'stop'], + timeoutMs: 60_000, + }, + ]; +} + +export function parseDaemonBaseUrl(stdout) { + const match = stdout.match(/^url: (http:\/\/127\.0\.0\.1:\d+)$/m); + if (!match) { + throw new Error(`Daemon URL was not printed by runtime start:\n${stdout}`); + } + return match[1]; +} + +export function validateEmbeddingResponse(raw, expectedDimensions) { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('Embedding response must be a JSON object'); + } + const embedding = raw.embedding; + if (!Array.isArray(embedding)) { + throw new Error('Embedding response must include an embedding array'); + } + if (embedding.length !== expectedDimensions) { + throw new Error(`Expected embedding dimension ${expectedDimensions}, got ${embedding.length}`); + } + for (const [index, value] of embedding.entries()) { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`Embedding value at index ${index} is not a finite number`); + } + } +} + +async function run(command, args, options = {}) { + process.stdout.write(`$ ${command} ${args.join(' ')}\n`); + try { + const result = await execFileAsync(command, args, { + cwd: options.cwd, + env: { ...process.env, ...options.env }, + encoding: 'utf8', + maxBuffer: 1024 * 1024 * 20, + timeout: options.timeoutMs ?? 120_000, + }); + if (result.stdout) { + process.stdout.write(result.stdout); + } + if (result.stderr) { + process.stderr.write(result.stderr); + } + return { code: 0, stdout: result.stdout, stderr: result.stderr }; + } catch (error) { + const stdout = typeof error.stdout === 'string' ? error.stdout : ''; + const stderr = typeof error.stderr === 'string' ? error.stderr : error.message; + if (stdout) { + process.stdout.write(stdout); + } + if (stderr) { + process.stderr.write(stderr); + } + return { + code: typeof error.code === 'number' ? error.code : 1, + stdout, + stderr, + }; + } +} + +function requireSuccess(label, result, options = {}) { + if (result.code !== 0) { + throw new Error(`${label} failed with code ${result.code}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); + } + if (options.stderrPattern && !options.stderrPattern.test(result.stderr)) { + throw new Error(`${label} stderr did not match ${options.stderrPattern}\nstderr:\n${result.stderr}`); + } +} + +function parseJsonStdout(label, result) { + requireSuccess(label, result); + try { + return JSON.parse(result.stdout); + } catch (error) { + throw new Error(`${label} did not write JSON stdout: ${error.message}\nstdout:\n${result.stdout}`); + } +} + +function requireOutput(label, result, pattern) { + if (!pattern.test(result.stdout)) { + throw new Error(`${label} stdout did not match ${pattern}\nstdout:\n${result.stdout}`); + } +} + +async function postJson(baseUrl, path, payload, timeoutMs) { + const response = await fetch(new URL(path, baseUrl), { + method: 'POST', + headers: { + accept: 'application/json', + 'content-type': 'application/json', + }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(timeoutMs), + }); + const text = await response.text(); + if (!response.ok) { + throw new Error(`POST ${path} failed with ${response.status}: ${text}`); + } + try { + return JSON.parse(text); + } catch (error) { + throw new Error(`POST ${path} returned non-JSON response: ${error.message}\n${text}`); + } +} + +async function writeSmokePackage(projectDir, tarballPath) { + await mkdir(projectDir, { recursive: true }); + await writeFile( + join(projectDir, 'package.json'), + `${JSON.stringify( + { + name: 'ktx-local-embeddings-runtime-smoke', + version: '0.0.0', + private: true, + type: 'module', + dependencies: { + '@kaelio/ktx': `file:${tarballPath}`, + }, + }, + null, + 2, + )}\n`, + ); +} + +export async function runLocalEmbeddingsRuntimeSmoke(options = {}) { + const rootDir = options.rootDir ?? DEFAULT_ROOT_DIR; + const tarballPath = options.tarballPath ?? (await selectPublicKtxTarball(rootDir)); + const root = await mkdtemp(join(tmpdir(), 'ktx-local-embeddings-smoke-')); + const keepTemp = options.keepTemp ?? process.env.KTX_KEEP_LOCAL_EMBEDDINGS_SMOKE === '1'; + const installDir = join(root, 'installed-package'); + const projectDir = join(root, 'project'); + const smokeEnv = buildLocalEmbeddingsSmokeEnv(root); + const commands = localEmbeddingsSmokeCommands({ projectDir }); + let daemonStarted = false; + + try { + await writeSmokePackage(installDir, tarballPath); + requireSuccess( + 'pnpm install public package', + await run('pnpm', ['install', '--ignore-scripts=false'], { + cwd: installDir, + env: smokeEnv, + timeoutMs: 300_000, + }), + ); + + const version = await run(commands[0].command, commands[0].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[0].timeoutMs, + }); + requireSuccess(commands[0].label, version); + requireOutput(commands[0].label, version, /@kaelio\/ktx 0\.0\.0-private/); + + const missingStatus = parseJsonStdout( + commands[1].label, + await run(commands[1].command, commands[1].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[1].timeoutMs, + }), + ); + if (missingStatus.kind !== 'missing') { + throw new Error(`Expected missing runtime before install, got ${JSON.stringify(missingStatus)}`); + } + + const install = await run(commands[2].command, commands[2].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[2].timeoutMs, + }); + requireSuccess(commands[2].label, install); + requireOutput(commands[2].label, install, /Installed KTX Python runtime/); + requireOutput(commands[2].label, install, /features: core, local-embeddings/); + + const readyStatus = parseJsonStdout( + commands[3].label, + await run(commands[3].command, commands[3].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[3].timeoutMs, + }), + ); + if (readyStatus.kind !== 'ready') { + throw new Error(`Expected ready runtime after install, got ${JSON.stringify(readyStatus)}`); + } + if (!readyStatus.manifest?.features?.includes('local-embeddings')) { + throw new Error(`Runtime manifest did not include local-embeddings: ${JSON.stringify(readyStatus.manifest)}`); + } + + const start = await run(commands[4].command, commands[4].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[4].timeoutMs, + }); + requireSuccess(commands[4].label, start); + daemonStarted = true; + const baseUrl = parseDaemonBaseUrl(start.stdout); + + const embeddingResponse = await postJson( + baseUrl, + '/embeddings/compute', + { text: 'KTX local embeddings release smoke' }, + 900_000, + ); + validateEmbeddingResponse(embeddingResponse, 384); + process.stdout.write('KTX local embeddings daemon computed a 384-dimensional embedding\n'); + + const setup = await run(commands[5].command, commands[5].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[5].timeoutMs, + }); + requireSuccess(commands[5].label, setup); + requireOutput(commands[5].label, setup, /Embeddings ready: yes \(all-MiniLM-L6-v2\)/); + + const config = await readFile(join(projectDir, 'ktx.yaml'), 'utf8'); + if (!config.includes('base_url: managed:local-embeddings')) { + throw new Error(`ktx.yaml did not contain managed local embeddings marker:\n${config}`); + } + process.stdout.write('KTX setup persisted managed local embeddings marker\n'); + + const stop = await run(commands[6].command, commands[6].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[6].timeoutMs, + }); + requireSuccess(commands[6].label, stop); + daemonStarted = false; + requireOutput(commands[6].label, stop, /Stopped KTX Python daemon/); + + process.stdout.write('KTX local embeddings runtime smoke verified\n'); + } finally { + if (daemonStarted) { + await run('pnpm', ['exec', 'ktx', 'runtime', 'stop'], { + cwd: installDir, + env: smokeEnv, + timeoutMs: 60_000, + }); + } + if (!keepTemp) { + await rm(root, { recursive: true, force: true }); + } else { + process.stdout.write(`Kept local embeddings smoke root: ${root}\n`); + } + } +} + +async function main() { + const args = process.argv.slice(2); + const optIn = localEmbeddingsSmokeOptIn(process.env, args); + if (!optIn.run) { + process.stdout.write(`Skipping KTX local embeddings runtime smoke. ${optIn.message}\n`); + if (args.includes('--require-opt-in')) { + process.exitCode = 1; + } + return; + } + + await runLocalEmbeddingsRuntimeSmoke(); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`); + process.exitCode = 1; + }); +} +``` + +- [ ] **Step 2: Run the smoke test** + +Run: + +```bash +node --test scripts/local-embeddings-runtime-smoke.test.mjs +``` + +Expected: FAIL only in the package script test because +`release:local-embeddings-smoke` is not registered yet. + +- [ ] **Step 3: Commit the smoke script** + +Run: + +```bash +git add scripts/local-embeddings-runtime-smoke.mjs +git commit -m "feat: add local embeddings runtime smoke" +``` + +### Task 3: Register the opt-in package script + +**Files:** + +- Modify: `package.json` +- Test: `scripts/local-embeddings-runtime-smoke.test.mjs` + +- [ ] **Step 1: Add the package script** + +In `package.json`, add this script immediately after +`"release:published-smoke"`: + +```json +"release:local-embeddings-smoke": "node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in", +``` + +The surrounding `scripts` section must contain this sequence after the edit: + +```json +"release:published-smoke": "node scripts/published-package-smoke.mjs --require-config", +"release:local-embeddings-smoke": "node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in", +"release:readiness": "node scripts/release-readiness.mjs", +``` + +- [ ] **Step 2: Run the focused test** + +Run: + +```bash +node --test scripts/local-embeddings-runtime-smoke.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 3: Verify the script stays opt-in** + +Run: + +```bash +pnpm run release:local-embeddings-smoke +``` + +Expected: FAIL with: + +```text +Skipping KTX local embeddings runtime smoke. Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke. +``` + +The command must exit non-zero because `--require-opt-in` is present. This +protects local and CI runs from downloading large dependencies by accident. + +- [ ] **Step 4: Commit the package script** + +Run: + +```bash +git add package.json +git commit -m "chore: register local embeddings smoke" +``` + +### Task 4: Verify the opt-in smoke path + +**Files:** + +- Verify: `scripts/local-embeddings-runtime-smoke.mjs` +- Verify: `scripts/local-embeddings-runtime-smoke.test.mjs` +- Verify: `package.json` + +- [ ] **Step 1: Run fast script tests** + +Run: + +```bash +node --test scripts/local-embeddings-runtime-smoke.test.mjs scripts/package-artifacts.test.mjs +``` + +Expected: PASS. Existing package artifact tests must still prove that the +default npm artifact smoke does not prepare an external Python environment or +run local embeddings downloads. + +- [ ] **Step 2: Build release artifacts for the smoke** + +Run: + +```bash +pnpm run artifacts:build +``` + +Expected: PASS and `dist/artifacts/npm/` contains exactly one +`kaelio-ktx-*.tgz` tarball. + +- [ ] **Step 3: Run the opt-in local embeddings smoke** + +Run this only in an environment where downloading `sentence-transformers`, +`torch`, and `all-MiniLM-L6-v2` is acceptable: + +```bash +KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 pnpm run release:local-embeddings-smoke +``` + +Expected: PASS with output containing: + +```text +KTX local embeddings daemon computed a 384-dimensional embedding +KTX setup persisted managed local embeddings marker +KTX local embeddings runtime smoke verified +``` + +- [ ] **Step 4: Run release readiness** + +Run: + +```bash +pnpm run release:readiness +``` + +Expected: PASS. The readiness report must not require +`release:local-embeddings-smoke`; that smoke remains a separately triggered +release job. + +- [ ] **Step 5: Run pre-commit for changed files when configured** + +Run: + +```bash +uv run pre-commit run --files scripts/local-embeddings-runtime-smoke.mjs scripts/local-embeddings-runtime-smoke.test.mjs package.json +``` + +Expected: PASS. If pre-commit is unavailable in the environment, record the +tooling failure and keep the previous verification output. + +- [ ] **Step 6: Commit verification fixes if needed** + +If verification required edits, run: + +```bash +git add scripts/local-embeddings-runtime-smoke.mjs scripts/local-embeddings-runtime-smoke.test.mjs package.json +git commit -m "fix: verify local embeddings smoke" +``` + +Skip this commit when no files changed after the previous commits. + +## Acceptance criteria + +- `node --test scripts/local-embeddings-runtime-smoke.test.mjs` passes. +- `pnpm run release:local-embeddings-smoke` fails fast without the opt-in + environment variable and prints the exact opt-in guidance. +- `KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 pnpm run release:local-embeddings-smoke` + installs the public `@kaelio/ktx` tarball into a clean project, isolates + `KTX_RUNTIME_ROOT` and model caches, installs `local-embeddings`, starts the + managed daemon, computes a 384-dimensional embedding through + `/embeddings/compute`, runs setup with `--embedding-backend + sentence-transformers`, verifies `base_url: managed:local-embeddings` in + `ktx.yaml`, and stops the daemon. +- The default `pnpm run artifacts:verify`, `pnpm run release:readiness`, and + `pnpm run check` paths do not run the local embeddings smoke. + +## Self-review + +- Spec coverage: this plan covers the remaining release-check item for local + embeddings in a separate job or opt-in check. Earlier implemented plans cover + the bundled wheel, managed runtime installer, `sl query` command integration, + daemon lifecycle, managed local embeddings runtime behavior, public npm + package assembly, and default core runtime release smoke. +- Placeholder scan: no steps contain placeholder implementation language. +- Type consistency: runtime feature names are consistently `core` and + `local-embeddings`; the public npm package name is `@kaelio/ktx`; the opt-in + environment variable is `KTX_RUN_LOCAL_EMBEDDINGS_SMOKE`; the managed local + embedding marker remains `managed:local-embeddings`. diff --git a/docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md b/docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md new file mode 100644 index 00000000..c2023c96 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md @@ -0,0 +1,1122 @@ +# Managed Local Embeddings Runtime Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make local `sentence-transformers` embedding setup use the +KTX-managed Python runtime and daemon instead of requiring users to start a +manual `ktx-daemon` process. + +**Architecture:** Add one managed local-embedding helper in the CLI that +prompts or fails according to the existing runtime install policy, starts the +managed daemon with the `local-embeddings` feature, and returns the daemon URL +for health checks. Store a stable managed-runtime marker in `ktx.yaml`, and +teach context embedding config resolution to turn that marker into a daemon URL +only when the CLI has provided one through the environment. + +**Tech Stack:** TypeScript, Vitest, Commander, `@clack/prompts`, KTX managed +Python runtime commands, `@ktx/llm` embedding health checks. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +Existing plans based on the spec: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` is + implemented. The worktree contains the runtime wheel builder, runtime wheel + packaging, the `kaelio-ktx` Python artifact policy entry, and matching + artifact tests. +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` is + implemented. The worktree contains `managed-python-runtime.ts`, the runtime + command runner, `runtime install`, `status`, `doctor`, and `prune` command + registration, and matching CLI tests. +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` + is implemented. The worktree contains `managed-python-command.ts`, `ktx sl + query` runtime policy flags, schema validation, and matching `sl` tests. +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` + is implemented. The worktree contains `managed-python-daemon.ts`, daemon + state paths in the runtime layout, `runtime start`, `runtime stop`, Python + `/health` version metadata, and matching TypeScript and Python tests. + +Spec requirements still outside this plan: + +- Public npm package surface rename from `@ktx/cli` to `@kaelio/ktx`. +- Managed runtime usage for non-embedding Python-backed command paths beyond + `ktx sl query`. +- Release smoke coverage for `npx @kaelio/ktx ...` invocation modes. + +This plan implements the next local-embedding runtime slice: + +- Selecting local embeddings installs only the `local-embeddings` runtime + feature. +- Local embedding setup starts or reuses the managed HTTP daemon. +- `--yes` installs and starts without prompting. +- `--no-input` fails with an exact preparation command when the managed local + embedding runtime is missing. +- Project config records a managed local embedding marker instead of a random + daemon port. +- Context embedding resolution only resolves the marker when the CLI provides + the active daemon URL. + +## File structure + +- Modify `packages/context/src/llm/local-config.ts`: define the managed local + embeddings marker and environment variable, and resolve that marker to a + runtime daemon URL. +- Modify `packages/context/src/llm/local-config.test.ts`: cover marker + resolution, missing daemon URL behavior, and provider construction. +- Modify `packages/context/src/llm/index.ts`: export the marker constants. +- Modify `packages/context/src/package-exports.test.ts`: assert root exports + expose the marker constants. +- Create `packages/cli/src/managed-local-embeddings.ts`: start or reuse the + managed daemon with `local-embeddings` and build health/project configs. +- Create `packages/cli/src/managed-local-embeddings.test.ts`: cover ready, + `--yes`, prompt, and `--no-input` behavior. +- Modify `packages/cli/src/setup-embeddings.ts`: use the managed helper for + local embeddings and persist the managed marker. +- Modify `packages/cli/src/setup-embeddings.test.ts`: update local embedding + setup expectations and no-input failure behavior. +- Modify `packages/cli/src/setup.ts`: pass CLI version and runtime install + policy into the embeddings step. +- Modify `packages/cli/src/commands/setup-commands.ts`: attach package version + to setup runs. +- Modify `packages/cli/src/cli-program.ts`: attach package version to the bare + interactive setup path. +- Modify `packages/cli/src/index.ts`: export the managed local embedding helper + for tests and programmatic use. +- Modify `packages/cli/src/index.test.ts` and `packages/cli/src/setup.test.ts`: + update setup argument expectations for `cliVersion`. + +### Task 1: Add managed embedding marker resolution in context + +**Files:** + +- Modify: `packages/context/src/llm/local-config.test.ts` +- Modify: `packages/context/src/llm/local-config.ts` +- Modify: `packages/context/src/llm/index.ts` +- Modify: `packages/context/src/package-exports.test.ts` + +- [ ] **Step 1: Write failing marker resolution tests** + +In `packages/context/src/llm/local-config.test.ts`, extend the import from +`./local-config.js` so it includes the new constants: + +```typescript +import { + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, + createLocalKtxEmbeddingProviderFromConfig, + createLocalKtxLlmProviderFromConfig, + resolveLocalKtxEmbeddingConfig, + resolveLocalKtxLlmConfig, +} from './local-config.js'; +``` + +Add these tests inside `describe('local KTX embedding config', () => { ... })` +after the existing `resolves sentence-transformers config` test: + +```typescript + it('resolves managed sentence-transformers config from the CLI-provided daemon URL', () => { + const config: KtxProjectEmbeddingConfig = { + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { + base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + pathPrefix: '', + }, + batchSize: 32, + }; + + expect( + resolveLocalKtxEmbeddingConfig(config, { + [MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: 'http://127.0.0.1:61234', + }), + ).toEqual({ + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, + batchSize: 32, + }); + }); + + it('returns null for managed sentence-transformers when no daemon URL is available', () => { + const config: KtxProjectEmbeddingConfig = { + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { + base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + pathPrefix: '', + }, + }; + + expect(resolveLocalKtxEmbeddingConfig(config, {})).toBeNull(); + }); +``` + +In `packages/context/src/package-exports.test.ts`, add these assertions after +the existing `root.createLocalKtxEmbeddingProviderFromConfig` assertion: + +```typescript + expect(root.MANAGED_SENTENCE_TRANSFORMERS_BASE_URL).toBe('managed:local-embeddings'); + expect(root.MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV).toBe( + 'KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL', + ); +``` + +- [ ] **Step 2: Run the failing context tests** + +Run: + +```bash +pnpm --filter @ktx/context run test -- src/llm/local-config.test.ts src/package-exports.test.ts +``` + +Expected: FAIL with missing exports for +`MANAGED_SENTENCE_TRANSFORMERS_BASE_URL` and +`MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV`. + +- [ ] **Step 3: Implement marker resolution** + +In `packages/context/src/llm/local-config.ts`, add these exports after the +`LocalConfigDeps` interface: + +```typescript +export const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL = 'managed:local-embeddings'; +export const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV = 'KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL'; +``` + +Add this helper before `resolveLocalKtxEmbeddingConfig`: + +```typescript +function resolveSentenceTransformersBaseUrl(value: string | undefined, env: NodeJS.ProcessEnv): string | undefined { + if (!value) { + return undefined; + } + if (value === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL) { + return resolveOptional(`env:${MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV}`, env); + } + return value; +} +``` + +Replace `resolveLocalKtxEmbeddingConfig` with this implementation: + +```typescript +export function resolveLocalKtxEmbeddingConfig( + config: KtxProjectEmbeddingConfig, + env: NodeJS.ProcessEnv, +): KtxEmbeddingConfig | null { + if (config.backend === 'none') { + return null; + } + if (config.backend === 'sentence-transformers') { + const baseURL = resolveSentenceTransformersBaseUrl(config.sentenceTransformers?.base_url, env); + if (!baseURL) { + return null; + } + return { + backend: config.backend, + model: config.model ?? 'all-MiniLM-L6-v2', + dimensions: config.dimensions, + sentenceTransformers: { + baseURL, + pathPrefix: config.sentenceTransformers?.pathPrefix, + }, + batchSize: config.batchSize, + }; + } + return { + backend: config.backend, + model: config.model ?? 'deterministic', + dimensions: config.dimensions, + ...(resolvedProviderConfig(config.openai, env) ? { openai: resolvedProviderConfig(config.openai, env) } : {}), + batchSize: config.batchSize, + }; +} +``` + +In `packages/context/src/llm/index.ts`, add the new constants to the existing +export from `./local-config.js`: + +```typescript +export { + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, + createLocalKtxEmbeddingProviderFromConfig, + createLocalKtxLlmProviderFromConfig, + resolveLocalKtxEmbeddingConfig, + resolveLocalKtxLlmConfig, +} from './local-config.js'; +``` + +- [ ] **Step 4: Verify the context marker tests pass** + +Run: + +```bash +pnpm --filter @ktx/context run test -- src/llm/local-config.test.ts src/package-exports.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add packages/context/src/llm/local-config.ts packages/context/src/llm/local-config.test.ts packages/context/src/llm/index.ts packages/context/src/package-exports.test.ts +git commit -m "feat: add managed local embeddings config marker" +``` + +### Task 2: Add the managed local embeddings CLI helper + +**Files:** + +- Create: `packages/cli/src/managed-local-embeddings.test.ts` +- Create: `packages/cli/src/managed-local-embeddings.ts` +- Modify: `packages/cli/src/index.ts` + +- [ ] **Step 1: Write the failing helper tests** + +Create `packages/cli/src/managed-local-embeddings.test.ts` with this content: + +```typescript +import { describe, expect, it, vi } from 'vitest'; +import { + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, +} from '@ktx/context'; +import { + ensureManagedLocalEmbeddingsDaemon, + managedLocalEmbeddingHealthConfig, + managedLocalEmbeddingProjectConfig, +} from './managed-local-embeddings.js'; +import type { ManagedPythonCommandRuntime } from './managed-python-command.js'; +import type { ManagedPythonDaemonStartResult } from './managed-python-daemon.js'; + +function makeIo() { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +function runtime(): ManagedPythonCommandRuntime { + return { + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', + }, + manifest: { + schemaVersion: 1, + cliVersion: '0.2.0', + installedAt: '2026-05-11T00:00:00.000Z', + asset: { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.2.0', + wheel: { + file: 'kaelio_ktx-0.2.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 123, + }, + }, + features: ['core', 'local-embeddings'], + python: { + executable: '/runtime/0.2.0/.venv/bin/python', + daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon', + }, + installLog: '/runtime/0.2.0/install.log', + }, + }; +} + +function daemonResult(status: 'started' | 'reused' = 'reused'): ManagedPythonDaemonStartResult { + return { + status, + layout: runtime().layout, + baseUrl: 'http://127.0.0.1:61234', + state: { + schemaVersion: 1, + pid: 12345, + host: '127.0.0.1', + port: 61234, + version: '0.2.0', + features: ['core', 'local-embeddings'], + startedAt: '2026-05-11T00:00:00.000Z', + stdoutLog: '/runtime/0.2.0/daemon.stdout.log', + stderrLog: '/runtime/0.2.0/daemon.stderr.log', + }, + }; +} + +describe('managedLocalEmbeddingProjectConfig', () => { + it('uses a stable managed runtime marker instead of a random daemon port', () => { + expect( + managedLocalEmbeddingProjectConfig({ + model: 'all-MiniLM-L6-v2', + dimensions: 384, + }), + ).toEqual({ + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { + base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + pathPrefix: '', + }, + }); + }); +}); + +describe('managedLocalEmbeddingHealthConfig', () => { + it('uses the active managed daemon URL for the immediate health check', () => { + expect( + managedLocalEmbeddingHealthConfig({ + baseUrl: 'http://127.0.0.1:61234', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + }), + ).toEqual({ + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, + }); + }); +}); + +describe('ensureManagedLocalEmbeddingsDaemon', () => { + it('ensures the local-embeddings feature and starts the managed daemon', async () => { + const io = makeIo(); + const ensureRuntime = vi.fn(async () => runtime()); + const startDaemon = vi.fn(async () => daemonResult('started')); + + await expect( + ensureManagedLocalEmbeddingsDaemon({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + ensureRuntime, + startDaemon, + }), + ).resolves.toEqual({ + baseUrl: 'http://127.0.0.1:61234', + env: { + [MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: 'http://127.0.0.1:61234', + }, + }); + + expect(ensureRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + feature: 'local-embeddings', + }); + expect(startDaemon).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['local-embeddings'], + force: false, + }); + expect(io.stderr()).toContain('Started KTX local embeddings daemon: http://127.0.0.1:61234'); + }); + + it('reuses an already running daemon without reporting a new start', async () => { + const io = makeIo(); + + await ensureManagedLocalEmbeddingsDaemon({ + cliVersion: '0.2.0', + installPolicy: 'prompt', + io: io.io, + ensureRuntime: vi.fn(async () => runtime()), + startDaemon: vi.fn(async () => daemonResult('reused')), + }); + + expect(io.stderr()).toContain('Using KTX local embeddings daemon: http://127.0.0.1:61234'); + }); +}); +``` + +- [ ] **Step 2: Run the failing helper tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-local-embeddings.test.ts +``` + +Expected: FAIL with an import error for +`./managed-local-embeddings.js`. + +- [ ] **Step 3: Implement the helper** + +Create `packages/cli/src/managed-local-embeddings.ts` with this content: + +```typescript +import { + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, +} from '@ktx/context'; +import type { KtxProjectEmbeddingConfig } from '@ktx/context/project'; +import type { KtxEmbeddingConfig } from '@ktx/llm'; +import type { KtxCliIo } from './cli-runtime.js'; +import { + ensureManagedPythonCommandRuntime, + type KtxManagedPythonInstallPolicy, + type ManagedPythonCommandRuntime, +} from './managed-python-command.js'; +import { startManagedPythonDaemon, type ManagedPythonDaemonStartResult } from './managed-python-daemon.js'; + +export interface ManagedLocalEmbeddingsDaemon { + baseUrl: string; + env: Record; +} + +export interface ManagedLocalEmbeddingsOptions { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + ensureRuntime?: (options: { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + feature: 'local-embeddings'; + }) => Promise; + startDaemon?: (options: { + cliVersion: string; + features: ['local-embeddings']; + force: boolean; + }) => Promise; +} + +export function managedLocalEmbeddingProjectConfig(input: { + model: string; + dimensions: number; +}): KtxProjectEmbeddingConfig { + return { + backend: 'sentence-transformers', + model: input.model, + dimensions: input.dimensions, + sentenceTransformers: { + base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + pathPrefix: '', + }, + }; +} + +export function managedLocalEmbeddingHealthConfig(input: { + baseUrl: string; + model: string; + dimensions: number; +}): KtxEmbeddingConfig { + return { + backend: 'sentence-transformers', + model: input.model, + dimensions: input.dimensions, + sentenceTransformers: { + baseURL: input.baseUrl, + pathPrefix: '', + }, + }; +} + +export async function ensureManagedLocalEmbeddingsDaemon( + options: ManagedLocalEmbeddingsOptions, +): Promise { + const ensureRuntime = options.ensureRuntime ?? ensureManagedPythonCommandRuntime; + const startDaemon = options.startDaemon ?? startManagedPythonDaemon; + + await ensureRuntime({ + cliVersion: options.cliVersion, + installPolicy: options.installPolicy, + io: options.io, + feature: 'local-embeddings', + }); + const daemon = await startDaemon({ + cliVersion: options.cliVersion, + features: ['local-embeddings'], + force: false, + }); + + const verb = daemon.status === 'started' ? 'Started' : 'Using'; + options.io.stderr.write(`${verb} KTX local embeddings daemon: ${daemon.baseUrl}\n`); + + return { + baseUrl: daemon.baseUrl, + env: { + [MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: daemon.baseUrl, + }, + }; +} +``` + +In `packages/cli/src/index.ts`, add this export after the existing +`managed-python-daemon.js` exports: + +```typescript +export { + ensureManagedLocalEmbeddingsDaemon, + managedLocalEmbeddingHealthConfig, + managedLocalEmbeddingProjectConfig, + type ManagedLocalEmbeddingsDaemon, + type ManagedLocalEmbeddingsOptions, +} from './managed-local-embeddings.js'; +``` + +- [ ] **Step 4: Verify helper tests pass** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-local-embeddings.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add packages/cli/src/managed-local-embeddings.ts packages/cli/src/managed-local-embeddings.test.ts packages/cli/src/index.ts +git commit -m "feat: add managed local embeddings daemon helper" +``` + +### Task 3: Wire setup embeddings to the managed runtime + +**Files:** + +- Modify: `packages/cli/src/setup-embeddings.ts` +- Modify: `packages/cli/src/setup-embeddings.test.ts` + +- [ ] **Step 1: Write failing setup tests for managed local embeddings** + +In `packages/cli/src/setup-embeddings.test.ts`, update the import from +`./setup-embeddings.js` so it also imports the managed install policy type: + +```typescript +import { + type KtxSetupEmbeddingsPromptAdapter, + runKtxSetupEmbeddingsStep, +} from './setup-embeddings.js'; +``` + +Add this helper near `makePromptAdapter`: + +```typescript +function managedDaemon(baseUrl = 'http://127.0.0.1:61234') { + return { + baseUrl, + env: { + KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL: baseUrl, + }, + }; +} +``` + +In every `runKtxSetupEmbeddingsStep` call that does not inject an `embeddingBackend: +'openai'`, add these arguments: + +```typescript + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', +``` + +In the test named `configures local sentence-transformers embeddings after +interactive selection`, add this dependency: + +```typescript + const ensureLocalEmbeddings = vi.fn(async () => managedDaemon()); +``` + +Pass it in the deps object: + +```typescript + { prompts, env: {}, healthCheck, ensureLocalEmbeddings }, +``` + +Replace the expected health check config in that test with: + +```typescript + expect(ensureLocalEmbeddings).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + }); + expect(healthCheck).toHaveBeenCalledWith({ + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, + }); +``` + +Replace the persisted local embedding expectation in that test with: + +```typescript + expect(config.ingest.embeddings).toMatchObject({ + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' }, + }); +``` + +Add this new test after the existing non-interactive local embeddings test: + +```typescript + it('fails non-interactive local setup when the managed local embeddings runtime is missing', async () => { + const io = makeIo(); + const ensureLocalEmbeddings = vi.fn(async () => { + throw new Error( + 'KTX Python runtime is required for this command. Run: ktx runtime install --feature local-embeddings --yes', + ); + }); + + const result = await runKtxSetupEmbeddingsStep( + { + projectDir: tempDir, + inputMode: 'disabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'never', + skipEmbeddings: false, + }, + io.io, + { env: {}, ensureLocalEmbeddings }, + ); + + expect(result.status).toBe('failed'); + expect(io.stderr()).toContain( + 'KTX Python runtime is required for this command. Run: ktx runtime install --feature local-embeddings --yes', + ); + }); +``` + +- [ ] **Step 2: Run the failing setup tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/setup-embeddings.test.ts +``` + +Expected: FAIL because `KtxSetupEmbeddingsArgs` has no `cliVersion` or +`runtimeInstallPolicy`, and `KtxSetupEmbeddingsDeps` has no +`ensureLocalEmbeddings`. + +- [ ] **Step 3: Update setup embeddings types and imports** + +In `packages/cli/src/setup-embeddings.ts`, add these imports: + +```typescript +import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; +import { + ensureManagedLocalEmbeddingsDaemon, + managedLocalEmbeddingHealthConfig, + managedLocalEmbeddingProjectConfig, + type ManagedLocalEmbeddingsDaemon, +} from './managed-local-embeddings.js'; +``` + +Add these fields to `KtxSetupEmbeddingsArgs` after `inputMode`: + +```typescript + cliVersion: string; + runtimeInstallPolicy: KtxManagedPythonInstallPolicy; +``` + +Add this dependency to `KtxSetupEmbeddingsDeps`: + +```typescript + ensureLocalEmbeddings?: (options: { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + }) => Promise; +``` + +- [ ] **Step 4: Replace manual local daemon messaging and config** + +In `packages/cli/src/setup-embeddings.ts`, remove these constants: + +```typescript +const LOCAL_EMBEDDING_DAEMON_COMMAND = 'ktx-daemon serve-http --host 127.0.0.1 --port 8765'; +const LOCAL_EMBEDDING_DAEMON_DEV_COMMAND = + 'cd ktx && source .venv/bin/activate && uv run ktx-daemon serve-http --host 127.0.0.1 --port 8765'; +``` + +Replace `localEmbeddingSetupMessage` with: + +```typescript +function localEmbeddingSetupMessage(message: string): string { + return [ + `Local embedding health check failed: ${message}`, + 'Local embeddings use the KTX-managed Python runtime.', + 'Prepare the runtime with: ktx runtime start --feature local-embeddings', + 'Use --yes with setup to install and start the runtime without prompting.', + 'The first run may download Python packages and the all-MiniLM-L6-v2 model.', + ].join('\n'); +} +``` + +Inside `runKtxSetupEmbeddingsStep`, before building `healthConfig`, add this +block after the OpenAI credential block: + +```typescript + let managedLocalEmbeddings: ManagedLocalEmbeddingsDaemon | undefined; + if (selectedBackend === LOCAL_EMBEDDING_BACKEND) { + const ensureLocalEmbeddings = deps.ensureLocalEmbeddings ?? ensureManagedLocalEmbeddingsDaemon; + try { + managedLocalEmbeddings = await ensureLocalEmbeddings({ + cliVersion: args.cliVersion, + installPolicy: args.runtimeInstallPolicy, + io, + }); + } catch (error) { + io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + return { status: 'failed', projectDir: args.projectDir }; + } + } +``` + +Replace the `healthConfig` assignment with: + +```typescript + const healthConfig = + selectedBackend === LOCAL_EMBEDDING_BACKEND && managedLocalEmbeddings + ? managedLocalEmbeddingHealthConfig({ + baseUrl: managedLocalEmbeddings.baseUrl, + model, + dimensions, + }) + : buildHealthConfig({ + backend: selectedBackend, + model, + dimensions, + credentialValue, + }); +``` + +Replace the successful local persistence call inside `if (health.ok) { ... }` +with: + +```typescript + await persistEmbeddingConfig( + args.projectDir, + selectedBackend === LOCAL_EMBEDDING_BACKEND + ? managedLocalEmbeddingProjectConfig({ model, dimensions }) + : buildProjectEmbeddingConfig({ + backend: selectedBackend, + model, + dimensions, + credentialRef, + }), + ); +``` + +- [ ] **Step 5: Verify setup embeddings tests pass** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/setup-embeddings.test.ts src/managed-local-embeddings.test.ts +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +Run: + +```bash +git add packages/cli/src/setup-embeddings.ts packages/cli/src/setup-embeddings.test.ts +git commit -m "feat: use managed runtime for local embedding setup" +``` + +### Task 4: Pass runtime policy and CLI version through setup commands + +**Files:** + +- Modify: `packages/cli/src/setup.ts` +- Modify: `packages/cli/src/commands/setup-commands.ts` +- Modify: `packages/cli/src/cli-program.ts` +- Modify: `packages/cli/src/setup.test.ts` +- Modify: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Write failing setup argument expectations** + +In `packages/cli/src/index.test.ts`, find the test that routes the main setup +command and add `cliVersion: '0.0.0-private'` to the expected setup run +argument object. + +Add this assertion to the same test when `--yes` is present: + +```typescript + yes: true, + cliVersion: '0.0.0-private', +``` + +In `packages/cli/src/setup.test.ts`, find the setup test that asserts the +embeddings runner arguments. Add these expected fields to the embeddings step +argument object: + +```typescript + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', +``` + +Add one focused unit test near the other setup flow tests: + +```typescript + it('passes no-input runtime policy to the embeddings step', async () => { + const io = makeIo(); + const embeddings = vi.fn(async () => ({ status: 'failed' as const, projectDir: tempDir })); + + await expect( + runKtxSetup( + { + command: 'run', + projectDir: tempDir, + mode: 'existing', + agents: false, + agentScope: 'project', + agentInstallMode: 'cli', + skipAgents: true, + inputMode: 'disabled', + yes: false, + cliVersion: '0.2.0', + skipLlm: true, + skipEmbeddings: false, + databaseSchemas: [], + skipDatabases: true, + skipSources: true, + }, + io.io, + { + project: { + run: vi.fn(async () => ({ status: 'ready' as const, projectDir: tempDir })), + }, + embeddings, + }, + ), + ).resolves.toBe(1); + + expect(embeddings).toHaveBeenCalledWith( + expect.objectContaining({ + cliVersion: '0.2.0', + runtimeInstallPolicy: 'never', + }), + io.io, + ); + }); +``` + +- [ ] **Step 2: Run the failing setup routing tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/setup.test.ts src/index.test.ts +``` + +Expected: FAIL because setup args do not carry `cliVersion` yet and embeddings +args do not derive `runtimeInstallPolicy`. + +- [ ] **Step 3: Add `cliVersion` to setup run args** + +In `packages/cli/src/setup.ts`, add this field to the run variant of +`KtxSetupArgs` immediately after `yes`: + +```typescript + cliVersion: string; +``` + +Add this helper near the other setup helpers: + +```typescript +function setupRuntimeInstallPolicy(args: Extract): 'prompt' | 'auto' | 'never' { + if (args.yes) { + return 'auto'; + } + return args.inputMode === 'disabled' ? 'never' : 'prompt'; +} +``` + +In the embeddings step call inside `runKtxSetupInner`, add: + +```typescript + cliVersion: args.cliVersion, + runtimeInstallPolicy: setupRuntimeInstallPolicy(args), +``` + +- [ ] **Step 4: Pass package version from Commander and bare setup** + +In `packages/cli/src/commands/setup-commands.ts`, add this field to the setup +run argument object: + +```typescript + cliVersion: context.packageInfo.version, +``` + +Place it immediately after `yes: options.yes === true`. + +In `packages/cli/src/cli-program.ts`, add this field to the bare interactive +setup argument object inside `runBareInteractiveCommand`: + +```typescript + cliVersion: context.packageInfo.version, +``` + +Place it immediately after `yes: false`. + +- [ ] **Step 5: Verify setup routing tests pass** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/setup.test.ts src/index.test.ts src/setup-embeddings.test.ts +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +Run: + +```bash +git add packages/cli/src/setup.ts packages/cli/src/commands/setup-commands.ts packages/cli/src/cli-program.ts packages/cli/src/setup.test.ts packages/cli/src/index.test.ts +git commit -m "feat: pass managed runtime policy through setup" +``` + +### Task 5: Final verification + +**Files:** + +- Verify: `packages/context/src/llm/local-config.ts` +- Verify: `packages/cli/src/managed-local-embeddings.ts` +- Verify: `packages/cli/src/setup-embeddings.ts` +- Verify: `packages/cli/src/setup.ts` + +- [ ] **Step 1: Run focused context tests** + +Run: + +```bash +pnpm --filter @ktx/context run test -- src/llm/local-config.test.ts src/package-exports.test.ts +``` + +Expected: PASS. + +- [ ] **Step 2: Run focused CLI tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-local-embeddings.test.ts src/setup-embeddings.test.ts src/setup.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 3: Run TypeScript checks for changed packages** + +Run: + +```bash +pnpm --filter @ktx/context run type-check +pnpm --filter @ktx/cli run type-check +``` + +Expected: PASS. + +- [ ] **Step 4: Run package-level tests if type-check changed public exports** + +Run: + +```bash +pnpm --filter @ktx/context run test +pnpm --filter @ktx/cli run test +``` + +Expected: PASS. + +- [ ] **Step 5: Run pre-commit for changed files** + +Run: + +```bash +uv run pre-commit run --files packages/context/src/llm/local-config.ts packages/context/src/llm/local-config.test.ts packages/context/src/llm/index.ts packages/context/src/package-exports.test.ts packages/cli/src/managed-local-embeddings.ts packages/cli/src/managed-local-embeddings.test.ts packages/cli/src/setup-embeddings.ts packages/cli/src/setup-embeddings.test.ts packages/cli/src/setup.ts packages/cli/src/commands/setup-commands.ts packages/cli/src/cli-program.ts packages/cli/src/setup.test.ts packages/cli/src/index.test.ts packages/cli/src/index.ts +``` + +Expected: PASS. If pre-commit is unavailable because local hook versions are +missing, run the focused tests and type-check commands from steps 1 through 3 +and record the pre-commit error. + +- [ ] **Step 6: Commit final verification adjustments** + +Run this only if final verification required small fixes: + +```bash +git add packages/context/src/llm/local-config.ts packages/context/src/llm/local-config.test.ts packages/context/src/llm/index.ts packages/context/src/package-exports.test.ts packages/cli/src/managed-local-embeddings.ts packages/cli/src/managed-local-embeddings.test.ts packages/cli/src/setup-embeddings.ts packages/cli/src/setup-embeddings.test.ts packages/cli/src/setup.ts packages/cli/src/commands/setup-commands.ts packages/cli/src/cli-program.ts packages/cli/src/setup.test.ts packages/cli/src/index.test.ts packages/cli/src/index.ts +git commit -m "test: verify managed local embeddings runtime setup" +``` + +## Acceptance criteria + +- `ktx setup --embedding-backend sentence-transformers --yes` installs the + `local-embeddings` runtime feature when needed, starts or reuses the managed + daemon, probes the active daemon URL, and writes `managed:local-embeddings` + to `ktx.yaml`. +- `ktx setup --embedding-backend sentence-transformers --no-input` fails with + the exact runtime preparation command when the runtime is missing. +- Existing OpenAI embedding setup behavior is unchanged. +- The project config no longer stores the daemon's random port. +- `resolveLocalKtxEmbeddingConfig` returns a usable `KtxEmbeddingConfig` for + managed local embeddings only when + `KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL` is present. +- Focused CLI and context tests pass. + +## Self-review + +- Spec coverage: This plan covers lazy `local-embeddings` installation after + local embeddings are selected, separate prompt/no-input behavior, and managed + daemon reuse for local embedding setup health checks. +- Placeholder scan: This plan contains concrete file paths, code snippets, + commands, expected outcomes, and commit commands. +- Type consistency: The new `ManagedLocalEmbeddingsDaemon` type, managed marker + constants, setup argument fields, and helper function names are used + consistently across tasks. diff --git a/docs/superpowers/plans/2026-05-11-managed-local-embeddings-smoke-public-version.md b/docs/superpowers/plans/2026-05-11-managed-local-embeddings-smoke-public-version.md new file mode 100644 index 00000000..0495ab97 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-local-embeddings-smoke-public-version.md @@ -0,0 +1,239 @@ +# Managed Local Embeddings Smoke Public Version Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the opt-in local embeddings release smoke validate the public +`@kaelio/ktx` package version instead of the private workspace version. + +**Architecture:** Reuse the public package constants from +`scripts/build-public-npm-package.mjs` inside the local embeddings smoke. Add a +small exported RegExp helper so the unit test can lock the version expectation +without running the expensive model-download smoke. + +**Tech Stack:** Node.js ESM scripts, `node:test`, pnpm release scripts. + +--- + +## Current State + +The npm-managed Python runtime spec is +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. +The current branch already contains implementation commits for each existing +plan derived from that spec. + +Implemented spec-derived plans: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` +- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` +- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md` +- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md` +- `docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md` +- `docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md` +- `docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md` + +The remaining gap is in +`scripts/local-embeddings-runtime-smoke.mjs`. The script selects and installs a +public tarball named `kaelio-ktx-*.tgz` and writes a smoke package dependency on +`@kaelio/ktx`, but line 267 still expects `@kaelio/ktx 0.0.0-private`. The +public package builder defines `PUBLIC_NPM_PACKAGE_VERSION = '0.1.0'`, and the +main packed-package smoke already expects `@kaelio/ktx 0.1.0`. + +## File Structure + +This change keeps the release version source of truth in one script and reuses +it from the opt-in smoke. + +- Modify `scripts/local-embeddings-runtime-smoke.mjs`: import the public package + constants, export `expectedPublicKtxVersionPattern()`, and use that pattern + for the smoke version assertion. +- Modify `scripts/local-embeddings-runtime-smoke.test.mjs`: import + `expectedPublicKtxVersionPattern()` and assert that it accepts + `@kaelio/ktx 0.1.0` and rejects `@kaelio/ktx 0.0.0-private`. + +### Task 1: Align the local embeddings smoke version assertion + +**Files:** +- Modify: `scripts/local-embeddings-runtime-smoke.mjs:1-267` +- Modify: `scripts/local-embeddings-runtime-smoke.test.mjs:5-118` +- Test: `scripts/local-embeddings-runtime-smoke.test.mjs` + +- [ ] **Step 1: Write the failing version-pattern test** + +In `scripts/local-embeddings-runtime-smoke.test.mjs`, update the import block +to include `expectedPublicKtxVersionPattern`: + +```js +import { + buildLocalEmbeddingsSmokeEnv, + expectedPublicKtxVersionPattern, + localEmbeddingsSmokeCommands, + localEmbeddingsSmokeOptIn, + parseDaemonBaseUrl, + publicKtxTarballName, + validateEmbeddingResponse, +} from './local-embeddings-runtime-smoke.mjs'; +``` + +Then add this test after the `publicKtxTarballName` describe block: + +```js +describe('expectedPublicKtxVersionPattern', () => { + it('matches the public package version and rejects the private workspace version', () => { + const pattern = expectedPublicKtxVersionPattern(); + + assert.match('@kaelio/ktx 0.1.0\n', pattern); + assert.doesNotMatch('@kaelio/ktx 0.0.0-private\n', pattern); + }); +}); +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: + +```bash +node --test scripts/local-embeddings-runtime-smoke.test.mjs +``` + +Expected: FAIL with an ESM export error that says +`expectedPublicKtxVersionPattern` is not exported from +`./local-embeddings-runtime-smoke.mjs`. + +- [ ] **Step 3: Import the public package constants** + +In `scripts/local-embeddings-runtime-smoke.mjs`, add this import after the +existing Node imports: + +```js +import { + PUBLIC_NPM_PACKAGE_NAME, + PUBLIC_NPM_PACKAGE_VERSION, +} from './build-public-npm-package.mjs'; +``` + +The top of the file becomes: + +```js +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +import { + PUBLIC_NPM_PACKAGE_NAME, + PUBLIC_NPM_PACKAGE_VERSION, +} from './build-public-npm-package.mjs'; +``` + +- [ ] **Step 4: Add the version-pattern helper** + +In `scripts/local-embeddings-runtime-smoke.mjs`, add these functions after the +`OPT_IN_MESSAGE` constant: + +```js +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +export function expectedPublicKtxVersionPattern() { + return new RegExp( + `${escapeRegExp(PUBLIC_NPM_PACKAGE_NAME)} ${escapeRegExp(PUBLIC_NPM_PACKAGE_VERSION)}`, + ); +} +``` + +- [ ] **Step 5: Use the helper in the smoke** + +In `scripts/local-embeddings-runtime-smoke.mjs`, replace this line: + +```js +requireOutput(commands[0].label, version, /@kaelio\/ktx 0\.0\.0-private/); +``` + +with: + +```js +requireOutput(commands[0].label, version, expectedPublicKtxVersionPattern()); +``` + +- [ ] **Step 6: Run the focused test** + +Run: + +```bash +node --test scripts/local-embeddings-runtime-smoke.test.mjs +``` + +Expected: PASS. The new test proves the smoke accepts `@kaelio/ktx 0.1.0` and +rejects `@kaelio/ktx 0.0.0-private`. + +- [ ] **Step 7: Run related release-script tests** + +Run: + +```bash +node --test scripts/local-embeddings-runtime-smoke.test.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs +``` + +Expected: PASS. These tests cover the public package constants, tarball name, +artifact smoke source, and local embeddings smoke helpers. + +- [ ] **Step 8: Run a stale-expectation search** + +Run: + +```bash +rg -n "@kaelio/ktx 0\\.0\\.0-private|0\\\\\\.0\\\\\\.0-private" scripts/local-embeddings-runtime-smoke.mjs +``` + +Expected: no output. The opt-in local embeddings smoke no longer contains the +private package version expectation. The test file still uses +`@kaelio/ktx 0.0.0-private` as a negative fixture. + +- [ ] **Step 9: Commit** + +Run: + +```bash +git add scripts/local-embeddings-runtime-smoke.mjs scripts/local-embeddings-runtime-smoke.test.mjs +git commit -m "fix: align local embeddings smoke with public version" +``` + +## Verification + +Run these checks before marking the plan complete: + +```bash +node --test scripts/local-embeddings-runtime-smoke.test.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs +rg -n "@kaelio/ktx 0\\.0\\.0-private|0\\\\\\.0\\\\\\.0-private" scripts/local-embeddings-runtime-smoke.mjs +``` + +Expected results: + +- `node --test ...` exits with code 0. +- `rg ...` prints no matches. +- No Python files changed, so the repository Python pre-commit requirement does + not apply. + +## Self-Review + +- Spec coverage: this plan fixes the opt-in local embeddings release smoke from + the npm-managed runtime spec so it validates the public npm package produced + by the current release artifact flow. +- Placeholder scan: the plan contains concrete file paths, code blocks, + commands, and expected outcomes. +- Type consistency: the helper name is consistently + `expectedPublicKtxVersionPattern`, and it uses + `PUBLIC_NPM_PACKAGE_NAME` plus `PUBLIC_NPM_PACKAGE_VERSION` from the public + package builder. diff --git a/docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md b/docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md new file mode 100644 index 00000000..fbba6289 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md @@ -0,0 +1,1650 @@ +# Managed Local Ingest Daemon Runtime Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make local ingest, scan, and MCP daemon-backed helper paths use the +KTX-managed core Python daemon instead of requiring `KTX_DAEMON_URL` or a +manually started daemon on `127.0.0.1:8765`. + +**Architecture:** Add lazy managed-daemon HTTP ports in the CLI package. Thread +those ports through CLI local ingest adapter creation and pull-config options so +Looker table identifier parsing, historic SQL analysis, and live-database daemon +fallbacks resolve the managed core daemon only when a request is made. + +**Tech Stack:** TypeScript, Vitest, Commander, KTX CLI managed Python runtime, +KTX context local ingest adapters, MCP local project ports. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +The following plans are based on that spec and are already implemented in this +worktree: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` +- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` + +Implementation evidence found before writing this plan includes: + +- `scripts/build-python-runtime-wheel.mjs` and + `packages/cli/assets/python/manifest.json`. +- `packages/cli/src/managed-python-runtime.ts`, + `packages/cli/src/runtime.ts`, and + `packages/cli/src/commands/runtime-commands.ts`. +- `packages/cli/src/managed-python-command.ts` and managed `ktx sl query`, + hidden agent SL query, and MCP semantic compute paths. +- `packages/cli/src/managed-python-daemon.ts` and `ktx runtime start` / + `ktx runtime stop`. +- `packages/cli/src/managed-local-embeddings.ts` and local embeddings setup + wiring. +- `scripts/build-public-npm-package.mjs`, release policy updates, release + smoke coverage, and opt-in local embeddings smoke coverage. +- `packages/cli/src/agent-runtime.ts` and `packages/cli/src/serve.ts` now + create managed semantic-layer compute when no explicit semantic HTTP URL is + provided. + +The remaining spec gap is local ingest daemon-backed helper behavior: + +- `packages/context/src/ingest/local-adapters.ts` still creates the Looker + table identifier parser from `options.looker.daemonBaseUrl`, + `KTX_DAEMON_URL`, or `http://127.0.0.1:8765`. +- `packages/cli/src/local-adapters.ts` still creates historic SQL analysis from + `options.sqlAnalysisUrl`, `KTX_SQL_ANALYSIS_URL`, `KTX_DAEMON_URL`, or + `http://127.0.0.1:8765`. +- `packages/cli/src/serve.ts` passes adapters to MCP local ingest, but + `LocalIngestMcpOptions` has no `pullConfigOptions`, so Looker pull-config + generation cannot receive CLI-managed daemon options. + +This plan closes that gap without changing explicit daemon URL behavior. +Explicit `--database-introspection-url`, explicit test dependency injection, +`KTX_SQL_ANALYSIS_URL`, and `KTX_DAEMON_URL` continue to win over the managed +daemon. + +## File structure + +- Create `packages/cli/src/managed-python-http.ts`: lazy managed core daemon + resolver, generic HTTP JSON runner, managed Looker table identifier parser, + managed SQL analysis port, and managed live-database daemon request options. +- Create `packages/cli/src/managed-python-http.test.ts`: verifies lazy daemon + resolution, install policy propagation, daemon reuse caching, and HTTP runner + delegation. +- Modify `packages/cli/src/local-adapters.ts`: accepts managed daemon options + and wires them into daemon-backed local ingest helpers only when no explicit + daemon URL is configured. +- Modify `packages/cli/src/ingest.ts`: adds runtime install policy fields to + run args and passes managed daemon options to both adapter creation and + local pull-config resolution. +- Modify `packages/cli/src/ingest.test.ts`: covers managed daemon option + threading and preserves explicit daemon URL behavior. +- Modify `packages/cli/src/commands/ingest-commands.ts`: adds `--yes` to + `ktx ingest run` and uses existing `--no-input` as the runtime noninteractive + mode. +- Modify `packages/cli/src/scan.ts`: adds runtime install policy fields and + passes managed daemon options to local ingest adapters used during scan. +- Modify `packages/cli/src/scan.test.ts`: covers managed daemon option + threading and explicit daemon URL behavior. +- Modify `packages/cli/src/commands/scan-commands.ts`: adds `--yes` and + `--no-input` to `ktx scan`. +- Modify `packages/context/src/ingest/local-ingest.ts`: adds + `pullConfigOptions` to `LocalIngestMcpOptions`. +- Modify `packages/context/src/mcp/local-project-ports.ts`: passes MCP local + ingest pull-config options into `runLocalIngest()`. +- Modify `packages/context/src/mcp/local-project-ports.test.ts`: covers MCP + pull-config option forwarding. +- Modify `packages/cli/src/serve.ts`: passes managed daemon options and + pull-config options to MCP local ingest. +- Modify `packages/cli/src/serve.test.ts`: covers MCP local ingest managed + daemon option wiring. +- Modify `packages/cli/src/index.test.ts`: updates Commander routing + expectations for ingest and scan runtime install policy flags. + +### Task 1: Add managed daemon HTTP helpers + +**Files:** + +- Create: `packages/cli/src/managed-python-http.test.ts` +- Create: `packages/cli/src/managed-python-http.ts` +- Test: `packages/cli/src/managed-python-http.test.ts` + +- [ ] **Step 1: Write failing tests for lazy daemon HTTP helpers** + +Create `packages/cli/src/managed-python-http.test.ts` with this content: + +```typescript +import { describe, expect, it, vi } from 'vitest'; +import { + createManagedDaemonHttpJsonRunner, + createManagedDaemonLookerTableIdentifierParser, + createManagedDaemonSqlAnalysisPort, + createManagedPythonDaemonBaseUrlResolver, + managedDaemonDatabaseIntrospectionOptions, +} from './managed-python-http.js'; + +function io() { + let stderr = ''; + return { + io: { + stdout: { write: vi.fn() }, + stderr: { write: (chunk: string) => (stderr += chunk) }, + }, + stderr: () => stderr, + }; +} + +describe('createManagedPythonDaemonBaseUrlResolver', () => { + it('ensures the core runtime, starts the daemon, reports the URL, and caches the result', async () => { + const testIo = io(); + const ensureRuntime = vi.fn(async () => ({ + layout: {} as never, + manifest: {} as never, + })); + const startDaemon = vi.fn(async () => ({ + status: 'started' as const, + layout: {} as never, + state: { pid: 1234 } as never, + baseUrl: 'http://127.0.0.1:61234', + })); + const resolveBaseUrl = createManagedPythonDaemonBaseUrlResolver({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: testIo.io, + ensureRuntime, + startDaemon, + }); + + await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); + await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); + + expect(ensureRuntime).toHaveBeenCalledTimes(1); + expect(ensureRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: testIo.io, + feature: 'core', + }); + expect(startDaemon).toHaveBeenCalledTimes(1); + expect(startDaemon).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['core'], + force: false, + }); + expect(testIo.stderr()).toContain('Started KTX Python daemon: http://127.0.0.1:61234'); + }); + + it('reports daemon reuse without reinstalling after the first resolved URL', async () => { + const testIo = io(); + const ensureRuntime = vi.fn(async () => ({ + layout: {} as never, + manifest: {} as never, + })); + const startDaemon = vi.fn(async () => ({ + status: 'reused' as const, + layout: {} as never, + state: { pid: 1234 } as never, + baseUrl: 'http://127.0.0.1:61234', + })); + const resolveBaseUrl = createManagedPythonDaemonBaseUrlResolver({ + cliVersion: '0.2.0', + installPolicy: 'never', + io: testIo.io, + ensureRuntime, + startDaemon, + }); + + await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); + await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); + + expect(ensureRuntime).toHaveBeenCalledTimes(1); + expect(startDaemon).toHaveBeenCalledTimes(1); + expect(testIo.stderr()).toContain('Using existing KTX Python daemon: http://127.0.0.1:61234'); + }); +}); + +describe('createManagedDaemonHttpJsonRunner', () => { + it('resolves the managed base URL lazily for each HTTP JSON request', async () => { + const postJson = vi.fn(async () => ({ ok: true })); + const runner = createManagedDaemonHttpJsonRunner({ + resolveBaseUrl: async () => 'http://127.0.0.1:61234', + postJson, + }); + + await expect(runner('/sql/parse-table-identifier', { items: [] })).resolves.toEqual({ ok: true }); + + expect(postJson).toHaveBeenCalledWith('http://127.0.0.1:61234', '/sql/parse-table-identifier', { items: [] }); + }); +}); + +describe('managed daemon ingest ports', () => { + it('creates a Looker table parser backed by the managed daemon runner', async () => { + const requestJson = vi.fn(async () => ({ + results: { + 'model.explore': { + ok: true, + catalog: 'warehouse', + schema: 'public', + name: 'orders', + canonical_table: 'public.orders', + }, + }, + })); + const parser = createManagedDaemonLookerTableIdentifierParser({ requestJson }); + + await expect( + parser.parse([{ key: 'model.explore', sql_table_name: 'public.orders', dialect: 'postgres' }]), + ).resolves.toEqual({ + 'model.explore': { + ok: true, + catalog: 'warehouse', + schema: 'public', + name: 'orders', + canonical_table: 'public.orders', + }, + }); + expect(requestJson).toHaveBeenCalledWith('/sql/parse-table-identifier', { + items: [{ key: 'model.explore', sql_table_name: 'public.orders', dialect: 'postgres' }], + }); + }); + + it('creates a SQL analysis port backed by the managed daemon runner', async () => { + const requestJson = vi.fn(async () => ({ + fingerprint: 'select-orders', + normalized_sql: 'SELECT * FROM public.orders WHERE id = ?', + tables_touched: ['public.orders'], + literal_slots: [{ position: 1, type: 'number', example_value: '42' }], + })); + const sqlAnalysis = createManagedDaemonSqlAnalysisPort({ requestJson }); + + await expect(sqlAnalysis.analyzeForFingerprint('SELECT * FROM public.orders WHERE id = 42', 'postgres')).resolves + .toEqual({ + fingerprint: 'select-orders', + normalizedSql: 'SELECT * FROM public.orders WHERE id = ?', + tablesTouched: ['public.orders'], + literalSlots: [{ position: 1, type: 'number', exampleValue: '42' }], + }); + expect(requestJson).toHaveBeenCalledWith('/api/sql/analyze-for-fingerprint', { + sql: 'SELECT * FROM public.orders WHERE id = 42', + dialect: 'postgres', + }); + }); + + it('returns live-database daemon request options backed by the managed runner', async () => { + const requestJson = vi.fn(async () => ({ + connection_id: 'warehouse', + tables: [], + })); + const options = managedDaemonDatabaseIntrospectionOptions({ requestJson }); + + await expect(options.requestJson('/database/introspect', { connection_id: 'warehouse' })).resolves.toEqual({ + connection_id: 'warehouse', + tables: [], + }); + expect(requestJson).toHaveBeenCalledWith('/database/introspect', { connection_id: 'warehouse' }); + }); +}); +``` + +- [ ] **Step 2: Run the failing helper tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-http.test.ts +``` + +Expected: FAIL with an import error for `./managed-python-http.js`. + +- [ ] **Step 3: Implement managed daemon HTTP helpers** + +Create `packages/cli/src/managed-python-http.ts` with this content: + +```typescript +import { request as httpRequest } from 'node:http'; +import { request as httpsRequest } from 'node:https'; +import { URL } from 'node:url'; +import { + createDaemonLookerTableIdentifierParser, + type DaemonLiveDatabaseIntrospectionOptions, + type KtxDaemonDatabaseHttpJsonRunner, + type KtxDaemonTableIdentifierHttpJsonRunner, + type LookerTableIdentifierParser, +} from '@ktx/context/ingest'; +import { + createHttpSqlAnalysisPort, + type KtxSqlAnalysisHttpJsonRunner, + type SqlAnalysisPort, +} from '@ktx/context/sql-analysis'; +import type { KtxCliIo } from './cli-runtime.js'; +import { + ensureManagedPythonCommandRuntime, + type KtxManagedPythonInstallPolicy, + type ManagedPythonCommandRuntime, +} from './managed-python-command.js'; +import { startManagedPythonDaemon, type ManagedPythonDaemonStartResult } from './managed-python-daemon.js'; + +export type ManagedPythonHttpJsonRunner = ( + path: string, + payload: Record, +) => Promise>; + +export type ManagedPythonHttpPostJson = ( + baseUrl: string, + path: string, + payload: Record, +) => Promise>; + +export interface ManagedPythonCoreDaemonOptions { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + ensureRuntime?: (options: { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + feature: 'core'; + }) => Promise; + startDaemon?: (options: { + cliVersion: string; + features: ['core']; + force: false; + }) => Promise; +} + +export type ManagedPythonDaemonHttpOptions = + | { + requestJson: ManagedPythonHttpJsonRunner; + } + | { + resolveBaseUrl: () => Promise; + postJson?: ManagedPythonHttpPostJson; + } + | (ManagedPythonCoreDaemonOptions & { + postJson?: ManagedPythonHttpPostJson; + }); + +function normalizedBaseUrl(baseUrl: string): string { + return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`; +} + +function parseJsonObject(raw: string, path: string): Record { + const parsed = JSON.parse(raw) as unknown; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`KTX managed daemon HTTP ${path} returned non-object JSON`); + } + return parsed as Record; +} + +export async function postManagedDaemonJson( + baseUrl: string, + path: string, + payload: Record, +): Promise> { + return await new Promise((resolve, reject) => { + const target = new URL(path.replace(/^\//, ''), normalizedBaseUrl(baseUrl)); + const body = JSON.stringify(payload); + const client = target.protocol === 'https:' ? httpsRequest : httpRequest; + const request = client( + target, + { + method: 'POST', + headers: { + accept: 'application/json', + 'content-type': 'application/json', + 'content-length': Buffer.byteLength(body), + }, + }, + (response) => { + const chunks: Buffer[] = []; + response.on('data', (chunk: Buffer) => chunks.push(chunk)); + response.on('end', () => { + const text = Buffer.concat(chunks).toString('utf8'); + const statusCode = response.statusCode ?? 0; + if (statusCode < 200 || statusCode >= 300) { + reject(new Error(`KTX managed daemon HTTP ${path} failed with ${statusCode}: ${text}`)); + return; + } + try { + resolve(parseJsonObject(text, path)); + } catch (error) { + reject(error); + } + }); + }, + ); + request.on('error', reject); + request.end(body); + }); +} + +export function createManagedPythonDaemonBaseUrlResolver( + options: ManagedPythonCoreDaemonOptions, +): () => Promise { + let cachedBaseUrl: string | undefined; + + return async () => { + if (cachedBaseUrl) { + return cachedBaseUrl; + } + + const ensureRuntime = options.ensureRuntime ?? ensureManagedPythonCommandRuntime; + const startDaemon = options.startDaemon ?? startManagedPythonDaemon; + await ensureRuntime({ + cliVersion: options.cliVersion, + installPolicy: options.installPolicy, + io: options.io, + feature: 'core', + }); + const daemon = await startDaemon({ + cliVersion: options.cliVersion, + features: ['core'], + force: false, + }); + const verb = daemon.status === 'started' ? 'Started' : 'Using existing'; + options.io.stderr.write(`${verb} KTX Python daemon: ${daemon.baseUrl}\n`); + cachedBaseUrl = daemon.baseUrl; + return cachedBaseUrl; + }; +} + +function isRequestJsonOnly(options: ManagedPythonDaemonHttpOptions): options is { requestJson: ManagedPythonHttpJsonRunner } { + return 'requestJson' in options; +} + +function isResolveBaseUrlOnly( + options: ManagedPythonDaemonHttpOptions, +): options is { resolveBaseUrl: () => Promise; postJson?: ManagedPythonHttpPostJson } { + return 'resolveBaseUrl' in options; +} + +export function createManagedDaemonHttpJsonRunner( + options: ManagedPythonDaemonHttpOptions, +): ManagedPythonHttpJsonRunner { + if (isRequestJsonOnly(options)) { + return options.requestJson; + } + const resolveBaseUrl = isResolveBaseUrlOnly(options) + ? options.resolveBaseUrl + : createManagedPythonDaemonBaseUrlResolver(options); + const postJson = options.postJson ?? postManagedDaemonJson; + + return async (path, payload) => postJson(await resolveBaseUrl(), path, payload); +} + +export function createManagedDaemonLookerTableIdentifierParser( + options: ManagedPythonDaemonHttpOptions, +): LookerTableIdentifierParser { + return createDaemonLookerTableIdentifierParser({ + baseUrl: 'http://127.0.0.1:0', + requestJson: createManagedDaemonHttpJsonRunner(options) as KtxDaemonTableIdentifierHttpJsonRunner, + }); +} + +export function createManagedDaemonSqlAnalysisPort(options: ManagedPythonDaemonHttpOptions): SqlAnalysisPort { + return createHttpSqlAnalysisPort({ + baseUrl: 'http://127.0.0.1:0', + requestJson: createManagedDaemonHttpJsonRunner(options) as KtxSqlAnalysisHttpJsonRunner, + }); +} + +export function managedDaemonDatabaseIntrospectionOptions( + options: ManagedPythonDaemonHttpOptions, +): Pick { + return { + requestJson: createManagedDaemonHttpJsonRunner(options) as KtxDaemonDatabaseHttpJsonRunner, + }; +} +``` + +- [ ] **Step 4: Verify the helper tests pass** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-http.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit the helper** + +Run: + +```bash +git add packages/cli/src/managed-python-http.ts packages/cli/src/managed-python-http.test.ts +git commit -m "feat(cli): add managed daemon HTTP helpers" +``` + +Expected: commit succeeds. + +### Task 2: Wire managed daemon options into CLI local adapters + +**Files:** + +- Modify: `packages/cli/src/local-adapters.ts` +- Test: `packages/cli/src/managed-python-http.test.ts` + +- [ ] **Step 1: Update local adapter imports** + +In `packages/cli/src/local-adapters.ts`, add this import after the +`createHttpSqlAnalysisPort` import: + +```typescript +import { + createManagedDaemonLookerTableIdentifierParser, + createManagedDaemonSqlAnalysisPort, + managedDaemonDatabaseIntrospectionOptions, + type ManagedPythonCoreDaemonOptions, +} from './managed-python-http.js'; +``` + +- [ ] **Step 2: Add managed daemon options to the local adapter option type** + +Replace this interface: + +```typescript +interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions { + historicSqlConnectionId?: string; + sqlAnalysisUrl?: string; +} +``` + +with this interface: + +```typescript +export interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions { + historicSqlConnectionId?: string; + sqlAnalysisUrl?: string; + managedDaemon?: ManagedPythonCoreDaemonOptions; +} +``` + +- [ ] **Step 3: Add helper functions for managed daemon adapter options** + +Add these helpers immediately after `hasSnowflakeDriver()`: + +```typescript +function ktxCliDaemonDatabaseIntrospectionOptions( + options: KtxCliLocalIngestAdaptersOptions, +): DefaultLocalIngestAdaptersOptions['databaseIntrospection'] { + if (options.databaseIntrospectionUrl || options.databaseIntrospection?.requestJson || !options.managedDaemon) { + return options.databaseIntrospection; + } + return { + ...(options.databaseIntrospection ?? {}), + ...managedDaemonDatabaseIntrospectionOptions(options.managedDaemon), + }; +} + +function ktxCliLookerOptions( + options: KtxCliLocalIngestAdaptersOptions, +): DefaultLocalIngestAdaptersOptions['looker'] { + const looker = options.looker; + if (looker?.parser || looker?.daemonBaseUrl || process.env.KTX_DAEMON_URL || !options.managedDaemon) { + return looker; + } + return { + ...(looker ?? {}), + parser: createManagedDaemonLookerTableIdentifierParser(options.managedDaemon), + }; +} + +function ktxCliHistoricSqlAnalysis(options: KtxCliLocalIngestAdaptersOptions) { + if (options.sqlAnalysisUrl) { + return createHttpSqlAnalysisPort({ baseUrl: options.sqlAnalysisUrl }); + } + if (process.env.KTX_SQL_ANALYSIS_URL) { + return createHttpSqlAnalysisPort({ baseUrl: process.env.KTX_SQL_ANALYSIS_URL }); + } + if (process.env.KTX_DAEMON_URL) { + return createHttpSqlAnalysisPort({ baseUrl: process.env.KTX_DAEMON_URL }); + } + if (options.managedDaemon) { + return createManagedDaemonSqlAnalysisPort(options.managedDaemon); + } + return createHttpSqlAnalysisPort({ baseUrl: 'http://127.0.0.1:8765' }); +} +``` + +- [ ] **Step 4: Use managed daemon request options for daemon live-database fallback** + +In `createKtxCliLiveDatabaseIntrospection()`, insert this line before the +`const daemon = createDaemonLiveDatabaseIntrospection({` statement: + +```typescript + const databaseIntrospection = ktxCliDaemonDatabaseIntrospectionOptions(options); +``` + +Then replace the daemon creation block: + +```typescript + const daemon = createDaemonLiveDatabaseIntrospection({ + connections: project.config.connections, + ...options.databaseIntrospection, + ...(options.databaseIntrospectionUrl ? { baseUrl: options.databaseIntrospectionUrl } : {}), + }); +``` + +with this block: + +```typescript + const daemon = createDaemonLiveDatabaseIntrospection({ + connections: project.config.connections, + ...databaseIntrospection, + ...(options.databaseIntrospectionUrl ? { baseUrl: options.databaseIntrospectionUrl } : {}), + }); +``` + +- [ ] **Step 5: Use managed daemon SQL analysis for historic SQL** + +In `historicSqlOptionsForLocalRun()`, replace this block: + +```typescript + return { + sqlAnalysis: createHttpSqlAnalysisPort({ + baseUrl: + options.sqlAnalysisUrl ?? + process.env.KTX_SQL_ANALYSIS_URL ?? + process.env.KTX_DAEMON_URL ?? + 'http://127.0.0.1:8765', + }), + postgresQueryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId), + postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'), + }; +``` + +with this block: + +```typescript + return { + sqlAnalysis: ktxCliHistoricSqlAnalysis(options), + postgresQueryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId), + postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'), + }; +``` + +- [ ] **Step 6: Pass managed Looker options into default local adapters** + +In `createKtxCliLocalIngestAdapters()`, replace: + +```typescript + const base = createDefaultLocalIngestAdapters(project, { + ...options, + ...(historicSql ? { historicSql } : {}), + }); +``` + +with: + +```typescript + const base = createDefaultLocalIngestAdapters(project, { + ...options, + databaseIntrospection: ktxCliDaemonDatabaseIntrospectionOptions(options), + looker: ktxCliLookerOptions(options), + ...(historicSql ? { historicSql } : {}), + }); +``` + +- [ ] **Step 7: Run the CLI type check for local adapter changes** + +Run: + +```bash +pnpm --filter @ktx/cli run type-check +``` + +Expected: PASS. + +- [ ] **Step 8: Commit local adapter wiring** + +Run: + +```bash +git add packages/cli/src/local-adapters.ts +git commit -m "feat(cli): route local adapters through managed daemon" +``` + +Expected: commit succeeds. + +### Task 3: Thread managed daemon options through ingest commands + +**Files:** + +- Modify: `packages/cli/src/ingest.ts` +- Modify: `packages/cli/src/ingest.test.ts` +- Modify: `packages/cli/src/commands/ingest-commands.ts` +- Test: `packages/cli/src/ingest.test.ts` +- Test: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Write failing ingest option-threading tests** + +In `packages/cli/src/ingest.test.ts`, add this test after +`passes daemon database introspection URL to default local ingest adapters`: + +```typescript + it('passes managed daemon options to adapters and pull-config options when no explicit daemon URL is set', async () => { + const projectDir = join(tempDir, 'managed-daemon-ingest-project'); + await initKtxProject({ projectDir, projectName: 'managed-daemon-ingest-project' }); + await writeWarehouseConfig(projectDir); + const createdAdapters: SourceAdapter[] = [ + { source: 'fake', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) }, + ]; + const createAdapters = vi.fn(() => createdAdapters as never); + const runLocal = vi.fn(async (input: RunLocalIngestOptions) => + completedLocalBundleRun(input, input.jobId ?? 'local-job-1'), + ); + const io = makeIo(); + + await expect( + runKtxIngest( + { + command: 'run', + projectDir, + connectionId: 'warehouse', + adapter: 'fake', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + outputMode: 'plain', + } satisfies KtxIngestArgs, + io.io, + { + createAdapters, + runLocalIngest: runLocal, + jobIdFactory: () => 'local-job-1', + }, + ), + ).resolves.toBe(0); + + const expectedManagedDaemon = { + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + }; + expect(createAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), { + managedDaemon: expectedManagedDaemon, + }); + expect(runLocal).toHaveBeenCalledWith( + expect.objectContaining({ + pullConfigOptions: { + managedDaemon: expectedManagedDaemon, + }, + }), + ); + }); +``` + +In the existing `passes daemon database introspection URL to default local ingest +adapters` test, add this assertion inside the existing `expect(runLocal)` block: + +```typescript + pullConfigOptions: { + databaseIntrospectionUrl: 'http://127.0.0.1:8765', + }, +``` + +- [ ] **Step 2: Run the failing ingest tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/ingest.test.ts +``` + +Expected: FAIL because `KtxIngestArgs` has no `cliVersion` or +`runtimeInstallPolicy`, and `runKtxIngest()` does not pass managed daemon +options into `createAdapters()` or `pullConfigOptions`. + +- [ ] **Step 3: Add runtime install policy fields to ingest args** + +In `packages/cli/src/ingest.ts`, add this import after the local adapters +import: + +```typescript +import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; +``` + +In the `KtxIngestArgs` `command: 'run'` branch, add these fields after +`databaseIntrospectionUrl?: string;`: + +```typescript + cliVersion?: string; + runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; +``` + +- [ ] **Step 4: Add a managed daemon option helper to ingest** + +In `packages/cli/src/ingest.ts`, add this helper after +`initialRunMemoryFlowInput()`: + +```typescript +function managedDaemonOptionsForIngestRun( + args: Extract, + io: KtxIngestIo, +) { + if (args.databaseIntrospectionUrl || !args.cliVersion || !args.runtimeInstallPolicy) { + return undefined; + } + return { + cliVersion: args.cliVersion, + installPolicy: args.runtimeInstallPolicy, + io, + }; +} +``` + +- [ ] **Step 5: Pass managed daemon options to adapters and pull-config resolution** + +In the `args.command === 'run'` branch of `runKtxIngest()`, replace the +`adapterOptions` block: + +```typescript + const adapterOptions = { + ...(localIngestOptions.pullConfigOptions ?? {}), + ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), + ...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}), + }; +``` + +with: + +```typescript + const managedDaemon = managedDaemonOptionsForIngestRun(args, io); + const adapterOptions = { + ...(localIngestOptions.pullConfigOptions ?? {}), + ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), + ...(managedDaemon ? { managedDaemon } : {}), + ...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}), + }; +``` + +In the non-Metabase `executeLocalIngest()` call, move `...localIngestOptions` +before `pullConfigOptions` and add `pullConfigOptions: adapterOptions`. +The call must contain this sequence after the edit: + +```typescript + const result = await executeLocalIngest({ + project, + adapters: createAdapters(project, adapterOptions), + adapter: args.adapter, + connectionId: args.connectionId, + sourceDir: args.sourceDir, + trigger: 'manual_resync', + jobId, + ...localIngestOptions, + pullConfigOptions: adapterOptions, + ...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}), + ...(memoryFlow ? { memoryFlow } : {}), + }); +``` + +- [ ] **Step 6: Add runtime flags to `ktx ingest run` routing** + +In `packages/cli/src/commands/ingest-commands.ts`, add this import after the +`KtxCliDeps` import: + +```typescript +import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; +``` + +In the `ingest run` command options, add this option immediately before +`.option('--no-input', ...)`: + +```typescript + .option('--yes', 'Install the managed Python runtime without prompting when required', false) +``` + +In the `KtxIngestArgs` object built for `ingest run`, add these fields after +`databaseIntrospectionUrl: options.databaseIntrospectionUrl || undefined,`: + +```typescript + cliVersion: context.packageInfo.version, + runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), +``` + +- [ ] **Step 7: Update Commander ingest routing expectations** + +In `packages/cli/src/index.test.ts`, in the test that routes +`dev ingest run`, add these expected fields after +`databaseIntrospectionUrl: undefined,`: + +```typescript + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'never', +``` + +Add this test after that existing routing test: + +```typescript + it('routes ingest managed runtime install policies', async () => { + const autoIo = makeIo(); + const conflictIo = makeIo(); + const ingest = vi.fn(async () => 0); + + await expect( + runKtxCli( + [ + 'dev', + 'ingest', + 'run', + '--project-dir', + tempDir, + '--connection-id', + 'warehouse', + '--adapter', + 'looker', + '--yes', + ], + autoIo.io, + { ingest }, + ), + ).resolves.toBe(0); + await expect( + runKtxCli( + [ + 'dev', + 'ingest', + 'run', + '--project-dir', + tempDir, + '--connection-id', + 'warehouse', + '--adapter', + 'looker', + '--yes', + '--no-input', + ], + conflictIo.io, + { ingest }, + ), + ).resolves.toBe(1); + + expect(ingest).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'run', + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'auto', + }), + autoIo.io, + ); + expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); + }); +``` + +- [ ] **Step 8: Run focused ingest and routing tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/ingest.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 9: Commit ingest runtime policy wiring** + +Run: + +```bash +git add packages/cli/src/ingest.ts packages/cli/src/ingest.test.ts packages/cli/src/commands/ingest-commands.ts packages/cli/src/index.test.ts +git commit -m "feat(cli): use managed daemon for ingest helpers" +``` + +Expected: commit succeeds. + +### Task 4: Thread managed daemon options through scan commands + +**Files:** + +- Modify: `packages/cli/src/scan.ts` +- Modify: `packages/cli/src/scan.test.ts` +- Modify: `packages/cli/src/commands/scan-commands.ts` +- Modify: `packages/cli/src/index.test.ts` +- Test: `packages/cli/src/scan.test.ts` +- Test: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Write failing scan option-threading test** + +In `packages/cli/src/scan.test.ts`, add this test after the test that passes +`databaseIntrospectionUrl`: + +```typescript + it('passes managed daemon options to local ingest adapters when no explicit daemon URL is set', async () => { + const report = minimalScanReport(); + const createLocalIngestAdapters = vi.fn(() => []); + const runLocalScan = vi.fn( + async (_input: RunLocalScanOptions): Promise => ({ + runId: 'scan-run-1', + status: 'done', + done: true, + connectionId: 'warehouse', + mode: 'structural', + dryRun: false, + syncId: 'sync-1', + report, + }), + ); + const io = makeIo(); + + await expect( + runKtxScan( + { + command: 'run', + projectDir: tempDir, + connectionId: 'warehouse', + mode: 'structural', + detectRelationships: false, + dryRun: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + io.io, + { runLocalScan, createLocalIngestAdapters }, + ), + ).resolves.toBe(0); + + expect(createLocalIngestAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir }), { + managedDaemon: { + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + }, + }); + }); +``` + +- [ ] **Step 2: Run the failing scan tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/scan.test.ts +``` + +Expected: FAIL because `KtxScanArgs` has no `cliVersion` or +`runtimeInstallPolicy`, and `runKtxScan()` does not pass managed daemon options +to adapter creation. + +- [ ] **Step 3: Add runtime install policy fields to scan args** + +In `packages/cli/src/scan.ts`, add this import after the local adapter import: + +```typescript +import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; +``` + +In the `KtxScanArgs` `command: 'run'` branch, add these fields after +`databaseIntrospectionUrl?: string;`: + +```typescript + cliVersion?: string; + runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; +``` + +- [ ] **Step 4: Add managed daemon option construction to scan** + +In `packages/cli/src/scan.ts`, add this helper after `warningLine()`: + +```typescript +function managedDaemonOptionsForScanRun(args: Extract, io: KtxCliIo) { + if (args.databaseIntrospectionUrl || !args.cliVersion || !args.runtimeInstallPolicy) { + return undefined; + } + return { + cliVersion: args.cliVersion, + installPolicy: args.runtimeInstallPolicy, + io, + }; +} +``` + +In the `runLocalScan()` call, replace this adapter creation block: + +```typescript + adapters: (deps.createLocalIngestAdapters ?? createKtxCliLocalIngestAdapters)(project, { + databaseIntrospectionUrl: args.databaseIntrospectionUrl, + }), +``` + +with: + +```typescript + adapters: (deps.createLocalIngestAdapters ?? createKtxCliLocalIngestAdapters)(project, { + ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), + ...(managedDaemonOptionsForScanRun(args, io) + ? { managedDaemon: managedDaemonOptionsForScanRun(args, io) } + : {}), + }), +``` + +Then replace the repeated helper call with a local constant to keep the code +single-pass. The final block must be: + +```typescript + const managedDaemon = managedDaemonOptionsForScanRun(args, io); + const connector = + args.mode !== 'structural' || args.detectRelationships + ? await createKtxCliScanConnector(project, args.connectionId) + : undefined; + const progress = createCliScanProgress(io); + try { + const result = await (deps.runLocalScan ?? runLocalScan)({ + project, + connectionId: args.connectionId, + mode: args.mode, + detectRelationships: args.detectRelationships, + dryRun: args.dryRun, + trigger: 'cli', + databaseIntrospectionUrl: args.databaseIntrospectionUrl, + connector, + adapters: (deps.createLocalIngestAdapters ?? createKtxCliLocalIngestAdapters)(project, { + ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), + ...(managedDaemon ? { managedDaemon } : {}), + }), + progress, + }); +``` + +- [ ] **Step 5: Add runtime flags to scan routing** + +In `packages/cli/src/commands/scan-commands.ts`, add this import after the +`cli-program.js` import: + +```typescript +import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; +``` + +In the top-level `scan` command options, add these options after +`--database-introspection-url`: + +```typescript + .option('--yes', 'Install the managed Python runtime without prompting when required', false) + .option('--no-input', 'Disable interactive managed runtime installation') +``` + +In the scan run action, add these fields after +`databaseIntrospectionUrl: options.databaseIntrospectionUrl,`: + +```typescript + cliVersion: context.packageInfo.version, + runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), +``` + +- [ ] **Step 6: Update Commander scan routing expectations** + +In `packages/cli/src/index.test.ts`, update the `routes low-level scan through +ktx dev with top-level project-dir` expected args by adding: + +```typescript + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', +``` + +Add this test after that routing test: + +```typescript + it('routes scan managed runtime install policies', async () => { + const autoIo = makeIo(); + const neverIo = makeIo(); + const conflictIo = makeIo(); + const scan = vi.fn().mockResolvedValue(0); + + await expect(runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--yes'], autoIo.io, { scan })) + .resolves.toBe(0); + await expect(runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--no-input'], neverIo.io, { scan })) + .resolves.toBe(0); + await expect( + runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--yes', '--no-input'], conflictIo.io, { + scan, + }), + ).resolves.toBe(1); + + expect(scan).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + command: 'run', + runtimeInstallPolicy: 'auto', + }), + autoIo.io, + ); + expect(scan).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + command: 'run', + runtimeInstallPolicy: 'never', + }), + neverIo.io, + ); + expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); + }); +``` + +- [ ] **Step 7: Run focused scan and routing tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/scan.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 8: Commit scan runtime policy wiring** + +Run: + +```bash +git add packages/cli/src/scan.ts packages/cli/src/scan.test.ts packages/cli/src/commands/scan-commands.ts packages/cli/src/index.test.ts +git commit -m "feat(cli): pass managed daemon options to scan" +``` + +Expected: commit succeeds. + +### Task 5: Pass pull-config options through MCP local ingest + +**Files:** + +- Modify: `packages/context/src/ingest/local-ingest.ts` +- Modify: `packages/context/src/mcp/local-project-ports.ts` +- Modify: `packages/context/src/mcp/local-project-ports.test.ts` +- Test: `packages/context/src/mcp/local-project-ports.test.ts` + +- [ ] **Step 1: Write failing MCP pull-config forwarding test** + +In `packages/context/src/mcp/local-project-ports.test.ts`, add this test in +the local ingest tool describe block, next to the existing local ingest tests: + +```typescript + it('passes local ingest pull-config options into runLocalIngest', async () => { + const runLocalIngest = vi.fn(async () => ({ + result: { ok: true }, + report: { + id: 'report-1', + runId: 'run-1', + jobId: 'job-1', + sourceKey: 'looker', + connectionId: 'warehouse', + body: { + syncId: 'sync-1', + workUnits: [], + failedWorkUnits: [], + diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 0 }, + provenanceRows: [], + }, + }, + } as never)); + const ports = createLocalProjectMcpContextPorts(project, { + localIngest: { + adapters: [{ source: 'looker', skillNames: [] }], + pullConfigOptions: { + looker: { + daemonBaseUrl: 'http://127.0.0.1:61234', + }, + }, + runLocalIngest, + }, + }); + + await expect( + ports.ingest.run({ + adapter: 'looker', + connectionId: 'warehouse', + trigger: 'manual_resync', + config: {}, + }), + ).resolves.toMatchObject({ + runId: 'run-1', + jobId: 'job-1', + reportId: 'report-1', + }); + + expect(runLocalIngest).toHaveBeenCalledWith( + expect.objectContaining({ + pullConfigOptions: { + looker: { + daemonBaseUrl: 'http://127.0.0.1:61234', + }, + }, + }), + ); + }); +``` + +- [ ] **Step 2: Run the failing MCP test** + +Run: + +```bash +pnpm --filter @ktx/context run test -- src/mcp/local-project-ports.test.ts +``` + +Expected: FAIL because `LocalIngestMcpOptions` does not accept +`pullConfigOptions`, and MCP local ingest does not pass it to +`runLocalIngest()`. + +- [ ] **Step 3: Add pull-config options to MCP local ingest options** + +In `packages/context/src/ingest/local-ingest.ts`, update +`LocalIngestMcpOptions` so the `Pick` includes +`'pullConfigOptions'`. The interface must contain this sequence after the edit: + +```typescript +export interface LocalIngestMcpOptions + extends Pick< + RunLocalIngestOptions, + | 'agentRunner' + | 'llmProvider' + | 'memoryModel' + | 'semanticLayerCompute' + | 'queryExecutor' + | 'logger' + | 'pullConfigOptions' + > { + adapters?: SourceAdapter[]; + jobIdFactory?: () => string; + runLocalMetabaseIngest?: (options: RunLocalMetabaseIngestOptions) => Promise; +} +``` + +- [ ] **Step 4: Pass pull-config options in MCP local ingest execution** + +In `packages/context/src/mcp/local-project-ports.ts`, in the +`runLocalIngest({ ... })` call, add this field after `sourceDir,`: + +```typescript + pullConfigOptions: options.localIngest?.pullConfigOptions, +``` + +- [ ] **Step 5: Run MCP tests** + +Run: + +```bash +pnpm --filter @ktx/context run test -- src/mcp/local-project-ports.test.ts +``` + +Expected: PASS. + +- [ ] **Step 6: Commit MCP pull-config forwarding** + +Run: + +```bash +git add packages/context/src/ingest/local-ingest.ts packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts +git commit -m "feat(context): pass MCP ingest pull config options" +``` + +Expected: commit succeeds. + +### Task 6: Wire managed daemon options through MCP serve + +**Files:** + +- Modify: `packages/cli/src/serve.ts` +- Modify: `packages/cli/src/serve.test.ts` +- Test: `packages/cli/src/serve.test.ts` +- Test: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Write failing serve managed daemon wiring test** + +In `packages/cli/src/serve.test.ts`, add this test after +`uses managed semantic compute when MCP semantic compute has no explicit HTTP +URL`: + +```typescript + it('passes managed daemon options to MCP local ingest adapters and pull-config options', async () => { + const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never; + const adapters = [{ source: 'looker', skillNames: [] }]; + const createIngestAdapters = vi.fn(() => adapters); + const createContextTools = vi.fn(() => ({ connections: { list: async () => [] } })); + const managedRuntimeIo = makeManagedRuntimeIo(); + + await expect( + runKtxServeStdio( + { + mcp: 'stdio', + projectDir: '/tmp/ktx-project', + userId: 'agent', + semanticCompute: false, + semanticComputeUrl: undefined, + databaseIntrospectionUrl: undefined, + executeQueries: false, + memoryCapture: false, + memoryModel: undefined, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + { + loadProject: async () => project, + createContextTools, + createIngestAdapters, + managedRuntimeIo: managedRuntimeIo.io, + createServer: vi.fn(() => ({ connect: vi.fn(async () => undefined) }) as never), + createTransport: vi.fn(() => ({}) as never), + stderr: { write: vi.fn() }, + }, + ), + ).resolves.toBe(0); + + const expectedManagedDaemon = { + cliVersion: '0.2.0', + installPolicy: 'auto', + io: managedRuntimeIo.io, + }; + expect(createIngestAdapters).toHaveBeenCalledWith(project, { + managedDaemon: expectedManagedDaemon, + }); + expect(createContextTools).toHaveBeenCalledWith( + project, + expect.objectContaining({ + localIngest: expect.objectContaining({ + adapters, + pullConfigOptions: { + managedDaemon: expectedManagedDaemon, + }, + }), + }), + ); + }); +``` + +Add this assertion to the existing test that passes +`databaseIntrospectionUrl: 'http://127.0.0.1:8765'`: + +```typescript + localIngest: expect.objectContaining({ + pullConfigOptions: { + databaseIntrospectionUrl: 'http://127.0.0.1:8765', + }, + }), +``` + +- [ ] **Step 2: Run the failing serve tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/serve.test.ts +``` + +Expected: FAIL because `runKtxServeStdio()` does not pass managed daemon +options or pull-config options into local ingest. + +- [ ] **Step 3: Add serve managed daemon option helper** + +In `packages/cli/src/serve.ts`, add this import after the managed command +import: + +```typescript +import type { ManagedPythonCoreDaemonOptions } from './managed-python-http.js'; +``` + +Add this helper after `requiredManagedRuntimeCliVersion()`: + +```typescript +function managedDaemonOptionsForServe( + args: KtxServeArgs, + deps: KtxServeDeps, +): ManagedPythonCoreDaemonOptions | undefined { + if (args.databaseIntrospectionUrl || !args.cliVersion) { + return undefined; + } + return { + cliVersion: args.cliVersion, + installPolicy: args.runtimeInstallPolicy ?? 'prompt', + io: deps.managedRuntimeIo ?? process, + }; +} +``` + +- [ ] **Step 4: Pass managed daemon options to serve local ingest** + +In `runKtxServeStdio()`, replace this block: + +```typescript + const createIngestAdapters = deps.createIngestAdapters ?? createKtxCliLocalIngestAdapters; + const localAdapters = createIngestAdapters(project, { + databaseIntrospectionUrl: args.databaseIntrospectionUrl, + }); +``` + +with: + +```typescript + const createIngestAdapters = deps.createIngestAdapters ?? createKtxCliLocalIngestAdapters; + const managedDaemon = managedDaemonOptionsForServe(args, deps); + const localAdapterOptions = { + ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), + ...(managedDaemon ? { managedDaemon } : {}), + }; + const localAdapters = createIngestAdapters(project, localAdapterOptions); +``` + +In the `localIngest` object, add this field after `adapters: localAdapters,`: + +```typescript + pullConfigOptions: localAdapterOptions, +``` + +- [ ] **Step 5: Run serve and routing tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/serve.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 6: Commit serve managed daemon wiring** + +Run: + +```bash +git add packages/cli/src/serve.ts packages/cli/src/serve.test.ts +git commit -m "feat(cli): pass managed daemon options to serve ingest" +``` + +Expected: commit succeeds. + +### Task 7: Verify managed local ingest daemon integration + +**Files:** + +- Verify: `packages/cli/src/managed-python-http.ts` +- Verify: `packages/cli/src/local-adapters.ts` +- Verify: `packages/cli/src/ingest.ts` +- Verify: `packages/cli/src/scan.ts` +- Verify: `packages/cli/src/serve.ts` +- Verify: `packages/context/src/ingest/local-ingest.ts` +- Verify: `packages/context/src/mcp/local-project-ports.ts` + +- [ ] **Step 1: Run focused CLI tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-http.test.ts src/ingest.test.ts src/scan.test.ts src/serve.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 2: Run focused context tests** + +Run: + +```bash +pnpm --filter @ktx/context run test -- src/mcp/local-project-ports.test.ts +``` + +Expected: PASS. + +- [ ] **Step 3: Run affected package type checks** + +Run: + +```bash +pnpm --filter @ktx/cli run type-check +pnpm --filter @ktx/context run type-check +``` + +Expected: both commands PASS. + +- [ ] **Step 4: Run the broader TypeScript test surface** + +Run: + +```bash +pnpm --filter @ktx/cli run test +pnpm --filter @ktx/context run test +``` + +Expected: both commands PASS. + +- [ ] **Step 5: Commit verification-only fixes if needed** + +If Step 1 through Step 4 require mechanical test expectation or type fixes, run: + +```bash +git add packages/cli/src packages/context/src +git commit -m "test: verify managed local ingest daemon runtime" +``` + +Expected: commit succeeds only when files changed during verification. If no +files changed, skip this commit. + +## Self-review + +Spec coverage: + +- The plan uses the managed core runtime and daemon for Python-backed local + ingest helper behavior. +- The plan preserves explicit daemon URLs and environment-variable override + behavior. +- The plan keeps the first-use installation policy aligned with existing + `--yes`, `--no-input`, and prompt semantics. +- The plan avoids local embedding dependency installation by requesting only + the `core` runtime feature. + +Placeholder scan: + +- No placeholder markers remain in the task steps. +- Every code-changing step includes the exact code block or replacement to use. + +Type consistency: + +- The new managed daemon option type is named `ManagedPythonCoreDaemonOptions`. +- CLI runtime policy fields use the existing + `KtxManagedPythonInstallPolicy` type. +- MCP local ingest reuses the existing `DefaultLocalIngestAdaptersOptions` + through `RunLocalIngestOptions['pullConfigOptions']`. diff --git a/docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md b/docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md new file mode 100644 index 00000000..733b6915 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md @@ -0,0 +1,935 @@ +# Managed Python Runtime Command Integration Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `ktx sl query` use the KTX-managed bundled Python runtime +instead of relying on a user-provided `python -m ktx_daemon`. + +**Architecture:** Add a small CLI helper that resolves the managed runtime, +installs the `core` feature when policy permits it, and creates the existing +`@ktx/context/daemon` one-shot semantic-layer compute port with the managed +`ktx-daemon` executable. Wire `ktx sl query` to pass an explicit runtime +install policy from `--yes`, `--no-input`, or the default interactive mode. + +**Tech Stack:** TypeScript, Commander, Vitest, `@clack/prompts`, +`@ktx/context/daemon`, existing KTX managed runtime installer. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +Existing plans based on the spec: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` is + implemented. The worktree contains + `scripts/build-python-runtime-wheel.mjs`, + `scripts/build-python-runtime-wheel.test.mjs`, runtime-wheel packaging in + `scripts/package-artifacts.mjs`, release-policy coverage, and matching + artifact tests. The targeted verification passes: + `node --test scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs`. +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` is + implemented. The worktree contains + `packages/cli/src/managed-python-runtime.ts`, + `packages/cli/src/runtime.ts`, + `packages/cli/src/commands/runtime-commands.ts`, CLI registration, and + matching Vitest coverage. The targeted CLI verification passes: + `pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts src/runtime.test.ts src/index.test.ts`. + +Spec requirements still outside this plan: + +- `ktx runtime start` and `ktx runtime stop`. +- Managed HTTP daemon state, health checks, reuse, and stale daemon repair. +- Lazy `local-embeddings` installation and local embedding daemon reuse. +- Public npm package rename from `@ktx/cli` to `@kaelio/ktx`. + +This plan implements the next runnable user path: `ktx sl query` installs or +uses the managed `core` Python runtime according to the command's input policy. + +## File structure + +- Create `packages/cli/src/managed-python-command.ts`: CLI helper for managed + runtime policy, optional prompt, runtime install, and managed semantic-layer + compute port creation. +- Create `packages/cli/src/managed-python-command.test.ts`: unit tests for + ready runtime reuse, `--no-input` failure, `--yes` installation, and + interactive prompt acceptance. +- Modify `packages/cli/src/sl.ts`: extend `KtxSlArgs` with CLI version and + runtime install policy for `query`, and use the managed helper when no test + compute port is injected. +- Modify `packages/cli/src/sl.test.ts`: update existing `query` arguments and + assert `runKtxSl` delegates default compute creation to the managed helper. +- Modify `packages/cli/src/commands/sl-commands.ts`: add `--yes` and + `--no-input` to `sl query`, derive the runtime install policy, and pass the + CLI package version. +- Modify `packages/cli/src/command-schemas.ts`: validate `cliVersion` and + `runtimeInstallPolicy` on parsed `sl query` arguments. +- Modify `packages/cli/src/index.test.ts`: assert Commander routes the new + `sl query` runtime policy flags. + +### Task 1: Add failing managed Python command helper tests + +**Files:** + +- Create: `packages/cli/src/managed-python-command.test.ts` +- Test: `packages/cli/src/managed-python-command.test.ts` + +- [ ] **Step 1: Write the failing test file** + +Create `packages/cli/src/managed-python-command.test.ts` with this content: + +```typescript +import { describe, expect, it, vi } from 'vitest'; +import { + createManagedPythonSemanticLayerComputePort, + managedRuntimeInstallCommand, +} from './managed-python-command.js'; +import type { + InstalledKtxRuntimeManifest, + KtxRuntimeFeature, + ManagedPythonRuntimeInstallResult, + ManagedPythonRuntimeLayout, + ManagedPythonRuntimeStatus, +} from './managed-python-runtime.js'; + +function makeIo() { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +function layout(): ManagedPythonRuntimeLayout { + return { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + }; +} + +function manifest(features: KtxRuntimeFeature[] = ['core']): InstalledKtxRuntimeManifest { + return { + schemaVersion: 1, + cliVersion: '0.2.0', + installedAt: '2026-05-11T00:00:00.000Z', + asset: { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.2.0', + wheel: { + file: 'kaelio_ktx-0.2.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 123, + }, + }, + features, + python: { + executable: '/runtime/0.2.0/.venv/bin/python', + daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon', + }, + installLog: '/runtime/0.2.0/install.log', + }; +} + +function readyStatus(features: KtxRuntimeFeature[] = ['core']): ManagedPythonRuntimeStatus { + return { + kind: 'ready', + detail: 'Runtime ready at /runtime/0.2.0', + layout: layout(), + manifest: manifest(features), + }; +} + +function missingStatus(): ManagedPythonRuntimeStatus { + return { + kind: 'missing', + detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', + layout: layout(), + }; +} + +function installResult(features: KtxRuntimeFeature[] = ['core']): ManagedPythonRuntimeInstallResult { + const installedManifest = manifest(features); + return { + status: 'installed', + layout: layout(), + asset: { + manifest: installedManifest.asset, + wheelPath: '/assets/python/kaelio_ktx-0.2.0-py3-none-any.whl', + }, + manifest: installedManifest, + }; +} + +describe('managedRuntimeInstallCommand', () => { + it('prints the exact command for each managed runtime feature', () => { + expect(managedRuntimeInstallCommand('core')).toBe('ktx runtime install --yes'); + expect(managedRuntimeInstallCommand('local-embeddings')).toBe( + 'ktx runtime install --feature local-embeddings --yes', + ); + }); +}); + +describe('createManagedPythonSemanticLayerComputePort', () => { + it('uses the managed ktx-daemon executable when the runtime is ready', async () => { + const io = makeIo(); + const compute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; + const createPythonCompute = vi.fn(() => compute); + + await expect( + createManagedPythonSemanticLayerComputePort({ + cliVersion: '0.2.0', + installPolicy: 'never', + io: io.io, + readStatus: vi.fn(async () => readyStatus()), + installRuntime: vi.fn(), + createPythonCompute, + }), + ).resolves.toBe(compute); + + expect(createPythonCompute).toHaveBeenCalledWith({ + command: '/runtime/0.2.0/.venv/bin/ktx-daemon', + args: [], + }); + expect(io.stderr()).toBe(''); + }); + + it('fails with a preparation command when input is disabled and the runtime is missing', async () => { + const io = makeIo(); + const installRuntime = vi.fn(); + + await expect( + createManagedPythonSemanticLayerComputePort({ + cliVersion: '0.2.0', + installPolicy: 'never', + io: io.io, + readStatus: vi.fn(async () => missingStatus()), + installRuntime, + }), + ).rejects.toThrow('KTX Python runtime is required for this command. Run: ktx runtime install --yes'); + + expect(installRuntime).not.toHaveBeenCalled(); + }); + + it('installs the core runtime without prompting when policy is auto', async () => { + const io = makeIo(); + const compute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; + const createPythonCompute = vi.fn(() => compute); + const installRuntime = vi.fn(async () => installResult()); + + await expect( + createManagedPythonSemanticLayerComputePort({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + readStatus: vi.fn(async () => missingStatus()), + installRuntime, + createPythonCompute, + }), + ).resolves.toBe(compute); + + expect(installRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['core'], + force: false, + }); + expect(io.stderr()).toContain('Installing KTX Python runtime (core) with uv'); + expect(io.stderr()).toContain('KTX Python runtime ready: /runtime/0.2.0'); + }); + + it('prompts before installing when policy is prompt', async () => { + const io = makeIo(); + const confirmInstall = vi.fn(async () => true); + const installRuntime = vi.fn(async () => installResult()); + + await createManagedPythonSemanticLayerComputePort({ + cliVersion: '0.2.0', + installPolicy: 'prompt', + io: io.io, + readStatus: vi.fn(async () => missingStatus()), + installRuntime, + createPythonCompute: vi.fn(() => ({ query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() })), + confirmInstall, + }); + + expect(confirmInstall).toHaveBeenCalledWith( + 'KTX needs to install the core Python runtime. This downloads Python dependencies with uv. Continue?', + ); + expect(installRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['core'], + force: false, + }); + }); +}); +``` + +- [ ] **Step 2: Run the failing test** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts +``` + +Expected: FAIL with an import error for +`./managed-python-command.js`. + +### Task 2: Implement the managed Python command helper + +**Files:** + +- Create: `packages/cli/src/managed-python-command.ts` +- Test: `packages/cli/src/managed-python-command.test.ts` + +- [ ] **Step 1: Create the helper** + +Create `packages/cli/src/managed-python-command.ts` with this content: + +```typescript +import { cancel, confirm, isCancel } from '@clack/prompts'; +import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon'; +import type { KtxCliIo } from './cli-runtime.js'; +import { + installManagedPythonRuntime, + readManagedPythonRuntimeStatus, + type InstalledKtxRuntimeManifest, + type KtxRuntimeFeature, + type ManagedPythonRuntimeInstallOptions, + type ManagedPythonRuntimeInstallResult, + type ManagedPythonRuntimeLayout, + type ManagedPythonRuntimeLayoutOptions, + type ManagedPythonRuntimeStatus, +} from './managed-python-runtime.js'; + +export type KtxManagedPythonInstallPolicy = 'prompt' | 'auto' | 'never'; + +export interface ManagedPythonCommandRuntime { + layout: ManagedPythonRuntimeLayout; + manifest: InstalledKtxRuntimeManifest; +} + +export interface ManagedPythonCommandDeps { + readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; + installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise; + confirmInstall?: (message: string) => Promise; +} + +export interface ManagedPythonCommandOptions extends ManagedPythonCommandDeps { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + feature?: KtxRuntimeFeature; +} + +export interface ManagedPythonSemanticLayerComputeOptions extends ManagedPythonCommandOptions { + createPythonCompute?: typeof createPythonSemanticLayerComputePort; +} + +export function managedRuntimeInstallCommand(feature: KtxRuntimeFeature): string { + return feature === 'local-embeddings' + ? 'ktx runtime install --feature local-embeddings --yes' + : 'ktx runtime install --yes'; +} + +function installPrompt(feature: KtxRuntimeFeature): string { + const label = feature === 'local-embeddings' ? 'local embeddings Python runtime' : 'core Python runtime'; + return `KTX needs to install the ${label}. This downloads Python dependencies with uv. Continue?`; +} + +function runtimeRequiredMessage(feature: KtxRuntimeFeature): string { + return `KTX Python runtime is required for this command. Run: ${managedRuntimeInstallCommand(feature)}`; +} + +function hasFeature(manifest: InstalledKtxRuntimeManifest, feature: KtxRuntimeFeature): boolean { + return manifest.features.includes(feature); +} + +async function defaultConfirmInstall(message: string): Promise { + if (process.stdin.isTTY !== true || process.stdout.isTTY !== true) { + return false; + } + const response = await confirm({ message, initialValue: true }); + if (isCancel(response)) { + cancel('Runtime installation cancelled.'); + return false; + } + return response === true; +} + +export async function ensureManagedPythonCommandRuntime( + options: ManagedPythonCommandOptions, +): Promise { + const feature = options.feature ?? 'core'; + const readStatus = options.readStatus ?? readManagedPythonRuntimeStatus; + const installRuntime = options.installRuntime ?? installManagedPythonRuntime; + const status = await readStatus({ cliVersion: options.cliVersion }); + + if (status.kind === 'ready' && status.manifest && hasFeature(status.manifest, feature)) { + return { layout: status.layout, manifest: status.manifest }; + } + + if (options.installPolicy === 'never') { + throw new Error(runtimeRequiredMessage(feature)); + } + + if (options.installPolicy === 'prompt') { + const confirmInstall = options.confirmInstall ?? defaultConfirmInstall; + const confirmed = await confirmInstall(installPrompt(feature)); + if (!confirmed) { + throw new Error(`KTX Python runtime installation was cancelled. Run: ${managedRuntimeInstallCommand(feature)}`); + } + } + + options.io.stderr.write(`Installing KTX Python runtime (${feature}) with uv...\n`); + const installed = await installRuntime({ + cliVersion: options.cliVersion, + features: [feature], + force: false, + }); + options.io.stderr.write(`KTX Python runtime ready: ${installed.layout.versionDir}\n`); + return { layout: installed.layout, manifest: installed.manifest }; +} + +export async function createManagedPythonSemanticLayerComputePort( + options: ManagedPythonSemanticLayerComputeOptions, +): Promise { + const runtime = await ensureManagedPythonCommandRuntime({ + cliVersion: options.cliVersion, + installPolicy: options.installPolicy, + io: options.io, + feature: 'core', + ...(options.readStatus ? { readStatus: options.readStatus } : {}), + ...(options.installRuntime ? { installRuntime: options.installRuntime } : {}), + ...(options.confirmInstall ? { confirmInstall: options.confirmInstall } : {}), + }); + const createPythonCompute = options.createPythonCompute ?? createPythonSemanticLayerComputePort; + return createPythonCompute({ + command: runtime.manifest.python.daemonExecutable, + args: [], + }); +} +``` + +- [ ] **Step 2: Run the helper test** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts +``` + +Expected: PASS. + +- [ ] **Step 3: Commit** + +Run: + +```bash +git add packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts +git commit -m "feat: add managed python command helper" +``` + +Expected: commit succeeds. + +### Task 3: Add failing `runKtxSl` managed runtime tests + +**Files:** + +- Modify: `packages/cli/src/sl.test.ts` +- Test: `packages/cli/src/sl.test.ts` + +- [ ] **Step 1: Add runtime fields to existing `query` test args** + +In each existing `runKtxSl` call whose argument object has +`command: 'query'`, add these properties: + +```typescript +cliVersion: '0.2.0', +runtimeInstallPolicy: 'auto', +``` + +For example, the first `query` argument object becomes: + +```typescript +{ + command: 'query', + projectDir: '/tmp/project', + connectionId: 'warehouse', + query: { measures: ['orders.order_count'], dimensions: [] }, + format: 'sql', + execute: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', +} +``` + +- [ ] **Step 2: Add the managed helper delegation test** + +In `packages/cli/src/sl.test.ts`, add this test inside +`describe('runKtxSl', () => { ... })` after the existing +`runs sl query and prints SQL output` test: + +```typescript + it('creates default sl query compute through the managed runtime helper', async () => { + const projectDir = join(tempDir, 'project'); + const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); + project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + await project.fileStore.writeFile( + 'semantic-layer/warehouse/orders.yaml', + `name: orders +table: public.orders +grain: [id] +columns: + - name: id + type: number +measures: + - name: order_count + expr: count(*) +joins: [] +`, + 'ktx', + 'ktx@example.com', + 'Add orders source', + ); + + const stdout = { write: vi.fn() }; + const stderr = { write: vi.fn() }; + const compute = { + query: vi.fn(async () => ({ + sql: 'select count(*) as order_count from public.orders', + dialect: 'postgres', + columns: [{ name: 'orders.order_count' }], + plan: {}, + })), + validateSources: vi.fn(), + generateSources: vi.fn(), + }; + const createManagedSemanticLayerCompute = vi.fn(async () => compute); + + await expect( + runKtxSl( + { + command: 'query', + projectDir, + connectionId: 'warehouse', + query: { measures: ['orders.order_count'], dimensions: [] }, + format: 'sql', + execute: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + { stdout, stderr }, + { createManagedSemanticLayerCompute }, + ), + ).resolves.toBe(0); + + expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: { stdout, stderr }, + }); + expect(stdout.write).toHaveBeenCalledWith('select count(*) as order_count from public.orders\n'); + }); +``` + +- [ ] **Step 3: Run the failing `sl` test** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/sl.test.ts +``` + +Expected: FAIL with a TypeScript/Vitest error because `runKtxSl` does not +accept `createManagedSemanticLayerCompute` yet. + +### Task 4: Wire `runKtxSl` to the managed helper + +**Files:** + +- Modify: `packages/cli/src/sl.ts` +- Test: `packages/cli/src/sl.test.ts` + +- [ ] **Step 1: Add the managed helper imports** + +In `packages/cli/src/sl.ts`, add this import after the existing imports: + +```typescript +import { + createManagedPythonSemanticLayerComputePort, + type KtxManagedPythonInstallPolicy, +} from './managed-python-command.js'; +``` + +- [ ] **Step 2: Extend the `query` args type** + +In the `KtxSlArgs` union, replace the current `query` object type with this +shape: + +```typescript + | { + command: 'query'; + projectDir: string; + connectionId?: string; + query: SemanticLayerQueryInput; + format: SlQueryFormat; + execute: boolean; + maxRows?: number; + cliVersion: string; + runtimeInstallPolicy: KtxManagedPythonInstallPolicy; + }; +``` + +- [ ] **Step 3: Extend `KtxSlDeps`** + +In `packages/cli/src/sl.ts`, replace `KtxSlDeps` with this interface: + +```typescript +interface KtxSlDeps { + loadProject?: typeof loadKtxProject; + createSemanticLayerCompute?: () => KtxSemanticLayerComputePort; + createManagedSemanticLayerCompute?: (options: { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxSlIo; + }) => Promise; + createQueryExecutor?: () => KtxSqlQueryExecutorPort; +} +``` + +- [ ] **Step 4: Use the managed helper in the `query` branch** + +In the `args.command === 'query'` branch, replace: + +```typescript + const compute = (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)(); +``` + +with: + +```typescript + const compute = deps.createSemanticLayerCompute + ? deps.createSemanticLayerCompute() + : await (deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort)({ + cliVersion: args.cliVersion, + installPolicy: args.runtimeInstallPolicy, + io, + }); +``` + +- [ ] **Step 5: Run the `sl` test** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/sl.test.ts +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +Run: + +```bash +git add packages/cli/src/sl.ts packages/cli/src/sl.test.ts +git commit -m "feat: use managed runtime for sl query compute" +``` + +Expected: commit succeeds. + +### Task 5: Add failing Commander routing tests for `sl query` + +**Files:** + +- Modify: `packages/cli/src/index.test.ts` +- Test: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Add routing tests** + +In `packages/cli/src/index.test.ts`, add this test near the other command +routing tests: + +```typescript + it('routes sl query managed runtime install policies', async () => { + const sl = vi.fn(async () => 0); + + const promptIo = makeIo(); + await expect( + runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count'], promptIo.io, { sl }), + ).resolves.toBe(0); + expect(sl).toHaveBeenLastCalledWith( + expect.objectContaining({ + command: 'query', + projectDir: tempDir, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', + query: expect.objectContaining({ measures: ['orders.order_count'], dimensions: [] }), + }), + promptIo.io, + ); + + const autoIo = makeIo(); + await expect( + runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes'], autoIo.io, { + sl, + }), + ).resolves.toBe(0); + expect(sl).toHaveBeenLastCalledWith( + expect.objectContaining({ + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'auto', + }), + autoIo.io, + ); + + const noInputIo = makeIo(); + await expect( + runKtxCli( + ['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--no-input'], + noInputIo.io, + { sl }, + ), + ).resolves.toBe(0); + expect(sl).toHaveBeenLastCalledWith( + expect.objectContaining({ + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'never', + }), + noInputIo.io, + ); + }); + + it('rejects conflicting sl query runtime install flags', async () => { + const io = makeIo(); + const sl = vi.fn(async () => 0); + + await expect( + runKtxCli( + ['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes', '--no-input'], + io.io, + { sl }, + ), + ).resolves.toBe(1); + + expect(sl).not.toHaveBeenCalled(); + expect(io.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); + }); +``` + +- [ ] **Step 2: Run the failing routing tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/index.test.ts +``` + +Expected: FAIL because `sl query` does not accept `--yes` or `--no-input` +and does not pass runtime policy fields yet. + +### Task 6: Wire `sl query` flags and schema validation + +**Files:** + +- Modify: `packages/cli/src/commands/sl-commands.ts` +- Modify: `packages/cli/src/command-schemas.ts` +- Test: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Add the runtime policy type import** + +In `packages/cli/src/commands/sl-commands.ts`, add this import: + +```typescript +import type { KtxManagedPythonInstallPolicy } from '../managed-python-command.js'; +``` + +- [ ] **Step 2: Add the runtime policy parser** + +In `packages/cli/src/commands/sl-commands.ts`, add this function near the +other option parsers: + +```typescript +function runtimeInstallPolicy(options: { yes?: boolean; input?: boolean }): KtxManagedPythonInstallPolicy { + if (options.yes === true && options.input === false) { + throw new Error('Choose only one runtime install mode: --yes or --no-input'); + } + if (options.yes === true) { + return 'auto'; + } + return options.input === false ? 'never' : 'prompt'; +} +``` + +- [ ] **Step 3: Add the command options** + +In the `sl.command('query')` option chain, add these options after +`.option('--execute', 'Execute the compiled query', false)`: + +```typescript + .option('--yes', 'Install the managed Python runtime without prompting when required', false) + .option('--no-input', 'Disable interactive managed runtime installation') +``` + +- [ ] **Step 4: Pass runtime fields into `slQueryCommandSchema.parse`** + +In the `sl.command('query')` action, add these properties to the parsed object: + +```typescript + cliVersion: context.packageInfo.version, + runtimeInstallPolicy: runtimeInstallPolicy(options), +``` + +The parsed object must include these fields next to `execute` and `format`: + +```typescript + const args = slQueryCommandSchema.parse({ + command: 'query', + projectDir: resolveCommandProjectDir(command), + connectionId: options.connectionId, + query: { + measures: options.measure, + dimensions: options.dimension, + ...(options.filter.length > 0 ? { filters: options.filter } : {}), + ...(options.segment.length > 0 ? { segments: options.segment } : {}), + ...(options.orderBy.length > 0 ? { order_by: options.orderBy } : {}), + ...(options.limit !== undefined ? { limit: options.limit } : {}), + ...(options.includeEmpty === true ? { include_empty: true } : {}), + }, + format: options.format, + execute: options.execute === true, + cliVersion: context.packageInfo.version, + runtimeInstallPolicy: runtimeInstallPolicy(options), + ...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}), + }); +``` + +- [ ] **Step 5: Extend the command schema** + +In `packages/cli/src/command-schemas.ts`, add these fields to +`slQueryCommandSchema` after `execute: z.boolean()`: + +```typescript + cliVersion: z.string().min(1), + runtimeInstallPolicy: z.enum(['prompt', 'auto', 'never']), +``` + +- [ ] **Step 6: Run the routing tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +Run: + +```bash +git add packages/cli/src/commands/sl-commands.ts packages/cli/src/command-schemas.ts packages/cli/src/index.test.ts +git commit -m "feat: route sl query managed runtime policy" +``` + +Expected: commit succeeds. + +### Task 7: Verify the full changed surface + +**Files:** + +- Verify: `packages/cli/src/managed-python-command.test.ts` +- Verify: `packages/cli/src/sl.test.ts` +- Verify: `packages/cli/src/index.test.ts` +- Verify: `packages/cli/src/managed-python-command.ts` +- Verify: `packages/cli/src/sl.ts` +- Verify: `packages/cli/src/commands/sl-commands.ts` +- Verify: `packages/cli/src/command-schemas.ts` + +- [ ] **Step 1: Run focused CLI tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts src/sl.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 2: Run CLI type checking** + +Run: + +```bash +pnpm --filter @ktx/cli run type-check +``` + +Expected: PASS. + +- [ ] **Step 3: Run pre-commit for changed TypeScript files** + +Run: + +```bash +uv run pre-commit run --files packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts packages/cli/src/sl.ts packages/cli/src/sl.test.ts packages/cli/src/commands/sl-commands.ts packages/cli/src/command-schemas.ts packages/cli/src/index.test.ts +``` + +Expected: PASS. If pre-commit is unavailable because the local `uv` version +does not satisfy `pyproject.toml`, record the version mismatch and run the +focused CLI tests plus type checking from Steps 1 and 2. + +- [ ] **Step 4: Commit verification fixes when needed** + +If Step 1, Step 2, or Step 3 changes files through formatting hooks, run: + +```bash +git add packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts packages/cli/src/sl.ts packages/cli/src/sl.test.ts packages/cli/src/commands/sl-commands.ts packages/cli/src/command-schemas.ts packages/cli/src/index.test.ts +git commit -m "test: verify managed runtime sl query integration" +``` + +Expected: commit succeeds only when verification changed files. If no files +changed, leave the branch with the commits from Tasks 2, 4, and 6. + +## Acceptance criteria + +When this plan is complete: + +- `ktx sl query` uses the managed runtime's installed `ktx-daemon` executable + for semantic-layer compilation when no test compute dependency is injected. +- `ktx sl query --yes` installs the `core` runtime feature without prompting + when the managed runtime is missing. +- `ktx sl query --no-input` fails with + `KTX Python runtime is required for this command. Run: ktx runtime install --yes` + when the managed runtime is missing. +- `ktx sl query` prompts before first managed runtime installation in an + interactive terminal. +- Existing injected-compute tests still bypass runtime installation. diff --git a/docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md b/docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md new file mode 100644 index 00000000..280ec728 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md @@ -0,0 +1,1546 @@ +# Managed Python Runtime Daemon Lifecycle Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add `ktx runtime start` and `ktx runtime stop` for the +KTX-managed Python HTTP daemon, including state files, health checks, reuse, +and stale daemon repair. + +**Architecture:** Keep daemon process management in a new CLI-owned module that +depends on the existing managed runtime installer. The module starts +`ktx-daemon serve-http` from the installed runtime on `127.0.0.1`, writes an +adjacent daemon state file, verifies `/health` before reuse, and removes stale +state when the process, port, version, or requested feature set no longer +matches. + +**Tech Stack:** TypeScript, Node 22 ESM, Commander, Vitest, `zod`, FastAPI, +`uvicorn`, `uv`, KTX managed runtime assets. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +Existing plans based on the spec: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` is + implemented. The worktree contains + `scripts/build-python-runtime-wheel.mjs`, + `scripts/build-python-runtime-wheel.test.mjs`, runtime-wheel packaging in + `scripts/package-artifacts.mjs`, release-policy coverage, and matching + artifact tests. +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` is + implemented. The worktree contains + `packages/cli/src/managed-python-runtime.ts`, + `packages/cli/src/runtime.ts`, + `packages/cli/src/commands/runtime-commands.ts`, CLI registration, and + matching Vitest coverage. +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` + is implemented. The worktree contains + `packages/cli/src/managed-python-command.ts`, `ktx sl query` runtime policy + flags, schema validation, and matching CLI tests. + +Implementation evidence collected before writing this plan: + +```bash +node --test scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs +``` + +Expected current result: + +```text +# pass 38 +# fail 0 +``` + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts src/runtime.test.ts src/index.test.ts src/managed-python-command.test.ts src/sl.test.ts +``` + +Expected current result: + +```text +Test Files 58 passed (58) +Tests 699 passed (699) +``` + +Spec requirements still outside this plan: + +- Lazy `local-embeddings` installation and daemon reuse from embedding setup, + embedding health checks, and ingest paths. +- Managed runtime usage for Python-backed operations beyond `ktx sl query`. +- Public npm package rename from `@ktx/cli` to `@kaelio/ktx`. + +This plan implements the daemon lifecycle requirement: + +- `ktx runtime start` +- `ktx runtime stop` +- A versioned daemon state file adjacent to the installed runtime manifest. +- Random localhost port allocation. +- Captured daemon stdout and stderr logs. +- `/health` validation before daemon reuse. +- Stale daemon cleanup when process, health, version, or features don't match. + +## File structure + +- Modify `python/ktx-daemon/src/ktx_daemon/app.py`: include a daemon version in + `/health`, supplied by `KTX_DAEMON_VERSION` for managed runtime starts. +- Modify `python/ktx-daemon/tests/test_app.py`: assert the health endpoint + returns the managed version when the environment variable is set. +- Modify `packages/cli/src/managed-python-runtime.ts`: add daemon state and log + paths to `ManagedPythonRuntimeLayout`. +- Modify `packages/cli/src/managed-python-runtime.test.ts`: assert the new + layout paths. +- Modify `packages/cli/src/runtime.test.ts` and + `packages/cli/src/managed-python-command.test.ts`: add daemon paths to + layout fixtures after the layout type changes. +- Create `packages/cli/src/managed-python-daemon.ts`: start, stop, status, + health-check, stale-state, and state-file logic for the managed HTTP daemon. +- Create `packages/cli/src/managed-python-daemon.test.ts`: unit tests for + stopped status, start, reuse, stale repair, and stop. +- Modify `packages/cli/src/runtime.ts`: route `runtime start` and + `runtime stop` through the daemon lifecycle module and print concise output. +- Modify `packages/cli/src/runtime.test.ts`: assert command runner behavior for + start and stop. +- Modify `packages/cli/src/commands/runtime-commands.ts`: register + `ktx runtime start` and `ktx runtime stop`, and accept `--yes` on + `ktx runtime install` so the preparation command printed by + `ktx sl query --no-input` is valid. +- Modify `packages/cli/src/index.test.ts`: assert Commander routes the new + runtime subcommands with the CLI package version. +- Modify `packages/cli/src/index.ts`: export the daemon lifecycle helpers for + tests and programmatic use. + +### Task 1: Add daemon metadata to runtime layout and Python health + +**Files:** + +- Modify: `packages/cli/src/managed-python-runtime.ts` +- Modify: `packages/cli/src/managed-python-runtime.test.ts` +- Modify: `packages/cli/src/runtime.test.ts` +- Modify: `packages/cli/src/managed-python-command.test.ts` +- Modify: `python/ktx-daemon/src/ktx_daemon/app.py` +- Modify: `python/ktx-daemon/tests/test_app.py` + +- [ ] **Step 1: Write failing TypeScript layout assertions** + +In `packages/cli/src/managed-python-runtime.test.ts`, update the first +`managedPythonRuntimeLayout` test so it includes these expectations after the +existing `daemonPath` assertion: + +```typescript + expect(layout.daemonStatePath).toBe( + '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.json', + ); + expect(layout.daemonStdoutPath).toBe( + '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.stdout.log', + ); + expect(layout.daemonStderrPath).toBe( + '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.stderr.log', + ); +``` + +- [ ] **Step 2: Run the failing layout test** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts +``` + +Expected: FAIL with TypeScript or assertion errors for missing +`daemonStatePath`, `daemonStdoutPath`, and `daemonStderrPath`. + +- [ ] **Step 3: Add daemon paths to the runtime layout type** + +In `packages/cli/src/managed-python-runtime.ts`, add these fields to +`ManagedPythonRuntimeLayout` immediately after `daemonPath`: + +```typescript + daemonStatePath: string; + daemonStdoutPath: string; + daemonStderrPath: string; +``` + +In `managedPythonRuntimeLayout`, add these properties to the returned object +immediately after `daemonPath`: + +```typescript + daemonStatePath: join(versionDir, 'daemon.json'), + daemonStdoutPath: join(versionDir, 'daemon.stdout.log'), + daemonStderrPath: join(versionDir, 'daemon.stderr.log'), +``` + +- [ ] **Step 4: Update layout fixtures used by existing tests** + +In `packages/cli/src/runtime.test.ts`, every object literal that represents a +`ManagedPythonRuntimeLayout` must include these fields: + +```typescript + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', +``` + +In `packages/cli/src/managed-python-command.test.ts`, update the `layout()` +helper to return these fields: + +```typescript + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', +``` + +- [ ] **Step 5: Verify the TypeScript layout change** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts src/runtime.test.ts src/managed-python-command.test.ts +``` + +Expected: PASS. + +- [ ] **Step 6: Write the failing Python health-version test** + +In `python/ktx-daemon/tests/test_app.py`, add this test after +`test_health_endpoint_returns_healthy`: + +```python +def test_health_endpoint_returns_managed_runtime_version(monkeypatch) -> None: + monkeypatch.setenv("KTX_DAEMON_VERSION", "0.2.0") + client = TestClient(create_app()) + + response = client.get("/health") + + assert response.status_code == 200 + assert response.json() == {"status": "healthy", "version": "0.2.0"} +``` + +- [ ] **Step 7: Run the failing Python health test** + +Run: + +```bash +source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_app.py::test_health_endpoint_returns_managed_runtime_version -q +``` + +Expected: FAIL because `/health` does not include `version`. + +- [ ] **Step 8: Include version metadata in daemon health** + +In `python/ktx-daemon/src/ktx_daemon/app.py`, add this import with the existing +imports: + +```python +import os +``` + +Replace the `health` endpoint with: + +```python + @app.get("/health") + async def health() -> dict[str, str]: + response = {"status": "healthy"} + version = os.environ.get("KTX_DAEMON_VERSION") + if version: + response["version"] = version + return response +``` + +- [ ] **Step 9: Verify Python health tests** + +Run: + +```bash +source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_app.py -q +``` + +Expected: PASS. + +- [ ] **Step 10: Run Python pre-commit for modified Python files** + +Run: + +```bash +source .venv/bin/activate && uv run pre-commit run --files python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_app.py +``` + +Expected: PASS. If pre-commit cannot run because hooks or tool versions are +missing, capture the error and run: + +```bash +source .venv/bin/activate && uv run ruff check python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_app.py +``` + +- [ ] **Step 11: Commit** + +Run: + +```bash +git add packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts packages/cli/src/runtime.test.ts packages/cli/src/managed-python-command.test.ts python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_app.py +git commit -m "feat: add managed runtime daemon metadata" +``` + +### Task 2: Implement managed daemon lifecycle library + +**Files:** + +- Create: `packages/cli/src/managed-python-daemon.test.ts` +- Create: `packages/cli/src/managed-python-daemon.ts` +- Test: `packages/cli/src/managed-python-daemon.test.ts` + +- [ ] **Step 1: Write the failing daemon lifecycle tests** + +Create `packages/cli/src/managed-python-daemon.test.ts` with this content: + +```typescript +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + readManagedPythonDaemonStatus, + startManagedPythonDaemon, + stopManagedPythonDaemon, + type ManagedPythonDaemonChild, + type ManagedPythonDaemonFetch, + type ManagedPythonDaemonSpawn, + type ManagedPythonDaemonState, +} from './managed-python-daemon.js'; +import type { + InstalledKtxRuntimeManifest, + ManagedPythonRuntimeInstallResult, + ManagedPythonRuntimeLayout, +} from './managed-python-runtime.js'; + +function layout(root: string): ManagedPythonRuntimeLayout { + return { + cliVersion: '0.2.0', + runtimeRoot: join(root, 'runtime'), + versionDir: join(root, 'runtime', '0.2.0'), + venvDir: join(root, 'runtime', '0.2.0', '.venv'), + manifestPath: join(root, 'runtime', '0.2.0', 'manifest.json'), + installLogPath: join(root, 'runtime', '0.2.0', 'install.log'), + assetDir: join(root, 'assets', 'python'), + assetManifestPath: join(root, 'assets', 'python', 'manifest.json'), + pythonPath: join(root, 'runtime', '0.2.0', '.venv', 'bin', 'python'), + daemonPath: join(root, 'runtime', '0.2.0', '.venv', 'bin', 'ktx-daemon'), + daemonStatePath: join(root, 'runtime', '0.2.0', 'daemon.json'), + daemonStdoutPath: join(root, 'runtime', '0.2.0', 'daemon.stdout.log'), + daemonStderrPath: join(root, 'runtime', '0.2.0', 'daemon.stderr.log'), + }; +} + +function manifest(root: string, features: Array<'core' | 'local-embeddings'> = ['core']): InstalledKtxRuntimeManifest { + const runtimeLayout = layout(root); + return { + schemaVersion: 1, + cliVersion: '0.2.0', + installedAt: '2026-05-11T00:00:00.000Z', + asset: { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.2.0', + wheel: { + file: 'kaelio_ktx-0.2.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 123, + }, + }, + features, + python: { + executable: runtimeLayout.pythonPath, + daemonExecutable: runtimeLayout.daemonPath, + }, + installLog: runtimeLayout.installLogPath, + }; +} + +function installResult(root: string, features: Array<'core' | 'local-embeddings'> = ['core']): ManagedPythonRuntimeInstallResult { + return { + status: 'ready', + layout: layout(root), + asset: { + manifest: manifest(root, features).asset, + wheelPath: join(root, 'assets', 'python', 'kaelio_ktx-0.2.0-py3-none-any.whl'), + }, + manifest: manifest(root, features), + }; +} + +function makeFetch(version = '0.2.0'): ManagedPythonDaemonFetch { + return vi.fn(async () => ({ + ok: true, + status: 200, + json: async () => ({ status: 'healthy', version }), + text: async () => '', + })); +} + +function makeSpawn(pid = 4242): ManagedPythonDaemonSpawn { + return vi.fn((_command, _args, _options): ManagedPythonDaemonChild => ({ + pid, + unref: vi.fn(), + })); +} + +function runningState(root: string, overrides: Partial = {}): ManagedPythonDaemonState { + const runtimeLayout = layout(root); + return { + schemaVersion: 1, + pid: 4242, + host: '127.0.0.1', + port: 58731, + version: '0.2.0', + features: ['core'], + startedAt: '2026-05-11T00:00:00.000Z', + stdoutLog: runtimeLayout.daemonStdoutPath, + stderrLog: runtimeLayout.daemonStderrPath, + ...overrides, + }; +} + +describe('managed Python daemon lifecycle', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-managed-daemon-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('reports stopped when no daemon state exists', async () => { + const status = await readManagedPythonDaemonStatus({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + processAlive: vi.fn(() => false), + fetch: makeFetch(), + }); + + expect(status.kind).toBe('stopped'); + expect(status.detail).toContain('No daemon state'); + }); + + it('starts ktx-daemon serve-http, waits for health, and writes state', async () => { + const spawnDaemon = makeSpawn(5555); + const installRuntime = vi.fn(async () => installResult(tempDir)); + + const result = await startManagedPythonDaemon({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + features: ['core'], + installRuntime, + spawnDaemon, + fetch: makeFetch(), + allocatePort: vi.fn(async () => 61234), + now: () => new Date('2026-05-11T00:00:00.000Z'), + pollIntervalMs: 1, + }); + + expect(result.status).toBe('started'); + expect(result.baseUrl).toBe('http://127.0.0.1:61234'); + expect(installRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + features: ['core'], + force: false, + }); + expect(spawnDaemon).toHaveBeenCalledWith( + layout(tempDir).daemonPath, + ['serve-http', '--host', '127.0.0.1', '--port', '61234'], + expect.objectContaining({ + detached: true, + env: expect.objectContaining({ KTX_DAEMON_VERSION: '0.2.0' }), + }), + ); + expect(JSON.parse(await readFile(layout(tempDir).daemonStatePath, 'utf8'))).toMatchObject({ + pid: 5555, + port: 61234, + version: '0.2.0', + features: ['core'], + stdoutLog: layout(tempDir).daemonStdoutPath, + stderrLog: layout(tempDir).daemonStderrPath, + }); + }); + + it('reuses a healthy daemon with the requested feature set', async () => { + await mkdir(layout(tempDir).versionDir, { recursive: true }); + await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`); + const spawnDaemon = makeSpawn(9999); + + const result = await startManagedPythonDaemon({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + features: ['core'], + installRuntime: vi.fn(async () => installResult(tempDir)), + spawnDaemon, + fetch: makeFetch(), + processAlive: vi.fn(() => true), + pollIntervalMs: 1, + }); + + expect(result.status).toBe('reused'); + expect(result.baseUrl).toBe('http://127.0.0.1:58731'); + expect(spawnDaemon).not.toHaveBeenCalled(); + }); + + it('starts a fresh daemon when the previous state is stale', async () => { + await mkdir(layout(tempDir).versionDir, { recursive: true }); + await writeFile( + layout(tempDir).daemonStatePath, + `${JSON.stringify(runningState(tempDir, { version: '0.1.0' }), null, 2)}\n`, + ); + + const result = await startManagedPythonDaemon({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + features: ['core'], + installRuntime: vi.fn(async () => installResult(tempDir)), + spawnDaemon: makeSpawn(6666), + fetch: makeFetch(), + processAlive: vi.fn(() => true), + killProcess: vi.fn(), + allocatePort: vi.fn(async () => 61235), + now: () => new Date('2026-05-11T00:00:00.000Z'), + pollIntervalMs: 1, + }); + + expect(result.status).toBe('started'); + expect(JSON.parse(await readFile(layout(tempDir).daemonStatePath, 'utf8'))).toMatchObject({ + pid: 6666, + port: 61235, + version: '0.2.0', + }); + }); + + it('stops a recorded daemon and removes the state file', async () => { + await mkdir(layout(tempDir).versionDir, { recursive: true }); + await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`); + const killProcess = vi.fn(); + + const result = await stopManagedPythonDaemon({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + processAlive: vi.fn(() => true), + killProcess, + }); + + expect(result.status).toBe('stopped'); + expect(killProcess).toHaveBeenCalledWith(4242); + await expect(readFile(layout(tempDir).daemonStatePath, 'utf8')).rejects.toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run the failing daemon lifecycle tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-daemon.test.ts +``` + +Expected: FAIL with an import error for `./managed-python-daemon.js`. + +- [ ] **Step 3: Implement the daemon lifecycle module** + +Create `packages/cli/src/managed-python-daemon.ts` with this content: + +```typescript +import { spawn } from 'node:child_process'; +import { mkdir, open, readFile, rm, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:net'; +import { setTimeout as delay } from 'node:timers/promises'; +import { z } from 'zod'; +import { + installManagedPythonRuntime, + managedPythonRuntimeLayout, + runtimeFeatureSchema, + type KtxRuntimeFeature, + type ManagedPythonRuntimeInstallOptions, + type ManagedPythonRuntimeInstallResult, + type ManagedPythonRuntimeLayout, + type ManagedPythonRuntimeLayoutOptions, +} from './managed-python-runtime.js'; + +export interface ManagedPythonDaemonState { + schemaVersion: 1; + pid: number; + host: '127.0.0.1'; + port: number; + version: string; + features: KtxRuntimeFeature[]; + startedAt: string; + stdoutLog: string; + stderrLog: string; +} + +export type ManagedPythonDaemonStatus = + | { kind: 'stopped'; detail: string; layout: ManagedPythonRuntimeLayout } + | { kind: 'running'; detail: string; layout: ManagedPythonRuntimeLayout; state: ManagedPythonDaemonState; baseUrl: string } + | { kind: 'stale'; detail: string; layout: ManagedPythonRuntimeLayout; state?: ManagedPythonDaemonState }; + +export interface ManagedPythonDaemonStartResult { + status: 'started' | 'reused'; + layout: ManagedPythonRuntimeLayout; + state: ManagedPythonDaemonState; + baseUrl: string; +} + +export interface ManagedPythonDaemonStopResult { + status: 'stopped' | 'already-stopped'; + layout: ManagedPythonRuntimeLayout; + state?: ManagedPythonDaemonState; +} + +export interface ManagedPythonDaemonChild { + pid?: number; + unref(): void; +} + +export type ManagedPythonDaemonSpawn = ( + command: string, + args: string[], + options: { + detached: boolean; + stdio: ['ignore', number, number]; + env: NodeJS.ProcessEnv; + }, +) => ManagedPythonDaemonChild; + +export type ManagedPythonDaemonFetch = ( + url: string, +) => Promise<{ + ok: boolean; + status: number; + json(): Promise; + text(): Promise; +}>; + +export interface ManagedPythonDaemonStartOptions extends ManagedPythonRuntimeLayoutOptions { + features: KtxRuntimeFeature[]; + force?: boolean; + installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise; + spawnDaemon?: ManagedPythonDaemonSpawn; + fetch?: ManagedPythonDaemonFetch; + allocatePort?: () => Promise; + processAlive?: (pid: number) => boolean; + killProcess?: (pid: number) => void; + now?: () => Date; + startupTimeoutMs?: number; + pollIntervalMs?: number; +} + +export interface ManagedPythonDaemonStatusOptions extends ManagedPythonRuntimeLayoutOptions { + fetch?: ManagedPythonDaemonFetch; + processAlive?: (pid: number) => boolean; +} + +export interface ManagedPythonDaemonStopOptions extends ManagedPythonRuntimeLayoutOptions { + processAlive?: (pid: number) => boolean; + killProcess?: (pid: number) => void; +} + +const daemonStateSchema = z.object({ + schemaVersion: z.literal(1), + pid: z.number().int().positive(), + host: z.literal('127.0.0.1'), + port: z.number().int().min(1).max(65535), + version: z.string().min(1), + features: z.array(runtimeFeatureSchema).min(1), + startedAt: z.string().min(1), + stdoutLog: z.string().min(1), + stderrLog: z.string().min(1), +}); + +function normalizeFeatures(features: KtxRuntimeFeature[]): KtxRuntimeFeature[] { + const requested = new Set(['core', ...features]); + return runtimeFeatureSchema.options.filter((feature) => requested.has(feature)); +} + +function hasFeatures(state: ManagedPythonDaemonState, features: KtxRuntimeFeature[]): boolean { + return normalizeFeatures(features).every((feature) => state.features.includes(feature)); +} + +function defaultFetch(url: string): ReturnType { + return fetch(url) as ReturnType; +} + +function defaultProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function defaultKillProcess(pid: number): void { + try { + process.kill(pid, 'SIGTERM'); + } catch (error) { + const code = (error as { code?: unknown }).code; + if (code !== 'ESRCH') { + throw error; + } + } +} + +function defaultSpawnDaemon( + command: string, + args: string[], + options: Parameters[2], +): ManagedPythonDaemonChild { + return spawn(command, args, options); +} + +function baseUrl(state: Pick): string { + return `http://${state.host}:${state.port}`; +} + +async function readState(path: string): Promise { + try { + return daemonStateSchema.parse(JSON.parse(await readFile(path, 'utf8')) as unknown); + } catch (error) { + const code = (error as { code?: unknown }).code; + if (code === 'ENOENT') { + return undefined; + } + throw error; + } +} + +async function writeState(path: string, state: ManagedPythonDaemonState): Promise { + await writeFile(path, `${JSON.stringify(state, null, 2)}\n`); +} + +async function healthOk(input: { + state: ManagedPythonDaemonState; + cliVersion: string; + fetch: ManagedPythonDaemonFetch; +}): Promise<{ ok: true } | { ok: false; detail: string }> { + try { + const response = await input.fetch(`${baseUrl(input.state)}/health`); + if (!response.ok) { + return { ok: false, detail: `Health check returned HTTP ${response.status}: ${await response.text()}` }; + } + const body = (await response.json()) as unknown; + if (!body || typeof body !== 'object' || Array.isArray(body)) { + return { ok: false, detail: 'Health check returned non-object JSON' }; + } + const record = body as Record; + if (record.status !== 'healthy') { + return { ok: false, detail: `Health check returned status ${String(record.status)}` }; + } + if (record.version !== input.cliVersion) { + return { + ok: false, + detail: `Daemon version ${String(record.version)} does not match CLI ${input.cliVersion}`, + }; + } + return { ok: true }; + } catch (error) { + return { ok: false, detail: error instanceof Error ? error.message : String(error) }; + } +} + +export async function readManagedPythonDaemonStatus( + options: ManagedPythonDaemonStatusOptions, +): Promise { + const layout = managedPythonRuntimeLayout(options); + let state: ManagedPythonDaemonState | undefined; + try { + state = await readState(layout.daemonStatePath); + } catch (error) { + return { + kind: 'stale', + detail: `Daemon state is invalid: ${error instanceof Error ? error.message : String(error)}`, + layout, + }; + } + if (!state) { + return { kind: 'stopped', detail: `No daemon state at ${layout.daemonStatePath}`, layout }; + } + if (state.version !== options.cliVersion) { + return { + kind: 'stale', + detail: `Daemon is for CLI ${state.version}, current CLI is ${options.cliVersion}`, + layout, + state, + }; + } + const processAlive = options.processAlive ?? defaultProcessAlive; + if (!processAlive(state.pid)) { + return { kind: 'stale', detail: `Daemon process ${state.pid} is not running`, layout, state }; + } + const health = await healthOk({ + state, + cliVersion: options.cliVersion, + fetch: options.fetch ?? defaultFetch, + }); + if (!health.ok) { + return { kind: 'stale', detail: health.detail, layout, state }; + } + return { kind: 'running', detail: `Daemon running at ${baseUrl(state)}`, layout, state, baseUrl: baseUrl(state) }; +} + +export async function allocateDaemonPort(): Promise { + return await new Promise((resolve, reject) => { + const server = createServer(); + server.on('error', reject); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + server.close(() => { + if (address && typeof address === 'object') { + resolve(address.port); + return; + } + reject(new Error('Failed to allocate a daemon port')); + }); + }); + }); +} + +async function waitForHealth(input: { + state: ManagedPythonDaemonState; + cliVersion: string; + fetch: ManagedPythonDaemonFetch; + timeoutMs: number; + pollIntervalMs: number; +}): Promise { + const deadline = Date.now() + input.timeoutMs; + let lastDetail = 'daemon did not answer health checks'; + while (Date.now() <= deadline) { + const health = await healthOk({ + state: input.state, + cliVersion: input.cliVersion, + fetch: input.fetch, + }); + if (health.ok) { + return; + } + lastDetail = health.detail; + await delay(input.pollIntervalMs); + } + throw new Error(`KTX Python daemon failed to start: ${lastDetail}. stderr: ${input.state.stderrLog}`); +} + +async function removeState(layout: ManagedPythonRuntimeLayout): Promise { + await rm(layout.daemonStatePath, { force: true }); +} + +async function stopRecordedDaemon(input: { + layout: ManagedPythonRuntimeLayout; + state: ManagedPythonDaemonState; + processAlive: (pid: number) => boolean; + killProcess: (pid: number) => void; +}): Promise { + if (input.processAlive(input.state.pid)) { + input.killProcess(input.state.pid); + } + await removeState(input.layout); +} + +export async function startManagedPythonDaemon( + options: ManagedPythonDaemonStartOptions, +): Promise { + const features = normalizeFeatures(options.features); + const installRuntime = options.installRuntime ?? installManagedPythonRuntime; + const layoutOverrides = { + ...(options.runtimeRoot !== undefined ? { runtimeRoot: options.runtimeRoot } : {}), + ...(options.assetDir !== undefined ? { assetDir: options.assetDir } : {}), + ...(options.platform !== undefined ? { platform: options.platform } : {}), + ...(options.env !== undefined ? { env: options.env } : {}), + ...(options.homeDir !== undefined ? { homeDir: options.homeDir } : {}), + }; + const layout = managedPythonRuntimeLayout({ cliVersion: options.cliVersion, ...layoutOverrides }); + const processAlive = options.processAlive ?? defaultProcessAlive; + const killProcess = options.killProcess ?? defaultKillProcess; + const fetchImpl = options.fetch ?? defaultFetch; + + const status = await readManagedPythonDaemonStatus({ + cliVersion: options.cliVersion, + ...layoutOverrides, + fetch: fetchImpl, + processAlive, + }); + if (options.force !== true && status.kind === 'running' && hasFeatures(status.state, features)) { + return { status: 'reused', layout, state: status.state, baseUrl: status.baseUrl }; + } + if (status.state) { + await stopRecordedDaemon({ layout, state: status.state, processAlive, killProcess }); + } else { + await removeState(layout); + } + + const installed = await installRuntime({ + cliVersion: options.cliVersion, + ...layoutOverrides, + features, + force: false, + }); + + await mkdir(layout.versionDir, { recursive: true }); + const stdout = await open(layout.daemonStdoutPath, 'a'); + const stderr = await open(layout.daemonStderrPath, 'a'); + try { + const port = await (options.allocatePort ?? allocateDaemonPort)(); + const spawnDaemon = options.spawnDaemon ?? defaultSpawnDaemon; + const child = spawnDaemon( + installed.manifest.python.daemonExecutable, + ['serve-http', '--host', '127.0.0.1', '--port', String(port)], + { + detached: true, + stdio: ['ignore', stdout.fd, stderr.fd], + env: { + ...process.env, + KTX_DAEMON_VERSION: options.cliVersion, + }, + }, + ); + child.unref(); + if (!child.pid) { + throw new Error(`KTX Python daemon did not report a pid. stderr: ${layout.daemonStderrPath}`); + } + const state: ManagedPythonDaemonState = { + schemaVersion: 1, + pid: child.pid, + host: '127.0.0.1', + port, + version: options.cliVersion, + features: installed.manifest.features, + startedAt: (options.now ?? (() => new Date()))().toISOString(), + stdoutLog: layout.daemonStdoutPath, + stderrLog: layout.daemonStderrPath, + }; + await waitForHealth({ + state, + cliVersion: options.cliVersion, + fetch: fetchImpl, + timeoutMs: options.startupTimeoutMs ?? 10_000, + pollIntervalMs: options.pollIntervalMs ?? 100, + }); + await writeState(layout.daemonStatePath, state); + return { status: 'started', layout, state, baseUrl: baseUrl(state) }; + } finally { + await stdout.close(); + await stderr.close(); + } +} + +export async function stopManagedPythonDaemon( + options: ManagedPythonDaemonStopOptions, +): Promise { + const layout = managedPythonRuntimeLayout(options); + const state = await readState(layout.daemonStatePath); + if (!state) { + return { status: 'already-stopped', layout }; + } + await stopRecordedDaemon({ + layout, + state, + processAlive: options.processAlive ?? defaultProcessAlive, + killProcess: options.killProcess ?? defaultKillProcess, + }); + return { status: 'stopped', layout, state }; +} +``` + +- [ ] **Step 4: Run daemon lifecycle tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-daemon.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add packages/cli/src/managed-python-daemon.ts packages/cli/src/managed-python-daemon.test.ts +git commit -m "feat: manage python daemon lifecycle" +``` + +### Task 3: Wire runtime start and stop commands + +**Files:** + +- Modify: `packages/cli/src/runtime.ts` +- Modify: `packages/cli/src/runtime.test.ts` +- Modify: `packages/cli/src/commands/runtime-commands.ts` +- Modify: `packages/cli/src/index.test.ts` +- Modify: `packages/cli/src/index.ts` + +- [ ] **Step 1: Write failing runtime command runner tests** + +In `packages/cli/src/runtime.test.ts`, add these imports: + +```typescript +import type { + ManagedPythonDaemonStartResult, + ManagedPythonDaemonStopResult, +} from './managed-python-daemon.js'; +``` + +Add these tests inside `describe('runKtxRuntime', () => { ... })` after the +install test: + +```typescript + it('starts the managed Python daemon and prints the base URL', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + startDaemon: vi.fn(async (): Promise => ({ + status: 'started', + baseUrl: 'http://127.0.0.1:61234', + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', + }, + state: { + schemaVersion: 1, + pid: 4242, + host: '127.0.0.1', + port: 61234, + version: '0.2.0', + features: ['core', 'local-embeddings'], + startedAt: '2026-05-11T00:00:00.000Z', + stdoutLog: '/runtime/0.2.0/daemon.stdout.log', + stderrLog: '/runtime/0.2.0/daemon.stderr.log', + }, + })), + }; + + await expect( + runKtxRuntime( + { command: 'start', cliVersion: '0.2.0', feature: 'local-embeddings', force: true }, + io.io, + deps, + ), + ).resolves.toBe(0); + + expect(deps.startDaemon).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['local-embeddings'], + force: true, + }); + expect(io.stdout()).toContain('Started KTX Python daemon'); + expect(io.stdout()).toContain('url: http://127.0.0.1:61234'); + expect(io.stdout()).toContain('pid: 4242'); + expect(io.stdout()).toContain('features: core, local-embeddings'); + expect(io.stdout()).toContain('stderr: /runtime/0.2.0/daemon.stderr.log'); + }); + + it('stops the managed Python daemon', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + stopDaemon: vi.fn(async (): Promise => ({ + status: 'stopped', + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', + }, + state: { + schemaVersion: 1, + pid: 4242, + host: '127.0.0.1', + port: 61234, + version: '0.2.0', + features: ['core'], + startedAt: '2026-05-11T00:00:00.000Z', + stdoutLog: '/runtime/0.2.0/daemon.stdout.log', + stderrLog: '/runtime/0.2.0/daemon.stderr.log', + }, + })), + }; + + await expect(runKtxRuntime({ command: 'stop', cliVersion: '0.2.0' }, io.io, deps)).resolves.toBe(0); + + expect(deps.stopDaemon).toHaveBeenCalledWith({ cliVersion: '0.2.0' }); + expect(io.stdout()).toContain('Stopped KTX Python daemon'); + expect(io.stdout()).toContain('pid: 4242'); + }); +``` + +- [ ] **Step 2: Run the failing command runner tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/runtime.test.ts +``` + +Expected: FAIL because `KtxRuntimeArgs` and `KtxRuntimeDeps` do not include +`start`, `stop`, `startDaemon`, or `stopDaemon`. + +- [ ] **Step 3: Update the runtime command runner** + +In `packages/cli/src/runtime.ts`, add these imports: + +```typescript +import { + startManagedPythonDaemon, + stopManagedPythonDaemon, + type ManagedPythonDaemonStartResult, + type ManagedPythonDaemonStopResult, +} from './managed-python-daemon.js'; +``` + +Extend `KtxRuntimeArgs` with: + +```typescript + | { command: 'start'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean } + | { command: 'stop'; cliVersion: string } +``` + +Extend `KtxRuntimeDeps` with: + +```typescript + startDaemon?: (options: { + cliVersion: string; + features: KtxRuntimeFeature[]; + force?: boolean; + }) => Promise; + stopDaemon?: (options: { cliVersion: string }) => Promise; +``` + +Add these writer helpers after `writeInstallResult`: + +```typescript +function writeDaemonStart(io: KtxCliIo, result: ManagedPythonDaemonStartResult): void { + const verb = result.status === 'reused' ? 'Using existing' : 'Started'; + io.stdout.write(`${verb} KTX Python daemon\n`); + io.stdout.write(`url: ${result.baseUrl}\n`); + io.stdout.write(`pid: ${result.state.pid}\n`); + io.stdout.write(`version: ${result.state.version}\n`); + io.stdout.write(`features: ${result.state.features.join(', ')}\n`); + io.stdout.write(`state: ${result.layout.daemonStatePath}\n`); + io.stdout.write(`stdout: ${result.state.stdoutLog}\n`); + io.stdout.write(`stderr: ${result.state.stderrLog}\n`); +} + +function writeDaemonStop(io: KtxCliIo, result: ManagedPythonDaemonStopResult): void { + if (result.status === 'already-stopped') { + io.stdout.write('KTX Python daemon already stopped\n'); + return; + } + io.stdout.write('Stopped KTX Python daemon\n'); + io.stdout.write(`pid: ${result.state?.pid ?? 'unknown'}\n`); + io.stdout.write(`state: ${result.layout.daemonStatePath}\n`); +} +``` + +Inside `runKtxRuntime`, add these branches after the install branch: + +```typescript + if (args.command === 'start') { + const startDaemon = deps.startDaemon ?? startManagedPythonDaemon; + const result = await startDaemon({ + cliVersion: args.cliVersion, + features: [args.feature], + force: args.force, + }); + writeDaemonStart(io, result); + return 0; + } + if (args.command === 'stop') { + const stopDaemon = deps.stopDaemon ?? stopManagedPythonDaemon; + const result = await stopDaemon({ cliVersion: args.cliVersion }); + writeDaemonStop(io, result); + return 0; + } +``` + +- [ ] **Step 4: Verify runtime command runner tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/runtime.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Write failing Commander routing tests** + +In `packages/cli/src/index.test.ts`, inside +`it('routes runtime management commands with the CLI package version', ...)`, +add two new IO handles after `installIo`: + +```typescript + const startIo = makeIo(); + const stopIo = makeIo(); +``` + +Replace the existing `runtime install` invocation with this version that also +passes `--yes`, then add the new `runtime start` and `runtime stop` +invocations immediately after it: + +```typescript + await expect( + runKtxCli(['runtime', 'install', '--feature', 'local-embeddings', '--force', '--yes'], installIo.io, { + runtime, + }), + ).resolves.toBe(0); + await expect( + runKtxCli(['runtime', 'start', '--feature', 'local-embeddings', '--force'], startIo.io, { runtime }), + ).resolves.toBe(0); + await expect(runKtxCli(['runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0); +``` + +Update the `expect(runtime).toHaveBeenNthCalledWith(...)` assertions so the +runtime calls are: + +```typescript + expect(runtime).toHaveBeenNthCalledWith( + 1, + { + command: 'install', + cliVersion: '0.0.0-private', + feature: 'local-embeddings', + force: true, + }, + installIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 2, + { + command: 'start', + cliVersion: '0.0.0-private', + feature: 'local-embeddings', + force: true, + }, + startIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 3, + { + command: 'stop', + cliVersion: '0.0.0-private', + }, + stopIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 4, + { + command: 'status', + cliVersion: '0.0.0-private', + json: true, + }, + statusIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 5, + { + command: 'doctor', + cliVersion: '0.0.0-private', + json: false, + }, + doctorIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 6, + { + command: 'prune', + cliVersion: '0.0.0-private', + dryRun: true, + yes: false, + }, + pruneIo.io, + ); +``` + +- [ ] **Step 6: Run the failing Commander routing test** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/index.test.ts +``` + +Expected: FAIL because `runtime install --yes` is not accepted and +`runtime start` and `runtime stop` are not registered. + +- [ ] **Step 7: Register start and stop subcommands** + +In `packages/cli/src/commands/runtime-commands.ts`, update the existing +runtime feature option to return a fresh Commander option per command: + +```typescript +function createRuntimeFeatureOption() { + return new Option('--feature ', 'Runtime feature level') + .choices(['core', 'local-embeddings']) + .default('core'); +} +``` + +Then update the existing `install` command so it accepts `--yes` without +changing behavior: + +```typescript + runtime + .command('install') + .description('Install the bundled Python runtime wheel into the managed runtime') + .addOption(createRuntimeFeatureOption()) + .option('--yes', 'Accept runtime installation without prompting', false) + .option('--force', 'Reinstall even when the runtime already looks ready', false) + .action(async (options: { feature: RuntimeFeature; yes?: boolean; force?: boolean }) => { + await runRuntimeArgs(context, { + command: 'install', + cliVersion: context.packageInfo.version, + feature: options.feature, + force: options.force === true, + }); + }); +``` + +Add this `start` command after the `install` command: + +```typescript + runtime + .command('start') + .description('Start the KTX-managed Python HTTP daemon') + .addOption(createRuntimeFeatureOption()) + .option('--force', 'Restart even when a matching daemon is already running', false) + .action(async (options: { feature: RuntimeFeature; force?: boolean }) => { + await runRuntimeArgs(context, { + command: 'start', + cliVersion: context.packageInfo.version, + feature: options.feature, + force: options.force === true, + }); + }); +``` + +Add this `stop` command after the `start` command: + +```typescript + runtime + .command('stop') + .description('Stop the KTX-managed Python HTTP daemon') + .action(async () => { + await runRuntimeArgs(context, { + command: 'stop', + cliVersion: context.packageInfo.version, + }); + }); +``` + +- [ ] **Step 8: Export daemon lifecycle helpers** + +In `packages/cli/src/index.ts`, add this export near the other public test and +programmatic exports: + +```typescript +export { + allocateDaemonPort, + readManagedPythonDaemonStatus, + startManagedPythonDaemon, + stopManagedPythonDaemon, +} from './managed-python-daemon.js'; +export type { + ManagedPythonDaemonStartResult, + ManagedPythonDaemonState, + ManagedPythonDaemonStatus, + ManagedPythonDaemonStopResult, +} from './managed-python-daemon.js'; +``` + +- [ ] **Step 9: Verify CLI routing tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/index.test.ts src/runtime.test.ts +``` + +Expected: PASS. + +- [ ] **Step 10: Commit** + +Run: + +```bash +git add packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts packages/cli/src/commands/runtime-commands.ts packages/cli/src/index.test.ts packages/cli/src/index.ts +git commit -m "feat: add runtime daemon start stop commands" +``` + +### Task 4: Verify daemon lifecycle end to end + +**Files:** + +- Verify: `packages/cli/src/managed-python-daemon.ts` +- Verify: `packages/cli/src/runtime.ts` +- Verify: `python/ktx-daemon/src/ktx_daemon/app.py` + +- [ ] **Step 1: Run focused CLI tests** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts src/managed-python-daemon.test.ts src/runtime.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 2: Run focused Python tests** + +Run: + +```bash +source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_app.py python/ktx-daemon/tests/test_cli.py -q +``` + +Expected: PASS. + +- [ ] **Step 3: Run TypeScript type-check** + +Run: + +```bash +pnpm --filter @ktx/cli run type-check +``` + +Expected: PASS. + +- [ ] **Step 4: Run Python pre-commit for modified files** + +Run: + +```bash +source .venv/bin/activate && uv run pre-commit run --files python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_app.py packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts packages/cli/src/managed-python-daemon.ts packages/cli/src/managed-python-daemon.test.ts packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts packages/cli/src/commands/runtime-commands.ts packages/cli/src/index.test.ts packages/cli/src/index.ts +``` + +Expected: PASS. If pre-commit rejects TypeScript file arguments because a hook +only handles Python, run the Python-only pre-commit command from Task 1 and +then run: + +```bash +pnpm --filter @ktx/cli run check +``` + +- [ ] **Step 5: Build the CLI package** + +Run: + +```bash +pnpm --filter @ktx/cli run build +``` + +Expected: PASS. + +- [ ] **Step 6: Build runtime wheel assets** + +Run: + +```bash +pnpm run artifacts:verify +``` + +Expected: PASS and `packages/cli/assets/python/manifest.json` exists with a +matching `kaelio_ktx-0.1.0-py3-none-any.whl`. + +- [ ] **Step 7: Smoke runtime install, start, reuse, and stop** + +Run: + +```bash +KTX_RUNTIME_ROOT="$(mktemp -d)" +KTX_RUNTIME_ROOT="$KTX_RUNTIME_ROOT" node packages/cli/dist/bin.js runtime install --yes +KTX_RUNTIME_ROOT="$KTX_RUNTIME_ROOT" node packages/cli/dist/bin.js runtime start +KTX_RUNTIME_ROOT="$KTX_RUNTIME_ROOT" node packages/cli/dist/bin.js runtime start +KTX_RUNTIME_ROOT="$KTX_RUNTIME_ROOT" node packages/cli/dist/bin.js runtime stop +rm -rf "$KTX_RUNTIME_ROOT" +``` + +Expected: + +```text +Installed KTX Python runtime +Started KTX Python daemon +Using existing KTX Python daemon +Stopped KTX Python daemon +``` + +If the existing runtime layout does not honor `KTX_RUNTIME_ROOT`, run the same +commands without that environment variable and clean up with: + +```bash +node packages/cli/dist/bin.js runtime stop +node packages/cli/dist/bin.js runtime prune --dry-run +``` + +- [ ] **Step 8: Commit verification-only fixes if needed** + +If verification exposed a small defect inside this plan's files, fix it and +commit only the touched files: + +```bash +git add packages/cli/src/managed-python-daemon.ts packages/cli/src/managed-python-daemon.test.ts packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts packages/cli/src/commands/runtime-commands.ts packages/cli/src/index.test.ts packages/cli/src/index.ts python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_app.py packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts packages/cli/src/managed-python-command.test.ts +git commit -m "fix: verify managed runtime daemon lifecycle" +``` + +Skip this step when there are no verification fixes. + +## Acceptance criteria + +- `ktx runtime start` installs or reuses the requested runtime feature level and + starts `ktx-daemon serve-http` on `127.0.0.1` with a random available port. +- `ktx runtime start` reuses a healthy matching daemon and starts a fresh daemon + when the recorded process, health response, version, or feature set is stale. +- `ktx runtime stop` terminates the recorded daemon process and removes the + daemon state file. +- The daemon state file records `pid`, `port`, `version`, `features`, + `startedAt`, stdout log path, and stderr log path. +- The daemon health endpoint returns `{"status": "healthy"}` by default and + includes `version` when `KTX_DAEMON_VERSION` is set. +- Daemon stdout and stderr are preserved under the versioned runtime directory. +- Focused TypeScript tests, focused Python tests, CLI type-check, and + Python-file pre-commit pass or have explicitly recorded environment blockers. + +## Self-review checklist + +- Spec coverage: this plan covers `ktx runtime start`, `ktx runtime stop`, + daemon state, random localhost port binding, health validation, version + matching, stale repair, and captured daemon logs. It leaves lazy embedding + command integration and public npm renaming for later plans. +- Placeholder scan: this plan contains no placeholder steps, deferred code + blocks, or undefined function names. +- Type consistency: runtime feature values are consistently `core` and + `local-embeddings`; daemon state uses `schemaVersion`, `pid`, `host`, `port`, + `version`, `features`, `startedAt`, `stdoutLog`, and `stderrLog`; command + runner types use `startDaemon` and `stopDaemon`. diff --git a/docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md b/docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md new file mode 100644 index 00000000..c6271cb1 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md @@ -0,0 +1,1750 @@ +# Managed Python Runtime Installer Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Install and inspect the bundled `kaelio-ktx` Python wheel in a +versioned KTX-managed runtime directory. + +**Architecture:** Add a CLI-owned managed-runtime module that knows where the +bundled wheel asset lives, verifies its checksum, creates a versioned virtual +environment with `uv`, installs the requested feature set, and writes an +installed-runtime manifest. Add `ktx runtime install`, `status`, `doctor`, and +`prune` commands that expose this behavior without changing normal +Python-backed commands yet. + +**Tech Stack:** TypeScript, Node 22 ESM, Commander, Vitest, `zod`, `uv`, npm +package assets. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +Plan 1, `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md`, +is implemented in this worktree. The implemented source includes +`scripts/build-python-runtime-wheel.mjs`, +`scripts/build-python-runtime-wheel.test.mjs`, runtime-wheel handling in +`scripts/package-artifacts.mjs`, test coverage in +`scripts/package-artifacts.test.mjs`, and the `kaelio-ktx` release-policy +entry. The targeted verification command passes: + +```bash +node --test scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs +``` + +Expected current result: + +```text +# pass 38 +# fail 0 +``` + +No other plan files currently reference the npm-managed Python runtime spec. + +This plan implements the next prerequisite: + +- Platform-specific managed runtime roots. +- Versioned runtime directories keyed by the CLI package version. +- Runtime asset manifest reading and wheel checksum verification. +- `uv` virtual environment creation. +- Core and `local-embeddings` feature installation levels. +- Installed-runtime manifest writing. +- `ktx runtime install`, `ktx runtime status`, `ktx runtime doctor`, and + `ktx runtime prune`. + +This plan intentionally leaves the following spec requirements for later +plans: + +- Lazy install from normal commands such as `ktx sl query`. +- `ktx runtime start` and `ktx runtime stop`. +- Daemon state, health checks, reuse, and stale-daemon repair. +- Public npm package renaming from `@ktx/cli` to `@kaelio/ktx`. + +## File structure + +- Create `packages/cli/src/managed-python-runtime.ts`: pure managed-runtime + library for path calculation, asset verification, install/status/doctor, and + pruning. +- Create `packages/cli/src/managed-python-runtime.test.ts`: unit tests for + runtime roots, manifest validation, install command shape, status checks, and + prune safety. +- Create `packages/cli/src/runtime.ts`: command runner that formats + `install`, `status`, `doctor`, and `prune` output. +- Create `packages/cli/src/runtime.test.ts`: command-runner tests with injected + managed-runtime dependencies. +- Create `packages/cli/src/commands/runtime-commands.ts`: Commander + registration for `ktx runtime ...`. +- Modify `packages/cli/src/cli-runtime.ts`: add the runtime command runner to + CLI dependency injection. +- Modify `packages/cli/src/cli-program.ts`: pass package info into command + registration and register the runtime command group. +- Modify `packages/cli/src/index.ts`: export runtime command types and the + runner for tests and programmatic use. +- Modify `packages/cli/src/index.test.ts`: assert root help exposes + `runtime` and Commander routes runtime subcommands correctly. + +### Task 1: Add failing managed-runtime library tests + +**Files:** + +- Create: `packages/cli/src/managed-python-runtime.test.ts` +- Test: `packages/cli/src/managed-python-runtime.test.ts` + +- [ ] **Step 1: Write the failing test file** + +Create `packages/cli/src/managed-python-runtime.test.ts` with this content: + +```typescript +import { createHash } from 'node:crypto'; +import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + doctorManagedPythonRuntime, + installManagedPythonRuntime, + managedPythonRuntimeLayout, + pruneManagedPythonRuntimes, + readManagedPythonRuntimeStatus, + verifyRuntimeAsset, + type ManagedPythonRuntimeExec, +} from './managed-python-runtime.js'; + +async function writeAsset(root: string, contents = 'wheel-bytes') { + const assetDir = join(root, 'assets', 'python'); + await mkdir(assetDir, { recursive: true }); + const wheelPath = join(assetDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'); + await writeFile(wheelPath, contents); + await writeFile( + join(assetDir, 'manifest.json'), + `${JSON.stringify( + { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.1.0', + wheel: { + file: 'kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: createHash('sha256').update(contents).digest('hex'), + bytes: Buffer.byteLength(contents), + }, + }, + null, + 2, + )}\n`, + ); + return { assetDir, wheelPath }; +} + +describe('managedPythonRuntimeLayout', () => { + it('uses the macOS application-support runtime root', () => { + const layout = managedPythonRuntimeLayout({ + cliVersion: '0.2.0', + platform: 'darwin', + env: {}, + homeDir: '/Users/alex', + assetDir: '/repo/packages/cli/assets/python', + }); + + expect(layout.runtimeRoot).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime'); + expect(layout.versionDir).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0'); + expect(layout.venvDir).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv'); + expect(layout.pythonPath).toBe( + '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv/bin/python', + ); + expect(layout.daemonPath).toBe( + '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv/bin/ktx-daemon', + ); + expect(layout.assetManifestPath).toBe('/repo/packages/cli/assets/python/manifest.json'); + }); + + it('honors XDG_DATA_HOME on Linux', () => { + const layout = managedPythonRuntimeLayout({ + cliVersion: '0.2.0', + platform: 'linux', + env: { XDG_DATA_HOME: '/var/xdg' }, + homeDir: '/home/alex', + assetDir: '/repo/packages/cli/assets/python', + }); + + expect(layout.runtimeRoot).toBe('/var/xdg/kaelio/ktx/runtime'); + expect(layout.versionDir).toBe('/var/xdg/kaelio/ktx/runtime/0.2.0'); + }); + + it('uses LocalAppData on Windows', () => { + const layout = managedPythonRuntimeLayout({ + cliVersion: '0.2.0', + platform: 'win32', + env: { LOCALAPPDATA: 'C:\\Users\\Alex\\AppData\\Local' }, + homeDir: 'C:\\Users\\Alex', + assetDir: 'C:\\repo\\packages\\cli\\assets\\python', + }); + + expect(layout.runtimeRoot).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime'); + expect(layout.pythonPath).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime/0.2.0/.venv/Scripts/python.exe'); + expect(layout.daemonPath).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime/0.2.0/.venv/Scripts/ktx-daemon.exe'); + }); +}); + +describe('verifyRuntimeAsset', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-asset-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('reads the manifest and verifies the wheel checksum', async () => { + const { assetDir, wheelPath } = await writeAsset(tempDir, 'valid-wheel'); + + const asset = await verifyRuntimeAsset({ assetDir }); + + expect(asset.manifest.distributionName).toBe('kaelio-ktx'); + expect(asset.manifest.normalizedName).toBe('kaelio_ktx'); + expect(asset.wheelPath).toBe(wheelPath); + }); + + it('rejects a wheel whose checksum does not match the manifest', async () => { + const { assetDir, wheelPath } = await writeAsset(tempDir, 'original'); + await writeFile(wheelPath, 'tampered'); + + await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow( + /Bundled Python runtime wheel checksum mismatch/, + ); + }); + + it('rejects an unsafe wheel filename in the manifest', async () => { + const { assetDir } = await writeAsset(tempDir, 'valid-wheel'); + await writeFile( + join(assetDir, 'manifest.json'), + `${JSON.stringify({ + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.1.0', + wheel: { + file: '../kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 1, + }, + })}\n`, + ); + + await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow(/Unsafe runtime wheel filename/); + }); +}); + +describe('installManagedPythonRuntime', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-install-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('creates a venv, installs the core wheel, and writes a manifest', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const commands: Array<{ command: string; args: string[] }> = []; + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { + commands.push({ command, args }); + return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' }; + }); + + const result = await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }); + + expect(result.status).toBe('installed'); + expect(commands).toEqual([ + { command: 'uv', args: ['--version'] }, + { command: 'uv', args: ['venv', result.layout.venvDir] }, + { + command: 'uv', + args: ['pip', 'install', '--python', result.layout.pythonPath, result.asset.wheelPath], + }, + ]); + const manifest = JSON.parse(await readFile(result.layout.manifestPath, 'utf8')) as { + cliVersion: string; + features: string[]; + python: { executable: string; daemonExecutable: string }; + }; + expect(manifest.cliVersion).toBe('0.2.0'); + expect(manifest.features).toEqual(['core']); + expect(manifest.python.executable).toBe(result.layout.pythonPath); + expect(manifest.python.daemonExecutable).toBe(result.layout.daemonPath); + }); + + it('installs the local-embeddings extra when requested', async () => { + const { assetDir } = await writeAsset(tempDir, 'embedding-wheel'); + const commands: Array<{ command: string; args: string[] }> = []; + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { + commands.push({ command, args }); + return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' }; + }); + + const result = await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['local-embeddings'], + exec, + }); + + expect(commands.at(-1)).toEqual({ + command: 'uv', + args: ['pip', 'install', '--python', result.layout.pythonPath, `${result.asset.wheelPath}[local-embeddings]`], + }); + const manifest = JSON.parse(await readFile(result.layout.manifestPath, 'utf8')) as { features: string[] }; + expect(manifest.features).toEqual(['core', 'local-embeddings']); + }); + + it('reuses an existing compatible runtime when force is false', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ + stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', + stderr: '', + })); + + const first = await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }); + await mkdir(join(first.layout.venvDir, 'bin'), { recursive: true }); + await writeFile(first.layout.pythonPath, '#!/usr/bin/env python\n'); + await writeFile(first.layout.daemonPath, '#!/usr/bin/env python\n'); + + const second = await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }); + + expect(second.status).toBe('ready'); + expect(exec).toHaveBeenCalledTimes(3); + }); + + it('keeps failed install logs in the versioned runtime directory', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { + if (command === 'uv' && args[0] === 'venv') { + throw Object.assign(new Error('uv venv failed'), { stdout: 'creating\n', stderr: 'bad python\n' }); + } + return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' }; + }); + + await expect( + installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }), + ).rejects.toThrow(/Python runtime install failed/); + + const log = await readFile(join(tempDir, 'runtime', '0.2.0', 'install.log'), 'utf8'); + expect(log).toContain('$ uv venv'); + expect(log).toContain('bad python'); + }); +}); + +describe('readManagedPythonRuntimeStatus', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-status-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('reports missing before install', async () => { + const status = await readManagedPythonRuntimeStatus({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir: join(tempDir, 'assets', 'python'), + }); + + expect(status.kind).toBe('missing'); + expect(status.detail).toContain('No runtime manifest'); + }); + + it('reports ready when manifest and executables exist', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ + stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', + stderr: '', + })); + const install = await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }); + await mkdir(join(install.layout.venvDir, 'bin'), { recursive: true }); + await writeFile(install.layout.pythonPath, '#!/usr/bin/env python\n'); + await writeFile(install.layout.daemonPath, '#!/usr/bin/env python\n'); + + const status = await readManagedPythonRuntimeStatus({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + }); + + expect(status.kind).toBe('ready'); + expect(status.manifest?.features).toEqual(['core']); + }); + + it('reports broken when an executable is missing', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ + stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', + stderr: '', + })); + await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }); + + const status = await readManagedPythonRuntimeStatus({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + }); + + expect(status.kind).toBe('broken'); + expect(status.detail).toContain('Missing Python executable'); + }); +}); + +describe('doctorManagedPythonRuntime', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-doctor-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('checks uv, bundled assets, and installed runtime status', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ + stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', + stderr: '', + })); + + const checks = await doctorManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + exec, + }); + + expect(checks.map((check) => [check.id, check.status])).toEqual([ + ['uv', 'pass'], + ['asset', 'pass'], + ['runtime', 'fail'], + ]); + expect(checks[2]?.fix).toBe('Run: ktx runtime install --yes'); + }); +}); + +describe('pruneManagedPythonRuntimes', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-prune-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('removes stale version directories and keeps the current version', async () => { + const runtimeRoot = join(tempDir, 'runtime'); + await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true }); + await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true }); + await writeFile(join(runtimeRoot, 'README.txt'), 'not a runtime directory\n'); + + const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot }); + + expect(result.removed).toEqual([join(runtimeRoot, '0.1.0')]); + expect(result.kept).toEqual([join(runtimeRoot, '0.2.0')]); + await expect(stat(join(runtimeRoot, '0.1.0'))).rejects.toThrow(); + expect(await readdir(runtimeRoot)).toEqual(['0.2.0', 'README.txt']); + }); + + it('supports dry-run without deleting stale directories', async () => { + const runtimeRoot = join(tempDir, 'runtime'); + await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true }); + await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true }); + + const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot, dryRun: true }); + + expect(result.removed).toEqual([]); + expect(result.stale).toEqual([join(runtimeRoot, '0.1.0')]); + expect(await readdir(runtimeRoot)).toEqual(['0.1.0', '0.2.0']); + }); +}); +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: + +```bash +pnpm --filter @ktx/cli exec vitest run src/managed-python-runtime.test.ts +``` + +Expected: FAIL with an import error for `./managed-python-runtime.js`. + +- [ ] **Step 3: Commit the failing tests** + +Run: + +```bash +git add packages/cli/src/managed-python-runtime.test.ts +git commit -m "test: cover managed python runtime lifecycle" +``` + +### Task 2: Implement the managed-runtime library + +**Files:** + +- Create: `packages/cli/src/managed-python-runtime.ts` +- Test: `packages/cli/src/managed-python-runtime.test.ts` + +- [ ] **Step 1: Create the managed-runtime implementation** + +Create `packages/cli/src/managed-python-runtime.ts` with this content: + +```typescript +import { createHash } from 'node:crypto'; +import { execFile } from 'node:child_process'; +import { access, appendFile, mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { basename, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; +import { z } from 'zod'; + +const execFileAsync = promisify(execFile); + +export const runtimeFeatureSchema = z.enum(['core', 'local-embeddings']); +export type KtxRuntimeFeature = z.infer; + +const runtimeAssetManifestSchema = z.object({ + schemaVersion: z.literal(1), + distributionName: z.literal('kaelio-ktx'), + normalizedName: z.literal('kaelio_ktx'), + version: z.string().min(1), + wheel: z.object({ + file: z.string().min(1), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + bytes: z.number().int().nonnegative(), + }), +}); + +export type KtxRuntimeAssetManifest = z.infer; + +const installedRuntimeManifestSchema = z.object({ + schemaVersion: z.literal(1), + cliVersion: z.string().min(1), + installedAt: z.string().min(1), + asset: runtimeAssetManifestSchema, + features: z.array(runtimeFeatureSchema).min(1), + python: z.object({ + executable: z.string().min(1), + daemonExecutable: z.string().min(1), + }), + installLog: z.string().min(1), +}); + +export type InstalledKtxRuntimeManifest = z.infer; + +export interface ManagedPythonRuntimeLayoutOptions { + cliVersion: string; + platform?: NodeJS.Platform; + env?: NodeJS.ProcessEnv; + homeDir?: string; + runtimeRoot?: string; + assetDir?: string; +} + +export interface ManagedPythonRuntimeLayout { + cliVersion: string; + runtimeRoot: string; + versionDir: string; + venvDir: string; + manifestPath: string; + installLogPath: string; + assetDir: string; + assetManifestPath: string; + pythonPath: string; + daemonPath: string; +} + +export interface ManagedRuntimeAsset { + manifest: KtxRuntimeAssetManifest; + wheelPath: string; +} + +export type ManagedPythonRuntimeExec = ( + command: string, + args: string[], + options?: { cwd?: string; env?: NodeJS.ProcessEnv }, +) => Promise<{ stdout: string; stderr: string }>; + +export interface ManagedPythonRuntimeInstallOptions extends ManagedPythonRuntimeLayoutOptions { + features: KtxRuntimeFeature[]; + force?: boolean; + exec?: ManagedPythonRuntimeExec; +} + +export interface ManagedPythonRuntimeInstallResult { + status: 'ready' | 'installed'; + layout: ManagedPythonRuntimeLayout; + asset: ManagedRuntimeAsset; + manifest: InstalledKtxRuntimeManifest; +} + +export type ManagedPythonRuntimeStatusKind = 'missing' | 'ready' | 'mismatched' | 'broken'; + +export interface ManagedPythonRuntimeStatus { + kind: ManagedPythonRuntimeStatusKind; + detail: string; + layout: ManagedPythonRuntimeLayout; + manifest?: InstalledKtxRuntimeManifest; +} + +export interface ManagedPythonRuntimeDoctorCheck { + id: 'uv' | 'asset' | 'runtime'; + label: string; + status: 'pass' | 'fail'; + detail: string; + fix?: string; +} + +export interface ManagedPythonRuntimePruneResult { + runtimeRoot: string; + stale: string[]; + kept: string[]; + removed: string[]; +} + +function defaultAssetDir(): string { + return fileURLToPath(new URL('../assets/python/', import.meta.url)); +} + +function runtimeRootFor(input: Required>): string { + if (input.platform === 'darwin') { + return join(input.homeDir, 'Library', 'Application Support', 'kaelio', 'ktx', 'runtime'); + } + if (input.platform === 'win32') { + return join(input.env.LOCALAPPDATA ?? join(input.homeDir, 'AppData', 'Local'), 'Kaelio', 'KTX', 'runtime'); + } + return join(input.env.XDG_DATA_HOME ?? join(input.homeDir, '.local', 'share'), 'kaelio', 'ktx', 'runtime'); +} + +function executablePath(venvDir: string, platform: NodeJS.Platform, name: string): string { + if (platform === 'win32') { + return join(venvDir, 'Scripts', `${name}.exe`); + } + return join(venvDir, 'bin', name); +} + +export function managedPythonRuntimeLayout(options: ManagedPythonRuntimeLayoutOptions): ManagedPythonRuntimeLayout { + const platform = options.platform ?? process.platform; + const env = options.env ?? process.env; + const homeDir = options.homeDir ?? homedir(); + const runtimeRoot = options.runtimeRoot ?? runtimeRootFor({ platform, env, homeDir }); + const versionDir = join(runtimeRoot, options.cliVersion); + const venvDir = join(versionDir, '.venv'); + const assetDir = options.assetDir ?? defaultAssetDir(); + + return { + cliVersion: options.cliVersion, + runtimeRoot, + versionDir, + venvDir, + manifestPath: join(versionDir, 'manifest.json'), + installLogPath: join(versionDir, 'install.log'), + assetDir, + assetManifestPath: join(assetDir, 'manifest.json'), + pythonPath: executablePath(venvDir, platform, 'python'), + daemonPath: executablePath(venvDir, platform, 'ktx-daemon'), + }; +} + +async function pathExists(path: string): Promise { + try { + await access(path); + return true; + } catch { + return false; + } +} + +function assertSafeWheelFilename(file: string): void { + if (file !== basename(file) || file.includes('/') || file.includes('\\')) { + throw new Error(`Unsafe runtime wheel filename in bundled manifest: ${file}`); + } +} + +async function readJsonFile(path: string): Promise { + return JSON.parse(await readFile(path, 'utf8')) as unknown; +} + +export async function verifyRuntimeAsset(input: { assetDir: string }): Promise { + const manifestPath = join(input.assetDir, 'manifest.json'); + const manifest = runtimeAssetManifestSchema.parse(await readJsonFile(manifestPath)); + assertSafeWheelFilename(manifest.wheel.file); + const wheelPath = join(input.assetDir, manifest.wheel.file); + const wheel = await readFile(wheelPath); + const sha256 = createHash('sha256').update(wheel).digest('hex'); + if (sha256 !== manifest.wheel.sha256 || wheel.byteLength !== manifest.wheel.bytes) { + throw new Error(`Bundled Python runtime wheel checksum mismatch: ${wheelPath}`); + } + return { manifest, wheelPath }; +} + +function normalizeFeatures(features: KtxRuntimeFeature[]): KtxRuntimeFeature[] { + const requested = new Set(['core', ...features]); + return runtimeFeatureSchema.options.filter((feature) => requested.has(feature)); +} + +async function readInstalledManifest(path: string): Promise { + if (!(await pathExists(path))) { + return undefined; + } + return installedRuntimeManifestSchema.parse(await readJsonFile(path)); +} + +function hasFeatures(manifest: InstalledKtxRuntimeManifest, features: KtxRuntimeFeature[]): boolean { + return normalizeFeatures(features).every((feature) => manifest.features.includes(feature)); +} + +async function defaultExec( + command: string, + args: string[], + options: { cwd?: string; env?: NodeJS.ProcessEnv } = {}, +): Promise<{ stdout: string; stderr: string }> { + const result = await execFileAsync(command, args, { + cwd: options.cwd, + env: options.env, + encoding: 'utf8', + maxBuffer: 1024 * 1024 * 20, + }); + return { stdout: result.stdout, stderr: result.stderr }; +} + +function errorOutput(error: unknown): { stdout: string; stderr: string } { + const value = error as { stdout?: unknown; stderr?: unknown }; + return { + stdout: typeof value.stdout === 'string' ? value.stdout : '', + stderr: typeof value.stderr === 'string' ? value.stderr : '', + }; +} + +async function runLogged(input: { + exec: ManagedPythonRuntimeExec; + logPath: string; + command: string; + args: string[]; + cwd?: string; +}): Promise<{ stdout: string; stderr: string }> { + await appendFile(input.logPath, `$ ${input.command} ${input.args.join(' ')}\n`); + try { + const result = await input.exec(input.command, input.args, { cwd: input.cwd }); + if (result.stdout) { + await appendFile(input.logPath, result.stdout.endsWith('\n') ? result.stdout : `${result.stdout}\n`); + } + if (result.stderr) { + await appendFile(input.logPath, result.stderr.endsWith('\n') ? result.stderr : `${result.stderr}\n`); + } + return result; + } catch (error) { + const output = errorOutput(error); + if (output.stdout) { + await appendFile(input.logPath, output.stdout.endsWith('\n') ? output.stdout : `${output.stdout}\n`); + } + if (output.stderr) { + await appendFile(input.logPath, output.stderr.endsWith('\n') ? output.stderr : `${output.stderr}\n`); + } + throw new Error(`Python runtime install failed. Install log: ${input.logPath}`); + } +} + +async function ensureUv(exec: ManagedPythonRuntimeExec): Promise { + try { + const result = await exec('uv', ['--version']); + return result.stdout.trim() || 'uv available'; + } catch { + throw new Error( + 'uv is required to install the KTX Python runtime. Install uv and retry: ktx runtime install --yes', + ); + } +} + +export async function installManagedPythonRuntime( + options: ManagedPythonRuntimeInstallOptions, +): Promise { + const layout = managedPythonRuntimeLayout(options); + const exec = options.exec ?? defaultExec; + const features = normalizeFeatures(options.features); + const asset = await verifyRuntimeAsset({ assetDir: layout.assetDir }); + const existing = await readInstalledManifest(layout.manifestPath); + if ( + options.force !== true && + existing && + existing.cliVersion === options.cliVersion && + existing.asset.wheel.sha256 === asset.manifest.wheel.sha256 && + hasFeatures(existing, features) && + (await pathExists(existing.python.executable)) && + (await pathExists(existing.python.daemonExecutable)) + ) { + return { status: 'ready', layout, asset, manifest: existing }; + } + + await rm(layout.versionDir, { recursive: true, force: true }); + await mkdir(layout.versionDir, { recursive: true }); + await writeFile(layout.installLogPath, ''); + await ensureUv(exec); + await runLogged({ exec, logPath: layout.installLogPath, command: 'uv', args: ['venv', layout.venvDir] }); + const wheelSpec = features.includes('local-embeddings') ? `${asset.wheelPath}[local-embeddings]` : asset.wheelPath; + await runLogged({ + exec, + logPath: layout.installLogPath, + command: 'uv', + args: ['pip', 'install', '--python', layout.pythonPath, wheelSpec], + }); + + const manifest: InstalledKtxRuntimeManifest = { + schemaVersion: 1, + cliVersion: options.cliVersion, + installedAt: new Date().toISOString(), + asset: asset.manifest, + features, + python: { + executable: layout.pythonPath, + daemonExecutable: layout.daemonPath, + }, + installLog: layout.installLogPath, + }; + await writeFile(layout.manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + return { status: 'installed', layout, asset, manifest }; +} + +export async function readManagedPythonRuntimeStatus( + options: ManagedPythonRuntimeLayoutOptions, +): Promise { + const layout = managedPythonRuntimeLayout(options); + let manifest: InstalledKtxRuntimeManifest | undefined; + try { + manifest = await readInstalledManifest(layout.manifestPath); + } catch (error) { + return { + kind: 'broken', + detail: `Runtime manifest is invalid: ${error instanceof Error ? error.message : String(error)}`, + layout, + }; + } + if (!manifest) { + return { kind: 'missing', detail: `No runtime manifest at ${layout.manifestPath}`, layout }; + } + if (manifest.cliVersion !== options.cliVersion) { + return { + kind: 'mismatched', + detail: `Runtime is for CLI ${manifest.cliVersion}, current CLI is ${options.cliVersion}`, + layout, + manifest, + }; + } + if (!(await pathExists(manifest.python.executable))) { + return { kind: 'broken', detail: `Missing Python executable: ${manifest.python.executable}`, layout, manifest }; + } + if (!(await pathExists(manifest.python.daemonExecutable))) { + return { kind: 'broken', detail: `Missing ktx-daemon executable: ${manifest.python.daemonExecutable}`, layout, manifest }; + } + return { kind: 'ready', detail: `Runtime ready at ${layout.versionDir}`, layout, manifest }; +} + +function check(status: ManagedPythonRuntimeDoctorCheck['status'], input: Omit) { + return { status, ...input }; +} + +export async function doctorManagedPythonRuntime( + options: ManagedPythonRuntimeLayoutOptions & { exec?: ManagedPythonRuntimeExec }, +): Promise { + const exec = options.exec ?? defaultExec; + const checks: ManagedPythonRuntimeDoctorCheck[] = []; + try { + const version = await ensureUv(exec); + checks.push(check('pass', { id: 'uv', label: 'uv', detail: version })); + } catch (error) { + checks.push( + check('fail', { + id: 'uv', + label: 'uv', + detail: error instanceof Error ? error.message : String(error), + fix: 'Install uv, then run: ktx runtime install --yes', + }), + ); + } + + try { + const asset = await verifyRuntimeAsset({ assetDir: managedPythonRuntimeLayout(options).assetDir }); + checks.push(check('pass', { id: 'asset', label: 'Bundled Python wheel', detail: asset.wheelPath })); + } catch (error) { + checks.push( + check('fail', { + id: 'asset', + label: 'Bundled Python wheel', + detail: error instanceof Error ? error.message : String(error), + fix: 'Run: pnpm run artifacts:check', + }), + ); + } + + const status = await readManagedPythonRuntimeStatus(options); + checks.push( + check(status.kind === 'ready' ? 'pass' : 'fail', { + id: 'runtime', + label: 'Managed Python runtime', + detail: status.detail, + ...(status.kind === 'ready' ? {} : { fix: 'Run: ktx runtime install --yes' }), + }), + ); + return checks; +} + +export async function pruneManagedPythonRuntimes(options: { + cliVersion: string; + runtimeRoot: string; + dryRun?: boolean; +}): Promise { + if (!(await pathExists(options.runtimeRoot))) { + return { runtimeRoot: options.runtimeRoot, stale: [], kept: [], removed: [] }; + } + const entries = await readdir(options.runtimeRoot); + const stale: string[] = []; + const kept: string[] = []; + for (const entry of entries) { + const path = join(options.runtimeRoot, entry); + const info = await stat(path); + if (!info.isDirectory()) { + continue; + } + if (entry === options.cliVersion) { + kept.push(path); + } else { + stale.push(path); + } + } + const removed: string[] = []; + if (options.dryRun !== true) { + for (const path of stale) { + await rm(path, { recursive: true, force: true }); + removed.push(path); + } + } + return { runtimeRoot: options.runtimeRoot, stale, kept, removed }; +} +``` + +- [ ] **Step 2: Run the managed-runtime tests** + +Run: + +```bash +pnpm --filter @ktx/cli exec vitest run src/managed-python-runtime.test.ts +``` + +Expected: PASS. + +- [ ] **Step 3: Run the CLI type checker** + +Run: + +```bash +pnpm --filter @ktx/cli run type-check +``` + +Expected: PASS. + +- [ ] **Step 4: Commit the implementation** + +Run: + +```bash +git add packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts +git commit -m "feat: add managed python runtime installer" +``` + +### Task 3: Add the runtime command runner + +**Files:** + +- Create: `packages/cli/src/runtime.ts` +- Create: `packages/cli/src/runtime.test.ts` +- Test: `packages/cli/src/runtime.test.ts` + +- [ ] **Step 1: Write the failing command-runner tests** + +Create `packages/cli/src/runtime.test.ts` with this content: + +```typescript +import { describe, expect, it, vi } from 'vitest'; +import { runKtxRuntime, type KtxRuntimeDeps } from './runtime.js'; + +function makeIo() { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +describe('runKtxRuntime', () => { + it('installs the requested runtime feature and prints the manifest path', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + installRuntime: vi.fn(async () => ({ + status: 'installed', + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + }, + asset: { + wheelPath: '/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl', + manifest: { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.1.0', + wheel: { + file: 'kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 10, + }, + }, + }, + manifest: { + schemaVersion: 1, + cliVersion: '0.2.0', + installedAt: '2026-05-11T00:00:00.000Z', + asset: { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.1.0', + wheel: { + file: 'kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 10, + }, + }, + features: ['core', 'local-embeddings'], + python: { + executable: '/runtime/0.2.0/.venv/bin/python', + daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon', + }, + installLog: '/runtime/0.2.0/install.log', + }, + })), + }; + + await expect( + runKtxRuntime( + { command: 'install', cliVersion: '0.2.0', feature: 'local-embeddings', force: true }, + io.io, + deps, + ), + ).resolves.toBe(0); + + expect(deps.installRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['local-embeddings'], + force: true, + }); + expect(io.stdout()).toContain('Installed KTX Python runtime'); + expect(io.stdout()).toContain('features: core, local-embeddings'); + expect(io.stdout()).toContain('manifest: /runtime/0.2.0/manifest.json'); + expect(io.stderr()).toBe(''); + }); + + it('prints runtime status as JSON', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + readStatus: vi.fn(async () => ({ + kind: 'missing', + detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + }, + })), + }; + + await expect(runKtxRuntime({ command: 'status', cliVersion: '0.2.0', json: true }, io.io, deps)).resolves.toBe(0); + + expect(JSON.parse(io.stdout())).toMatchObject({ + kind: 'missing', + detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', + layout: { runtimeRoot: '/runtime' }, + }); + }); + + it('returns failure for doctor when any check fails', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + doctorRuntime: vi.fn(async () => [ + { id: 'uv', label: 'uv', status: 'pass', detail: 'uv 0.9.5' }, + { + id: 'runtime', + label: 'Managed Python runtime', + status: 'fail', + detail: 'No runtime manifest', + fix: 'Run: ktx runtime install --yes', + }, + ]), + }; + + await expect(runKtxRuntime({ command: 'doctor', cliVersion: '0.2.0', json: false }, io.io, deps)).resolves.toBe(1); + + expect(io.stdout()).toContain('PASS uv: uv 0.9.5'); + expect(io.stdout()).toContain('FAIL Managed Python runtime: No runtime manifest'); + expect(io.stdout()).toContain('Fix: Run: ktx runtime install --yes'); + }); + + it('requires --yes before pruning stale runtime directories', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + pruneRuntime: vi.fn(async () => { + throw new Error('should not prune without --yes'); + }), + }; + + await expect(runKtxRuntime({ command: 'prune', cliVersion: '0.2.0', dryRun: false, yes: false }, io.io, deps)) + .resolves.toBe(1); + + expect(io.stderr()).toContain('Refusing to prune without --yes'); + expect(deps.pruneRuntime).not.toHaveBeenCalled(); + }); + + it('prints stale directories during prune dry-run', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + readStatus: vi.fn(async () => ({ + kind: 'missing', + detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + }, + })), + pruneRuntime: vi.fn(async () => ({ + runtimeRoot: '/runtime', + stale: ['/runtime/0.1.0'], + kept: ['/runtime/0.2.0'], + removed: [], + })), + }; + + await expect(runKtxRuntime({ command: 'prune', cliVersion: '0.2.0', dryRun: true, yes: false }, io.io, deps)) + .resolves.toBe(0); + + expect(io.stdout()).toContain('Stale KTX Python runtimes'); + expect(io.stdout()).toContain('/runtime/0.1.0'); + }); +}); +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: + +```bash +pnpm --filter @ktx/cli exec vitest run src/runtime.test.ts +``` + +Expected: FAIL with an import error for `./runtime.js`. + +- [ ] **Step 3: Create the command runner** + +Create `packages/cli/src/runtime.ts` with this content: + +```typescript +import { + doctorManagedPythonRuntime, + installManagedPythonRuntime, + pruneManagedPythonRuntimes, + readManagedPythonRuntimeStatus, + type KtxRuntimeFeature, + type ManagedPythonRuntimeDoctorCheck, + type ManagedPythonRuntimeInstallOptions, + type ManagedPythonRuntimeInstallResult, + type ManagedPythonRuntimeLayoutOptions, + type ManagedPythonRuntimePruneResult, + type ManagedPythonRuntimeStatus, +} from './managed-python-runtime.js'; +import type { KtxCliIo } from './cli-runtime.js'; + +export type KtxRuntimeArgs = + | { command: 'install'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean } + | { command: 'status'; cliVersion: string; json: boolean } + | { command: 'doctor'; cliVersion: string; json: boolean } + | { command: 'prune'; cliVersion: string; dryRun: boolean; yes: boolean }; + +export interface KtxRuntimeDeps { + installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise; + readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; + doctorRuntime?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; + pruneRuntime?: (options: { cliVersion: string; runtimeRoot: string; dryRun?: boolean }) => Promise; +} + +function writeJson(io: KtxCliIo, value: unknown): void { + io.stdout.write(`${JSON.stringify(value, null, 2)}\n`); +} + +function writeInstallResult(io: KtxCliIo, result: ManagedPythonRuntimeInstallResult): void { + const verb = result.status === 'ready' ? 'Using existing' : 'Installed'; + io.stdout.write(`${verb} KTX Python runtime\n`); + io.stdout.write(`version: ${result.manifest.cliVersion}\n`); + io.stdout.write(`features: ${result.manifest.features.join(', ')}\n`); + io.stdout.write(`python: ${result.manifest.python.executable}\n`); + io.stdout.write(`daemon: ${result.manifest.python.daemonExecutable}\n`); + io.stdout.write(`manifest: ${result.layout.manifestPath}\n`); + io.stdout.write(`install log: ${result.layout.installLogPath}\n`); +} + +function writeStatus(io: KtxCliIo, status: ManagedPythonRuntimeStatus): void { + io.stdout.write('KTX Python runtime\n'); + io.stdout.write(`status: ${status.kind}\n`); + io.stdout.write(`detail: ${status.detail}\n`); + io.stdout.write(`runtime root: ${status.layout.runtimeRoot}\n`); + io.stdout.write(`version dir: ${status.layout.versionDir}\n`); + if (status.manifest) { + io.stdout.write(`features: ${status.manifest.features.join(', ')}\n`); + io.stdout.write(`python: ${status.manifest.python.executable}\n`); + io.stdout.write(`daemon: ${status.manifest.python.daemonExecutable}\n`); + } +} + +function writeDoctor(io: KtxCliIo, checks: ManagedPythonRuntimeDoctorCheck[]): void { + io.stdout.write('KTX Python runtime doctor\n'); + for (const check of checks) { + io.stdout.write(`${check.status.toUpperCase()} ${check.label}: ${check.detail}\n`); + if (check.fix) { + io.stdout.write(` Fix: ${check.fix}\n`); + } + } +} + +function writePrune(io: KtxCliIo, result: ManagedPythonRuntimePruneResult, dryRun: boolean): void { + if (result.stale.length === 0) { + io.stdout.write(`No stale KTX Python runtimes found under ${result.runtimeRoot}\n`); + return; + } + io.stdout.write(dryRun ? 'Stale KTX Python runtimes\n' : 'Removed stale KTX Python runtimes\n'); + for (const path of dryRun ? result.stale : result.removed) { + io.stdout.write(`${path}\n`); + } +} + +export async function runKtxRuntime( + args: KtxRuntimeArgs, + io: KtxCliIo = process, + deps: KtxRuntimeDeps = {}, +): Promise { + try { + if (args.command === 'install') { + const installRuntime = deps.installRuntime ?? installManagedPythonRuntime; + const result = await installRuntime({ + cliVersion: args.cliVersion, + features: [args.feature], + force: args.force, + }); + writeInstallResult(io, result); + return 0; + } + if (args.command === 'status') { + const readStatus = deps.readStatus ?? readManagedPythonRuntimeStatus; + const status = await readStatus({ cliVersion: args.cliVersion }); + if (args.json) { + writeJson(io, status); + } else { + writeStatus(io, status); + } + return 0; + } + if (args.command === 'doctor') { + const doctorRuntime = deps.doctorRuntime ?? doctorManagedPythonRuntime; + const checks = await doctorRuntime({ cliVersion: args.cliVersion }); + if (args.json) { + writeJson(io, { checks }); + } else { + writeDoctor(io, checks); + } + return checks.some((check) => check.status === 'fail') ? 1 : 0; + } + if (!args.dryRun && !args.yes) { + io.stderr.write('Refusing to prune without --yes. Preview with: ktx runtime prune --dry-run\n'); + return 1; + } + const status = await (deps.readStatus ?? readManagedPythonRuntimeStatus)({ cliVersion: args.cliVersion }); + const pruneRuntime = deps.pruneRuntime ?? pruneManagedPythonRuntimes; + const result = await pruneRuntime({ + cliVersion: args.cliVersion, + runtimeRoot: status.layout.runtimeRoot, + dryRun: args.dryRun, + }); + writePrune(io, result, args.dryRun); + return 0; + } catch (error) { + io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } +} +``` + +- [ ] **Step 4: Run the command-runner tests** + +Run: + +```bash +pnpm --filter @ktx/cli exec vitest run src/runtime.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit the command runner** + +Run: + +```bash +git add packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts +git commit -m "feat: add runtime command runner" +``` + +### Task 4: Register `ktx runtime` commands + +**Files:** + +- Create: `packages/cli/src/commands/runtime-commands.ts` +- Modify: `packages/cli/src/cli-runtime.ts` +- Modify: `packages/cli/src/cli-program.ts` +- Modify: `packages/cli/src/index.ts` +- Modify: `packages/cli/src/index.test.ts` +- Test: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Create the runtime command registration** + +Create `packages/cli/src/commands/runtime-commands.ts` with this content: + +```typescript +import { type Command, Option } from '@commander-js/extra-typings'; +import type { KtxCliCommandContext } from '../cli-program.js'; +import type { KtxRuntimeArgs } from '../runtime.js'; + +type RuntimeFeature = Extract['feature']; + +const runtimeFeatureOption = new Option('--feature ', 'Runtime feature level') + .choices(['core', 'local-embeddings']) + .default('core'); + +async function runRuntimeArgs(context: KtxCliCommandContext, args: KtxRuntimeArgs): Promise { + const runner = context.deps.runtime ?? (await import('../runtime.js')).runKtxRuntime; + context.setExitCode(await runner(args, context.io)); +} + +export function registerRuntimeCommands(program: Command, context: KtxCliCommandContext): void { + const runtime = program + .command('runtime') + .description('Install, inspect, and prune the KTX-managed Python runtime') + .showHelpAfterError(); + + runtime + .command('install') + .description('Install the bundled Python runtime wheel into the managed runtime') + .addOption(runtimeFeatureOption) + .option('--force', 'Reinstall even when the runtime already looks ready', false) + .action(async (options: { feature: RuntimeFeature; force?: boolean }) => { + await runRuntimeArgs(context, { + command: 'install', + cliVersion: context.packageInfo.version, + feature: options.feature, + force: options.force === true, + }); + }); + + runtime + .command('status') + .description('Show managed Python runtime status') + .option('--json', 'Print JSON output', false) + .action(async (options: { json?: boolean }) => { + await runRuntimeArgs(context, { + command: 'status', + cliVersion: context.packageInfo.version, + json: options.json === true, + }); + }); + + runtime + .command('doctor') + .description('Check managed Python runtime prerequisites and installation') + .option('--json', 'Print JSON output', false) + .action(async (options: { json?: boolean }) => { + await runRuntimeArgs(context, { + command: 'doctor', + cliVersion: context.packageInfo.version, + json: options.json === true, + }); + }); + + runtime + .command('prune') + .description('Remove stale managed Python runtimes for older CLI versions') + .option('--dry-run', 'List stale runtimes without deleting them', false) + .option('--yes', 'Confirm deletion of stale runtime directories', false) + .action(async (options: { dryRun?: boolean; yes?: boolean }) => { + await runRuntimeArgs(context, { + command: 'prune', + cliVersion: context.packageInfo.version, + dryRun: options.dryRun === true, + yes: options.yes === true, + }); + }); +} +``` + +- [ ] **Step 2: Add runtime dependency injection to CLI runtime** + +In `packages/cli/src/cli-runtime.ts`, add this import after the existing +`KtxPublicIngestArgs` import: + +```typescript +import type { KtxRuntimeArgs } from './runtime.js'; +``` + +Then add this property to `KtxCliDeps` after `publicIngest`: + +```typescript + runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise; +``` + +- [ ] **Step 3: Add package info to command context and register the command** + +In `packages/cli/src/cli-program.ts`, add this import after the +`registerPublicIngestCommands` import: + +```typescript +import { registerRuntimeCommands } from './commands/runtime-commands.js'; +``` + +Add this property to `KtxCliCommandContext` after `deps`: + +```typescript + packageInfo: KtxCliPackageInfo; +``` + +Add this property to the `context` object inside `runCommanderKtxCli` after +`deps`: + +```typescript + packageInfo: info, +``` + +Register the runtime commands after `registerSlCommands(program, context);`: + +```typescript + registerRuntimeCommands(program, context); + profileMark('commander:register-runtime'); +``` + +- [ ] **Step 4: Export runtime APIs from the CLI package** + +In `packages/cli/src/index.ts`, add this export after the setup exports: + +```typescript +export { runKtxRuntime, type KtxRuntimeArgs, type KtxRuntimeDeps } from './runtime.js'; +``` + +- [ ] **Step 5: Update root help and routing tests** + +In `packages/cli/src/index.test.ts`, update the root help command list in the +test named `prints the May 6 public command surface in root help` from: + +```typescript + for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'serve', 'status']) { +``` + +to: + +```typescript + for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'runtime', 'serve', 'status']) { +``` + +Then add this test after the root help test: + +```typescript + it('routes runtime management commands with the CLI package version', async () => { + const runtime = vi.fn(async () => 0); + const installIo = makeIo(); + const statusIo = makeIo(); + const doctorIo = makeIo(); + const pruneIo = makeIo(); + + await expect( + runKtxCli(['runtime', 'install', '--feature', 'local-embeddings', '--force'], installIo.io, { runtime }), + ).resolves.toBe(0); + await expect(runKtxCli(['runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['runtime', 'doctor'], doctorIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(0); + + expect(runtime).toHaveBeenNthCalledWith( + 1, + { + command: 'install', + cliVersion: '0.0.0-private', + feature: 'local-embeddings', + force: true, + }, + installIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 2, + { + command: 'status', + cliVersion: '0.0.0-private', + json: true, + }, + statusIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 3, + { + command: 'doctor', + cliVersion: '0.0.0-private', + json: false, + }, + doctorIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 4, + { + command: 'prune', + cliVersion: '0.0.0-private', + dryRun: true, + yes: false, + }, + pruneIo.io, + ); + }); +``` + +- [ ] **Step 6: Run the CLI routing tests** + +Run: + +```bash +pnpm --filter @ktx/cli exec vitest run src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 7: Commit the command registration** + +Run: + +```bash +git add packages/cli/src/commands/runtime-commands.ts packages/cli/src/cli-runtime.ts packages/cli/src/cli-program.ts packages/cli/src/index.ts packages/cli/src/index.test.ts +git commit -m "feat: expose runtime management commands" +``` + +### Task 5: Verify the managed runtime installer end to end + +**Files:** + +- Verify: `packages/cli/src/managed-python-runtime.ts` +- Verify: `packages/cli/src/runtime.ts` +- Verify: `packages/cli/src/commands/runtime-commands.ts` +- Verify: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Run focused Vitest coverage** + +Run: + +```bash +pnpm --filter @ktx/cli exec vitest run src/managed-python-runtime.test.ts src/runtime.test.ts src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 2: Run the CLI type checker** + +Run: + +```bash +pnpm --filter @ktx/cli run type-check +``` + +Expected: PASS. + +- [ ] **Step 3: Build CLI artifacts so bundled Python assets exist** + +Run: + +```bash +pnpm run artifacts:check +``` + +Expected: PASS. The command must leave these generated files: + +```text +packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl +packages/cli/assets/python/manifest.json +``` + +- [ ] **Step 4: Smoke the status command without installing** + +Run: + +```bash +pnpm --filter @ktx/cli run build +node packages/cli/dist/bin.js runtime status --json +``` + +Expected: PASS with JSON containing `"kind": "missing"` or `"kind": "ready"`. +Both are valid because a developer machine might already have a runtime for +the current CLI version. + +- [ ] **Step 5: Smoke the doctor command** + +Run: + +```bash +node packages/cli/dist/bin.js runtime doctor +``` + +Expected: command exits `0` if the runtime is ready and exits `1` if the +runtime is missing. In both cases, stdout must include: + +```text +KTX Python runtime doctor +``` + +- [ ] **Step 6: Run pre-commit for changed files** + +Run: + +```bash +uv run pre-commit run --files packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts packages/cli/src/commands/runtime-commands.ts packages/cli/src/cli-runtime.ts packages/cli/src/cli-program.ts packages/cli/src/index.ts packages/cli/src/index.test.ts +``` + +Expected: PASS. If pre-commit cannot run because this checkout lacks a +compatible pre-commit environment, record the exact failure and keep the +Vitest, type-check, and build results. + +- [ ] **Step 7: Commit final verification fixes** + +If verification required edits, run: + +```bash +git add packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts packages/cli/src/commands/runtime-commands.ts packages/cli/src/cli-runtime.ts packages/cli/src/cli-program.ts packages/cli/src/index.ts packages/cli/src/index.test.ts +git commit -m "test: verify managed python runtime commands" +``` + +If no verification edits were needed, do not create an empty commit. + +## Self-review + +Spec coverage: + +- Covers runtime root selection for macOS, Linux, and Windows. +- Covers versioned runtime directories based on the CLI package version. +- Covers locating `uv`, creating a virtual environment, installing the bundled + wheel, and writing a runtime manifest. +- Covers feature levels by installing `core` by default and + `local-embeddings` through the wheel extra when requested. +- Covers focused errors for missing `uv`, failed install logs, status output, + doctor output, and stale runtime pruning. +- Leaves lazy install from normal commands, daemon start/stop/reuse, and + public npm renaming for later plans. + +Placeholder scan: + +- The plan contains no placeholder markers and no unspecified implementation + steps. + +Type and name consistency: + +- Runtime feature strings are consistently `core` and `local-embeddings`. +- Runtime command args use `cliVersion`, `feature`, `force`, `json`, `dryRun`, + and `yes` consistently across command registration, tests, and runner code. +- Asset manifest names are consistently `kaelio-ktx`, `kaelio_ktx`, and + `manifest.json`. diff --git a/docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md b/docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md new file mode 100644 index 00000000..23959596 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md @@ -0,0 +1,585 @@ +# Managed Python Runtime Release Smoke Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the public `@kaelio/ktx` artifact smoke prove that the npm +package installs and uses its own managed Python runtime without an externally +prepared Python environment. + +**Architecture:** Keep the release smoke black-box: install the packed public +npm tarball into a clean project, isolate `KTX_RUNTIME_ROOT`, and exercise the +installed `ktx` binary. The first `ktx sl query --yes` performs the lazy core +runtime install from bundled package assets, then the smoke verifies +`runtime status`, `runtime doctor`, daemon start/reuse, and daemon stop. + +**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, uv, KTX CLI managed +Python runtime assets. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +Existing plans based on the spec: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` +- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` + +All six are implemented in this worktree. Evidence found before writing this +plan includes: + +- `scripts/build-python-runtime-wheel.mjs` and + `scripts/build-python-runtime-wheel.test.mjs`. +- `packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl` and + `packages/cli/assets/python/manifest.json`. +- `packages/cli/src/managed-python-runtime.ts`, + `packages/cli/src/runtime.ts`, and + `packages/cli/src/commands/runtime-commands.ts`. +- `packages/cli/src/managed-python-command.ts` and `ktx sl query` runtime + install policy flags. +- `packages/cli/src/managed-python-daemon.ts`, daemon state paths, and + `ktx runtime start` / `ktx runtime stop`. +- `packages/cli/src/managed-local-embeddings.ts`, + `packages/context/src/llm/local-config.ts` managed marker constants, and + setup wiring in `packages/cli/src/setup-embeddings.ts`. +- `scripts/build-public-npm-package.mjs`, + `scripts/build-public-npm-package.test.mjs`, `release-policy.json` listing + `@kaelio/ktx`, and published smoke command construction for the required + `@kaelio/ktx` invocation modes. + +The remaining release-smoke gap is in `scripts/package-artifacts.mjs`: + +- `verifyNpmArtifacts()` creates a smoke `.venv`, installs the built Python + runtime wheel into it, and runs installed CLI smoke scripts with that venv at + the front of `PATH`. +- The installed CLI smoke does run `ktx sl query --yes`, but it does not + isolate `KTX_RUNTIME_ROOT`, does not assert that the first query installed + the managed runtime from bundled npm assets, and does not exercise + `ktx runtime status`, `doctor`, `start`, reuse, and `stop`. + +This plan closes that release-flow gap without changing the separate Python +artifact smoke. `verifyPythonArtifacts()` must continue to install the built +Python wheel directly because it verifies the Python artifact itself. + +## File structure + +- Modify `scripts/package-artifacts.test.mjs`: remove the npm-smoke venv test, + add a source-level guard that npm artifact verification does not prepare an + external Python venv, and assert that the installed CLI smoke exercises the + managed runtime lifecycle. +- Modify `scripts/package-artifacts.mjs`: remove npm-smoke Python venv PATH + setup, isolate `KTX_RUNTIME_ROOT` inside `npmRuntimeSmokeSource()`, assert + first-run lazy install, and add runtime status/doctor/start/reuse/stop smoke + commands. + +### Task 1: Add failing release-smoke tests + +**Files:** + +- Modify: `scripts/package-artifacts.test.mjs` +- Test: `scripts/package-artifacts.test.mjs` + +- [ ] **Step 1: Remove the stale npm-smoke venv import** + +In `scripts/package-artifacts.test.mjs`, delete `npmSmokePythonEnv` from the +import list. The surrounding import block must contain this sequence after the +edit: + +```javascript + npmDemoSmokeSource, + npmRuntimeSmokeSource, + npmSmokePackageJson, + npmVerifySource, +``` + +- [ ] **Step 2: Replace the npm-smoke venv test with a source guard** + +Delete this entire test block: + +```javascript +describe('npmSmokePythonEnv', () => { + it('prepends the npm smoke virtualenv bin directory to PATH', () => { + const env = npmSmokePythonEnv('/tmp/ktx-npm-smoke', { PATH: '/usr/bin' }); + + assert.match(env.PATH, /^\/tmp\/ktx-npm-smoke\/\.venv\/(bin|Scripts)/); + assert.match(env.PATH, /\/usr\/bin$/); + }); +}); +``` + +Insert this block in the same location: + +```javascript +describe('verifyNpmArtifacts', () => { + it('does not prepare an external Python environment for the npm smoke', async () => { + const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8'); + const start = source.indexOf('async function verifyNpmArtifacts'); + const end = source.indexOf('async function verifyNpmDemoArtifacts'); + assert.ok(start > 0, 'verifyNpmArtifacts function must exist'); + assert.ok(end > start, 'verifyNpmDemoArtifacts must follow verifyNpmArtifacts'); + + const body = source.slice(start, end); + assert.doesNotMatch(body, /uv', \['venv', '\.venv'\]/); + assert.doesNotMatch(body, /pythonArtifactInstallArgs/); + assert.doesNotMatch(body, /npmSmokePythonEnv/); + }); +}); +``` + +- [ ] **Step 3: Extend the installed CLI smoke assertions** + +In the `it('runs installed CLI commands through the public package runtime', +...)` test, add these assertions after the existing +`assert.match(source, /ktx sl query sqlite execute/);` assertion: + +```javascript + assert.match(source, /import Database from 'better-sqlite3'/); + assert.doesNotMatch(source, /run\('python'/); + assert.match(source, /KTX_RUNTIME_ROOT/); + assert.match(source, /managed-runtime/); + assert.match(source, /ktx runtime status missing/); + assert.match(source, /runtimeStatusBefore\.kind, 'missing'/); + assert.match(source, /Installing KTX Python runtime \(core\) with uv/); + assert.match(source, /KTX Python runtime ready:/); + assert.match(source, /ktx runtime status ready/); + assert.match(source, /runtimeStatusAfter\.kind, 'ready'/); + assert.match(source, /runtimeStatusAfter\.manifest\.features/); + assert.match(source, /ktx runtime doctor/); + assert.match(source, /PASS Managed Python runtime/); + assert.match(source, /ktx runtime start/); + assert.match(source, /ktx runtime start reuse/); + assert.match(source, /Using existing KTX Python daemon/); + assert.match(source, /ktx runtime stop/); +``` + +- [ ] **Step 4: Run the failing package artifact tests** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs +``` + +Expected: FAIL. The guard fails because `verifyNpmArtifacts()` still creates +the npm-smoke `.venv`, and the installed CLI smoke assertions fail because +`npmRuntimeSmokeSource()` does not yet isolate or verify the managed runtime. + +### Task 2: Make the npm smoke use only the managed runtime + +**Files:** + +- Modify: `scripts/package-artifacts.mjs` +- Modify: `scripts/package-artifacts.test.mjs` +- Test: `scripts/package-artifacts.test.mjs` + +- [ ] **Step 1: Remove the npm-smoke PATH helper** + +In `scripts/package-artifacts.mjs`, change the path import from: + +```javascript +import { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +``` + +to: + +```javascript +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +``` + +Then delete this exported helper: + +```javascript +export function npmSmokePythonEnv(projectDir, baseEnv = process.env) { + const binDir = process.platform === 'win32' ? join(projectDir, '.venv', 'Scripts') : join(projectDir, '.venv', 'bin'); + const existingPath = baseEnv.PATH ?? ''; + + return Object.assign({}, baseEnv, { + PATH: existingPath ? `${binDir}${delimiter}${existingPath}` : binDir, + }); +} +``` + +- [ ] **Step 2: Add runtime-smoke helpers to `npmRuntimeSmokeSource()`** + +Inside the template string returned by `npmRuntimeSmokeSource()`, add this +helper immediately after `requireSuccess()`: + +```javascript +function requireSuccessWithStderr(label, result, stderrPattern) { + assert.equal( + result.code, + 0, + label + ' failed with code ' + result.code + '\\nstdout:\\n' + result.stdout + '\\nstderr:\\n' + result.stderr, + ); + assert.match(result.stderr, stderrPattern, label + ' stderr did not match ' + stderrPattern); +} +``` + +Then replace the smoke root setup: + +```javascript +const root = await mkdtemp(join(tmpdir(), 'ktx-installed-cli-smoke-')); +try { + const projectDir = join(root, 'project'); + const sourceDir = join(root, 'source'); +``` + +with: + +```javascript +const root = await mkdtemp(join(tmpdir(), 'ktx-installed-cli-smoke-')); +const previousRuntimeRoot = process.env.KTX_RUNTIME_ROOT; +process.env.KTX_RUNTIME_ROOT = join(root, 'managed-runtime'); +let daemonStarted = false; +try { + const projectDir = join(root, 'project'); + const sourceDir = join(root, 'source'); +``` + +Finally replace the existing `finally` block at the end of +`npmRuntimeSmokeSource()`: + +```javascript +} finally { + await rm(root, { recursive: true, force: true }); +} +``` + +with: + +```javascript +} finally { + if (daemonStarted) { + await run('pnpm', ['exec', 'ktx', 'runtime', 'stop']); + } + if (previousRuntimeRoot === undefined) { + delete process.env.KTX_RUNTIME_ROOT; + } else { + process.env.KTX_RUNTIME_ROOT = previousRuntimeRoot; + } + await rm(root, { recursive: true, force: true }); +} +``` + +- [ ] **Step 3: Create the sqlite smoke warehouse without Python** + +Inside the template string returned by `npmRuntimeSmokeSource()`, add this +import after the `assert` import: + +```javascript +import Database from 'better-sqlite3'; +``` + +Then replace the current `writeSqliteWarehouse()` function: + +```javascript +async function writeSqliteWarehouse(projectDir) { + const createDb = await run('python', [ + '-c', + [ + 'import sqlite3', + 'import sys', + 'db_path = sys.argv[1]', + 'conn = sqlite3.connect(db_path)', + 'conn.executescript("""', + 'DROP TABLE IF EXISTS orders;', + 'CREATE TABLE orders (', + ' id INTEGER PRIMARY KEY,', + ' status TEXT NOT NULL,', + ' amount INTEGER NOT NULL', + ');', + "INSERT INTO orders (status, amount) VALUES ('paid', 20), ('paid', 30), ('open', 10);", + '""")', + 'conn.close()', + ].join('\\n'), + join(projectDir, 'warehouse.db'), + ]); + requireSuccess('create sqlite warehouse', createDb); +} +``` + +with: + +```javascript +async function writeSqliteWarehouse(projectDir) { + const database = new Database(join(projectDir, 'warehouse.db')); + try { + database.exec(` +DROP TABLE IF EXISTS orders; +CREATE TABLE orders ( + id INTEGER PRIMARY KEY, + status TEXT NOT NULL, + amount INTEGER NOT NULL +); +INSERT INTO orders (status, amount) VALUES ('paid', 20), ('paid', 30), ('open', 10); +`); + } finally { + database.close(); + } +} +``` + +- [ ] **Step 4: Assert the isolated runtime is initially missing** + +In `npmRuntimeSmokeSource()`, insert this block immediately after the public +package version assertion: + +```javascript + const runtimeStatusBefore = parseJsonResult( + 'ktx runtime status missing', + await run('pnpm', ['exec', 'ktx', 'runtime', 'status', '--json']), + ); + assert.equal(runtimeStatusBefore.kind, 'missing'); + assert.equal(runtimeStatusBefore.layout.runtimeRoot, process.env.KTX_RUNTIME_ROOT); + process.stdout.write('ktx managed runtime starts missing in isolated root\\n'); +``` + +- [ ] **Step 5: Assert first `sl query --yes` performs lazy managed install** + +In `npmRuntimeSmokeSource()`, replace the current `slQuery` verification block: + +```javascript + const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', + '--connection-id', + 'warehouse', + '--measure', + 'orders.order_count', + '--format', + 'json', + '--yes', + '--project-dir', + projectDir, + ]); + requireSuccess('ktx sl query', slQuery); + requireOutput('ktx sl query', slQuery, /"mode": "compile_only"/); + requireOutput('ktx sl query', slQuery, /orders/); +``` + +with: + +```javascript + const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', + '--connection-id', + 'warehouse', + '--measure', + 'orders.order_count', + '--format', + 'json', + '--yes', + '--project-dir', + projectDir, + ]); + requireSuccessWithStderr( + 'ktx sl query first managed runtime install', + slQuery, + /Installing KTX Python runtime \(core\) with uv[\s\S]*KTX Python runtime ready:/, + ); + requireOutput('ktx sl query first managed runtime install', slQuery, /"mode": "compile_only"/); + requireOutput('ktx sl query first managed runtime install', slQuery, /orders/); + + const runtimeStatusAfter = parseJsonResult( + 'ktx runtime status ready', + await run('pnpm', ['exec', 'ktx', 'runtime', 'status', '--json']), + ); + assert.equal(runtimeStatusAfter.kind, 'ready'); + assert.deepEqual(runtimeStatusAfter.manifest.features, ['core']); + assert.equal(runtimeStatusAfter.layout.runtimeRoot, process.env.KTX_RUNTIME_ROOT); + process.stdout.write('ktx managed runtime lazy install verified\\n'); +``` + +- [ ] **Step 6: Add runtime doctor and daemon lifecycle smoke** + +In `npmRuntimeSmokeSource()`, insert this block immediately after the +`sqliteSlQuery` verification block: + +```javascript + const runtimeDoctor = await run('pnpm', ['exec', 'ktx', 'runtime', 'doctor']); + requireSuccess('ktx runtime doctor', runtimeDoctor); + requireOutput('ktx runtime doctor', runtimeDoctor, /PASS uv/); + requireOutput('ktx runtime doctor', runtimeDoctor, /PASS Bundled Python wheel/); + requireOutput('ktx runtime doctor', runtimeDoctor, /PASS Managed Python runtime/); + process.stdout.write('ktx runtime doctor verified\\n'); + + const runtimeStart = await run('pnpm', ['exec', 'ktx', 'runtime', 'start']); + requireSuccess('ktx runtime start', runtimeStart); + daemonStarted = true; + requireOutput('ktx runtime start', runtimeStart, /Started KTX Python daemon/); + requireOutput('ktx runtime start', runtimeStart, /url: http:\/\/127\.0\.0\.1:\d+/); + requireOutput('ktx runtime start', runtimeStart, /features: core/); + + const runtimeStartReuse = await run('pnpm', ['exec', 'ktx', 'runtime', 'start']); + requireSuccess('ktx runtime start reuse', runtimeStartReuse); + requireOutput('ktx runtime start reuse', runtimeStartReuse, /Using existing KTX Python daemon/); + requireOutput('ktx runtime start reuse', runtimeStartReuse, /features: core/); + + const runtimeStop = await run('pnpm', ['exec', 'ktx', 'runtime', 'stop']); + requireSuccess('ktx runtime stop', runtimeStop); + daemonStarted = false; + requireOutput('ktx runtime stop', runtimeStop, /Stopped KTX Python daemon/); + process.stdout.write('ktx runtime daemon lifecycle verified\\n'); +``` + +- [ ] **Step 7: Remove npm-smoke Python preparation from artifact verification** + +In `scripts/package-artifacts.mjs`, replace `verifyNpmArtifacts()` with this +implementation: + +```javascript +async function verifyNpmArtifacts(layout, tmpRoot) { + for (const packageInfo of NPM_ARTIFACT_PACKAGES) { + await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`); + } + + const projectDir = join(tmpRoot, 'npm-clean-install'); + await mkdir(projectDir, { recursive: true }); + await writeFile( + join(projectDir, 'package.json'), + `${JSON.stringify(npmSmokePackageJson(layout), null, 2)}\n`, + ); + await writeFile(join(projectDir, 'verify-npm.mjs'), npmVerifySource()); + await writeFile(join(projectDir, 'verify-installed-cli.mjs'), npmRuntimeSmokeSource()); + await writeFile(join(projectDir, 'verify-installed-demo.mjs'), npmDemoSmokeSource()); + + await runCommand('pnpm', ['install'], { cwd: projectDir }); + await runCommand('pnpm', ['rebuild', 'better-sqlite3'], { cwd: projectDir }); + await runCommand('node', ['verify-npm.mjs'], { cwd: projectDir }); + await runCommand('pnpm', ['exec', 'ktx', '--version'], { cwd: projectDir }); + await runCommand('node', ['verify-installed-cli.mjs'], { cwd: projectDir }); + await runCommand('node', ['verify-installed-demo.mjs'], { cwd: projectDir }); +} +``` + +- [ ] **Step 8: Run the focused package artifact tests** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 9: Commit the release-smoke implementation** + +Run: + +```bash +git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs +git commit -m "test: verify managed runtime in public package smoke" +``` + +### Task 3: Verify the release-smoke surface + +**Files:** + +- Test: `scripts/package-artifacts.test.mjs` +- Test: `scripts/package-artifacts.mjs` + +- [ ] **Step 1: Run script unit tests that cover artifact packaging** + +Run: + +```bash +node --test scripts/build-python-runtime-wheel.test.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 2: Run the public package artifact smoke** + +Run: + +```bash +pnpm run artifacts:verify +``` + +Expected: PASS. The `verify-installed-cli.mjs` output must include: + +```text +ktx managed runtime starts missing in isolated root +ktx managed runtime lazy install verified +ktx runtime doctor verified +ktx runtime daemon lifecycle verified +``` + +- [ ] **Step 3: Run release readiness** + +Run: + +```bash +pnpm run release:readiness +``` + +Expected: PASS. The report must still list `@kaelio/ktx` as the only npm +package and must still report registry publishing as disabled by +`release-policy.json`. + +- [ ] **Step 4: Run pre-commit for changed files** + +Run: + +```bash +if [ -d .venv ]; then source .venv/bin/activate; fi +uv run pre-commit run --files scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs +``` + +Expected: PASS. If pre-commit cannot run because the local environment lacks a +compatible hook version, record the exact failure and keep the passing +`node --test` and artifact smoke results. + +- [ ] **Step 5: Commit verification fixes if needed** + +If Step 1, Step 2, Step 3, or Step 4 required edits, run: + +```bash +git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs +git commit -m "test: finalize managed runtime release smoke" +``` + +If no files changed after Task 2, do not create an empty commit. + +## Acceptance criteria + +- `verifyNpmArtifacts()` no longer creates a Python `.venv`, no longer calls + `pythonArtifactInstallArgs()`, and no longer runs npm smoke scripts with a + custom Python venv at the front of `PATH`. +- The installed public npm smoke creates its sqlite warehouse with + `better-sqlite3` and does not shell out to `python`. +- The installed public npm smoke sets an isolated `KTX_RUNTIME_ROOT` and + confirms that `ktx runtime status --json` starts as `missing`. +- The first installed `ktx sl query --yes` installs the `core` managed Python + runtime from bundled npm package assets and still returns compile-only SQL. +- A second semantic query executes against sqlite using the installed managed + runtime. +- `ktx runtime doctor` passes after lazy install. +- `ktx runtime start` starts a core daemon, a second `ktx runtime start` reuses + the daemon, and `ktx runtime stop` stops it. +- The separate Python artifact verification still installs and tests the + Python wheel directly. +- Focused script tests, `pnpm run artifacts:verify`, release readiness, and + pre-commit pass or have explicitly recorded environment blockers. + +## Self-review + +- Spec coverage: the previous six plans cover the bundled wheel, runtime + installer, `sl query` command integration, daemon lifecycle, local embeddings, + and public npm package surface. This plan covers release-flow checks for clean + install of the packed npm package, first-run managed runtime install from the + bundled wheel, one-shot semantic-layer query through the managed runtime, + runtime status and doctor output, and daemon start/reuse/stop. +- Remaining intentional gap: optional `local-embeddings` smoke remains outside + the default release artifact smoke because the spec permits it in a separate + job or opt-in check and the dependency downloads are large. +- Placeholder scan: no steps contain placeholder implementation language. +- Type consistency: runtime feature names remain `core` and + `local-embeddings`; the public npm package name remains `@kaelio/ktx`; the + runtime root environment variable is `KTX_RUNTIME_ROOT`. diff --git a/docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md b/docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md new file mode 100644 index 00000000..fe28270d --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md @@ -0,0 +1,657 @@ +# Managed Runtime Docs and Postgres Smoke Cleanup Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Remove the remaining manual Python service guidance from the Postgres +historic SQL smoke and update public docs so the npm-managed Python runtime is +the documented path. + +**Architecture:** Keep the existing managed-runtime code unchanged. Add source +and docs guards first, then make the Postgres historic smoke use the +CLI-managed core daemon through `createKtxCliLocalIngestAdapters()`, and update +the README files that still describe internal package artifacts, manual +`ktx-daemon` startup, or `python-service/`. + +**Tech Stack:** Bash, Node 22 ESM, `node:test`, Markdown, pnpm, uv, KTX CLI +managed Python runtime. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +The following plans are based on that spec and are already implemented in this +worktree: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` +- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` + +Implementation evidence found before writing this plan includes: + +- `scripts/build-python-runtime-wheel.mjs` and + `packages/cli/assets/python/manifest.json`. +- `packages/cli/src/managed-python-runtime.ts`, + `packages/cli/src/runtime.ts`, and + `packages/cli/src/commands/runtime-commands.ts`. +- `packages/cli/src/managed-python-command.ts` and managed `ktx sl query` + runtime policy flags. +- `packages/cli/src/managed-python-daemon.ts` and `ktx runtime start` / + `ktx runtime stop`. +- `packages/cli/src/managed-local-embeddings.ts` and local embeddings setup + wiring. +- `scripts/build-public-npm-package.mjs`, release policy updates, release + smoke coverage, and opt-in local embeddings smoke coverage. +- `packages/cli/src/agent-runtime.ts` and `packages/cli/src/serve.ts` now + create managed semantic-layer compute when no explicit semantic HTTP URL is + provided. +- `packages/cli/src/managed-python-http.ts`, + `packages/cli/src/local-adapters.ts`, `packages/cli/src/ingest.ts`, + `packages/cli/src/scan.ts`, and `packages/cli/src/serve.ts` wire local ingest + helper paths to the managed core daemon. + +The remaining drift is documentation and one example smoke script: + +- `examples/postgres-historic/scripts/smoke.sh` still checks for + `python-service/.venv`, starts `uvicorn app.main:app`, and exports + `KTX_SQL_ANALYSIS_URL`. +- `examples/postgres-historic/README.md` still documents + `python-service/.venv` or `KTX_SQL_ANALYSIS_URL` as a prerequisite. +- `examples/package-artifacts/README.md` still says the npm smoke installs + generated `@ktx/context` and `@ktx/cli` tarballs. +- `README.md` still presents source-tree `pnpm run ktx -- ...` commands as the + quick start and tells users to start `ktx-daemon` manually for MCP. + +This plan closes that drift. It does not rename internal workspace packages and +does not remove explicit daemon URL override behavior from production code. + +## File structure + +- Modify `scripts/examples-docs.test.mjs`: add regression coverage for managed + runtime docs, public npm package docs, and the Postgres smoke script. +- Modify `examples/postgres-historic/scripts/smoke.sh`: remove + `python-service/` startup and pass managed daemon options into stage-only + historic SQL ingest. +- Modify `examples/postgres-historic/README.md`: document the managed runtime + and remove old SQL-analysis service instructions. +- Modify `examples/package-artifacts/README.md`: describe the single public + `@kaelio/ktx` npm artifact and managed runtime smoke. +- Modify `README.md`: make public `@kaelio/ktx` invocation modes and managed + runtime commands visible while keeping source-tree development commands in + the development section. + +### Task 1: Add failing docs and smoke guards + +**Files:** + +- Modify: `scripts/examples-docs.test.mjs` +- Test: `scripts/examples-docs.test.mjs` + +- [ ] **Step 1: Add public runtime README assertions** + +In `scripts/examples-docs.test.mjs`, insert this test after the existing +`walks through ktx connection list and ktx connection test in the README +quickstart` test: + +```javascript + it('documents public npm and managed runtime usage in the README', async () => { + const rootReadme = await readText('README.md'); + + assert.match(rootReadme, /npx @kaelio\/ktx setup demo --no-input/); + assert.match(rootReadme, /npx @kaelio\/ktx sl query/); + assert.match(rootReadme, /npm install @kaelio\/ktx/); + assert.match(rootReadme, /npm install -g @kaelio\/ktx/); + assert.match(rootReadme, /ktx runtime install/); + assert.match(rootReadme, /ktx runtime status/); + assert.match(rootReadme, /ktx runtime doctor/); + assert.match(rootReadme, /ktx runtime start/); + assert.match(rootReadme, /ktx runtime stop/); + assert.match(rootReadme, /ktx serve --mcp stdio/); + assert.doesNotMatch(rootReadme, /uv run ktx-daemon serve-http/); + assert.doesNotMatch(rootReadme, /--semantic-compute-url http:\/\/127\.0\.0\.1:8765/); + }); +``` + +- [ ] **Step 2: Add package artifact README assertions** + +In `scripts/examples-docs.test.mjs`, insert this test after the new public +runtime README test: + +```javascript + it('documents the public package artifact smoke shape', async () => { + const readme = await readText('examples/package-artifacts/README.md'); + + assert.match(readme, /@kaelio\/ktx/); + assert.match(readme, /managed Python runtime/); + assert.match(readme, /ktx runtime status/); + assert.match(readme, /ktx runtime doctor/); + assert.doesNotMatch(readme, /@ktx\/context/); + assert.doesNotMatch(readme, /@ktx\/cli/); + assert.doesNotMatch(readme, /python -m ktx_daemon semantic-validate/); + }); +``` + +- [ ] **Step 3: Extend Postgres smoke assertions** + +In the existing `documents the Postgres historic SQL smoke example` test in +`scripts/examples-docs.test.mjs`, add these assertions after +`assert.match(smoke, /pg_stat_statements_reset/);`: + +```javascript + assert.match(smoke, /KTX_RUNTIME_ROOT/); + assert.match(smoke, /managedDaemon/); + assert.match(smoke, /installPolicy: 'auto'/); + assert.match(smoke, /getKtxCliPackageInfo/); + assert.doesNotMatch(smoke, /python-service/); + assert.doesNotMatch(smoke, /PYTHON_SERVICE/); + assert.doesNotMatch(smoke, /uvicorn app\.main:app/); + assert.doesNotMatch(smoke, /export KTX_SQL_ANALYSIS_URL/); + assert.doesNotMatch(readme, /python-service/); + assert.doesNotMatch(readme, /KTX_SQL_ANALYSIS_URL/); +``` + +- [ ] **Step 4: Run the docs test to verify it fails** + +Run: + +```bash +node --test scripts/examples-docs.test.mjs +``` + +Expected: FAIL. The failure includes missing `@kaelio/ktx` README matches and +the existing `python-service` / `KTX_SQL_ANALYSIS_URL` references in the +Postgres smoke files. + +### Task 2: Move the Postgres historic smoke to the managed runtime + +**Files:** + +- Modify: `examples/postgres-historic/scripts/smoke.sh` +- Test: `scripts/examples-docs.test.mjs` + +- [ ] **Step 1: Remove Python service process state** + +In `examples/postgres-historic/scripts/smoke.sh`, replace the variable block: + +```bash +KTX_BIN="$KTX_ROOT/packages/cli/dist/bin.js" +PYTHON_SERVICE_LOG="$PROJECT_PARENT/python-service.log" +PYTHON_SERVICE_PID="" +``` + +with: + +```bash +KTX_BIN="$KTX_ROOT/packages/cli/dist/bin.js" +export KTX_RUNTIME_ROOT="$PROJECT_PARENT/managed-runtime" +unset KTX_DAEMON_URL +unset KTX_SQL_ANALYSIS_URL +``` + +- [ ] **Step 2: Replace cleanup** + +In `examples/postgres-historic/scripts/smoke.sh`, replace the `cleanup()` +function with: + +```bash +cleanup() { + if [[ -f "$KTX_BIN" ]]; then + node "$KTX_BIN" runtime stop >/dev/null 2>&1 || true + fi + if [[ "${KTX_POSTGRES_HISTORIC_KEEP_DOCKER:-0}" != "1" ]]; then + docker compose -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT +``` + +- [ ] **Step 3: Delete the old SQL analysis service starter** + +Delete the entire `start_sql_analysis_if_needed()` function from +`examples/postgres-historic/scripts/smoke.sh`. The deleted function begins with +this line: + +```bash +start_sql_analysis_if_needed() { +``` + +and ends with this line: + +```bash +} +``` + +immediately before the `latest_manifest()` function. + +- [ ] **Step 4: Pass managed daemon options to stage-only ingest** + +In the Node heredoc inside `run_historic_stage_only()`, replace this block: + +```javascript +const { createKtxCliLocalIngestAdapters } = await import(join(ktxRoot, 'packages/cli/dist/local-adapters.js')); + +const project = await loadKtxProject({ projectDir }); +const adapters = createKtxCliLocalIngestAdapters(project, { historicSqlConnectionId: 'warehouse' }); +``` + +with: + +```javascript +const { createKtxCliLocalIngestAdapters } = await import(join(ktxRoot, 'packages/cli/dist/local-adapters.js')); +const { getKtxCliPackageInfo } = await import(join(ktxRoot, 'packages/cli/dist/index.js')); + +const project = await loadKtxProject({ projectDir }); +const cliVersion = getKtxCliPackageInfo().version; +const managedRuntimeIo = { stdout: process.stdout, stderr: process.stderr }; +const adapters = createKtxCliLocalIngestAdapters(project, { + historicSqlConnectionId: 'warehouse', + managedDaemon: { + cliVersion, + installPolicy: 'auto', + io: managedRuntimeIo, + }, +}); +``` + +- [ ] **Step 5: Remove the old starter call** + +Delete this line from the bottom half of +`examples/postgres-historic/scripts/smoke.sh`: + +```bash +start_sql_analysis_if_needed +``` + +- [ ] **Step 6: Run the docs test to verify the script guards pass** + +Run: + +```bash +node --test scripts/examples-docs.test.mjs +``` + +Expected: FAIL remains because README files have not been updated yet. The +Postgres smoke script assertions now pass. + +### Task 3: Update Postgres historic and artifact docs + +**Files:** + +- Modify: `examples/postgres-historic/README.md` +- Modify: `examples/package-artifacts/README.md` +- Test: `scripts/examples-docs.test.mjs` + +- [ ] **Step 1: Replace Postgres prerequisites** + +In `examples/postgres-historic/README.md`, replace the `## Prerequisites` +section with: + +```markdown +## Prerequisites + +- Docker with Compose v2 +- Node and pnpm matching the KTX workspace +- `uv` on `PATH` so the KTX-managed Python runtime can install the bundled + runtime wheel +``` + +- [ ] **Step 2: Replace the smoke run description** + +In `examples/postgres-historic/README.md`, replace the paragraph after the +`examples/postgres-historic/scripts/smoke.sh` command with: + +```markdown +The smoke creates a temporary KTX project, isolates the managed Python runtime +under the temporary project parent, starts Postgres on `127.0.0.1:55432`, and +uses this connection URL: +``` + +- [ ] **Step 3: Update the full ingest command** + +In `examples/postgres-historic/README.md`, replace the manual ingest command: + +```bash +node packages/cli/dist/bin.js --project-dir /tmp/ktx-postgres-historic dev ingest run \ + --connection-id warehouse \ + --adapter historic-sql \ + --plain \ + --no-input +``` + +with: + +```bash +pnpm run ktx -- dev ingest run --project-dir /tmp/ktx-postgres-historic \ + --connection-id warehouse \ + --adapter historic-sql \ + --plain \ + --yes \ + --no-input +``` + +- [ ] **Step 4: Replace SQL-analysis troubleshooting** + +In `examples/postgres-historic/README.md`, replace the final troubleshooting +bullet: + +```markdown +- SQL-analysis failures: set `KTX_SQL_ANALYSIS_URL` to the running service URL + or create `python-service/.venv` before running `scripts/smoke.sh`. +``` + +with: + +```markdown +- SQL-analysis failures: run `pnpm run ktx -- runtime doctor` from the KTX + repository root and confirm `uv`, the bundled Python wheel, and the managed + runtime all pass. +``` + +- [ ] **Step 5: Replace package artifact README body** + +Replace the full contents of `examples/package-artifacts/README.md` with: + +````markdown +# Package artifact smoke checks + +The package artifact smoke checks create temporary projects instead of storing +sample projects in this directory. Run the checks from `ktx/`: + +```bash +pnpm run artifacts:check +``` + +The npm smoke project installs the generated public `@kaelio/ktx` tarball, +imports the package entry point, and runs installed `ktx` commands against a +generated local project. + +The managed runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies +`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from +the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed +daemon, and stops it. + +The Python smoke project still installs the Python artifacts directly because +it verifies the standalone Python distributions that feed the bundled runtime +wheel. +```` + +- [ ] **Step 6: Run the docs test to verify these docs pass** + +Run: + +```bash +node --test scripts/examples-docs.test.mjs +``` + +Expected: FAIL remains because `README.md` still lacks the public npm managed +runtime documentation. The Postgres and package artifact assertions now pass. + +### Task 4: Update the root README public runtime path + +**Files:** + +- Modify: `README.md` +- Test: `scripts/examples-docs.test.mjs` + +- [ ] **Step 1: Replace quick start** + +In `README.md`, replace the `## Quick start` section through the end of the +full-demo paragraph with: + +````markdown +## Quick start + +Run the pre-seeded demo through the public npm package: + +```bash +npx @kaelio/ktx setup demo --no-input +npx @kaelio/ktx setup demo inspect +``` + +The default demo uses packaged sample data and prebuilt context. It does not +require API keys, network access, or an LLM provider. + +To replay the packaged ingest run, use: + +```bash +npx @kaelio/ktx setup demo --mode replay --no-input +``` + +To run the full agentic demo with an LLM provider, set a provider key for the +current process: + +```bash +ANTHROPIC_API_KEY=$YOUR_ANTHROPIC_API_KEY \ + npx @kaelio/ktx setup demo --mode full --no-input +``` + +Interactive full-demo setup can prompt for a provider key without writing the +key to `ktx.yaml`. + +You can also install the CLI in a project or globally: + +```bash +npm install @kaelio/ktx +npx ktx --help +npm install -g @kaelio/ktx +ktx --help +``` +```` + +- [ ] **Step 2: Replace local project setup command** + +In the `## Build a local project` section of `README.md`, replace: + +```bash +uv sync --all-packages +source .venv/bin/activate + +PROJECT_DIR="$(mktemp -d)/ktx-demo" +pnpm run ktx -- init "$PROJECT_DIR" --name ktx-demo +``` + +with: + +```bash +npm install @kaelio/ktx +PROJECT_DIR="$(mktemp -d)/ktx-demo" +npx ktx init "$PROJECT_DIR" --name ktx-demo +``` + +- [ ] **Step 3: Replace README command prefixes** + +In `README.md`, replace the source-tree command prefix `pnpm run ktx --` with +`npx ktx` in all user workflow commands under `## Build a local project`, +`### Scan the demo warehouse`, and `## Serve MCP`. Keep `pnpm run ktx --` in +the `## Development` section. + +For example, this command: + +```bash +pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \ +``` + +becomes: + +```bash +npx ktx sl query --project-dir "$PROJECT_DIR" \ +``` + +- [ ] **Step 4: Add managed runtime section** + +Insert this section after the scan walkthrough in `README.md`: + +````markdown +## Managed Python runtime + +KTX installs its Python runtime only when a Python-backed command needs it. +The runtime lives outside the npm cache, is versioned by the installed CLI +version, and is managed by `ktx runtime` commands: + +```bash +npx ktx runtime install --yes +npx ktx runtime status +npx ktx runtime doctor +npx ktx runtime start +npx ktx runtime stop +``` + +Commands such as `npx @kaelio/ktx sl query ... --yes` can install the core +runtime lazily from the bundled wheel. Local embeddings remain lazy; prepare +them only when you select local `sentence-transformers` embeddings: + +```bash +npx ktx runtime install --feature local-embeddings --yes +npx ktx runtime start --feature local-embeddings +``` +```` + +- [ ] **Step 5: Replace Serve MCP section** + +In `README.md`, replace the full `## Serve MCP` section with: + +````markdown +## Serve MCP + +Start the stdio MCP server from the project directory: + +```bash +npx ktx serve --mcp stdio --project-dir "$PROJECT_DIR" \ + --user-id local \ + --semantic-compute \ + --execute-queries \ + --yes +``` + +The `--semantic-compute` flag uses the managed Python runtime when no explicit +semantic compute URL is provided. KTX starts or reuses the managed runtime as +needed. + +The MCP server exposes `connection_list`, `knowledge_search`, +`knowledge_read`, `knowledge_write`, `sl_list_sources`, `sl_read_source`, +`sl_write_source`, `sl_validate`, `sl_query`, `ingest_trigger`, +`ingest_status`, `ingest_report`, and `ingest_replay`. +```` + +- [ ] **Step 6: Update release status wording** + +In `README.md`, replace this sentence in `## Release status`: + +```markdown +This repository is prepared for source publication. Package publishing is still +disabled by `release-policy.json`; registry names, public versions, package +visibility, and provenance policy must be chosen before publishing artifacts to +npm or Python package indexes. +``` + +with: + +```markdown +This repository builds a single public npm artifact named `@kaelio/ktx`. +Package publishing is still disabled by `release-policy.json`; registry +credentials, public versions, release tags, and provenance policy must be +chosen before publishing artifacts to npm or Python package indexes. +``` + +- [ ] **Step 7: Run the docs test to verify the README passes** + +Run: + +```bash +node --test scripts/examples-docs.test.mjs +``` + +Expected: PASS. + +### Task 5: Final verification and commit + +**Files:** + +- Verify: `scripts/examples-docs.test.mjs` +- Verify: `examples/postgres-historic/scripts/smoke.sh` +- Verify: `examples/postgres-historic/README.md` +- Verify: `examples/package-artifacts/README.md` +- Verify: `README.md` + +- [ ] **Step 1: Run the script test suite affected by docs** + +Run: + +```bash +node --test scripts/examples-docs.test.mjs scripts/check-boundaries.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 2: Run the boundary check** + +Run: + +```bash +node scripts/check-boundaries.mjs +``` + +Expected: + +```text +ktx boundary check passed +``` + +- [ ] **Step 3: Search for removed external runtime references** + +Run: + +```bash +rg -n "python-service|uvicorn app\\.main:app|export KTX_SQL_ANALYSIS_URL|uv run ktx-daemon serve-http|@ktx/context.*@ktx/cli" README.md examples/postgres-historic/README.md examples/postgres-historic/scripts/smoke.sh examples/package-artifacts/README.md +``` + +Expected: no matches. + +- [ ] **Step 4: Commit** + +```bash +git add scripts/examples-docs.test.mjs \ + examples/postgres-historic/scripts/smoke.sh \ + examples/postgres-historic/README.md \ + examples/package-artifacts/README.md \ + README.md +git commit -m "docs: align managed runtime examples" +``` + +## Acceptance criteria + +- The Postgres historic SQL smoke no longer references `python-service/`, + `uvicorn app.main:app`, or `export KTX_SQL_ANALYSIS_URL`. +- The stage-only Postgres historic smoke uses `createKtxCliLocalIngestAdapters` + with managed daemon options and `installPolicy: 'auto'`. +- The root README documents `npx @kaelio/ktx`, local `npx ktx`, global `ktx`, + `ktx runtime ...`, and MCP `--semantic-compute --yes` managed-runtime usage. +- Package artifact docs describe the single public `@kaelio/ktx` tarball and + the managed runtime smoke. +- `node --test scripts/examples-docs.test.mjs scripts/check-boundaries.test.mjs` + passes. +- `node scripts/check-boundaries.mjs` passes. + +## Self-review + +- Spec coverage: This plan covers the remaining user-facing drift from the + npm-managed runtime spec by removing manual Python service guidance, + documenting public `@kaelio/ktx` invocation modes, and making the Postgres + example smoke use the managed core daemon. +- Placeholder scan: The plan contains exact files, edits, commands, expected + outcomes, and commit instructions. +- Type consistency: The plan uses the existing `managedDaemon` option shape + from `packages/cli/src/local-adapters.ts` and the existing + `installPolicy: 'auto'` value from `packages/cli/src/managed-python-command.ts`. diff --git a/docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md b/docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md new file mode 100644 index 00000000..b6cbd405 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md @@ -0,0 +1,377 @@ +# Managed Runtime Prune Smoke and Docs Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Prove and document `ktx runtime prune` as part of the npm-managed +Python runtime release contract. + +**Architecture:** The prune command already exists in the CLI runtime layer, so +this plan adds black-box package smoke coverage and public documentation only. +The smoke creates an isolated stale versioned runtime directory, previews it, +verifies confirmation is required, and removes it through the installed +`@kaelio/ktx` package. + +**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, Markdown, KTX CLI +managed Python runtime. + +--- + +## Current state + +This plan follows +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +The following plan files are based on that spec and are implemented in the +current tree: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` +- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` +- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md` +- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md` + +Implementation evidence found before writing this plan includes: + +- `packages/cli/assets/python/manifest.json` and + `packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl`. +- `packages/cli/src/managed-python-runtime.ts`, including + `installManagedPythonRuntime()`, `doctorManagedPythonRuntime()`, and + `pruneManagedPythonRuntimes()`. +- `packages/cli/src/runtime.ts`, including the `install`, `status`, + `doctor`, `start`, `stop`, and `prune` runtime command runner branches. +- `packages/cli/src/commands/runtime-commands.ts`, including the + `runtime prune --dry-run` and `runtime prune --yes` Commander wiring. +- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`, + `scripts/published-package-smoke.mjs`, `scripts/local-embeddings-runtime-smoke.mjs`, + `scripts/publish-public-npm-package.mjs`, `release-policy.json`, and + `.github/workflows/release.yml`. +- `README.md` and `examples/package-artifacts/README.md` document the managed + runtime but do not mention `ktx runtime prune`. + +The remaining gap is narrow: the spec lists `ktx runtime prune` as part of the +runtime management command family, but public docs and installed package smoke +coverage only prove `install`, `status`, `doctor`, `start`, and `stop`. + +## File structure + +- Modify `scripts/package-artifacts.test.mjs`: assert that the generated + installed npm smoke covers `ktx runtime prune --dry-run`, confirmation + failure, and confirmed deletion. +- Modify `scripts/package-artifacts.mjs`: extend `npmRuntimeSmokeSource()` to + create a stale runtime directory and exercise `ktx runtime prune`. +- Modify `scripts/examples-docs.test.mjs`: require public docs to mention + `ktx runtime prune --dry-run` and `ktx runtime prune --yes`. +- Modify `README.md`: add prune commands and one sentence describing preview + and confirmed deletion. +- Modify `examples/package-artifacts/README.md`: describe prune coverage in the + package artifact smoke. + +### Task 1: Add installed package prune smoke coverage + +**Files:** + +- Modify: `scripts/package-artifacts.test.mjs` +- Modify: `scripts/package-artifacts.mjs` + +- [ ] **Step 1: Add failing smoke-source assertions** + +In `scripts/package-artifacts.test.mjs`, inside +`it('runs installed CLI commands through the public package runtime', () => {` +and immediately after the existing assertions for `ktx runtime stop`, add: + +```javascript + assert.match(source, /ktx runtime prune dry run/); + assert.match(source, /0\.0\.0/); + assert.match(source, /ktx runtime prune needs confirmation/); + assert.match(source, /Refusing to prune without --yes/); + assert.match(source, /ktx runtime prune confirmed/); + assert.match(source, /Removed stale KTX Python runtimes/); + assert.match(source, /assert\.rejects\(\(\) => access\(staleRuntimeDir\)\)/); +``` + +- [ ] **Step 2: Run the package artifact test and verify failure** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs +``` + +Expected: FAIL in the installed CLI smoke source test because +`npmRuntimeSmokeSource()` does not yet contain the prune labels, confirmation +guard, or stale runtime removal assertion. + +- [ ] **Step 3: Extend the generated installed CLI smoke** + +In `scripts/package-artifacts.mjs`, inside `npmRuntimeSmokeSource()`, add this +block immediately after: + +```javascript + process.stdout.write('ktx runtime daemon lifecycle verified\n'); +``` + +Add: + +```javascript + const staleRuntimeDir = join(process.env.KTX_RUNTIME_ROOT, '0.0.0'); + await mkdir(staleRuntimeDir, { recursive: true }); + + const runtimePruneDryRun = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune', '--dry-run']); + requireSuccess('ktx runtime prune dry run', runtimePruneDryRun); + requireOutput('ktx runtime prune dry run', runtimePruneDryRun, /Stale KTX Python runtimes/); + requireOutput('ktx runtime prune dry run', runtimePruneDryRun, /0\.0\.0/); + await access(staleRuntimeDir); + + const runtimePruneNeedsConfirmation = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune']); + assert.equal(runtimePruneNeedsConfirmation.code, 1, 'ktx runtime prune without --yes must fail'); + assert.equal(runtimePruneNeedsConfirmation.stdout, '', 'ktx runtime prune confirmation failure wrote stdout'); + assert.match(runtimePruneNeedsConfirmation.stderr, /Refusing to prune without --yes/); + + const runtimePruneConfirmed = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune', '--yes']); + requireSuccess('ktx runtime prune confirmed', runtimePruneConfirmed); + requireOutput('ktx runtime prune confirmed', runtimePruneConfirmed, /Removed stale KTX Python runtimes/); + requireOutput('ktx runtime prune confirmed', runtimePruneConfirmed, /0\.0\.0/); + await assert.rejects(() => access(staleRuntimeDir)); + process.stdout.write('ktx runtime prune verified\n'); +``` + +No import changes are needed because the generated smoke already imports +`assert`, `access`, `mkdir`, and `join`. + +- [ ] **Step 4: Run the package artifact test and verify pass** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs +``` + +Expected: PASS. The source assertions now find prune dry-run coverage, +confirmation failure coverage, confirmed prune coverage, and stale directory +deletion verification. + +- [ ] **Step 5: Commit the smoke coverage** + +Run: + +```bash +git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs +git commit -m "test: cover managed runtime prune in package smoke" +``` + +### Task 2: Document runtime prune in public docs + +**Files:** + +- Modify: `scripts/examples-docs.test.mjs` +- Modify: `README.md` +- Modify: `examples/package-artifacts/README.md` + +- [ ] **Step 1: Add failing docs assertions** + +In `scripts/examples-docs.test.mjs`, inside +`it('documents public npm and managed runtime usage in the README', async () => {` +and immediately after: + +```javascript + assert.match(rootReadme, /ktx runtime stop/); +``` + +Add: + +```javascript + assert.match(rootReadme, /ktx runtime prune --dry-run/); + assert.match(rootReadme, /ktx runtime prune --yes/); +``` + +In the same file, inside +`it('documents the public package artifact smoke shape', async () => {` and +immediately after: + +```javascript + assert.match(readme, /ktx runtime doctor/); +``` + +Add: + +```javascript + assert.match(readme, /ktx runtime prune --dry-run/); + assert.match(readme, /ktx runtime prune --yes/); +``` + +- [ ] **Step 2: Run the docs test and verify failure** + +Run: + +```bash +node --test scripts/examples-docs.test.mjs +``` + +Expected: FAIL because `README.md` and +`examples/package-artifacts/README.md` do not yet mention `ktx runtime prune`. + +- [ ] **Step 3: Update the root README runtime section** + +In `README.md`, in the `## Managed Python runtime` command block, replace: + +```bash +npx ktx runtime install --yes +npx ktx runtime status +npx ktx runtime doctor +npx ktx runtime start +npx ktx runtime stop +``` + +with: + +```bash +npx ktx runtime install --yes +npx ktx runtime status +npx ktx runtime doctor +npx ktx runtime start +npx ktx runtime stop +npx ktx runtime prune --dry-run +npx ktx runtime prune --yes +``` + +Immediately after that command block, add: + +```markdown +Use `runtime prune --dry-run` to preview stale runtime directories from older +CLI versions. Add `--yes` to remove those stale directories after daemon +processes are stopped. +``` + +- [ ] **Step 4: Update package artifact smoke docs** + +In `examples/package-artifacts/README.md`, replace: + +```markdown +The managed Python runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies +`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from +the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed +daemon, and stops it. +``` + +with: + +```markdown +The managed Python runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies +`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from +the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed +daemon, stops it, previews a stale runtime with `ktx runtime prune --dry-run`, +verifies confirmation is required, and removes the stale runtime with +`ktx runtime prune --yes`. +``` + +- [ ] **Step 5: Run the docs test and verify pass** + +Run: + +```bash +node --test scripts/examples-docs.test.mjs +``` + +Expected: PASS. The public README and package artifact README now document +runtime prune alongside the other managed runtime commands. + +- [ ] **Step 6: Commit the docs coverage** + +Run: + +```bash +git add scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md +git commit -m "docs: document managed runtime prune" +``` + +### Task 3: Verify the completed prune release surface + +**Files:** + +- Verify: `scripts/package-artifacts.mjs` +- Verify: `scripts/package-artifacts.test.mjs` +- Verify: `scripts/examples-docs.test.mjs` +- Verify: `README.md` +- Verify: `examples/package-artifacts/README.md` + +- [ ] **Step 1: Run focused tests** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs scripts/examples-docs.test.mjs +``` + +Expected: PASS. The source-level tests cover generated package smoke behavior +and docs assertions. + +- [ ] **Step 2: Run the installed package artifact smoke** + +Run: + +```bash +pnpm run artifacts:check +``` + +Expected: PASS. The generated installed CLI smoke prints: + +```text +ktx runtime prune verified +``` + +and removes the temporary `0.0.0` directory from the isolated +`KTX_RUNTIME_ROOT`. + +- [ ] **Step 3: Inspect git status** + +Run: + +```bash +git status --short +``` + +Expected: only the five planned files are modified before the final commit, or +no modified files remain after the task commits. + +- [ ] **Step 4: Commit verification fixes if needed** + +If verification required small corrections, commit only those intended files: + +```bash +git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md +git commit -m "test: verify managed runtime prune release surface" +``` + +## Acceptance criteria + +- The generated installed npm package smoke creates a stale versioned runtime + directory under the isolated `KTX_RUNTIME_ROOT`. +- `ktx runtime prune --dry-run` lists the stale runtime and leaves it on disk. +- `ktx runtime prune` without `--yes` exits nonzero and prints the existing + confirmation guidance. +- `ktx runtime prune --yes` removes the stale runtime directory. +- `README.md` lists `ktx runtime prune --dry-run` and + `ktx runtime prune --yes` with the other managed runtime commands. +- `examples/package-artifacts/README.md` describes prune coverage in the + package artifact smoke. + +## Self-review + +- Spec coverage: this plan covers the remaining visible gap for the runtime + management command family in the npm-managed Python runtime spec. The prune + implementation already exists, and this plan adds release smoke and public + docs coverage. +- Placeholder scan: no placeholder steps, deferred implementation notes, or + unspecified behavior gaps remain. +- Type consistency: the plan uses existing labels and functions: + `npmRuntimeSmokeSource()`, `requireSuccess()`, `requireOutput()`, + `KTX_RUNTIME_ROOT`, `ktx runtime prune --dry-run`, and + `ktx runtime prune --yes`. diff --git a/docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md b/docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md new file mode 100644 index 00000000..b3e1e0f9 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md @@ -0,0 +1,647 @@ +# Managed Runtime uv Prerequisite Contract Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the remaining npm-managed Python runtime open decision by +making `uv` a documented, release-policy-checked prerequisite. + +**Architecture:** Keep the runtime installer behavior simple: the CLI locates +`uv` on `PATH` and prints a focused error when it is missing. Encode that +decision in `release-policy.json`, validate it during release readiness, use one +shared runtime error message, and document the prerequisite in public docs. + +**Tech Stack:** Node 22 ESM scripts, `node:test`, TypeScript, Vitest, JSON +release policy, Markdown. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +The following plan files are based on that spec and are already implemented in +this worktree: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` +- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` +- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md` +- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md` +- `docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md` + +Implementation evidence found before writing this plan includes: + +- `packages/cli/assets/python/manifest.json` and the bundled + `kaelio_ktx-0.1.0-py3-none-any.whl`. +- `packages/cli/src/managed-python-runtime.ts`, including runtime roots, + bundled wheel verification, install, status, doctor, and prune behavior. +- `packages/cli/src/managed-python-command.ts`, + `packages/cli/src/managed-python-daemon.ts`, + `packages/cli/src/managed-local-embeddings.ts`, and + `packages/cli/src/managed-python-http.ts`. +- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`, + `scripts/published-package-smoke.mjs`, + `scripts/local-embeddings-runtime-smoke.mjs`, and + `scripts/publish-public-npm-package.mjs`. +- `release-policy.json` is already in `npm-public-release-ready` mode for + `@kaelio/ktx` `0.1.0` and keeps Python package publishing disabled. +- `README.md` and `examples/package-artifacts/README.md` document the managed + runtime command family, including `runtime prune`. + +The remaining spec gap is the open decision in +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`: + +```text +KTX still needs a final decision on whether uv is a hard prerequisite or a +bootstrap dependency that KTX downloads automatically. +``` + +This plan chooses the hard-prerequisite path for the first public release. KTX +will not download `uv` automatically in this release. + +## File structure + +- Modify `release-policy.json`: add a `runtimeInstaller` policy section that + records the hard `uv` prerequisite decision. +- Modify `scripts/release-readiness.mjs`: validate the runtime installer + policy, include it in readiness reports, and print it in text output. +- Modify `scripts/release-readiness.test.mjs`: cover the accepted policy and + rejection paths for missing or bootstrap-style `uv` policies. +- Modify `packages/cli/src/managed-python-runtime.ts`: export one shared + missing-`uv` message and use it for install and doctor output. +- Modify `packages/cli/src/managed-python-runtime.test.ts`: cover install and + doctor behavior when `uv` is missing. +- Modify `scripts/examples-docs.test.mjs`: require public docs to state the + hard `uv` prerequisite. +- Modify `README.md`: document that `uv` must be on `PATH` and KTX does not + download it automatically. +- Modify `examples/package-artifacts/README.md`: document the artifact smoke + `uv` prerequisite. + +### Task 1: Encode the runtime installer policy + +**Files:** + +- Modify: `release-policy.json` +- Modify: `scripts/release-readiness.test.mjs` +- Modify: `scripts/release-readiness.mjs` +- Test: `scripts/release-readiness.test.mjs` + +- [ ] **Step 1: Add failing release policy tests** + +In `scripts/release-readiness.test.mjs`, inside the `releasePolicy()` helper +return value, add the `runtimeInstaller` object immediately after +`publishedPackageSmoke`: + +```javascript + runtimeInstaller: { + uvStrategy: 'path-prerequisite', + bootstrapUv: false, + missingUvBehavior: 'focused-error', + }, +``` + +In the three `assert.deepEqual(report, { ... })` expectations, add this field +immediately after `publishedPackageSmokeGate`: + +```javascript + runtimeInstaller: { + uvStrategy: 'path-prerequisite', + bootstrapUv: false, + missingUvBehavior: 'focused-error', + }, +``` + +Add these tests immediately after the +`it('accepts the npm public release ready policy', async () => { ... })` block: + +```javascript + it('rejects npm public release ready mode without a runtime installer policy', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-policy-missing-test-')); + try { + await writeReadyFixture(root, { + policy: releasePolicy({ + releaseMode: 'npm-public-release-ready', + npm: { + publish: true, + registry: null, + access: 'public', + tag: 'latest', + }, + publishedPackageSmoke: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + runtimeInstaller: undefined, + requiredBeforePublishing: [], + }), + }); + + await assert.rejects( + () => releaseReadinessReport(root), + /Release policy runtimeInstaller must be a JSON object/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it('rejects uv bootstrap download policy for the first public npm release', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-policy-bootstrap-test-')); + try { + await writeReadyFixture(root, { + policy: releasePolicy({ + releaseMode: 'npm-public-release-ready', + npm: { + publish: true, + registry: null, + access: 'public', + tag: 'latest', + }, + publishedPackageSmoke: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + runtimeInstaller: { + uvStrategy: 'bootstrap-download', + bootstrapUv: true, + missingUvBehavior: 'download', + }, + requiredBeforePublishing: [], + }), + }); + + await assert.rejects( + () => releaseReadinessReport(root), + /Release policy runtimeInstaller\.uvStrategy must be path-prerequisite/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +``` + +- [ ] **Step 2: Run the release readiness tests and verify failure** + +Run: + +```bash +node --test scripts/release-readiness.test.mjs +``` + +Expected: FAIL because `releaseReadinessReport()` does not include +`runtimeInstaller`, and `validateReleasePolicy()` does not validate the new +policy section. + +- [ ] **Step 3: Validate the runtime installer policy** + +In `scripts/release-readiness.mjs`, add this function immediately after the +`assertRequiredBeforePublishing(policy)` function definition: + +```javascript +function assertRuntimeInstallerPolicy(policy) { + assertPlainObject(policy.runtimeInstaller, 'Release policy runtimeInstaller'); + assertString(policy.runtimeInstaller.uvStrategy, 'Release policy runtimeInstaller.uvStrategy'); + assertBoolean(policy.runtimeInstaller.bootstrapUv, 'Release policy runtimeInstaller.bootstrapUv'); + assertString( + policy.runtimeInstaller.missingUvBehavior, + 'Release policy runtimeInstaller.missingUvBehavior', + ); + + if (policy.runtimeInstaller.uvStrategy !== 'path-prerequisite') { + throw new Error('Release policy runtimeInstaller.uvStrategy must be path-prerequisite'); + } + if (policy.runtimeInstaller.bootstrapUv !== false) { + throw new Error('Release policy runtimeInstaller.bootstrapUv must be false'); + } + if (policy.runtimeInstaller.missingUvBehavior !== 'focused-error') { + throw new Error('Release policy runtimeInstaller.missingUvBehavior must be focused-error'); + } +} +``` + +In `validateReleasePolicy(policy)`, add this call immediately after +`assertRequiredBeforePublishing(policy);`: + +```javascript + assertRuntimeInstallerPolicy(policy); +``` + +In `releaseReadinessReport(rootDir = scriptRootDir())`, add +`runtimeInstaller` to the returned object immediately after +`publishedPackageSmokeGate`: + +```javascript + runtimeInstaller: policy.runtimeInstaller, +``` + +In `main()`, add these lines immediately after the published package smoke +registry line: + +```javascript + process.stdout.write(`Runtime uv strategy: ${report.runtimeInstaller.uvStrategy}\n`); + process.stdout.write( + `Runtime uv bootstrap: ${report.runtimeInstaller.bootstrapUv ? 'enabled' : 'disabled'}\n`, + ); +``` + +- [ ] **Step 4: Encode the policy in `release-policy.json`** + +Replace `release-policy.json` with this exact content: + +```json +{ + "schemaVersion": 1, + "releaseMode": "npm-public-release-ready", + "npm": { + "publish": true, + "registry": null, + "access": "public", + "tag": "latest", + "packages": ["@kaelio/ktx"] + }, + "python": { + "publish": false, + "repository": null, + "packages": ["ktx-sl", "ktx-daemon", "kaelio-ktx"] + }, + "publishedPackageSmoke": { + "packageName": "@kaelio/ktx", + "version": "0.1.0", + "registry": null + }, + "runtimeInstaller": { + "uvStrategy": "path-prerequisite", + "bootstrapUv": false, + "missingUvBehavior": "focused-error" + }, + "requiredBeforePublishing": [] +} +``` + +- [ ] **Step 5: Run the release readiness tests and verify success** + +Run: + +```bash +node --test scripts/release-readiness.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 6: Commit the release policy contract** + +```bash +git add release-policy.json scripts/release-readiness.mjs scripts/release-readiness.test.mjs +git commit -m "chore: encode uv runtime prerequisite policy" +``` + +### Task 2: Centralize missing-uv runtime output + +**Files:** + +- Modify: `packages/cli/src/managed-python-runtime.test.ts` +- Modify: `packages/cli/src/managed-python-runtime.ts` +- Test: `packages/cli/src/managed-python-runtime.test.ts` + +- [ ] **Step 1: Add failing missing-uv runtime tests** + +In `packages/cli/src/managed-python-runtime.test.ts`, add +`MISSING_UV_RUNTIME_INSTALL_MESSAGE` to the import from +`./managed-python-runtime.js`: + +```typescript +import { + MISSING_UV_RUNTIME_INSTALL_MESSAGE, + doctorManagedPythonRuntime, + installManagedPythonRuntime, + managedPythonRuntimeLayout, + pruneManagedPythonRuntimes, + readManagedPythonRuntimeStatus, + verifyRuntimeAsset, + type ManagedPythonRuntimeExec, +} from './managed-python-runtime.js'; +``` + +Inside `describe('installManagedPythonRuntime', () => { ... })`, add this test +after the local embeddings test: + +```typescript + it('fails with the hard-prerequisite message when uv is missing', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const commands: Array<{ command: string; args: string[] }> = []; + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { + commands.push({ command, args }); + throw new Error('spawn uv ENOENT'); + }); + + await expect( + installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }), + ).rejects.toThrow(MISSING_UV_RUNTIME_INSTALL_MESSAGE); + + expect(commands).toEqual([{ command: 'uv', args: ['--version'] }]); + }); +``` + +Inside `describe('doctorManagedPythonRuntime', () => { ... })`, add this test +after the existing doctor test: + +```typescript + it('reports uv as a hard prerequisite when uv is missing', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async () => { + throw new Error('spawn uv ENOENT'); + }); + + const checks = await doctorManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + exec, + }); + + expect(checks[0]).toEqual({ + id: 'uv', + label: 'uv', + status: 'fail', + detail: MISSING_UV_RUNTIME_INSTALL_MESSAGE, + fix: 'Install uv, make sure it is on PATH, and run: ktx runtime install --yes', + }); + }); +``` + +- [ ] **Step 2: Run the runtime tests and verify failure** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts +``` + +Expected: FAIL because the shared message constant does not exist and the +doctor fix text still uses the older message. + +- [ ] **Step 3: Add the shared missing-uv message** + +In `packages/cli/src/managed-python-runtime.ts`, add this export immediately +after the `ManagedPythonRuntimePruneResult` interface: + +```typescript +export const MISSING_UV_RUNTIME_INSTALL_MESSAGE = + 'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx runtime install --yes'; +``` + +Replace the body of the `catch` block in `ensureUv()` with: + +```typescript + throw new Error(MISSING_UV_RUNTIME_INSTALL_MESSAGE); +``` + +In `doctorManagedPythonRuntime()`, replace the `fix` value for the `uv` check +with: + +```typescript + fix: 'Install uv, make sure it is on PATH, and run: ktx runtime install --yes', +``` + +- [ ] **Step 4: Run the runtime tests and verify success** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit the runtime output contract** + +```bash +git add packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts +git commit -m "fix: clarify missing uv runtime error" +``` + +### Task 3: Document the hard uv prerequisite + +**Files:** + +- Modify: `scripts/examples-docs.test.mjs` +- Modify: `README.md` +- Modify: `examples/package-artifacts/README.md` +- Test: `scripts/examples-docs.test.mjs` + +- [ ] **Step 1: Add failing docs assertions** + +In `scripts/examples-docs.test.mjs`, inside +`it('documents public npm and managed runtime usage in the README', ... )`, add +these assertions immediately after the existing `ktx runtime prune --yes` +assertion: + +```javascript + assert.match(rootReadme, /KTX requires `uv` on `PATH`/); + assert.match(rootReadme, /KTX doesn't download `uv` automatically/); +``` + +Inside `it('documents the public package artifact smoke shape', ... )`, add +this assertion immediately after the `managed Python runtime` assertion: + +```javascript + assert.match(readme, /requires `uv` on `PATH`/); +``` + +- [ ] **Step 2: Run the docs test and verify failure** + +Run: + +```bash +node --test scripts/examples-docs.test.mjs +``` + +Expected: FAIL because the README files do not state the hard `uv` +prerequisite. + +- [ ] **Step 3: Update the root README runtime section** + +In `README.md`, in the `## Managed Python runtime` section, replace this +paragraph: + +```markdown +KTX installs its Python runtime only when a Python-backed command needs it. +The runtime lives outside the npm cache, is versioned by the installed CLI +version, and is managed by `ktx runtime` commands: +``` + +With: + +```markdown +KTX installs its Python runtime only when a Python-backed command needs it. +The runtime lives outside the npm cache, is versioned by the installed CLI +version, and is managed by `ktx runtime` commands. + +KTX requires `uv` on `PATH` to create the managed runtime. Install `uv` with +your system package manager or the official installer before running Python- +backed KTX commands. KTX doesn't download `uv` automatically; run +`ktx runtime doctor` if runtime installation fails: +``` + +- [ ] **Step 4: Update the package artifact smoke README** + +In `examples/package-artifacts/README.md`, replace this paragraph: + +```markdown +The managed Python runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies +`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from +the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed +daemon, stops it, previews a stale runtime with `ktx runtime prune --dry-run`, +verifies confirmation is required, and removes the stale runtime with +`ktx runtime prune --yes`. +``` + +With: + +```markdown +The managed Python runtime smoke requires `uv` on `PATH`, isolates +`KTX_RUNTIME_ROOT`, verifies `ktx runtime status`, runs `ktx sl query --yes` to +install the core runtime from the bundled wheel, checks `ktx runtime doctor`, +starts and reuses the managed daemon, stops it, previews a stale runtime with +`ktx runtime prune --dry-run`, verifies confirmation is required, and removes +the stale runtime with `ktx runtime prune --yes`. +``` + +- [ ] **Step 5: Run the docs test and verify success** + +Run: + +```bash +node --test scripts/examples-docs.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 6: Commit the public docs update** + +```bash +git add README.md examples/package-artifacts/README.md scripts/examples-docs.test.mjs +git commit -m "docs: document uv runtime prerequisite" +``` + +### Task 4: Verify the completed contract + +**Files:** + +- Verify: `release-policy.json` +- Verify: `scripts/release-readiness.mjs` +- Verify: `scripts/release-readiness.test.mjs` +- Verify: `packages/cli/src/managed-python-runtime.ts` +- Verify: `packages/cli/src/managed-python-runtime.test.ts` +- Verify: `scripts/examples-docs.test.mjs` +- Verify: `README.md` +- Verify: `examples/package-artifacts/README.md` + +- [ ] **Step 1: Run focused verification** + +Run: + +```bash +node --test scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs +pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts +``` + +Expected: PASS. + +- [ ] **Step 2: Verify release readiness text output** + +Run: + +```bash +pnpm run release:readiness +``` + +Expected output includes: + +```text +KTX release mode: npm-public-release-ready +Runtime uv strategy: path-prerequisite +Runtime uv bootstrap: disabled +NPM publish target: @kaelio/ktx@0.1.0 (latest) +``` + +- [ ] **Step 3: Verify no pre-commit config is required** + +Run: + +```bash +rg --files -g '.pre-commit-config.yaml' -g 'pre-commit-config.yaml' +``` + +Expected: no output and exit code 1. No Python files changed, so the repository +Python pre-commit requirement does not apply. + +- [ ] **Step 4: Review the final diff** + +Run: + +```bash +git diff --stat +git diff -- release-policy.json scripts/release-readiness.mjs scripts/release-readiness.test.mjs packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md +``` + +Expected: only the runtime installer policy, missing-`uv` message/tests, and +public docs changed. + +- [ ] **Step 5: Commit final verification notes if needed** + +If Task 4 produces only verification output and no file changes, skip this +step. If a correction was made during verification, commit it: + +```bash +git add release-policy.json scripts/release-readiness.mjs scripts/release-readiness.test.mjs packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md +git commit -m "chore: finish uv prerequisite release contract" +``` + +## Self-review + +Spec coverage: + +- The earlier implemented plans cover the single public npm package, bundled + Python wheel, managed runtime installer, runtime commands, daemon lifecycle, + local embeddings, Python-backed command integration, release smoke, published + smoke, docs cleanup, release handoff, and prune coverage. +- This plan closes the spec's remaining `uv` open decision by choosing + `path-prerequisite`, recording that decision in release policy, validating it + in release readiness, using one CLI error message, and documenting it. +- The plan keeps Python package publication disabled and keeps KTX-owned Python + code bundled in the npm package. + +Placeholder scan: + +- No task contains deferred implementation markers. +- Each code-changing step names exact files and includes the concrete code to + add or replace. + +Type consistency: + +- The release policy field is consistently named `runtimeInstaller`. +- The chosen strategy is consistently `path-prerequisite`. +- The shared CLI message constant is consistently + `MISSING_UV_RUNTIME_INSTALL_MESSAGE`. diff --git a/docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md b/docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md new file mode 100644 index 00000000..5cbe3cf4 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md @@ -0,0 +1,1904 @@ +# Public Kaelio KTX npm Package Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Produce one installable public npm package, `@kaelio/ktx`, whose +`ktx` binary includes the bundled Python runtime wheel and does not require +users to install any `@ktx/*` workspace packages directly. + +**Architecture:** Keep the internal pnpm workspace package names unchanged for +development, then assemble a release package under `dist/public-npm-package`. +The release package copies the CLI `dist/` and assets, vendors built internal +`@ktx/*` packages as bundled dependencies, writes a public `@kaelio/ktx` +`package.json`, and packs exactly one npm tarball. Release and smoke scripts +then treat `@kaelio/ktx` as the only npm artifact while preserving internal +workspace builds. + +**Tech Stack:** Node 22 ESM scripts, pnpm, TypeScript, Vitest, `node:test`, +npm bundled dependencies, KTX managed Python runtime assets. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +Existing plans based on the spec: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` + +All five are implemented in this worktree. Evidence found before writing this +plan includes: + +- `scripts/build-python-runtime-wheel.mjs` and + `scripts/build-python-runtime-wheel.test.mjs`. +- `packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl` and + `packages/cli/assets/python/manifest.json`. +- `packages/cli/src/managed-python-runtime.ts`, + `packages/cli/src/runtime.ts`, and + `packages/cli/src/commands/runtime-commands.ts`. +- `packages/cli/src/managed-python-command.ts` and `ktx sl query` runtime + install policy flags. +- `packages/cli/src/managed-python-daemon.ts`, daemon state paths, and + `ktx runtime start` / `ktx runtime stop`. +- `packages/cli/src/managed-local-embeddings.ts`, + `packages/context/src/llm/local-config.ts` managed marker constants, and + setup wiring in `packages/cli/src/setup-embeddings.ts`. + +Spec requirements still outside those plans: + +- The visible npm package is still `@ktx/cli`, not `@kaelio/ktx`. +- Release artifacts still model multiple npm workspace packages instead of one + public npm package. +- Installed-package smoke coverage still relies on installing internal + `@ktx/*` tarballs. +- Published-package smoke coverage does not yet exercise the required + `@kaelio/ktx` invocation modes: + `npx @kaelio/ktx setup demo`, `npx @kaelio/ktx sl query ...`, local + `npm install @kaelio/ktx` plus `npx ktx ...`, and global + `npm install -g @kaelio/ktx` plus `ktx ...`. + +This plan implements the public npm package surface and local tarball smoke +coverage. It intentionally keeps internal package imports such as +`@ktx/context` in source code so development stays compatible with the existing +workspace. + +## File structure + +- Modify `packages/cli/src/cli-runtime.ts`: read the package name and version + from the installed package root so the same `dist/` reports `@ktx/cli` in the + workspace and `@kaelio/ktx` in the assembled public package. +- Modify `packages/cli/src/index.test.ts`: cover dynamic package metadata. +- Create `scripts/build-public-npm-package.mjs`: assemble and pack the + `@kaelio/ktx` release package with bundled internal `@ktx/*` packages. +- Create `scripts/build-public-npm-package.test.mjs`: test dependency union, + bundled package copying, public `package.json` generation, and pack command + shape. +- Modify `scripts/package-artifacts.mjs`: build internal packages, build Python + artifacts, build the public package, and write a manifest with exactly one + npm artifact named `@kaelio/ktx`. +- Modify `scripts/package-artifacts.test.mjs`: update artifact layout, + release metadata, npm smoke package, and manifest expectations for the + single public npm artifact. +- Modify `scripts/published-package-smoke-config.mjs`: add the public-package + invocation commands needed by the runtime spec. +- Modify `scripts/published-package-smoke.mjs`: validate the new command list. +- Modify `scripts/published-package-smoke.test.mjs`: expect `@kaelio/ktx` and + the supported `npx`, local install, and global install invocation modes. +- Modify `scripts/release-readiness.mjs`: allow the one public npm artifact + while `release-policy.json` still disables publishing. +- Modify `scripts/release-readiness.test.mjs`: expect only `@kaelio/ktx` in + npm release metadata and policy checks. +- Modify `release-policy.json`: list `@kaelio/ktx` as the only npm package and + set the published smoke package to `@kaelio/ktx`. +- Modify `scripts/precommit-check.test.mjs` only if package filter assertions + expect the public package name after artifact-script changes. Keep + `scripts/precommit-check.mjs` using internal workspace package names. + +### Task 1: Read CLI package metadata from the installed package root + +**Files:** + +- Modify: `packages/cli/src/cli-runtime.ts` +- Modify: `packages/cli/src/index.test.ts` + +- [ ] **Step 1: Add a failing dynamic metadata test** + +In `packages/cli/src/index.test.ts`, extend the import from `./index.js` so it +includes `packageInfoFromJson`: + +```typescript +import { + getKtxCliPackageInfo, + packageInfoFromJson, + rendererUnavailableVizFallback, + renderMemoryFlowTui, + resolveVizFallback, + runKtxCli, + sanitizeMemoryFlowTuiError, + startLiveMemoryFlowTui, + warnVizFallbackOnce, +} from './index.js'; +``` + +Add this test inside `describe('getKtxCliPackageInfo', () => { ... })` after the +existing metadata tests: + +```typescript + it('normalizes public package metadata from package.json contents', () => { + expect( + packageInfoFromJson({ + name: '@kaelio/ktx', + version: '0.1.0', + }), + ).toEqual({ + name: '@kaelio/ktx', + version: '0.1.0', + contextPackageName: '@ktx/context', + }); + }); +``` + +- [ ] **Step 2: Run the failing metadata test** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/index.test.ts +``` + +Expected: FAIL with a missing export for `packageInfoFromJson`. + +- [ ] **Step 3: Implement dynamic package metadata** + +In `packages/cli/src/cli-runtime.ts`, add this import at the top of the file: + +```typescript +import { createRequire } from 'node:module'; +``` + +Replace the `KtxCliPackageInfo` interface and `getKtxCliPackageInfo()` with +this code: + +```typescript +const requirePackageJson = createRequire(import.meta.url); + +export interface KtxCliPackageInfo { + name: string; + version: string; + contextPackageName: '@ktx/context'; +} + +export function packageInfoFromJson(packageJson: unknown): KtxCliPackageInfo { + if ( + typeof packageJson !== 'object' || + packageJson === null || + !('name' in packageJson) || + !('version' in packageJson) || + typeof packageJson.name !== 'string' || + typeof packageJson.version !== 'string' + ) { + throw new Error('Invalid KTX CLI package metadata'); + } + + return { + name: packageJson.name, + version: packageJson.version, + contextPackageName: '@ktx/context', + }; +} + +export function getKtxCliPackageInfo(): KtxCliPackageInfo { + return packageInfoFromJson(requirePackageJson('../package.json')); +} +``` + +In `packages/cli/src/index.ts`, add `packageInfoFromJson` to the export from +`./cli-runtime.js`: + +```typescript +export { + getKtxCliPackageInfo, + packageInfoFromJson, + runInitForCommander, + runKtxCli, + type KtxCliDeps, + type KtxCliIo, + type KtxCliPackageInfo, +} from './cli-runtime.js'; +``` + +- [ ] **Step 4: Verify CLI metadata tests pass** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add packages/cli/src/cli-runtime.ts packages/cli/src/index.ts packages/cli/src/index.test.ts +git commit -m "feat: read CLI package metadata dynamically" +``` + +### Task 2: Add the public npm package assembly script + +**Files:** + +- Create: `scripts/build-public-npm-package.test.mjs` +- Create: `scripts/build-public-npm-package.mjs` + +- [ ] **Step 1: Write failing tests for the public package builder** + +Create `scripts/build-public-npm-package.test.mjs` with this content: + +```javascript +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { + PUBLIC_BUNDLED_WORKSPACE_PACKAGES, + PUBLIC_NPM_PACKAGE_NAME, + collectPublicDependencies, + createPublicNpmPackageTree, + publicNpmPackageJson, + publicNpmPackageLayout, + publicNpmPackCommand, +} from './build-public-npm-package.mjs'; + +async function writeJson(path, value) { + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); +} + +async function writePackage(root, packageRoot, packageJson, files = {}) { + const absoluteRoot = join(root, packageRoot); + await mkdir(absoluteRoot, { recursive: true }); + await writeJson(join(absoluteRoot, 'package.json'), packageJson); + + for (const [relativePath, contents] of Object.entries(files)) { + const target = join(absoluteRoot, relativePath); + await mkdir(join(target, '..'), { recursive: true }); + await writeFile(target, contents); + } +} + +async function writeWorkspaceFixture(root) { + await writePackage( + root, + 'packages/cli', + { + name: '@ktx/cli', + version: '0.0.0-private', + description: 'CLI wrapper for KTX', + type: 'module', + engines: { node: '>=22.0.0' }, + bin: { ktx: './dist/bin.js' }, + main: 'dist/index.js', + types: 'dist/index.d.ts', + exports: { + '.': { + types: './dist/index.d.ts', + import: './dist/index.js', + default: './dist/index.js', + }, + './package.json': './package.json', + }, + files: ['dist', 'assets'], + dependencies: { + '@clack/prompts': '1.3.0', + '@ktx/context': 'workspace:*', + commander: '14.0.3', + }, + license: 'Apache-2.0', + repository: { + type: 'git', + url: 'git+https://github.com/kaelio/ktx.git', + directory: 'packages/cli', + }, + }, + { + 'dist/bin.js': '#!/usr/bin/env node\n', + 'dist/index.js': 'export const cli = true;\n', + 'dist/index.d.ts': 'export declare const cli: true;\n', + 'assets/python/manifest.json': '{"schemaVersion":1}\n', + }, + ); + + await writePackage( + root, + 'packages/context', + { + name: '@ktx/context', + version: '0.0.0-private', + type: 'module', + main: 'dist/index.js', + exports: { '.': './dist/index.js' }, + files: ['dist', 'prompts', 'skills'], + dependencies: { + '@ktx/llm': 'workspace:*', + yaml: '^2.8.2', + }, + }, + { + 'dist/index.js': 'export const context = true;\n', + 'prompts/system.md': 'prompt\n', + 'skills/sl/SKILL.md': 'skill\n', + }, + ); + + await writePackage( + root, + 'packages/llm', + { + name: '@ktx/llm', + version: '0.0.0-private', + type: 'module', + main: 'dist/index.js', + exports: { '.': './dist/index.js' }, + files: ['dist'], + dependencies: { + ai: '^6.0.168', + }, + }, + { + 'dist/index.js': 'export const llm = true;\n', + }, + ); + + for (const packageName of PUBLIC_BUNDLED_WORKSPACE_PACKAGES.filter( + (name) => name.startsWith('@ktx/connector-'), + )) { + const directory = packageName.replace('@ktx/', ''); + await writePackage( + root, + `packages/${directory}`, + { + name: packageName, + version: '0.0.0-private', + type: 'module', + main: 'dist/index.js', + exports: { '.': './dist/index.js' }, + files: ['dist'], + dependencies: { + '@ktx/context': 'workspace:*', + }, + }, + { + 'dist/index.js': `export const name = ${JSON.stringify(packageName)};\n`, + }, + ); + } +} + +describe('publicNpmPackageLayout', () => { + it('uses stable public package build and tarball paths', () => { + const layout = publicNpmPackageLayout('/repo/ktx'); + + assert.equal(layout.rootDir, '/repo/ktx'); + assert.equal(layout.packRoot, '/repo/ktx/dist/public-npm-package'); + assert.equal(layout.npmDir, '/repo/ktx/dist/artifacts/npm'); + assert.equal(layout.tarballPath, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.0.0-private.tgz'); + }); +}); + +describe('collectPublicDependencies', () => { + it('unions external runtime dependencies and omits workspace packages', () => { + assert.deepEqual( + collectPublicDependencies([ + { + name: '@ktx/cli', + dependencies: { + '@ktx/context': 'workspace:*', + commander: '14.0.3', + zod: '^4.4.3', + }, + }, + { + name: '@ktx/context', + dependencies: { + '@ktx/llm': 'workspace:*', + commander: '14.0.3', + yaml: '^2.8.2', + zod: '^4.1.13', + }, + }, + ]), + { + commander: '14.0.3', + yaml: '^2.8.2', + zod: '^4.4.3', + }, + ); + }); + + it('fails on incompatible external dependency ranges', () => { + assert.throws( + () => + collectPublicDependencies([ + { name: '@ktx/cli', dependencies: { zod: '^4.4.3' } }, + { name: '@ktx/context', dependencies: { zod: '^3.25.0' } }, + ]), + /Incompatible dependency versions for zod/, + ); + }); +}); + +describe('publicNpmPackageJson', () => { + it('describes the public @kaelio/ktx binary package', () => { + const packageJson = publicNpmPackageJson( + { + name: '@ktx/cli', + version: '0.0.0-private', + engines: { node: '>=22.0.0' }, + bin: { ktx: './dist/bin.js' }, + main: 'dist/index.js', + types: 'dist/index.d.ts', + exports: { '.': './dist/index.js', './package.json': './package.json' }, + license: 'Apache-2.0', + }, + { commander: '14.0.3' }, + ); + + assert.equal(packageJson.name, PUBLIC_NPM_PACKAGE_NAME); + assert.equal(packageJson.private, false); + assert.deepEqual(packageJson.bin, { ktx: './dist/bin.js' }); + assert.deepEqual(packageJson.dependencies, { commander: '14.0.3' }); + assert.deepEqual(packageJson.bundledDependencies, PUBLIC_BUNDLED_WORKSPACE_PACKAGES); + assert.deepEqual(packageJson.files, ['dist', 'assets']); + }); +}); + +describe('createPublicNpmPackageTree', () => { + it('copies CLI files, assets, and bundled internal workspace packages', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-public-npm-test-')); + try { + await writeWorkspaceFixture(root); + const layout = publicNpmPackageLayout(root); + + const result = await createPublicNpmPackageTree(layout); + + assert.equal(result.packageJson.name, '@kaelio/ktx'); + assert.equal(result.packageJson.dependencies.commander, '14.0.3'); + assert.equal(result.packageJson.dependencies.yaml, '^2.8.2'); + assert.equal(result.packageJson.dependencies.ai, '^6.0.168'); + assert.equal( + await readFile(join(layout.packRoot, 'assets', 'python', 'manifest.json'), 'utf8'), + '{"schemaVersion":1}\n', + ); + assert.equal( + await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'dist', 'index.js'), 'utf8'), + 'export const context = true;\n', + ); + assert.equal( + await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'prompts', 'system.md'), 'utf8'), + 'prompt\n', + ); + + const bundledContextJson = JSON.parse( + await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'package.json'), 'utf8'), + ); + assert.equal(bundledContextJson.private, true); + assert.deepEqual(bundledContextJson.dependencies, { yaml: '^2.8.2' }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); + +describe('publicNpmPackCommand', () => { + it('packs the assembled public package with pnpm', () => { + const layout = publicNpmPackageLayout('/repo/ktx'); + + assert.deepEqual(publicNpmPackCommand(layout), { + command: 'pnpm', + args: ['pack', '--out', '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.0.0-private.tgz'], + cwd: '/repo/ktx/dist/public-npm-package', + }); + }); +}); +``` + +- [ ] **Step 2: Run the failing builder tests** + +Run: + +```bash +node --test scripts/build-public-npm-package.test.mjs +``` + +Expected: FAIL with an import error for +`./build-public-npm-package.mjs`. + +- [ ] **Step 3: Implement the public package builder** + +Create `scripts/build-public-npm-package.mjs` with this content: + +```javascript +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +export const PUBLIC_NPM_PACKAGE_NAME = '@kaelio/ktx'; +export const PUBLIC_NPM_PACKAGE_VERSION = '0.0.0-private'; +export const PUBLIC_NPM_PACKAGE_TARBALL = 'kaelio-ktx-0.0.0-private.tgz'; + +export const PUBLIC_BUNDLED_WORKSPACE_PACKAGES = [ + '@ktx/llm', + '@ktx/context', + '@ktx/connector-bigquery', + '@ktx/connector-clickhouse', + '@ktx/connector-mysql', + '@ktx/connector-postgres', + '@ktx/connector-posthog', + '@ktx/connector-snowflake', + '@ktx/connector-sqlite', + '@ktx/connector-sqlserver', +]; + +export const PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS = { + '@ktx/llm': 'packages/llm', + '@ktx/context': 'packages/context', + '@ktx/connector-bigquery': 'packages/connector-bigquery', + '@ktx/connector-clickhouse': 'packages/connector-clickhouse', + '@ktx/connector-mysql': 'packages/connector-mysql', + '@ktx/connector-postgres': 'packages/connector-postgres', + '@ktx/connector-posthog': 'packages/connector-posthog', + '@ktx/connector-snowflake': 'packages/connector-snowflake', + '@ktx/connector-sqlite': 'packages/connector-sqlite', + '@ktx/connector-sqlserver': 'packages/connector-sqlserver', +}; + +function scriptRootDir() { + return resolve(dirname(fileURLToPath(import.meta.url)), '..'); +} + +export function publicNpmPackageLayout(rootDir = scriptRootDir()) { + return { + rootDir, + cliPackageRoot: join(rootDir, 'packages', 'cli'), + packRoot: join(rootDir, 'dist', 'public-npm-package'), + npmDir: join(rootDir, 'dist', 'artifacts', 'npm'), + tarballPath: join(rootDir, 'dist', 'artifacts', 'npm', PUBLIC_NPM_PACKAGE_TARBALL), + }; +} + +async function readJson(path) { + return JSON.parse(await readFile(path, 'utf8')); +} + +async function writeJson(path, value) { + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); +} + +function sortedObject(entries) { + return Object.fromEntries([...entries].sort(([left], [right]) => left.localeCompare(right))); +} + +function isWorkspacePackageName(name) { + return name.startsWith('@ktx/'); +} + +function parseCaretVersion(value) { + const match = /^\^(\d+)\.(\d+)\.(\d+)$/.exec(value); + if (!match) { + return null; + } + return { + major: Number(match[1]), + minor: Number(match[2]), + patch: Number(match[3]), + }; +} + +function compareParsedVersions(left, right) { + return left.major - right.major || left.minor - right.minor || left.patch - right.patch; +} + +function mergeDependencyVersion(name, previous, next) { + if (previous === next) { + return previous; + } + + const previousCaret = parseCaretVersion(previous); + const nextCaret = parseCaretVersion(next); + if (previousCaret && nextCaret && previousCaret.major === nextCaret.major) { + return compareParsedVersions(previousCaret, nextCaret) >= 0 ? previous : next; + } + + throw new Error(`Incompatible dependency versions for ${name}: ${previous} and ${next}`); +} + +export function collectPublicDependencies(packageJsons) { + const dependencies = new Map(); + + for (const packageJson of packageJsons) { + for (const [name, version] of Object.entries(packageJson.dependencies ?? {})) { + if (isWorkspacePackageName(name)) { + continue; + } + const previous = dependencies.get(name); + dependencies.set(name, previous ? mergeDependencyVersion(name, previous, version) : version); + } + } + + return sortedObject(dependencies); +} + +export function publicNpmPackageJson(cliPackageJson, dependencies) { + return { + name: PUBLIC_NPM_PACKAGE_NAME, + version: cliPackageJson.version ?? PUBLIC_NPM_PACKAGE_VERSION, + description: 'Standalone KTX context layer for database agents', + private: false, + type: 'module', + engines: cliPackageJson.engines ?? { node: '>=22.0.0' }, + bin: { ktx: './dist/bin.js' }, + main: cliPackageJson.main ?? 'dist/index.js', + types: cliPackageJson.types ?? 'dist/index.d.ts', + exports: cliPackageJson.exports ?? { + '.': { + types: './dist/index.d.ts', + import: './dist/index.js', + default: './dist/index.js', + }, + './package.json': './package.json', + }, + files: ['dist', 'assets'], + dependencies, + bundledDependencies: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, + license: cliPackageJson.license ?? 'Apache-2.0', + repository: { + type: 'git', + url: 'git+https://github.com/kaelio/ktx.git', + }, + bugs: { + url: 'https://github.com/kaelio/ktx/issues', + }, + homepage: 'https://github.com/kaelio/ktx#readme', + }; +} + +function bundledWorkspacePackageJson(packageJson) { + const dependencies = Object.fromEntries( + Object.entries(packageJson.dependencies ?? {}).filter(([name]) => !isWorkspacePackageName(name)), + ); + + return { + name: packageJson.name, + version: packageJson.version ?? PUBLIC_NPM_PACKAGE_VERSION, + private: true, + type: packageJson.type ?? 'module', + main: packageJson.main, + types: packageJson.types, + exports: packageJson.exports, + files: packageJson.files, + dependencies: sortedObject(Object.entries(dependencies)), + license: packageJson.license ?? 'Apache-2.0', + }; +} + +async function copyPackageFileEntries(sourceRoot, targetRoot, packageJson) { + for (const entry of packageJson.files ?? ['dist']) { + await cp(join(sourceRoot, entry), join(targetRoot, entry), { + recursive: true, + force: true, + }); + } +} + +async function copyCliPackage(layout, cliPackageJson, dependencies) { + await copyPackageFileEntries(layout.cliPackageRoot, layout.packRoot, cliPackageJson); + await writeJson(join(layout.packRoot, 'package.json'), publicNpmPackageJson(cliPackageJson, dependencies)); +} + +async function copyBundledWorkspacePackage(rootDir, packageName, packageJson) { + const packageRoot = PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS[packageName]; + if (!packageRoot) { + throw new Error(`Missing bundled workspace package root for ${packageName}`); + } + + const sourceRoot = join(rootDir, packageRoot); + const targetRoot = join(rootDir, 'dist', 'public-npm-package', 'node_modules', ...packageName.split('/')); + await mkdir(targetRoot, { recursive: true }); + await copyPackageFileEntries(sourceRoot, targetRoot, packageJson); + await writeJson(join(targetRoot, 'package.json'), bundledWorkspacePackageJson(packageJson)); +} + +export async function createPublicNpmPackageTree(layout = publicNpmPackageLayout()) { + const cliPackageJson = await readJson(join(layout.cliPackageRoot, 'package.json')); + const bundledPackageJsons = await Promise.all( + PUBLIC_BUNDLED_WORKSPACE_PACKAGES.map(async (packageName) => { + const packageRoot = PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS[packageName]; + const packageJson = await readJson(join(layout.rootDir, packageRoot, 'package.json')); + if (packageJson.name !== packageName) { + throw new Error(`Unexpected package name in ${packageRoot}/package.json: ${packageJson.name}`); + } + return packageJson; + }), + ); + const dependencies = collectPublicDependencies([cliPackageJson, ...bundledPackageJsons]); + + await rm(layout.packRoot, { recursive: true, force: true }); + await mkdir(layout.packRoot, { recursive: true }); + await mkdir(layout.npmDir, { recursive: true }); + await copyCliPackage(layout, cliPackageJson, dependencies); + + for (const packageJson of bundledPackageJsons) { + await copyBundledWorkspacePackage(layout.rootDir, packageJson.name, packageJson); + } + + return { + layout, + packageJson: publicNpmPackageJson(cliPackageJson, dependencies), + bundledPackages: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, + }; +} + +export function publicNpmPackCommand(layout = publicNpmPackageLayout()) { + return { + command: 'pnpm', + args: ['pack', '--out', layout.tarballPath], + cwd: layout.packRoot, + }; +} + +export async function buildPublicNpmPackage(layout = publicNpmPackageLayout()) { + await createPublicNpmPackageTree(layout); + const pack = publicNpmPackCommand(layout); + await execFileAsync(pack.command, pack.args, { + cwd: pack.cwd, + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024, + }); + return layout.tarballPath; +} + +async function main() { + const tarball = await buildPublicNpmPackage(); + process.stdout.write(`Built ${PUBLIC_NPM_PACKAGE_NAME} package: ${tarball}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + try { + await main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} +``` + +- [ ] **Step 4: Verify builder tests pass** + +Run: + +```bash +node --test scripts/build-public-npm-package.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add scripts/build-public-npm-package.mjs scripts/build-public-npm-package.test.mjs +git commit -m "feat: assemble public kaelio ktx npm package" +``` + +### Task 3: Make release artifacts use only `@kaelio/ktx` as the npm artifact + +**Files:** + +- Modify: `scripts/package-artifacts.mjs` +- Modify: `scripts/package-artifacts.test.mjs` +- Modify: `scripts/release-readiness.mjs` +- Modify: `scripts/release-readiness.test.mjs` +- Modify: `release-policy.json` + +- [ ] **Step 1: Add failing artifact expectations for the public package** + +In `scripts/package-artifacts.test.mjs`, update the import from +`./package-artifacts.mjs` so it also imports +`INTERNAL_NPM_WORKSPACE_PACKAGES`: + +```javascript +import { + CLI_PYTHON_ASSET_MANIFEST, + INTERNAL_NPM_WORKSPACE_PACKAGES, + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_NORMALIZED_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, + artifactManifestPath, + buildArtifactCommands, + copyRuntimeWheelAssets, + findPythonArtifacts, + NPM_ARTIFACT_PACKAGES, + npmDemoSmokeSource, + npmRuntimeSmokeSource, + npmSmokePackageJson, + npmSmokePythonEnv, + npmVerifySource, + packageArtifactLayout, + packageReleaseMetadata, + pythonArtifactInstallArgs, + pythonVerifySource, + verifyArtifactManifest, + writeArtifactManifest, +} from './package-artifacts.mjs'; +``` + +Replace the top-level `NPM_BUILD_PACKAGE_ORDER` declaration with: + +```javascript +const INTERNAL_BUILD_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => packageInfo.name); +const CONNECTOR_PACKAGE_NAMES = INTERNAL_BUILD_PACKAGE_NAMES.filter((packageName) => + packageName.startsWith('@ktx/connector-'), +); +const NPM_BUILD_PACKAGE_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli']; +``` + +Replace `expectedNpmArtifactPath` with: + +```javascript +function expectedNpmArtifactPath(packageName) { + if (packageName === '@kaelio/ktx') { + return 'npm/kaelio-ktx-0.0.0-private.tgz'; + } + return `npm/${packageName.replace('@ktx/', 'ktx-')}-0.0.0-private.tgz`; +} +``` + +Replace `writeReleaseMetadataInputs` with: + +```javascript +async function writeReleaseMetadataInputs(root) { + for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { + await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); + await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { + name: packageInfo.name, + version: '0.0.0-private', + private: true, + }); + } + + await mkdir(join(root, 'python', 'ktx-sl'), { recursive: true }); + await mkdir(join(root, 'python', 'ktx-daemon'), { recursive: true }); + await writeFile( + join(root, 'python', 'ktx-sl', 'pyproject.toml'), + ['[project]', 'name = "ktx-sl"', 'version = "0.1.0"', ''].join('\n'), + ); + await writeFile( + join(root, 'python', 'ktx-daemon', 'pyproject.toml'), + ['[project]', 'name = "ktx-daemon"', 'version = "0.1.0"', ''].join('\n'), + ); +} +``` + +Update the `packageArtifactLayout` test so the npm assertions are: + +```javascript + assert.equal(layout.cliTarball, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.0.0-private.tgz'); + assert.deepEqual(Object.keys(layout.npmTarballs), ['@kaelio/ktx']); +``` + +Update the `buildArtifactCommands` test so it expects one public package build +command instead of per-package `pnpm pack` commands: + +```javascript + assert.deepEqual( + commands.slice(0, NPM_BUILD_PACKAGE_ORDER.length).map((command) => [command.command, command.args]), + NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]), + ); + assert.deepEqual( + commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 3).map((command) => [ + command.command, + command.args, + ]), + [ + [process.execPath, ['scripts/build-python-runtime-wheel.mjs']], + ['uv', ['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python']], + ['uv', ['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python']], + ], + ); + assert.deepEqual(commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 3).map((command) => [command.command, command.args]), [ + [process.execPath, ['scripts/build-public-npm-package.mjs']], + ]); +``` + +In the `packageReleaseMetadata` test, replace the expected npm metadata entries +with: + +```javascript + { + ecosystem: 'npm', + packageName: '@kaelio/ktx', + packageRoot: 'packages/cli', + packageVersion: '0.0.0-private', + private: false, + releaseMode: 'ci-artifact-only', + }, +``` + +In the artifact manifest test, replace the npm package expectations with: + +```javascript + [ + { + ecosystem: 'npm', + packageName: '@kaelio/ktx', + packageRoot: 'packages/cli', + packageVersion: '0.0.0-private', + private: false, + releaseMode: 'ci-artifact-only', + }, + ], +``` + +Also replace the expected npm file entries with: + +```javascript + [ + { + artifactKind: 'tarball', + ecosystem: 'npm', + packageName: '@kaelio/ktx', + packageVersion: '0.0.0-private', + path: 'npm/kaelio-ktx-0.0.0-private.tgz', + }, + ], +``` + +- [ ] **Step 2: Run failing artifact tests** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs +``` + +Expected: FAIL because `INTERNAL_NPM_WORKSPACE_PACKAGES` is missing and the +artifact layout still points at `@ktx/cli`. + +- [ ] **Step 3: Wire `scripts/package-artifacts.mjs` to the public builder** + +In `scripts/package-artifacts.mjs`, add this import after the runtime wheel +import: + +```javascript +import { + PUBLIC_NPM_PACKAGE_NAME, + PUBLIC_NPM_PACKAGE_TARBALL, +} from './build-public-npm-package.mjs'; +``` + +Replace the `NPM_ARTIFACT_PACKAGES` declaration with: + +```javascript +export const INTERNAL_NPM_WORKSPACE_PACKAGES = [ + { name: '@ktx/context', packageRoot: 'packages/context' }, + { name: '@ktx/llm', packageRoot: 'packages/llm' }, + { name: '@ktx/connector-bigquery', packageRoot: 'packages/connector-bigquery' }, + { name: '@ktx/connector-clickhouse', packageRoot: 'packages/connector-clickhouse' }, + { name: '@ktx/connector-mysql', packageRoot: 'packages/connector-mysql' }, + { name: '@ktx/connector-postgres', packageRoot: 'packages/connector-postgres' }, + { name: '@ktx/connector-posthog', packageRoot: 'packages/connector-posthog' }, + { name: '@ktx/connector-snowflake', packageRoot: 'packages/connector-snowflake' }, + { name: '@ktx/connector-sqlite', packageRoot: 'packages/connector-sqlite' }, + { name: '@ktx/connector-sqlserver', packageRoot: 'packages/connector-sqlserver' }, + { name: '@ktx/cli', packageRoot: 'packages/cli' }, +]; + +export const NPM_ARTIFACT_PACKAGES = [{ name: PUBLIC_NPM_PACKAGE_NAME, packageRoot: 'packages/cli' }]; +``` + +Replace the `CONNECTOR_PACKAGE_NAMES` calculation with: + +```javascript +const CONNECTOR_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES + .map((packageInfo) => packageInfo.name) + .filter((packageName) => packageName.startsWith('@ktx/connector-')); +``` + +Replace `npmPackageTarballName` with: + +```javascript +function npmPackageTarballName(packageName) { + if (packageName === PUBLIC_NPM_PACKAGE_NAME) { + return PUBLIC_NPM_PACKAGE_TARBALL; + } + return `${packageName.replace('@ktx/', 'ktx-')}-${PACKAGE_VERSION}.tgz`; +} +``` + +In `packageArtifactLayout`, keep `contextTarball` for compatibility but make +`cliTarball` point at the public package: + +```javascript + contextTarball: npmTarballs[PUBLIC_NPM_PACKAGE_NAME], + cliTarball: npmTarballs[PUBLIC_NPM_PACKAGE_NAME], +``` + +In `buildArtifactCommands`, replace `packagesByName` and `npmBuildCommands` +with: + +```javascript + const packagesByName = new Map(INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => [packageInfo.name, packageInfo])); + const npmBuildCommands = NPM_ARTIFACT_BUILD_ORDER.map((packageName) => { +``` + +Replace `npmPackCommands` and the final returned pack commands with the public +builder command: + +```javascript + const publicPackageCommand = { + command: process.execPath, + args: ['scripts/build-public-npm-package.mjs'], + cwd: layout.rootDir, + }; +``` + +Return: + +```javascript + return [ + ...npmBuildCommands, + { + command: process.execPath, + args: ['scripts/build-python-runtime-wheel.mjs'], + cwd: layout.rootDir, + }, + { + command: 'uv', + args: ['build', '--package', 'ktx-sl', '--out-dir', layout.pythonDir], + cwd: layout.rootDir, + }, + { + command: 'uv', + args: ['build', '--package', 'ktx-daemon', '--out-dir', layout.pythonDir], + cwd: layout.rootDir, + }, + publicPackageCommand, + ]; +``` + +Replace `readNpmPackageMetadata` with: + +```javascript +async function readNpmPackageMetadata(rootDir, packageInfo) { + const packageJson = await readJson(join(rootDir, packageInfo.packageRoot, 'package.json')); + const expectedSourceName = packageInfo.name === PUBLIC_NPM_PACKAGE_NAME ? '@ktx/cli' : packageInfo.name; + if (packageJson.name !== expectedSourceName) { + throw new Error( + `Unexpected package name in ${packageInfo.packageRoot}/package.json: expected ${expectedSourceName}, got ${packageJson.name}`, + ); + } + return releaseMetadataEntry({ + ecosystem: 'npm', + packageName: packageInfo.name, + packageRoot: packageInfo.packageRoot, + packageVersion: packageJson.version, + privatePackage: packageInfo.name === PUBLIC_NPM_PACKAGE_NAME ? false : packageJson.private === true, + }); +} +``` + +In `buildArtifacts`, replace the command-slicing counters with: + +```javascript + const npmBuildCount = NPM_ARTIFACT_BUILD_ORDER.length; + const npmPackStart = commands.length - 1; +``` + +Keep the existing three loops after those counters. This makes the first loop +build all internal workspace packages, the second loop build and copy Python +runtime artifacts, and the final loop run only +`scripts/build-public-npm-package.mjs`. + +- [ ] **Step 4: Update release policy for one npm package** + +Replace the `release-policy.json` `npm.packages` value with: + +```json +["@kaelio/ktx"] +``` + +Replace `publishedPackageSmoke.packageName` with: + +```json +"@kaelio/ktx" +``` + +Replace `requiredBeforePublishing` with: + +```json +[ + "Choose public release version.", + "Configure registry credentials outside source control.", + "Choose release tag and provenance policy." +] +``` + +- [ ] **Step 5: Allow one public npm artifact while publishing remains disabled** + +In `scripts/release-readiness.mjs`, replace the npm portion of +`assertNonPublishingArtifactPolicy` with: + +```javascript + if (entry.ecosystem === 'npm') { + const isPublicKtxPackage = entry.packageName === '@kaelio/ktx'; + if (isPublicKtxPackage) { + if (entry.private !== false) { + throw new Error(`${policyLabel} npm package @kaelio/ktx must be publishable when npm.publish is false`); + } + } else if (entry.private !== true) { + throw new Error(`${policyLabel} npm package ${entry.packageName} must remain private`); + } + if (!entry.packageVersion.endsWith('-private')) { + throw new Error(`${policyLabel} npm package ${entry.packageName} must use a private version suffix`); + } + } +``` + +In `scripts/release-readiness.test.mjs`, update the import from +`./package-artifacts.mjs` so it includes `INTERNAL_NPM_WORKSPACE_PACKAGES`: + +```javascript +import { + INTERNAL_NPM_WORKSPACE_PACKAGES, + NPM_ARTIFACT_PACKAGES, + packageArtifactLayout, + writeArtifactManifest, +} from './package-artifacts.mjs'; +``` + +Replace `writeReleaseMetadataInputs` with: + +```javascript +async function writeReleaseMetadataInputs(root) { + for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { + await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); + await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { + name: packageInfo.name, + version: '0.0.0-private', + private: true, + }); + } + + await mkdir(join(root, 'python', 'ktx-sl'), { recursive: true }); + await mkdir(join(root, 'python', 'ktx-daemon'), { recursive: true }); + + await writeFile( + join(root, 'python', 'ktx-sl', 'pyproject.toml'), + ['[project]', 'name = "ktx-sl"', 'version = "0.1.0"', ''].join('\n'), + ); + await writeFile( + join(root, 'python', 'ktx-daemon', 'pyproject.toml'), + ['[project]', 'name = "ktx-daemon"', 'version = "0.1.0"', ''].join('\n'), + ); +} +``` + +Update `releasePolicy()` so `npm.packages` defaults to: + +```javascript +packages: ['@kaelio/ktx'], +``` + +Update expected `packageNames` arrays so the npm section is only: + +```javascript +'@kaelio/ktx', +``` + +Update published smoke fixture package names from `@ktx/cli-public` to +`@kaelio/ktx`. + +Replace the stale public-npm rejection test with this policy mismatch test: + +```javascript + it('rejects release policy that still lists internal npm packages', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-release-stale-internal-npm-policy-test-')); + try { + await writeReadyFixture(root, { + policy: releasePolicy({ + npm: { + packages: ['@kaelio/ktx', '@ktx/context'], + }, + }), + }); + + await assert.rejects( + () => releaseReadinessReport(root), + /Release policy npm\.packages mismatch/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +``` + +- [ ] **Step 6: Verify artifact and release readiness tests** + +Run: + +```bash +node --test scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +Run: + +```bash +git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.mjs scripts/release-readiness.test.mjs release-policy.json +git commit -m "feat: release one public kaelio ktx npm artifact" +``` + +### Task 4: Add public package invocation smoke coverage + +**Files:** + +- Modify: `scripts/published-package-smoke-config.mjs` +- Modify: `scripts/published-package-smoke.mjs` +- Modify: `scripts/published-package-smoke.test.mjs` +- Modify: `scripts/package-artifacts.mjs` +- Modify: `scripts/package-artifacts.test.mjs` + +- [ ] **Step 1: Add failing published smoke command expectations** + +In `scripts/published-package-smoke.test.mjs`, change all fixture package names +from `@ktx/cli-public`, `@ktx/cli-from-env`, and `@ktx/cli-from-policy` to +`@kaelio/ktx`. + +In the `builds the full hybrid-search smoke command list` test, replace the +expected command list with: + +```javascript + assert.deepEqual(buildPublishedPackageSmokeCommands(config, '/tmp/ktx-smoke/demo', '/tmp/ktx-smoke/empty'), [ + { + label: 'published package version', + command: 'npx', + args: ['--yes', '@kaelio/ktx@latest', '--version'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package setup demo', + command: 'npx', + args: [ + '--yes', + '@kaelio/ktx@latest', + 'setup', + 'demo', + '--project-dir', + '/tmp/ktx-smoke/demo', + '--no-input', + '--plain', + ], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package sl query', + command: 'npx', + args: [ + '--yes', + '@kaelio/ktx@latest', + 'sl', + 'query', + '--project-dir', + '/tmp/ktx-smoke/demo', + '--connection-id', + 'orbit_demo', + '--measure', + 'contracts.contract_count', + '--format', + 'sql', + '--yes', + ], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package local install', + command: 'pnpm', + args: ['add', '@kaelio/ktx@latest'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package local binary', + command: 'pnpm', + args: ['exec', 'ktx', '--version'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package global install', + command: 'pnpm', + args: ['add', '--global', '@kaelio/ktx@latest'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package global binary', + command: 'ktx', + args: ['--version'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + ]); +``` + +- [ ] **Step 2: Run failing published smoke tests** + +Run: + +```bash +node --test scripts/published-package-smoke.test.mjs +``` + +Expected: FAIL because the command list still contains the old hybrid-search +commands and package names. + +- [ ] **Step 3: Update published package smoke commands** + +In `scripts/published-package-smoke-config.mjs`, replace +`buildPublishedPackageSmokeCommands` with: + +```javascript +export function buildPublishedPackageSmokeCommands(config, projectDir) { + return [ + buildPublishedPackageNpxCommand(config, ['--version'], 'published package version'), + buildPublishedPackageNpxCommand( + config, + ['setup', 'demo', '--project-dir', projectDir, '--no-input', '--plain'], + 'published package setup demo', + ), + buildPublishedPackageNpxCommand( + config, + [ + 'sl', + 'query', + '--project-dir', + projectDir, + '--connection-id', + 'orbit_demo', + '--measure', + 'contracts.contract_count', + '--format', + 'sql', + '--yes', + ], + 'published package sl query', + ), + { + label: 'published package local install', + command: 'pnpm', + args: ['add', publishedPackageSpec(config)], + env: config.registry ? { npm_config_registry: config.registry } : {}, + }, + { + label: 'published package local binary', + command: 'pnpm', + args: ['exec', 'ktx', '--version'], + env: config.registry ? { npm_config_registry: config.registry } : {}, + }, + { + label: 'published package global install', + command: 'pnpm', + args: ['add', '--global', publishedPackageSpec(config)], + env: config.registry ? { npm_config_registry: config.registry } : {}, + }, + { + label: 'published package global binary', + command: 'ktx', + args: ['--version'], + env: config.registry ? { npm_config_registry: config.registry } : {}, + }, + ]; +} +``` + +In `scripts/published-package-smoke.mjs`, replace the command execution loop in +`runPublishedPackageSmoke` with: + +```javascript + const commands = buildPublishedPackageSmokeCommands(config, projectDir, emptyProjectDir); + const pnpmHome = join(root, 'pnpm-home'); + const globalEnv = { + PNPM_HOME: pnpmHome, + PATH: `${pnpmHome}${process.platform === 'win32' ? ';' : ':'}${process.env.PATH ?? ''}`, + }; + for (const command of commands) { + const isGlobalCommand = command.label.includes('global'); + const result = await runCommand(command.command, command.args, { + cwd: command.label.includes('local') || isGlobalCommand ? root : undefined, + env: isGlobalCommand ? { ...globalEnv, ...command.env } : command.env, + }); + requireSuccess(command.label, result); + if ( + command.label === 'published package version' || + command.label === 'published package local binary' || + command.label === 'published package global binary' + ) { + assert.match(result.stdout, /@kaelio\/ktx /); + } + if (command.label === 'published package sl query') { + assert.match(result.stdout, /SELECT/i); + assert.match(result.stdout, /contracts/i); + } + } + + process.stdout.write('published package invocation smoke verified\n'); +``` + +Remove `assertHybridWikiSearch`, `assertHybridSlSearch`, and +`assertMissingProjectReadiness` if they are no longer used. + +- [ ] **Step 4: Add local tarball public package smoke to artifact verification** + +In `scripts/package-artifacts.mjs`, replace `npmSmokePackageJson(layout)` with: + +```javascript +export function npmSmokePackageJson(layout) { + return { + name: 'ktx-artifact-npm-smoke', + version: '0.0.0', + private: true, + type: 'module', + dependencies: { + '@kaelio/ktx': `file:${layout.cliTarball}`, + }, + pnpm: { + onlyBuiltDependencies: ['better-sqlite3'], + }, + }; +} +``` + +Replace the top of `npmVerifySource()` with this smaller public-package check: + +```javascript +export function npmVerifySource() { + return ` +const cli = await import('@kaelio/ktx'); + +if (cli.getKtxCliPackageInfo().name !== '@kaelio/ktx') { + throw new Error('Unexpected @kaelio/ktx package info'); +} +if (typeof cli.runKtxCli !== 'function') { + throw new Error('Missing runKtxCli export'); +} +`; +} +``` + +In `npmRuntimeSmokeSource()`, add this assertion after `const root = ...`: + +```javascript +const version = await run('pnpm', ['exec', 'ktx', '--version']); +requireSuccess('ktx public package version', version); +requireOutput('ktx public package version', version, /@kaelio\\/ktx 0\\.0\\.0-private/); +``` + +In `npmRuntimeSmokeSource()`, remove these direct imports because the smoke +project no longer installs internal workspace packages directly: + +```javascript +import { spawn, execFile } from 'node:child_process'; +import { once } from 'node:events'; +import { request as httpRequest } from 'node:http'; +import { createServer } from 'node:net'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import { + createDaemonLookerTableIdentifierParser, + LocalLookerRuntimeStore, +} from '@ktx/context/ingest'; +``` + +Replace them with: + +```javascript +import { execFile } from 'node:child_process'; +``` + +Still inside `npmRuntimeSmokeSource()`, delete these helper functions because +the public tarball smoke must exercise the CLI-managed runtime instead of +manually wiring an internal daemon: + +```javascript +function requireToolNames(tools, expectedNames) { + const names = tools.tools.map((tool) => tool.name).sort(); + for (const expectedName of expectedNames) { + assert.ok(names.includes(expectedName), 'MCP tool list did not include ' + expectedName + ': ' + names.join(', ')); + } +} + +function structuredContent(result) { + assert.ok(result.structuredContent, 'MCP result did not include structuredContent'); + return result.structuredContent; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function getAvailablePort() { + const server = createServer(); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') { + server.close(); + throw new Error('expected TCP server address for daemon smoke'); + } + const port = address.port; + server.close(); + await once(server, 'close'); + return port; +} + +function httpGetOk(url) { + return new Promise((resolve, reject) => { + const request = httpRequest(url, { method: 'GET' }, (response) => { + response.resume(); + response.on('end', () => resolve((response.statusCode ?? 0) >= 200 && (response.statusCode ?? 0) < 300)); + }); + request.on('error', reject); + request.end(); + }); +} + +function spawnLogged(command, args, options = {}) { + const stdout = []; + const stderr = []; + let spawnError; + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env ?? process.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stdout.on('data', (chunk) => stdout.push(chunk)); + child.stderr.on('data', (chunk) => stderr.push(chunk)); + child.on('error', (error) => { + spawnError = error; + }); + return { + child, + error() { + return spawnError; + }, + output() { + return { + stdout: Buffer.concat(stdout).toString('utf8'), + stderr: Buffer.concat(stderr).toString('utf8'), + }; + }, + }; +} + +async function waitForHttpHealth(url, daemon) { + const deadline = Date.now() + 15_000; + while (Date.now() < deadline) { + if (daemon.error()) { + const output = daemon.output(); + throw new Error( + 'Failed to start ktx-daemon serve-http: ' + + daemon.error().message + + '\nstdout:\n' + + output.stdout + + '\nstderr:\n' + + output.stderr, + ); + } + if (daemon.child.exitCode !== null || daemon.child.signalCode !== null) { + const output = daemon.output(); + throw new Error( + 'ktx-daemon serve-http exited before health check passed\nstdout:\n' + + output.stdout + + '\nstderr:\n' + + output.stderr, + ); + } + try { + if (await httpGetOk(url)) { + return; + } + } catch { + await sleep(100); + continue; + } + await sleep(100); + } + const output = daemon.output(); + throw new Error('Timed out waiting for ' + url + '\nstdout:\n' + output.stdout + '\nstderr:\n' + output.stderr); +} + +async function startSemanticDaemon(port) { + const daemon = spawnLogged('ktx-daemon', [ + 'serve-http', + '--host', + '127.0.0.1', + '--port', + String(port), + '--log-level', + 'warning', + ]); + await waitForHttpHealth('http://127.0.0.1:' + port + '/health', daemon); + return daemon; +} + +async function stopSemanticDaemon(daemon) { + if (daemon.child.exitCode !== null || daemon.child.signalCode !== null) { + return; + } + daemon.child.kill('SIGTERM'); + const closed = once(daemon.child, 'close').then(() => true); + const timedOut = sleep(5_000).then(() => false); + if (!(await Promise.race([closed, timedOut]))) { + daemon.child.kill('SIGKILL'); + await once(daemon.child, 'close'); + } +} +``` + +Replace both `ktx agent sl query` smoke commands with top-level `ktx sl query` +commands so the installed public tarball verifies the managed Python runtime +path: + +```javascript + const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', + '--connection-id', + 'warehouse', + '--measure', + 'orders.order_count', + '--format', + 'json', + '--yes', + '--project-dir', + projectDir, + ]); + requireSuccess('ktx sl query', slQuery); + requireOutput('ktx sl query', slQuery, /"mode": "compile_only"/); + requireOutput('ktx sl query', slQuery, /orders/); + + const sqliteSlQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', + '--connection-id', + 'warehouse', + '--measure', + 'orders.order_count', + '--format', + 'json', + '--execute', + '--max-rows', + '100', + '--yes', + '--project-dir', + projectDir, + ]); + requireSuccess('ktx sl query sqlite execute', sqliteSlQuery); + requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"dialect": "sqlite"/); + requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"mode": "executed"/); + requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"driver": "sqlite"/); + requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"rows": \\[\\s*\\[\\s*3\\s*\\]\\s*\\]/); + process.stdout.write('ktx sl query sqlite execute verified\n'); +``` + +In `npmRuntimeSmokeSource()`, delete the MCP smoke block that starts with: + +```javascript + const daemonPort = await getAvailablePort(); +``` + +and ends after this cleanup block: + +```javascript + } finally { + await client.close(); + await stopSemanticDaemon(daemon); + } +``` + +In `npmDemoSmokeSource()`, keep the existing `pnpm exec ktx` flow. Add an +assertion that the public package is the only direct dependency: + +```javascript + assert.deepEqual(Object.keys(packageJson.dependencies), ['@kaelio/ktx']); +``` + +- [ ] **Step 5: Update artifact smoke tests** + +In `scripts/package-artifacts.test.mjs`, update the +`npmSmokePackageJson` expectations so they assert: + +```javascript + assert.deepEqual(npmSmokePackageJson(layout).dependencies, { + '@kaelio/ktx': `file:${layout.cliTarball}`, + }); +``` + +Replace installed export assertions that import `@ktx/context`, `@ktx/llm`, or +connector packages with these assertions: + +```javascript + assert.match(verifySource, /const cli = await import\('@kaelio\/ktx'\);/); + assert.match(verifySource, /getKtxCliPackageInfo/); + assert.match(verifySource, /runKtxCli/); + assert.doesNotMatch(verifySource, /@ktx\/context/); + assert.doesNotMatch(verifySource, /@ktx\/llm/); + assert.doesNotMatch(verifySource, /@ktx\/connector-/); +``` + +Add runtime smoke assertions: + +```javascript + assert.match(runtimeSource, /ktx public package version/); + assert.match(runtimeSource, /@kaelio\\\\\/ktx 0\\\\.0\\\\.0-private/); + assert.match(runtimeSource, /'ktx', 'sl', 'query'/); + assert.doesNotMatch(runtimeSource, /@ktx\/context/); + assert.doesNotMatch(runtimeSource, /@modelcontextprotocol/); + assert.doesNotMatch(runtimeSource, /startSemanticDaemon/); +``` + +- [ ] **Step 6: Verify smoke command tests** + +Run: + +```bash +node --test scripts/published-package-smoke.test.mjs scripts/package-artifacts.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +Run: + +```bash +git add scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs +git commit -m "test: cover public kaelio ktx package invocations" +``` + +### Task 5: Run focused verification and artifact smoke + +**Files:** + +- Verify: `scripts/build-public-npm-package.mjs` +- Verify: `scripts/package-artifacts.mjs` +- Verify: `scripts/published-package-smoke.mjs` +- Verify: `scripts/release-readiness.mjs` +- Verify: `packages/cli/src/cli-runtime.ts` + +- [ ] **Step 1: Run script unit tests** + +Run: + +```bash +node --test scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 2: Run CLI package tests touched by metadata changes** + +Run: + +```bash +pnpm --filter @ktx/cli run test -- src/index.test.ts +``` + +Expected: PASS. + +- [ ] **Step 3: Build artifacts from source** + +Run: + +```bash +pnpm run artifacts:build +``` + +Expected: PASS and create: + +```text +dist/artifacts/npm/kaelio-ktx-0.0.0-private.tgz +dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl +dist/artifacts/manifest.json +``` + +- [ ] **Step 4: Verify artifact manifest** + +Run: + +```bash +pnpm run artifacts:verify-manifest +``` + +Expected: PASS. + +- [ ] **Step 5: Verify installed public tarball smoke** + +Run: + +```bash +pnpm run artifacts:verify +``` + +Expected: PASS. The installed npm smoke must install only +`@kaelio/ktx` directly and must not require direct `@ktx/*` dependencies in the +smoke project. + +- [ ] **Step 6: Run release readiness** + +Run: + +```bash +pnpm run release:readiness +``` + +Expected: PASS. The report must list `@kaelio/ktx` as the only npm package and +must still state that registry publishing remains disabled by +`release-policy.json`. + +- [ ] **Step 7: Run pre-commit for changed files** + +Run: + +```bash +source .venv/bin/activate && uv run pre-commit run --files packages/cli/src/cli-runtime.ts packages/cli/src/index.ts packages/cli/src/index.test.ts scripts/build-public-npm-package.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.mjs scripts/release-readiness.test.mjs release-policy.json +``` + +Expected: PASS. If pre-commit is unavailable because hook tooling is missing, +run these fallback checks: + +```bash +node --test scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs +pnpm --filter @ktx/cli run type-check +pnpm --filter @ktx/cli run test -- src/index.test.ts +``` + +- [ ] **Step 8: Commit verification fixes** + +If verification required fixes, commit only the changed files from this plan: + +```bash +git status --short +git add packages/cli/src/cli-runtime.ts packages/cli/src/index.ts packages/cli/src/index.test.ts scripts/build-public-npm-package.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.mjs scripts/release-readiness.test.mjs release-policy.json +git commit -m "chore: verify public kaelio ktx package artifacts" +``` + +## Self-review notes + +- Spec coverage: this plan implements the `@kaelio/ktx` npm package name, one + visible `ktx` binary, bundled JavaScript CLI output, packaged demo assets, + bundled Python runtime wheel assets, and smoke coverage for the required + public invocation modes. +- Remaining after this plan: managed runtime use in deeper Python-backed + paths, such as MCP `serve` defaults and Looker table identifier parsing, + still needs a separate plan if those paths must stop accepting externally + supplied daemon URLs. +- Placeholder scan: this plan uses exact paths, exact commands, concrete code + blocks, and no deferred implementation markers. +- Type consistency: public npm package names are consistently `@kaelio/ktx`; + internal workspace package names remain `@ktx/*`. diff --git a/docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md b/docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md new file mode 100644 index 00000000..11100184 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md @@ -0,0 +1,1332 @@ +# Public NPM Release Handoff Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Turn the remaining npm-managed Python runtime release gap into a +guarded public `@kaelio/ktx` npm release handoff for version `0.1.0`. + +**Architecture:** Keep one public npm package and keep Python packages +unpublished. The public package builder stamps the assembled `@kaelio/ktx` +package as `0.1.0`, release readiness accepts a publish-ready policy only when +all blocking decisions are encoded, and a new publish script performs a dry-run +by default before any live registry publish. + +**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm 10 publish, JSON release +policy, GitHub Actions workflow validation. + +--- + +## Spec trace and current state + +This plan follows +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +The existing plan files that reference that spec are: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` +- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` +- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md` + +All twelve are implemented in the current tree: their referenced source and +test files exist, and the runtime command, daemon, package artifact, +published-package smoke, local-embedding smoke, and README markers are present. + +The remaining release gap is explicit in `release-policy.json`: the repository +still uses `ci-artifact-only`, `npm.publish` is `false`, and the README states +that registry publishing is disabled. This plan changes that to a guarded +handoff for the first public npm release while leaving Python registry +publication disabled because the spec says KTX-owned Python code ships inside +the npm package as a bundled wheel for this release. + +## File structure + +- Modify `scripts/build-public-npm-package.mjs`: make the assembled public npm + package version and tarball name `0.1.0` instead of `0.0.0-private`. +- Modify `scripts/build-public-npm-package.test.mjs`: cover public version + stamping and the versioned tarball path. +- Modify `scripts/package-artifacts.mjs`: make artifact metadata report + `@kaelio/ktx` as version `0.1.0`. +- Modify `scripts/package-artifacts.test.mjs`: update artifact manifest, + metadata, runtime smoke, and demo smoke expectations for the public tarball. +- Modify `scripts/local-embeddings-runtime-smoke.test.mjs`: update public + tarball selection coverage for `kaelio-ktx-0.1.0.tgz`. +- Modify `scripts/release-readiness.mjs`: add the + `npm-public-release-ready` release mode and policy validation. +- Modify `scripts/release-readiness.test.mjs`: cover the publish-ready policy + and validation failures. +- Modify `release-policy.json`: encode the first public npm release handoff. +- Create `scripts/publish-public-npm-package.mjs`: verify readiness and run + `pnpm publish` in dry-run mode by default. +- Create `scripts/publish-public-npm-package.test.mjs`: cover publish command + construction and policy gating. +- Modify `package.json`: add `release:npm-publish`. +- Create `.github/workflows/release.yml`: add a manual dry-run/live publish + workflow for the public npm tarball. +- Create `scripts/release-workflow.test.mjs`: validate that the release + workflow is manual, uses pnpm, runs readiness checks, and gates live publish. +- Modify `README.md`: replace the disabled publishing note with the guarded + handoff commands. + +### Task 1: Stamp public npm artifacts as `0.1.0` + +**Files:** + +- Modify: `scripts/build-public-npm-package.mjs` +- Modify: `scripts/build-public-npm-package.test.mjs` +- Modify: `scripts/package-artifacts.mjs` +- Modify: `scripts/package-artifacts.test.mjs` +- Modify: `scripts/local-embeddings-runtime-smoke.test.mjs` + +- [ ] **Step 1: Write failing public version tests** + +In `scripts/build-public-npm-package.test.mjs`, extend the import from +`./build-public-npm-package.mjs` so it includes `PUBLIC_NPM_PACKAGE_VERSION` +and `publicNpmPackageTarballName`: + +```js +import { + PUBLIC_BUNDLED_WORKSPACE_PACKAGES, + PUBLIC_NPM_PACKAGE_NAME, + PUBLIC_NPM_PACKAGE_VERSION, + collectPublicDependencies, + createPublicNpmPackageTree, + publicNpmPackageJson, + publicNpmPackageLayout, + publicNpmPackageTarballName, + publicNpmPackCommand, +} from './build-public-npm-package.mjs'; +``` + +Replace the `publicNpmPackageLayout` test expectation with: + +```js +describe('publicNpmPackageLayout', () => { + it('uses the first public npm release version for the tarball name', () => { + const layout = publicNpmPackageLayout('/repo/ktx'); + + assert.equal(PUBLIC_NPM_PACKAGE_VERSION, '0.1.0'); + assert.equal(publicNpmPackageTarballName(), 'kaelio-ktx-0.1.0.tgz'); + assert.equal(layout.tarballPath, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz'); + }); +}); +``` + +In the `publicNpmPackageJson` test, add this assertion after the package name +assertion: + +```js +assert.equal(packageJson.version, '0.1.0'); +``` + +In the `publicNpmPackCommand` test, replace the tarball assertion block with: + +```js +assert.deepEqual(publicNpmPackCommand(layout), { + command: 'pnpm', + args: [ + '--config.node-linker=hoisted', + 'pack', + '--out', + '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', + ], + cwd: '/repo/ktx/dist/public-npm-package', +}); +``` + +- [ ] **Step 2: Run public package tests to verify failure** + +Run: + +```bash +node --test scripts/build-public-npm-package.test.mjs +``` + +Expected: FAIL. The failure mentions at least one stale +`kaelio-ktx-0.0.0-private.tgz` or `0.0.0-private` public package version +expectation. + +- [ ] **Step 3: Implement public version stamping** + +In `scripts/build-public-npm-package.mjs`, replace the current public version +constants and layout helper with: + +```js +export const PUBLIC_NPM_PACKAGE_NAME = '@kaelio/ktx'; +export const PUBLIC_NPM_PACKAGE_VERSION = '0.1.0'; + +export function publicNpmPackageTarballName(version = PUBLIC_NPM_PACKAGE_VERSION) { + return `kaelio-ktx-${version}.tgz`; +} +``` + +Replace `publicNpmPackageLayout` with: + +```js +export function publicNpmPackageLayout(rootDir = scriptRootDir(), version = PUBLIC_NPM_PACKAGE_VERSION) { + return { + rootDir, + packageVersion: version, + cliPackageRoot: join(rootDir, 'packages', 'cli'), + packRoot: join(rootDir, 'dist', 'public-npm-package'), + npmDir: join(rootDir, 'dist', 'artifacts', 'npm'), + tarballPath: join(rootDir, 'dist', 'artifacts', 'npm', publicNpmPackageTarballName(version)), + }; +} +``` + +Change `publicNpmPackageJson` so it accepts the public version explicitly: + +```js +export function publicNpmPackageJson(cliPackageJson, dependencies, version = PUBLIC_NPM_PACKAGE_VERSION) { + return { + name: PUBLIC_NPM_PACKAGE_NAME, + version, + description: 'Standalone KTX context layer for database agents', + private: false, + type: 'module', + engines: cliPackageJson.engines ?? { node: '>=22.0.0' }, + bin: { ktx: './dist/bin.js' }, + main: cliPackageJson.main ?? 'dist/index.js', + types: cliPackageJson.types ?? 'dist/index.d.ts', + exports: cliPackageJson.exports ?? { + '.': { + types: './dist/index.d.ts', + import: './dist/index.js', + default: './dist/index.js', + }, + './package.json': './package.json', + }, + files: ['dist', 'assets'], + dependencies, + bundledDependencies: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, + license: cliPackageJson.license ?? 'Apache-2.0', + repository: { + type: 'git', + url: 'git+https://github.com/kaelio/ktx.git', + }, + bugs: { + url: 'https://github.com/kaelio/ktx/issues', + }, + homepage: 'https://github.com/kaelio/ktx#readme', + }; +} +``` + +In `copyCliPackage`, pass the layout version: + +```js +await writeJson( + join(layout.packRoot, 'package.json'), + publicNpmPackageJson(cliPackageJson, dependencies, layout.packageVersion), +); +``` + +In `createPublicNpmPackageTree`, return the versioned package JSON: + +```js +return { + layout, + packageJson: publicNpmPackageJson(cliPackageJson, dependencies, layout.packageVersion), + bundledPackages: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, +}; +``` + +- [ ] **Step 4: Run public package tests to verify pass** + +Run: + +```bash +node --test scripts/build-public-npm-package.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 5: Write failing artifact metadata tests** + +In `scripts/package-artifacts.test.mjs`, replace expectations that use the +public npm tarball or package version: + +```js +assert.equal(layout.cliTarball, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz'); +``` + +```js +{ + ecosystem: 'npm', + packageName: '@kaelio/ktx', + packageRoot: 'packages/cli', + packageVersion: '0.1.0', + private: false, + releaseMode: 'ci-artifact-only', +} +``` + +```js +{ + ecosystem: 'npm', + packageName: '@kaelio/ktx', + packageVersion: '0.1.0', + path: 'npm/kaelio-ktx-0.1.0.tgz', + bytes: Buffer.byteLength('@kaelio/ktx-tarball'), + sha256: createHash('sha256').update('@kaelio/ktx-tarball').digest('hex'), +} +``` + +In the runtime smoke source expectation, replace: + +```js +requireOutput('ktx public package version', version, /@kaelio\/ktx 0\.1\.0/); +``` + +In `scripts/local-embeddings-runtime-smoke.test.mjs`, replace the public +tarball selection assertion with: + +```js +assert.equal( + publicKtxTarballName(['kaelio-ktx-0.1.0.tgz', 'ignore-me.tgz']), + 'kaelio-ktx-0.1.0.tgz', +); +``` + +- [ ] **Step 6: Run artifact tests to verify failure** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs scripts/local-embeddings-runtime-smoke.test.mjs +``` + +Expected: FAIL. The failure mentions stale artifact metadata or tarball +expectations for `0.0.0-private`. + +- [ ] **Step 7: Implement artifact metadata versioning** + +In `scripts/package-artifacts.mjs`, change the build-public import to: + +```js +import { + PUBLIC_NPM_PACKAGE_NAME, + PUBLIC_NPM_PACKAGE_VERSION, + publicNpmPackageTarballName, +} from './build-public-npm-package.mjs'; +``` + +Replace `npmPackageTarballName` with: + +```js +function npmPackageTarballName(packageName) { + if (packageName === PUBLIC_NPM_PACKAGE_NAME) { + return publicNpmPackageTarballName(PUBLIC_NPM_PACKAGE_VERSION); + } + return `${packageName.replace('@ktx/', 'ktx-')}-${PACKAGE_VERSION}.tgz`; +} +``` + +In `readNpmPackageMetadata`, return the public package version for +`@kaelio/ktx`: + +```js + const isPublicKtxPackage = packageInfo.name === PUBLIC_NPM_PACKAGE_NAME; + return releaseMetadataEntry({ + ecosystem: 'npm', + packageName: packageInfo.name, + packageRoot: packageInfo.packageRoot, + packageVersion: isPublicKtxPackage ? PUBLIC_NPM_PACKAGE_VERSION : packageJson.version, + privatePackage: isPublicKtxPackage ? false : packageJson.private === true, + }); +``` + +In `npmRuntimeSmokeSource`, replace the version output regex with: + +```js +requireOutput('ktx public package version', version, /@kaelio\/ktx 0\.1\.0/); +``` + +- [ ] **Step 8: Run artifact tests to verify pass** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs scripts/local-embeddings-runtime-smoke.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 9: Commit public version stamping** + +Run: + +```bash +git add scripts/build-public-npm-package.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/local-embeddings-runtime-smoke.test.mjs +git commit -m "build: stamp public npm package version" +``` + +Expected: commit created. + +### Task 2: Add publish-ready release policy validation + +**Files:** + +- Modify: `scripts/release-readiness.mjs` +- Modify: `scripts/release-readiness.test.mjs` +- Modify: `release-policy.json` + +- [ ] **Step 1: Write failing release readiness tests** + +In `scripts/release-readiness.test.mjs`, add `PUBLIC_NPM_PACKAGE_VERSION` to +the imports from `./build-public-npm-package.mjs`: + +```js +import { PUBLIC_NPM_PACKAGE_VERSION } from './build-public-npm-package.mjs'; +``` + +Update `releasePolicy()` so the default npm block includes publish settings: + +```js +npm: { + publish: false, + registry: null, + access: 'public', + tag: 'latest', + packages: ['@kaelio/ktx'], + ...npmOverrides, +}, +``` + +In each existing `releaseReadinessReport` expected object for +`ci-artifact-only` and `published-package-smoke-required`, add: + +```js +npmPublish: null, +``` + +Place it after `publishedPackageSmokeGate` and before +`blockedPublishingDecisions`. + +In `writeReleaseMetadataInputs`, keep internal workspace package versions +private. The public package version comes from artifact metadata: + +```js +version: '0.0.0-private', +private: true, +``` + +Add this test after the existing +`reports required published package smoke when release mode requires it` test: + +```js +it('accepts the npm public release ready policy', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-test-')); + try { + await writeReadyFixture(root, { + policy: releasePolicy({ + releaseMode: 'npm-public-release-ready', + npm: { + publish: true, + registry: null, + access: 'public', + tag: 'latest', + }, + publishedPackageSmoke: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + requiredBeforePublishing: [], + }), + }); + + const report = await releaseReadinessReport(root); + + assert.deepEqual(report, { + schemaVersion: 1, + releaseMode: 'npm-public-release-ready', + sourceRevision: 'abc123', + npmPublishEnabled: true, + pythonPublishEnabled: false, + packageNames: ['@kaelio/ktx', 'ktx-sl', 'ktx-daemon', 'kaelio-ktx'], + publishedPackageSmokeGate: { + status: 'required', + script: 'pnpm run release:published-smoke', + reason: 'Run the published package smoke after the npm package is published.', + configSource: 'release-policy', + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + npmPublish: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + access: 'public', + tag: 'latest', + registry: null, + }, + blockedPublishingDecisions: [], + }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); +``` + +Add this validation test: + +```js +it('rejects npm public release ready mode when npm publish is disabled', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-disabled-test-')); + try { + await writeReadyFixture(root, { + policy: releasePolicy({ + releaseMode: 'npm-public-release-ready', + npm: { + publish: false, + registry: null, + access: 'public', + tag: 'latest', + }, + publishedPackageSmoke: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + requiredBeforePublishing: [], + }), + }); + + await assert.rejects( + () => releaseReadinessReport(root), + /npm-public-release-ready policy requires npm.publish true/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); +``` + +Add this validation test: + +```js +it('rejects npm public release ready mode when Python publishing is enabled', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-python-test-')); + try { + await writeReadyFixture(root, { + policy: releasePolicy({ + releaseMode: 'npm-public-release-ready', + npm: { + publish: true, + registry: null, + access: 'public', + tag: 'latest', + }, + python: { + publish: true, + repository: 'pypi', + }, + publishedPackageSmoke: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + requiredBeforePublishing: [], + }), + }); + + await assert.rejects( + () => releaseReadinessReport(root), + /npm-public-release-ready policy keeps python.publish false/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); +``` + +- [ ] **Step 2: Run release readiness tests to verify failure** + +Run: + +```bash +node --test scripts/release-readiness.test.mjs +``` + +Expected: FAIL with `Unsupported release policy releaseMode: +npm-public-release-ready` or missing `npm.access` validation. + +- [ ] **Step 3: Implement publish-ready policy validation** + +In `scripts/release-readiness.mjs`, import the public package version: + +```js +import { PUBLIC_NPM_PACKAGE_VERSION } from './build-public-npm-package.mjs'; +``` + +Add the release mode constant and include it in `SUPPORTED_RELEASE_MODES`: + +```js +const NPM_PUBLIC_RELEASE_READY_MODE = 'npm-public-release-ready'; +const SUPPORTED_RELEASE_MODES = new Set([ + CI_ARTIFACT_ONLY_RELEASE_MODE, + PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE, + NPM_PUBLIC_RELEASE_READY_MODE, +]); +``` + +Add string validators for the npm publish settings: + +```js +function assertNpmAccess(value) { + if (value !== 'public') { + throw new Error('Release policy npm.access must be public'); + } +} + +function assertNpmTag(value) { + assertString(value, 'Release policy npm.tag'); + if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(value)) { + throw new Error(`Invalid Release policy npm.tag: ${value}`); + } +} +``` + +In `validateReleasePolicy`, validate the new npm fields after +`assertNullableString(policy.npm.registry, 'Release policy npm.registry');`: + +```js + assertNpmAccess(policy.npm.access); + assertNpmTag(policy.npm.tag); +``` + +Replace `assertRequiredBeforePublishing` with: + +```js +function assertRequiredBeforePublishing(policy) { + assertStringArray(policy.requiredBeforePublishing, 'Release policy requiredBeforePublishing'); + + if (policy.releaseMode === CI_ARTIFACT_ONLY_RELEASE_MODE && policy.requiredBeforePublishing.length === 0) { + throw new Error('Release policy requiredBeforePublishing must list the remaining publishing decisions'); + } + + if ( + (policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE || + policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) && + policy.requiredBeforePublishing.length > 0 + ) { + throw new Error(`${policy.releaseMode} release mode requires requiredBeforePublishing to be empty`); + } +} +``` + +Replace `publishedPackageSmokeGate` with: + +```js +function publishedPackageSmokeGate(policy) { + const config = readPublishedPackageSmokeConfig({}, [], policy.publishedPackageSmoke); + + if ( + (policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE || + policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) && + !config.enabled + ) { + throw new Error(`${policy.releaseMode} release mode requires release-policy.json publishedPackageSmoke.packageName`); + } + + const base = + policy.releaseMode === CI_ARTIFACT_ONLY_RELEASE_MODE + ? { + status: 'not_required', + reason: 'Published package smoke remains pending until release-policy.json enables npm registry publishing.', + } + : policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE + ? { + status: 'required', + reason: 'Run the published package smoke after the npm package is published.', + } + : { + status: 'required', + reason: 'Run the published package smoke before accepting the hybrid-search release.', + }; + + return { + ...base, + script: 'pnpm run release:published-smoke', + configSource: config.enabled ? config.configSource : null, + packageName: config.enabled ? config.packageName : null, + version: config.enabled ? config.packageVersion : policy.publishedPackageSmoke.version, + registry: config.enabled ? (config.registry ?? null) : policy.publishedPackageSmoke.registry, + }; +} +``` + +Add this function below `assertNonPublishingArtifactPolicy`: + +```js +function assertNpmPublicReleaseReadyPolicy(policy, metadata) { + if (policy.npm.publish !== true) { + throw new Error('npm-public-release-ready policy requires npm.publish true'); + } + if (policy.python.publish !== false) { + throw new Error('npm-public-release-ready policy keeps python.publish false'); + } + if (policy.python.repository !== null) { + throw new Error('npm-public-release-ready policy keeps python.repository null'); + } + + assertSameMembers(policy.npm.packages, ['@kaelio/ktx'], 'Release policy npm.packages'); + assertSameMembers(policy.python.packages, metadataNames(metadata, 'python'), 'Release policy python.packages'); + + const npmMetadata = metadata.find((entry) => entry.ecosystem === 'npm' && entry.packageName === '@kaelio/ktx'); + if (!npmMetadata) { + throw new Error('npm-public-release-ready policy requires @kaelio/ktx artifact metadata'); + } + if (npmMetadata.private !== false) { + throw new Error('npm-public-release-ready policy requires @kaelio/ktx to be publishable'); + } + if (npmMetadata.packageVersion !== PUBLIC_NPM_PACKAGE_VERSION) { + throw new Error( + `npm-public-release-ready policy expected @kaelio/ktx ${PUBLIC_NPM_PACKAGE_VERSION}, got ${npmMetadata.packageVersion}`, + ); + } + if (policy.publishedPackageSmoke.packageName !== '@kaelio/ktx') { + throw new Error('npm-public-release-ready policy requires publishedPackageSmoke.packageName @kaelio/ktx'); + } + if (policy.publishedPackageSmoke.version !== PUBLIC_NPM_PACKAGE_VERSION) { + throw new Error( + `npm-public-release-ready policy requires publishedPackageSmoke.version ${PUBLIC_NPM_PACKAGE_VERSION}`, + ); + } +} +``` + +Inside `assertNonPublishingArtifactPolicy`, replace the npm package version +suffix check with public-package-aware validation: + +```js + if (isPublicKtxPackage) { + if (entry.packageVersion !== PUBLIC_NPM_PACKAGE_VERSION) { + throw new Error( + `${policyLabel} npm package @kaelio/ktx must use public version ${PUBLIC_NPM_PACKAGE_VERSION}`, + ); + } + } else if (!entry.packageVersion.endsWith('-private')) { + throw new Error(`${policyLabel} npm package ${entry.packageName} must use a private version suffix`); + } +``` + +In `releaseReadinessReport`, replace the unconditional +`assertNonPublishingArtifactPolicy(policy, metadata);` call with: + +```js + if (policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) { + assertNpmPublicReleaseReadyPolicy(policy, metadata); + } else { + assertNonPublishingArtifactPolicy(policy, metadata); + } +``` + +Add `npmPublish` to the returned report: + +```js + npmPublish: + policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE + ? { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + access: policy.npm.access, + tag: policy.npm.tag, + registry: policy.npm.registry, + } + : null, +``` + +Update the text output so it prints the npm publish target when present: + +```js + if (report.npmPublish) { + process.stdout.write( + `NPM publish target: ${report.npmPublish.packageName}@${report.npmPublish.version} (${report.npmPublish.tag})\n`, + ); + } else { + process.stdout.write('Registry publishing remains disabled by release-policy.json.\n'); + } +``` + +- [ ] **Step 4: Update release policy** + +Replace `release-policy.json` with: + +```json +{ + "schemaVersion": 1, + "releaseMode": "npm-public-release-ready", + "npm": { + "publish": true, + "registry": null, + "access": "public", + "tag": "latest", + "packages": ["@kaelio/ktx"] + }, + "python": { + "publish": false, + "repository": null, + "packages": ["ktx-sl", "ktx-daemon", "kaelio-ktx"] + }, + "publishedPackageSmoke": { + "packageName": "@kaelio/ktx", + "version": "0.1.0", + "registry": null + }, + "requiredBeforePublishing": [] +} +``` + +- [ ] **Step 5: Run release readiness tests to verify pass** + +Run: + +```bash +node --test scripts/release-readiness.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 6: Commit release policy validation** + +Run: + +```bash +git add scripts/release-readiness.mjs scripts/release-readiness.test.mjs release-policy.json +git commit -m "release: add npm public release policy" +``` + +Expected: commit created. + +### Task 3: Add guarded npm publish script + +**Files:** + +- Create: `scripts/publish-public-npm-package.test.mjs` +- Create: `scripts/publish-public-npm-package.mjs` +- Modify: `package.json` + +- [ ] **Step 1: Write failing publish script tests** + +Create `scripts/publish-public-npm-package.test.mjs` with: + +```js +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { describe, it } from 'node:test'; + +import { + buildNpmPublishCommand, + requireNpmPublicReleaseReady, + resolvePublishMode, +} from './publish-public-npm-package.mjs'; + +const readyReport = { + releaseMode: 'npm-public-release-ready', + npmPublishEnabled: true, + npmPublish: { + packageName: '@kaelio/ktx', + version: '0.1.0', + access: 'public', + tag: 'latest', + registry: null, + }, +}; + +describe('resolvePublishMode', () => { + it('dry-runs by default', () => { + assert.deepEqual(resolvePublishMode([]), { live: false }); + }); + + it('requires an explicit flag for live publish', () => { + assert.deepEqual(resolvePublishMode(['--publish']), { live: true }); + }); +}); + +describe('requireNpmPublicReleaseReady', () => { + it('accepts the npm public release ready report', () => { + assert.equal(requireNpmPublicReleaseReady(readyReport), readyReport.npmPublish); + }); + + it('rejects artifact-only reports', () => { + assert.throws( + () => + requireNpmPublicReleaseReady({ + releaseMode: 'ci-artifact-only', + npmPublishEnabled: false, + npmPublish: null, + }), + /release-policy.json must use npm-public-release-ready before publishing/, + ); + }); +}); + +describe('buildNpmPublishCommand', () => { + it('builds a dry-run pnpm publish command by default', () => { + assert.deepEqual(buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', readyReport.npmPublish, { live: false }), { + command: 'pnpm', + args: [ + 'publish', + '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', + '--access', + 'public', + '--tag', + 'latest', + '--dry-run', + ], + env: {}, + }); + }); + + it('omits dry-run only for explicit live publish', () => { + assert.deepEqual(buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', readyReport.npmPublish, { live: true }).args, [ + 'publish', + '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', + '--access', + 'public', + '--tag', + 'latest', + ]); + }); + + it('uses npm_config_registry when a registry is configured', () => { + const publish = { + ...readyReport.npmPublish, + registry: 'https://registry.npmjs.org/', + }; + + assert.deepEqual( + buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', publish, { live: false }).env, + { npm_config_registry: 'https://registry.npmjs.org/' }, + ); + }); +}); + +describe('package script', () => { + it('registers release:npm-publish', async () => { + const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); + + assert.equal(packageJson.scripts['release:npm-publish'], 'node scripts/publish-public-npm-package.mjs'); + }); +}); +``` + +- [ ] **Step 2: Run publish script tests to verify failure** + +Run: + +```bash +node --test scripts/publish-public-npm-package.test.mjs +``` + +Expected: FAIL with `Cannot find module` for +`scripts/publish-public-npm-package.mjs`. + +- [ ] **Step 3: Implement publish script** + +Create `scripts/publish-public-npm-package.mjs` with: + +```js +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { access } from 'node:fs/promises'; +import { promisify } from 'node:util'; +import { pathToFileURL } from 'node:url'; + +import { packageArtifactLayout } from './package-artifacts.mjs'; +import { releaseReadinessReport } from './release-readiness.mjs'; + +const execFileAsync = promisify(execFile); + +export function resolvePublishMode(args = process.argv.slice(2)) { + return { live: args.includes('--publish') }; +} + +export function requireNpmPublicReleaseReady(report) { + if (report.releaseMode !== 'npm-public-release-ready' || report.npmPublishEnabled !== true || !report.npmPublish) { + throw new Error('release-policy.json must use npm-public-release-ready before publishing'); + } + return report.npmPublish; +} + +export function buildNpmPublishCommand(tarballPath, publish, mode) { + return { + command: 'pnpm', + args: [ + 'publish', + tarballPath, + '--access', + publish.access, + '--tag', + publish.tag, + ...(mode.live ? [] : ['--dry-run']), + ], + env: publish.registry ? { npm_config_registry: publish.registry } : {}, + }; +} + +async function assertFileExists(path) { + try { + await access(path); + } catch { + throw new Error(`Missing npm tarball: ${path}. Run pnpm run artifacts:check first.`); + } +} + +async function runPublishCommand(command) { + process.stdout.write(`$ ${command.command} ${command.args.join(' ')}\n`); + await execFileAsync(command.command, command.args, { + env: { ...process.env, ...command.env }, + encoding: 'utf8', + maxBuffer: 1024 * 1024 * 20, + }); +} + +export async function publishPublicNpmPackage(options = {}) { + const rootDir = options.rootDir; + const mode = options.mode ?? resolvePublishMode(options.args); + const report = await releaseReadinessReport(rootDir); + const publish = requireNpmPublicReleaseReady(report); + const layout = packageArtifactLayout(rootDir); + const tarballPath = layout.cliTarball; + + await assertFileExists(tarballPath); + const command = buildNpmPublishCommand(tarballPath, publish, mode); + await runPublishCommand(command); + + process.stdout.write( + mode.live + ? `Published ${publish.packageName}@${publish.version} with tag ${publish.tag}\n` + : `Dry-run verified ${publish.packageName}@${publish.version} with tag ${publish.tag}\n`, + ); +} + +async function main() { + await publishPublicNpmPackage({ args: process.argv.slice(2) }); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + try { + await main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} +``` + +- [ ] **Step 4: Add the package script** + +In root `package.json`, add this script after `release:local-embeddings-smoke`: + +```json +"release:npm-publish": "node scripts/publish-public-npm-package.mjs", +``` + +- [ ] **Step 5: Run publish script tests to verify pass** + +Run: + +```bash +node --test scripts/publish-public-npm-package.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 6: Run a dry-run publish after artifacts are built** + +Run: + +```bash +pnpm run artifacts:check +pnpm run release:npm-publish +``` + +Expected: PASS. The publish command includes `--dry-run`, and the final line is: + +```text +Dry-run verified @kaelio/ktx@0.1.0 with tag latest +``` + +- [ ] **Step 7: Commit publish script** + +Run: + +```bash +git add scripts/publish-public-npm-package.mjs scripts/publish-public-npm-package.test.mjs package.json +git commit -m "release: add guarded npm publish script" +``` + +Expected: commit created. + +### Task 4: Add manual release workflow and docs + +**Files:** + +- Create: `.github/workflows/release.yml` +- Create: `scripts/release-workflow.test.mjs` +- Modify: `README.md` + +- [ ] **Step 1: Write failing workflow tests** + +Create `scripts/release-workflow.test.mjs` with: + +```js +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { describe, it } from 'node:test'; + +describe('release workflow', () => { + it('publishes only from manual dispatch with an explicit live input', async () => { + const workflow = await readFile(new URL('../.github/workflows/release.yml', import.meta.url), 'utf8'); + + assert.match(workflow, /^name: KTX Release$/m); + assert.match(workflow, /^ workflow_dispatch:$/m); + assert.match(workflow, /publish_live:/); + assert.match(workflow, /default: false/); + assert.match(workflow, /pnpm run artifacts:check/); + assert.match(workflow, /pnpm run release:readiness/); + assert.match(workflow, /pnpm run release:npm-publish$/m); + assert.match(workflow, /pnpm run release:npm-publish -- --publish/); + assert.match(workflow, /NODE_AUTH_TOKEN: \$\{\{ secrets.NPM_TOKEN \}\}/); + assert.doesNotMatch(workflow, /^ push:/m); + assert.doesNotMatch(workflow, /^ pull_request:/m); + }); +}); +``` + +- [ ] **Step 2: Run workflow tests to verify failure** + +Run: + +```bash +node --test scripts/release-workflow.test.mjs +``` + +Expected: FAIL because `.github/workflows/release.yml` does not exist. + +- [ ] **Step 3: Add the release workflow** + +Create `.github/workflows/release.yml` with: + +```yaml +name: KTX Release + +on: + workflow_dispatch: + inputs: + publish_live: + description: "Publish @kaelio/ktx to npm instead of running a dry-run" + required: true + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: ktx-release-${{ github.ref }} + cancel-in-progress: false + +jobs: + npm-public-release: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + with: + node-version: "24" + cache: "pnpm" + cache-dependency-path: "pnpm-lock.yaml" + + - name: Install TypeScript dependencies + run: pnpm install --frozen-lockfile + + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Setup uv + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install Python dependencies + run: uv sync --all-packages + + - name: Build and verify artifacts + run: pnpm run artifacts:check + + - name: Check release readiness + run: pnpm run release:readiness + + - name: Dry-run npm publish + if: ${{ !inputs.publish_live }} + run: pnpm run release:npm-publish + + - name: Publish npm package + if: ${{ inputs.publish_live }} + run: pnpm run release:npm-publish -- --publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} +``` + +- [ ] **Step 4: Update release docs** + +In `README.md`, replace the current `## Release status` section with: + +```markdown +## Release status + +This repository builds one public npm artifact named `@kaelio/ktx`. The first +public npm handoff is policy-gated through `release-policy.json`, which keeps +Python package publishing disabled because KTX-owned Python code ships inside +the npm package as a bundled wheel. + +Build local package artifacts and verify the guarded dry-run publish path with: + +```bash +source .venv/bin/activate +pnpm run artifacts:check +pnpm run release:readiness +pnpm run release:npm-publish +``` + +Run the live npm publish only from the manual `KTX Release` workflow with the +`publish_live` input enabled after the `NPM_TOKEN` secret is configured. +``` + +- [ ] **Step 5: Run workflow and README checks** + +Run: + +```bash +node --test scripts/release-workflow.test.mjs scripts/examples-docs.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 6: Commit workflow and docs** + +Run: + +```bash +git add .github/workflows/release.yml scripts/release-workflow.test.mjs README.md +git commit -m "release: document public npm release handoff" +``` + +Expected: commit created. + +### Task 5: Final verification + +**Files:** + +- Verify: `scripts/*.test.mjs` +- Verify: `packages/cli/src/*` +- Verify: `README.md` +- Verify: `release-policy.json` + +- [ ] **Step 1: Run focused script tests** + +Run: + +```bash +node --test scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs scripts/local-embeddings-runtime-smoke.test.mjs scripts/release-readiness.test.mjs scripts/publish-public-npm-package.test.mjs scripts/published-package-smoke.test.mjs scripts/release-workflow.test.mjs scripts/examples-docs.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 2: Run workspace type and package checks** + +Run: + +```bash +pnpm run type-check +pnpm run artifacts:check +``` + +Expected: PASS. The artifact build creates +`dist/artifacts/npm/kaelio-ktx-0.1.0.tgz`. + +- [ ] **Step 3: Run release readiness and dry-run publish** + +Run: + +```bash +pnpm run release:readiness +pnpm run release:npm-publish +``` + +Expected: PASS. `release:readiness` prints `KTX release mode: +npm-public-release-ready`, and `release:npm-publish` prints `Dry-run verified +@kaelio/ktx@0.1.0 with tag latest`. + +- [ ] **Step 4: Run pre-commit for changed files** + +Run: + +```bash +uv run pre-commit run --files scripts/build-public-npm-package.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/local-embeddings-runtime-smoke.test.mjs scripts/release-readiness.mjs scripts/release-readiness.test.mjs scripts/publish-public-npm-package.mjs scripts/publish-public-npm-package.test.mjs scripts/release-workflow.test.mjs release-policy.json package.json README.md .github/workflows/release.yml +``` + +Expected: PASS. If pre-commit is unavailable because the local `uv` version or +pre-commit environment is missing, report that explicitly and keep the script +tests, `pnpm run type-check`, `pnpm run artifacts:check`, `pnpm run +release:readiness`, and `pnpm run release:npm-publish` results as the closest +checks. + +- [ ] **Step 5: Confirm the worktree is clean** + +Run: + +```bash +git status --short +``` + +Expected: no output. If there are uncommitted tracked changes, inspect them and +commit only files from this plan with the exact task commit commands above. + +## Success criteria + +- `@kaelio/ktx` artifact metadata and tarball names use version `0.1.0`. +- `release-policy.json` encodes `npm-public-release-ready`, + `npm.publish: true`, and `python.publish: false`. +- `pnpm run release:npm-publish` performs a dry-run by default. +- Live npm publishing requires `pnpm run release:npm-publish -- --publish` or + the manual `KTX Release` workflow with `publish_live` enabled. +- Published-package smoke remains the post-publication proof for `npx + @kaelio/ktx`, local `npx ktx`, and global `ktx` invocation modes. +- No Python package publication is added for this release. + +## Self-review + +- Spec coverage: this plan covers the remaining public npm handoff gap while + preserving the bundled Python wheel model and single npm package surface. +- Placeholder scan: no open placeholders or deferred implementation notes are + present. +- Type consistency: the release mode name is consistently + `npm-public-release-ready`; the public npm version is consistently `0.1.0`; + the publish script consumes the `npmPublish` report shape produced by + `release-readiness.mjs`. diff --git a/docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md b/docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md new file mode 100644 index 00000000..ad581827 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md @@ -0,0 +1,602 @@ +# Published Package Managed Runtime Smoke Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the post-publication smoke prove that the published +`@kaelio/ktx` package uses the same isolated managed Python runtime across +`npx @kaelio/ktx`, local `npx ktx`, and global `ktx` invocation modes. + +**Architecture:** Keep the smoke black-box and network-gated. Strengthen the +command builder so every Python-backed published-package command receives the +same temporary `KTX_RUNTIME_ROOT`, then run a real semantic-layer query through +the direct `npx`, local install, and global install paths instead of checking +only `--version` for local and global binaries. + +**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, npx, KTX managed Python +runtime, published `@kaelio/ktx` package smoke. + +--- + +## Existing status + +This plan is based on +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +The following plans are based on that spec and are implemented in this +worktree: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` +- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` + +Implementation evidence found before writing this plan includes: + +- `scripts/build-python-runtime-wheel.mjs` and + `packages/cli/assets/python/manifest.json`. +- `packages/cli/src/managed-python-runtime.ts`, + `packages/cli/src/runtime.ts`, + `packages/cli/src/commands/runtime-commands.ts`, + `packages/cli/src/managed-python-command.ts`, + `packages/cli/src/managed-python-daemon.ts`, + `packages/cli/src/managed-local-embeddings.ts`, and + `packages/cli/src/managed-python-http.ts`. +- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`, + `scripts/local-embeddings-runtime-smoke.mjs`, and + `scripts/published-package-smoke.mjs`. +- `packages/cli/src/agent-runtime.ts`, `packages/cli/src/serve.ts`, + `packages/cli/src/ingest.ts`, and `packages/cli/src/scan.ts` thread managed + runtime policy through the Python-backed CLI paths. +- `examples/postgres-historic/scripts/smoke.sh`, + `examples/postgres-historic/README.md`, + `examples/package-artifacts/README.md`, and `README.md` now document the + managed runtime instead of a manual `python-service/` process. + +The remaining release-confidence gap is in the post-publication smoke: + +- `scripts/published-package-smoke-config.mjs` runs `npx @kaelio/ktx setup + demo` and `npx @kaelio/ktx sl query ... --yes`, but it does not isolate + `KTX_RUNTIME_ROOT` for those commands. +- The same smoke installs `@kaelio/ktx` locally and globally, but local and + global verification only run `--version`. +- The design spec requires the direct `npx @kaelio/ktx`, local `npx ktx`, and + global `ktx` modes to work for real KTX commands. A semantic-layer query is + the smallest Python-backed command that proves the bundled managed runtime is + usable in each mode. + +## File structure + +- Modify `scripts/published-package-smoke.test.mjs`: expect a shared + `KTX_RUNTIME_ROOT` in the published smoke commands, expect local and global + semantic query commands, and cover label classification used by the runner. +- Modify `scripts/published-package-smoke-config.mjs`: derive a temporary + runtime root from the smoke project directory, merge it with registry + environment settings, and add local and global `sl query` commands. +- Modify `scripts/published-package-smoke.mjs`: validate the renamed version + labels and semantic query labels when the smoke runs. + +### Task 1: Isolate runtime roots and add real local/global command coverage + +**Files:** + +- Modify: `scripts/published-package-smoke.test.mjs` +- Modify: `scripts/published-package-smoke-config.mjs` +- Test: `scripts/published-package-smoke.test.mjs` + +- [ ] **Step 1: Write the failing command-list test** + +In `scripts/published-package-smoke.test.mjs`, replace the existing +`it('builds the full public package smoke command list', ...)` block with this +test: + +```javascript + it('builds the full public package smoke command list', () => { + assert.deepEqual( + buildPublishedPackageSmokeCommands( + config, + '/tmp/ktx-smoke/demo', + '/tmp/ktx-smoke/managed-runtime', + ), + [ + { + label: 'published package npx version', + command: 'npx', + args: ['--yes', '@kaelio/ktx@latest', '--version'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package setup demo', + command: 'npx', + args: [ + '--yes', + '@kaelio/ktx@latest', + 'setup', + 'demo', + '--project-dir', + '/tmp/ktx-smoke/demo', + '--no-input', + '--plain', + ], + env: { + npm_config_registry: 'https://registry.npmjs.org/', + KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', + }, + }, + { + label: 'published package npx sl query', + command: 'npx', + args: [ + '--yes', + '@kaelio/ktx@latest', + 'sl', + 'query', + '--project-dir', + '/tmp/ktx-smoke/demo', + '--connection-id', + 'orbit_demo', + '--measure', + 'contracts.contract_count', + '--format', + 'sql', + '--yes', + ], + env: { + npm_config_registry: 'https://registry.npmjs.org/', + KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', + }, + }, + { + label: 'published package local install', + command: 'pnpm', + args: ['add', '@kaelio/ktx@latest'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package local version', + command: 'npx', + args: ['ktx', '--version'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package local sl query', + command: 'npx', + args: [ + 'ktx', + 'sl', + 'query', + '--project-dir', + '/tmp/ktx-smoke/demo', + '--connection-id', + 'orbit_demo', + '--measure', + 'contracts.contract_count', + '--format', + 'sql', + '--yes', + ], + env: { + npm_config_registry: 'https://registry.npmjs.org/', + KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', + }, + }, + { + label: 'published package global install', + command: 'pnpm', + args: ['add', '--global', '@kaelio/ktx@latest'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package global version', + command: 'ktx', + args: ['--version'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package global sl query', + command: 'ktx', + args: [ + 'sl', + 'query', + '--project-dir', + '/tmp/ktx-smoke/demo', + '--connection-id', + 'orbit_demo', + '--measure', + 'contracts.contract_count', + '--format', + 'sql', + '--yes', + ], + env: { + npm_config_registry: 'https://registry.npmjs.org/', + KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', + }, + }, + ], + ); + }); +``` + +- [ ] **Step 2: Run the test and verify it fails** + +Run: + +```bash +node --test scripts/published-package-smoke.test.mjs +``` + +Expected: FAIL with an `AssertionError` showing that the actual command list +still uses `published package version`, lacks `KTX_RUNTIME_ROOT`, and lacks the +local/global `sl query` commands. + +- [ ] **Step 3: Implement the command builder changes** + +In `scripts/published-package-smoke-config.mjs`, add this import before the +existing `node:assert/strict` import: + +```javascript +import { dirname, join } from 'node:path'; +``` + +In the same file, add these helper functions after +`assertHttpRegistry(registry, label)`: + +```javascript +function registryEnv(config) { + return config.registry ? { npm_config_registry: config.registry } : {}; +} + +function runtimeCommandEnv(config, runtimeRoot) { + return { ...registryEnv(config), KTX_RUNTIME_ROOT: runtimeRoot }; +} + +function semanticQueryArgs(projectDir) { + return [ + 'sl', + 'query', + '--project-dir', + projectDir, + '--connection-id', + 'orbit_demo', + '--measure', + 'contracts.contract_count', + '--format', + 'sql', + '--yes', + ]; +} +``` + +Replace `buildPublishedPackageNpxCommand()` and +`buildPublishedPackageSmokeCommands()` with this implementation: + +```javascript +export function buildPublishedPackageNpxCommand(config, args, label = 'published package command', extraEnv = {}) { + return { + label, + command: 'npx', + args: ['--yes', publishedPackageSpec(config), ...args], + env: { ...registryEnv(config), ...extraEnv }, + }; +} + +export function buildPublishedPackageSmokeCommands( + config, + projectDir, + runtimeRoot = join(dirname(projectDir), 'managed-runtime'), +) { + const runtimeEnv = runtimeCommandEnv(config, runtimeRoot); + const packageEnv = registryEnv(config); + const queryArgs = semanticQueryArgs(projectDir); + + return [ + buildPublishedPackageNpxCommand(config, ['--version'], 'published package npx version'), + buildPublishedPackageNpxCommand( + config, + ['setup', 'demo', '--project-dir', projectDir, '--no-input', '--plain'], + 'published package setup demo', + { KTX_RUNTIME_ROOT: runtimeRoot }, + ), + buildPublishedPackageNpxCommand(config, queryArgs, 'published package npx sl query', { + KTX_RUNTIME_ROOT: runtimeRoot, + }), + { + label: 'published package local install', + command: 'pnpm', + args: ['add', publishedPackageSpec(config)], + env: packageEnv, + }, + { + label: 'published package local version', + command: 'npx', + args: ['ktx', '--version'], + env: packageEnv, + }, + { + label: 'published package local sl query', + command: 'npx', + args: ['ktx', ...queryArgs], + env: runtimeEnv, + }, + { + label: 'published package global install', + command: 'pnpm', + args: ['add', '--global', publishedPackageSpec(config)], + env: packageEnv, + }, + { + label: 'published package global version', + command: 'ktx', + args: ['--version'], + env: packageEnv, + }, + { + label: 'published package global sl query', + command: 'ktx', + args: queryArgs, + env: runtimeEnv, + }, + ]; +} +``` + +- [ ] **Step 4: Run the command-list test and verify it passes** + +Run: + +```bash +node --test scripts/published-package-smoke.test.mjs +``` + +Expected: PASS for the command construction tests, with remaining failures only +if the runner label validation test from Task 2 has already been added. + +- [ ] **Step 5: Commit the command-builder change** + +Run: + +```bash +git add scripts/published-package-smoke-config.mjs scripts/published-package-smoke.test.mjs +git commit -m "test: cover published package runtime smoke commands" +``` + +### Task 2: Validate smoke runner labels for the new command list + +**Files:** + +- Modify: `scripts/published-package-smoke.test.mjs` +- Modify: `scripts/published-package-smoke.mjs` +- Test: `scripts/published-package-smoke.test.mjs` + +- [ ] **Step 1: Write the failing label classification test** + +In `scripts/published-package-smoke.test.mjs`, replace the import from +`./published-package-smoke.mjs` with this import: + +```javascript +import { + buildPublishedPackageNpxCommand, + buildPublishedPackageSmokeCommands, + isPublishedPackageSemanticQueryLabel, + isPublishedPackageVersionLabel, + publishedPackageSpec, + readPublishedPackageSmokeConfig, +} from './published-package-smoke.mjs'; +``` + +Add this test after the `describe('published package smoke command +construction', ...)` block: + +```javascript +describe('published package smoke output validation labels', () => { + it('classifies version and semantic query commands', () => { + assert.equal(isPublishedPackageVersionLabel('published package npx version'), true); + assert.equal(isPublishedPackageVersionLabel('published package local version'), true); + assert.equal(isPublishedPackageVersionLabel('published package global version'), true); + assert.equal(isPublishedPackageVersionLabel('published package setup demo'), false); + + assert.equal(isPublishedPackageSemanticQueryLabel('published package npx sl query'), true); + assert.equal(isPublishedPackageSemanticQueryLabel('published package local sl query'), true); + assert.equal(isPublishedPackageSemanticQueryLabel('published package global sl query'), true); + assert.equal(isPublishedPackageSemanticQueryLabel('published package local install'), false); + }); +}); +``` + +- [ ] **Step 2: Run the test and verify it fails** + +Run: + +```bash +node --test scripts/published-package-smoke.test.mjs +``` + +Expected: FAIL with an import error because +`isPublishedPackageSemanticQueryLabel` and `isPublishedPackageVersionLabel` are +not exported yet. + +- [ ] **Step 3: Implement label classification and runner validation** + +In `scripts/published-package-smoke.mjs`, add these constants and exports after +`const SMOKE_TIMEOUT_MS = 180_000;`: + +```javascript +const VERSION_LABELS = new Set([ + 'published package npx version', + 'published package local version', + 'published package global version', +]); + +const SEMANTIC_QUERY_LABELS = new Set([ + 'published package npx sl query', + 'published package local sl query', + 'published package global sl query', +]); + +export function isPublishedPackageVersionLabel(label) { + return VERSION_LABELS.has(label); +} + +export function isPublishedPackageSemanticQueryLabel(label) { + return SEMANTIC_QUERY_LABELS.has(label); +} +``` + +In `runPublishedPackageSmoke(config)`, replace this block: + +```javascript + if ( + command.label === 'published package version' || + command.label === 'published package local binary' || + command.label === 'published package global binary' + ) { + assert.match(result.stdout, /@kaelio\/ktx /); + } + if (command.label === 'published package sl query') { + assert.match(result.stdout, /SELECT/i); + assert.match(result.stdout, /contracts/i); + } +``` + +with this block: + +```javascript + if (isPublishedPackageVersionLabel(command.label)) { + assert.match(result.stdout, /@kaelio\/ktx /); + } + if (isPublishedPackageSemanticQueryLabel(command.label)) { + assert.match(result.stdout, /SELECT/i); + assert.match(result.stdout, /contracts/i); + } +``` + +- [ ] **Step 4: Run the label tests and verify they pass** + +Run: + +```bash +node --test scripts/published-package-smoke.test.mjs +``` + +Expected: PASS. + +- [ ] **Step 5: Commit the runner-label change** + +Run: + +```bash +git add scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs +git commit -m "test: validate published package smoke outputs" +``` + +### Task 3: Verify release-script compatibility + +**Files:** + +- Verify: `scripts/published-package-smoke-config.mjs` +- Verify: `scripts/published-package-smoke.mjs` +- Verify: `scripts/published-package-smoke.test.mjs` +- Verify: `scripts/release-readiness.test.mjs` +- Verify: `package.json` + +- [ ] **Step 1: Run the focused Node tests** + +Run: + +```bash +node --test scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs +``` + +Expected: PASS. The release-readiness tests must continue to report the +published package smoke gate without executing the network smoke. + +- [ ] **Step 2: Run release readiness** + +Run: + +```bash +pnpm run release:readiness +``` + +Expected: PASS and output containing these lines: + +```text +Release mode: ci-artifact-only +NPM publish enabled: false +Published package smoke: pending +Published package smoke script: pnpm run release:published-smoke +``` + +- [ ] **Step 3: Confirm the network smoke stays explicit** + +Run: + +```bash +rg -n '"release:published-smoke": "node scripts/published-package-smoke.mjs --require-config"' package.json +``` + +Expected: PASS with one match in `package.json`. Do not run +`pnpm run release:published-smoke` in normal CI before the package is published +to the configured registry. + +- [ ] **Step 4: Check pre-commit availability** + +Run: + +```bash +test ! -f .pre-commit-config.yaml +``` + +Expected: PASS in the current worktree. If a pre-commit config exists when this +plan is executed, run this instead after activating `.venv`: + +```bash +source .venv/bin/activate +uv run pre-commit run --files scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs +``` + +- [ ] **Step 5: Commit verification-only fixes if needed** + +If Step 1 or Step 2 required additional source changes, commit them with: + +```bash +git add scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs package.json +git commit -m "chore: verify published package runtime smoke" +``` + +If no files changed after Task 2, do not create an empty commit. + +## Acceptance criteria + +- `buildPublishedPackageSmokeCommands()` derives + `/managed-runtime` from the demo project directory by default. +- Direct `npx @kaelio/ktx`, local `npx ktx`, and global `ktx` semantic query + commands all receive the same `KTX_RUNTIME_ROOT`. +- Local and global post-publication smoke coverage runs `sl query ... --yes`, + not only `--version`. +- `runPublishedPackageSmoke()` validates version output for all version labels + and validates generated SQL output for all semantic query labels. +- `node --test scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs` + passes. +- `pnpm run release:readiness` still reports the published-package smoke as a + pending explicit release gate while registry publishing is disabled. + +## Self-review notes + +- Spec coverage: this plan covers the remaining invocation-mode confidence gap + from the spec by proving the published package uses an isolated managed + runtime across direct `npx`, local binary, and global binary paths. +- Placeholder scan: the plan contains concrete file paths, exact code blocks, + exact commands, and exact expected outcomes. +- Type consistency: the command label strings are consistent across tests, + command construction, and smoke-runner output validation. diff --git a/docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md b/docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md new file mode 100644 index 00000000..a9098867 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md @@ -0,0 +1,978 @@ +# Single Public Runtime Artifact Cleanup Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make release artifacts match the npm-managed Python runtime design: +one public `@kaelio/ktx` npm tarball plus one bundled `kaelio-ktx` runtime +wheel, with no standalone `ktx-sl` or `ktx-daemon` release artifacts. + +**Architecture:** Keep `python/ktx-sl` and `python/ktx-daemon` as source +packages used to assemble the bundled runtime wheel. Remove direct standalone +Python wheel and source-distribution builds from the release artifact path, +manifest, readiness policy, and artifact smoke docs. The packed npm package +remains the only user-visible package; Python-backed verification continues +through the managed runtime installed from the bundled wheel. + +**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, uv-built bundled +runtime wheel, JSON release policy, Markdown. + +--- + +## Current state + +This plan follows +`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. + +The following plan files are based on that spec and are implemented in the +current tree: + +- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` +- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` +- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` +- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` +- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` +- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md` +- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md` +- `docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md` +- `docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md` + +Implementation evidence found before writing this plan includes: + +- `packages/cli/assets/python/manifest.json` and + `packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl`. +- `packages/cli/src/managed-python-runtime.ts`, + `packages/cli/src/managed-python-command.ts`, + `packages/cli/src/managed-python-daemon.ts`, + `packages/cli/src/managed-local-embeddings.ts`, + `packages/cli/src/managed-python-http.ts`, and `packages/cli/src/runtime.ts`. +- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`, + `scripts/published-package-smoke.mjs`, + `scripts/local-embeddings-runtime-smoke.mjs`, + `scripts/publish-public-npm-package.mjs`, and + `.github/workflows/release.yml`. +- `release-policy.json` is in `npm-public-release-ready` mode, publishes + `@kaelio/ktx`, disables Python package publishing, and encodes the hard + `uv` prerequisite. +- `README.md` and `examples/package-artifacts/README.md` document public npm + usage, managed runtime commands, `runtime prune`, and the `uv` prerequisite. + +The remaining mismatch is in the artifact release surface: + +- `scripts/package-artifacts.mjs` still runs `uv build --package ktx-sl` and + `uv build --package ktx-daemon`. +- `scripts/package-artifacts.mjs` still adds `ktx-sl` and `ktx-daemon` wheel + and source-distribution files to the artifact manifest. +- `scripts/package-artifacts.mjs` still runs a direct Python clean-install + smoke, even though the npm artifact smoke already proves Python-backed + commands through the managed runtime. +- `release-policy.json` still lists `ktx-sl` and `ktx-daemon` under + `python.packages`. +- `examples/package-artifacts/README.md` says the Python smoke installs + standalone Python artifacts directly. + +This plan removes those release artifacts. It does not delete the Python source +packages because the bundled runtime wheel builder still copies from +`python/ktx-sl/semantic_layer` and `python/ktx-daemon/src/ktx_daemon`. + +## File structure + +- Modify `scripts/package-artifacts.test.mjs`: make artifact tests expect only + `@kaelio/ktx` plus the `kaelio-ktx` bundled runtime wheel, and add a guard + that direct standalone Python artifact smoke code is gone. +- Modify `scripts/package-artifacts.mjs`: stop building standalone Python + artifacts, stop looking for their wheel and source-distribution files, remove + their release metadata, and remove the direct Python artifact verification + path. +- Modify `scripts/release-readiness.test.mjs`: update release policy fixtures + and readiness reports so the only Python release metadata is `kaelio-ktx`. +- Modify `release-policy.json`: set `python.packages` to `["kaelio-ktx"]`. +- Modify `scripts/examples-docs.test.mjs`: require docs to describe the single + npm tarball plus runtime wheel artifact shape and reject the old direct + Python-artifact smoke wording. +- Modify `README.md`: clarify that `python/ktx-sl` and `python/ktx-daemon` are + source packages, not release artifacts for the first npm release. +- Modify `examples/package-artifacts/README.md`: replace the stale standalone + Python smoke paragraph with the managed-runtime artifact contract. + +### Task 1: Make package artifact tests expect one runtime wheel + +**Files:** + +- Modify: `scripts/package-artifacts.test.mjs` +- Test: `scripts/package-artifacts.test.mjs` + +- [ ] **Step 1: Update package artifact imports** + +In `scripts/package-artifacts.test.mjs`, replace the import from +`./package-artifacts.mjs` with this import: + +```javascript +import { + CLI_PYTHON_ASSET_MANIFEST, + INTERNAL_NPM_WORKSPACE_PACKAGES, + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_NORMALIZED_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, + artifactManifestPath, + buildArtifactCommands, + copyRuntimeWheelAssets, + findPythonArtifacts, + NPM_ARTIFACT_PACKAGES, + npmDemoSmokeSource, + npmRuntimeSmokeSource, + npmSmokePackageJson, + npmVerifySource, + packageArtifactLayout, + packageReleaseMetadata, + verifyArtifactManifest, + writeArtifactManifest, +} from './package-artifacts.mjs'; +``` + +- [ ] **Step 2: Remove standalone Python fixture setup** + +In `scripts/package-artifacts.test.mjs`, replace `writeReleaseMetadataInputs` +with this function: + +```javascript +async function writeReleaseMetadataInputs(root) { + for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { + await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); + await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { + name: packageInfo.name, + version: '0.0.0-private', + private: true, + }); + } +} +``` + +Replace `writeUploadableArtifactFixtures` with this function: + +```javascript +async function writeUploadableArtifactFixtures(layout) { + await mkdir(layout.npmDir, { recursive: true }); + await mkdir(layout.pythonDir, { recursive: true }); + + const fileContents = new Map([ + ...NPM_ARTIFACT_PACKAGES.map((packageInfo) => [ + layout.npmTarballs[packageInfo.name], + `${packageInfo.name}-tarball`, + ]), + [ + join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + 'kaelio-ktx-runtime-wheel', + ], + ]); + + for (const [path, contents] of fileContents) { + await writeFile(path, contents); + } +} +``` + +- [ ] **Step 3: Change build command expectations** + +In the `buildArtifactCommands` test, replace the body with this code: + +```javascript + it('builds TypeScript packages and the runtime wheel before packing npm artifacts', () => { + const layout = packageArtifactLayout('/repo/ktx'); + const commands = buildArtifactCommands(layout); + + assert.deepEqual( + commands.slice(0, NPM_BUILD_PACKAGE_ORDER.length).map((command) => [command.command, command.args]), + NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]), + ); + assert.deepEqual( + commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [ + command.command, + command.args, + ]), + [[process.execPath, ['scripts/build-python-runtime-wheel.mjs']]], + ); + assert.deepEqual( + commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [command.command, command.args]), + [[process.execPath, ['scripts/build-public-npm-package.mjs']]], + ); + }); +``` + +- [ ] **Step 4: Change release metadata expectations** + +In the `packageReleaseMetadata` test, replace the expected array with this +array: + +```javascript + assert.deepEqual(await packageReleaseMetadata(root), [ + { + ecosystem: 'npm', + packageName: '@kaelio/ktx', + packageRoot: 'packages/cli', + packageVersion: '0.1.0', + private: false, + releaseMode: 'ci-artifact-only', + }, + { + ecosystem: 'python', + packageName: 'kaelio-ktx', + packageRoot: 'python/runtime-wheel', + packageVersion: '0.1.0', + private: false, + releaseMode: 'ci-artifact-only', + }, + ]); +``` + +- [ ] **Step 5: Change Python artifact discovery expectations** + +Replace the `findPythonArtifacts` success test with this test: + +```javascript + it('finds the bundled runtime wheel only', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-')); + try { + await writeFile(join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), ''); + + assert.deepEqual(await findPythonArtifacts(root), { + runtimeWheel: join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +``` + +- [ ] **Step 6: Change artifact manifest expectations** + +Inside the artifact manifest test, replace the Python package assertion with: + +```javascript + assert.deepEqual( + manifest.packages.filter((entry) => entry.ecosystem === 'python'), + [ + { + ecosystem: 'python', + packageName: 'kaelio-ktx', + packageRoot: 'python/runtime-wheel', + packageVersion: '0.1.0', + private: false, + releaseMode: 'ci-artifact-only', + }, + ], + ); +``` + +Replace the Python file assertion with: + +```javascript + assert.deepEqual( + manifest.files + .filter((file) => file.ecosystem === 'python') + .map((file) => ({ + artifactKind: file.artifactKind, + ecosystem: file.ecosystem, + packageName: file.packageName, + packageVersion: file.packageVersion, + path: file.path, + })), + [ + { + artifactKind: 'wheel', + ecosystem: 'python', + packageName: 'kaelio-ktx', + packageVersion: '0.1.0', + path: 'python/kaelio_ktx-0.1.0-py3-none-any.whl', + }, + ], + ); +``` + +In the `verifyArtifactManifest` success test, replace the file-count assertion +with: + +```javascript + assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 1); +``` + +- [ ] **Step 7: Replace direct Python smoke tests with a dead-code guard** + +Remove the whole `describe('pythonArtifactInstallArgs', ...)` block. + +In `describe('verification snippets', ...)`, remove the test named +`asserts the Python modules that clean installs must expose`. + +Add this test after the `verifyNpmArtifacts` test: + +```javascript +describe('standalone Python artifact cleanup', () => { + it('does not build or verify standalone Python package artifacts', async () => { + const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8'); + + assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-sl'/); + assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-daemon'/); + assert.doesNotMatch(source, /async function verifyPythonArtifacts/); + assert.doesNotMatch(source, /pythonArtifactInstallArgs/); + assert.doesNotMatch(source, /pythonVerifySource/); + assert.doesNotMatch(source, /ktx_sl-0\.1\.0/); + assert.doesNotMatch(source, /ktx_daemon-0\.1\.0/); + }); +}); +``` + +- [ ] **Step 8: Run package artifact tests and verify failure** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs +``` + +Expected: FAIL. The failures mention the extra `ktx-sl` and `ktx-daemon` +artifact commands, metadata entries, manifest files, or direct Python smoke +helpers. + +### Task 2: Remove standalone Python artifacts from package artifacts + +**Files:** + +- Modify: `scripts/package-artifacts.mjs` +- Test: `scripts/package-artifacts.test.mjs` + +- [ ] **Step 1: Remove dead constants and imports** + +In `scripts/package-artifacts.mjs`, replace the `node:path` import with this +import: + +```javascript +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +``` + +Remove these constants: + +```javascript +const PACKAGE_VERSION = '0.0.0-private'; +const PYTHON_PACKAGE_VERSION = '0.1.0'; +``` + +Remove the whole `ordersSource` constant block. + +- [ ] **Step 2: Make npm artifact names public-package only** + +Replace `npmPackageTarballName` with this function: + +```javascript +function npmPackageTarballName(packageName) { + if (packageName !== PUBLIC_NPM_PACKAGE_NAME) { + throw new Error(`Unsupported npm artifact package: ${packageName}`); + } + return publicNpmPackageTarballName(PUBLIC_NPM_PACKAGE_VERSION); +} +``` + +- [ ] **Step 3: Remove standalone Python build commands** + +Replace `buildArtifactCommands` with this function: + +```javascript +export function buildArtifactCommands(layout) { + const packagesByName = new Map(INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => [packageInfo.name, packageInfo])); + const npmBuildCommands = NPM_ARTIFACT_BUILD_ORDER.map((packageName) => { + const packageInfo = packagesByName.get(packageName); + if (!packageInfo) { + throw new Error(`Unknown npm artifact build package: ${packageName}`); + } + return { + command: 'pnpm', + args: ['--filter', packageInfo.name, 'run', 'build'], + cwd: layout.rootDir, + }; + }); + const publicPackageCommand = { + command: process.execPath, + args: ['scripts/build-public-npm-package.mjs'], + cwd: layout.rootDir, + }; + + return [ + ...npmBuildCommands, + { + command: process.execPath, + args: ['scripts/build-python-runtime-wheel.mjs'], + cwd: layout.rootDir, + }, + publicPackageCommand, + ]; +} +``` + +- [ ] **Step 4: Discover only the bundled runtime wheel** + +Replace `findOne` and `findPythonArtifacts` with these functions: + +```javascript +function findOne(files, distributionName, suffix, label, pythonDir, version) { + const normalized = normalizePythonDistributionName(distributionName); + const found = files.find((file) => file.startsWith(`${normalized}-${version}`) && file.endsWith(suffix)); + if (!found) { + throw new Error(`Missing Python artifact: ${label}`); + } + return join(pythonDir, found); +} + +export async function findPythonArtifacts(pythonDir) { + const files = await readdir(pythonDir); + + return { + runtimeWheel: findOne( + files, + RUNTIME_WHEEL_DISTRIBUTION_NAME, + '.whl', + 'kaelio-ktx runtime wheel', + pythonDir, + RUNTIME_WHEEL_PACKAGE_VERSION, + ), + }; +} +``` + +- [ ] **Step 5: Emit release metadata only for npm and runtime wheel** + +Replace `packageReleaseMetadata` with this function: + +```javascript +export async function packageReleaseMetadata(rootDir = scriptRootDir()) { + const npmPackages = await Promise.all( + NPM_ARTIFACT_PACKAGES.map((packageInfo) => readNpmPackageMetadata(rootDir, packageInfo)), + ); + + return [ + ...npmPackages, + releaseMetadataEntry({ + ecosystem: 'python', + packageName: RUNTIME_WHEEL_DISTRIBUTION_NAME, + packageRoot: 'python/runtime-wheel', + packageVersion: RUNTIME_WHEEL_PACKAGE_VERSION, + privatePackage: false, + }), + ]; +} +``` + +- [ ] **Step 6: Remove dead TOML metadata helpers** + +Delete these helper functions from `scripts/package-artifacts.mjs` because +release metadata no longer reads standalone Python `pyproject.toml` files: + +```javascript +function readProjectBlock(toml, sourcePath) { + const lines = toml.split(/\r?\n/); + const block = []; + let inProject = false; + + for (const line of lines) { + if (/^\[project\]\s*$/.test(line)) { + inProject = true; + continue; + } + if (inProject && /^\[.*\]\s*$/.test(line)) { + break; + } + if (inProject) { + block.push(line); + } + } + + if (!inProject) { + throw new Error(`Missing [project] table in ${sourcePath}`); + } + return block.join('\n'); +} +``` + +```javascript +function readTomlStringField(projectBlock, fieldName, sourcePath) { + const match = projectBlock.match(new RegExp(`^${fieldName}\\s*=\\s*"([^"]+)"\\s*$`, 'm')); + if (!match) { + throw new Error(`Missing project.${fieldName} in ${sourcePath}`); + } + return match[1]; +} +``` + +```javascript +async function readPyprojectMetadata(path) { + const toml = await readFile(path, 'utf-8'); + const projectBlock = readProjectBlock(toml, path); + return { + name: readTomlStringField(projectBlock, 'name', path), + version: readTomlStringField(projectBlock, 'version', path), + }; +} +``` + +- [ ] **Step 7: Emit manifest records only for npm and runtime wheel** + +Replace `artifactPackageRecords` with this function: + +```javascript +function artifactPackageRecords(layout, pythonArtifacts, packages) { + const packagesByName = packageMetadataByName(packages); + const npmRecords = NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({ + artifactKind: 'tarball', + artifactPath: layout.npmTarballs[packageInfo.name], + metadata: requirePackageMetadata(packagesByName, packageInfo.name), + })); + + return [ + ...npmRecords, + { + artifactKind: 'wheel', + artifactPath: pythonArtifacts.runtimeWheel, + metadata: requirePackageMetadata(packagesByName, RUNTIME_WHEEL_DISTRIBUTION_NAME), + }, + ]; +} +``` + +- [ ] **Step 8: Remove direct Python artifact verification helpers** + +Delete these exports and functions from `scripts/package-artifacts.mjs`: + +```javascript +export function pythonArtifactInstallArgs(python, pythonArtifacts) { + return ['pip', 'install', '--python', python, pythonArtifacts.runtimeWheel]; +} +``` + +```javascript +export function pythonVerifySource() { + return ` +import importlib.metadata + +import semantic_layer +import ktx_daemon + +assert importlib.metadata.version("kaelio-ktx") == "0.1.0" +assert semantic_layer is not None +assert ktx_daemon.PACKAGE_NAME == "ktx-daemon" +`; +} +``` + +```javascript +function pythonExecutable(projectDir) { + if (process.platform === 'win32') { + return join(projectDir, '.venv', 'Scripts', 'python.exe'); + } + return join(projectDir, '.venv', 'bin', 'python'); +} +``` + +```javascript +export function npmSmokePythonEnv(projectDir, baseEnv = process.env) { + const binDir = process.platform === 'win32' ? join(projectDir, '.venv', 'Scripts') : join(projectDir, '.venv', 'bin'); + const existingPath = baseEnv.PATH ?? ''; + + return { + ...baseEnv, + PATH: existingPath ? `${binDir}${delimiter}${existingPath}` : binDir, + }; +} +``` + +```javascript +async function verifyPythonArtifacts(layout, tmpRoot) { + const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); + + const projectDir = join(tmpRoot, 'python-clean-install'); + await mkdir(projectDir, { recursive: true }); + const python = pythonExecutable(projectDir); + await writeFile(join(projectDir, 'verify_python.py'), pythonVerifySource()); + + await runCommand('uv', ['venv', '.venv'], { cwd: projectDir }); + await runCommand('uv', pythonArtifactInstallArgs(python, pythonArtifacts), { + cwd: projectDir, + }); + await runCommand(python, ['verify_python.py'], { cwd: projectDir }); + await runCommand(python, ['-m', 'ktx_daemon', 'semantic-validate'], { + cwd: projectDir, + input: `${JSON.stringify({ sources: [ordersSource], dialect: 'postgres' })}\n`, + }); +} +``` + +- [ ] **Step 9: Verify artifacts through npm only** + +Replace `verifyArtifacts` with this function: + +```javascript +async function verifyArtifacts(layout) { + await verifyArtifactManifest(layout); + + const tmpRoot = await mkdtemp(join(tmpdir(), 'ktx-artifacts-')); + try { + await verifyNpmArtifacts(layout, tmpRoot); + } finally { + await rm(tmpRoot, { recursive: true, force: true }); + } +} +``` + +- [ ] **Step 10: Run package artifact tests and verify pass** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs +``` + +Expected: PASS. The output includes `# fail 0`. + +- [ ] **Step 11: Commit package artifact cleanup** + +Run: + +```bash +git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs +git commit -m "refactor: limit release artifacts to public package runtime" +``` + +### Task 3: Align release policy and readiness reports + +**Files:** + +- Modify: `release-policy.json` +- Modify: `scripts/release-readiness.test.mjs` +- Test: `scripts/release-readiness.test.mjs` + +- [ ] **Step 1: Update release readiness fixtures** + +In `scripts/release-readiness.test.mjs`, replace +`writeReleaseMetadataInputs` with: + +```javascript +async function writeReleaseMetadataInputs(root) { + for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { + await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); + await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { + name: packageInfo.name, + version: '0.0.0-private', + private: true, + }); + } +} +``` + +Replace `writeUploadableArtifactFixtures` with: + +```javascript +async function writeUploadableArtifactFixtures(layout) { + await mkdir(layout.npmDir, { recursive: true }); + await mkdir(layout.pythonDir, { recursive: true }); + + const fileContents = new Map([ + ...NPM_ARTIFACT_PACKAGES.map((packageInfo) => [ + layout.npmTarballs[packageInfo.name], + `${packageInfo.name}-tarball`, + ]), + [join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), 'kaelio-ktx-runtime-wheel'], + ]); + + for (const [path, contents] of fileContents) { + await writeFile(path, contents); + } +} +``` + +In `releasePolicy`, replace the `python` object with: + +```javascript + python: { + publish: false, + repository: null, + packages: ['kaelio-ktx'], + ...pythonOverrides, + }, +``` + +- [ ] **Step 2: Update readiness report expectations** + +In `scripts/release-readiness.test.mjs`, replace every expected +`packageNames` array with: + +```javascript + packageNames: ['@kaelio/ktx', 'kaelio-ktx'], +``` + +There are three report assertions to update: + +- `accepts the current ci-artifact-only policy, package metadata, and artifact manifest` +- `reports required published package smoke when release mode requires it` +- `accepts the npm public release ready policy` + +- [ ] **Step 3: Update checked release policy** + +In `release-policy.json`, replace the `python.packages` value with: + +```json + "packages": ["kaelio-ktx"] +``` + +- [ ] **Step 4: Run readiness tests and verify pass** + +Run: + +```bash +node --test scripts/release-readiness.test.mjs +``` + +Expected: PASS. The output includes `# fail 0`. + +- [ ] **Step 5: Commit release policy cleanup** + +Run: + +```bash +git add release-policy.json scripts/release-readiness.test.mjs +git commit -m "chore: align release policy with bundled runtime wheel" +``` + +### Task 4: Document the single release artifact surface + +**Files:** + +- Modify: `scripts/examples-docs.test.mjs` +- Modify: `README.md` +- Modify: `examples/package-artifacts/README.md` +- Test: `scripts/examples-docs.test.mjs` + +- [ ] **Step 1: Add failing docs assertions** + +In `scripts/examples-docs.test.mjs`, inside +`it('documents the public package artifact smoke shape', ...)`, add these +assertions after the existing `assert.match(readme, /managed Python runtime/);` +line: + +```javascript + assert.match(readme, /public `@kaelio\/ktx` npm tarball and the bundled `kaelio-ktx` runtime wheel/); + assert.match(readme, /does not install standalone Python packages directly/); + assert.doesNotMatch(readme, /standalone Python distributions/); + assert.doesNotMatch(readme, /installs the Python artifacts directly/); +``` + +In `it('documents public npm and managed runtime usage in the README', ...)`, +add these assertions after the existing `uv` assertions: + +```javascript + assert.match(rootReadme, /release artifact manifest contains the public npm tarball and the bundled `kaelio-ktx` runtime wheel/); + assert.match(rootReadme, /source packages for development, not public release artifacts/); +``` + +- [ ] **Step 2: Run docs tests and verify failure** + +Run: + +```bash +node --test scripts/examples-docs.test.mjs +``` + +Expected: FAIL. The failure mentions the missing single-artifact wording in +`README.md` or `examples/package-artifacts/README.md`. + +- [ ] **Step 3: Update the package artifact example README** + +In `examples/package-artifacts/README.md`, replace: + +```markdown +The Python smoke project still installs the Python artifacts directly because +it verifies the standalone Python distributions that feed the bundled runtime +wheel. +``` + +with: + +```markdown +The artifact manifest contains the public `@kaelio/ktx` npm tarball and the +bundled `kaelio-ktx` runtime wheel. The smoke does not install standalone +Python packages directly; Python-backed behavior is verified through the +managed runtime installed from the npm package. +``` + +- [ ] **Step 4: Update the root README release status** + +In `README.md`, in the `## Release status` section, replace this paragraph: + +```markdown +This repository builds one public npm artifact named `@kaelio/ktx`. The first +public npm handoff is policy-gated through `release-policy.json`, which keeps +Python package publishing disabled because KTX-owned Python code ships inside +the npm package as a bundled wheel. +``` + +with: + +```markdown +This repository builds one public npm artifact named `@kaelio/ktx`. The release +artifact manifest contains the public npm tarball and the bundled `kaelio-ktx` +runtime wheel. The first public npm handoff is policy-gated through +`release-policy.json`, which keeps Python package publishing disabled because +KTX-owned Python code ships inside the npm package as a bundled wheel. The +`python/ktx-sl` and `python/ktx-daemon` directories remain source packages for +development, not public release artifacts. +``` + +- [ ] **Step 5: Run docs tests and verify pass** + +Run: + +```bash +node --test scripts/examples-docs.test.mjs +``` + +Expected: PASS. The output includes `# fail 0`. + +- [ ] **Step 6: Commit docs cleanup** + +Run: + +```bash +git add README.md examples/package-artifacts/README.md scripts/examples-docs.test.mjs +git commit -m "docs: describe single public runtime artifact surface" +``` + +### Task 5: Verify the cleaned release artifact contract + +**Files:** + +- Verify: `scripts/package-artifacts.mjs` +- Verify: `scripts/package-artifacts.test.mjs` +- Verify: `scripts/release-readiness.test.mjs` +- Verify: `scripts/examples-docs.test.mjs` +- Verify: `release-policy.json` +- Verify: `README.md` +- Verify: `examples/package-artifacts/README.md` + +- [ ] **Step 1: Run focused tests** + +Run: + +```bash +node --test scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs +``` + +Expected: PASS. The output includes `# fail 0`. + +- [ ] **Step 2: Verify stale artifact strings are gone from production/docs files** + +Run (scans only production and docs files, not test files — test files keep guard assertions that reference the removed strings): + +```bash +rg -n "uv', \\['build', '--package', 'ktx-sl'|uv', \\['build', '--package', 'ktx-daemon'|ktx_sl-0\\.1\\.0|ktx_daemon-0\\.1\\.0|pythonArtifactInstallArgs|pythonVerifySource|verifyPythonArtifacts|standalone Python distributions|installs the Python artifacts directly" scripts/package-artifacts.mjs scripts/release-readiness.mjs README.md examples/package-artifacts/README.md release-policy.json +``` + +Expected: no matches. + +- [ ] **Step 3: Verify release readiness against the current artifact manifest** + +Run: + +```bash +pnpm run release:readiness -- --json +``` + +Expected: PASS when `dist/artifacts/manifest.json` has been rebuilt after this +change. The JSON output contains: + +```json +{ + "releaseMode": "npm-public-release-ready", + "packageNames": ["@kaelio/ktx", "kaelio-ktx"], + "pythonPublishEnabled": false +} +``` + +If this command fails because the local artifact manifest was generated before +the cleanup, run: + +```bash +pnpm run artifacts:check +pnpm run release:readiness -- --json +``` + +Expected: both commands pass. The rebuilt manifest contains only +`npm/kaelio-ktx-0.1.0.tgz` and +`python/kaelio_ktx-0.1.0-py3-none-any.whl` under `files`. + +- [ ] **Step 4: Run pre-commit on changed files when configured** + +Run: + +```bash +uv run pre-commit run --files scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs release-policy.json README.md examples/package-artifacts/README.md +``` + +Expected: PASS. If pre-commit is not installed or no pre-commit config exists, +record the exact error and keep the focused Node test output from Step 1. + +- [ ] **Step 5: Commit final verification fixes if needed** + +If Step 1, Step 2, Step 3, or Step 4 required code or docs fixes, commit them: + +```bash +git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs release-policy.json README.md examples/package-artifacts/README.md +git commit -m "test: verify single public runtime artifact contract" +``` + +If no fixes were required after the previous commits, do not create an empty +commit. + +## Acceptance criteria + +- `scripts/package-artifacts.mjs` builds TypeScript packages, builds the + bundled `kaelio-ktx` runtime wheel, copies it into CLI assets, and packs the + public `@kaelio/ktx` npm tarball. +- `scripts/package-artifacts.mjs` no longer builds `ktx-sl` or `ktx-daemon` + standalone wheel or source-distribution artifacts. +- Artifact manifests contain release metadata for `@kaelio/ktx` and + `kaelio-ktx` only. +- `release-policy.json` lists only `@kaelio/ktx` under `npm.packages` and only + `kaelio-ktx` under `python.packages`. +- The artifact smoke verifies Python-backed behavior through the installed + public npm package and managed runtime, not by installing standalone Python + artifacts directly. +- Public docs state that `python/ktx-sl` and `python/ktx-daemon` remain source + packages for development, not public release artifacts. + +## Self-review + +Spec coverage: + +- The plan preserves the single public npm package requirement. +- The plan preserves the bundled KTX-owned Python wheel requirement. +- The plan keeps Python package publishing disabled. +- The plan removes the only remaining artifact path that treated KTX-owned + Python source packages as standalone release artifacts. + +Placeholder scan: + +- No steps contain placeholder implementation text. +- Every code-changing step names exact files and provides concrete replacement + snippets. + +Type and name consistency: + +- Public npm package name remains `@kaelio/ktx`. +- Bundled runtime distribution name remains `kaelio-ktx`. +- Runtime wheel filename remains `kaelio_ktx-0.1.0-py3-none-any.whl`. +- Removed standalone Python artifact names are consistently `ktx-sl` and + `ktx-daemon`. diff --git a/docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md b/docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md new file mode 100644 index 00000000..bc089765 --- /dev/null +++ b/docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md @@ -0,0 +1,234 @@ +# npm-managed Python runtime design + +This spec defines how KTX ships as one visible npm package while still using +Python for sqlglot, semantic-layer planning, database-agent compute, and local +embeddings. The goal is a user experience where users install or run only +`@kaelio/ktx`, and KTX manages its Python runtime automatically when a command +needs it. + +## Goals + +KTX must be usable through the npm package `@kaelio/ktx` with a `ktx` binary. +Users can run KTX without learning about the Python packages that power parts of +the system. + +The first release must support these invocation modes: + +- `npx @kaelio/ktx setup demo` +- `npx @kaelio/ktx sl query ...` +- `npm install @kaelio/ktx`, followed by `npx ktx ...` +- `npm install -g @kaelio/ktx`, followed by `ktx ...` + +KTX-owned Python code must ship inside the npm package as a bundled wheel. KTX +doesn't need to publish its own Python code to PyPI for this release. + +## Non-goals + +This release does not need to provide a public TypeScript SDK split across +multiple npm packages. The internal workspace package layout can remain useful +for development, but the public npm surface is a single package. + +This release does not need a fully offline install. KTX's own Python wheel is +bundled, but third-party Python dependencies can come from PyPI through `uv`. + +This release does not install local embedding dependencies by default. Local +embeddings remain lazy because `sentence-transformers`, `torch`, and model +downloads are large. + +## Package model + +KTX publishes one public npm package: + +```text +@kaelio/ktx +``` + +That package exposes one binary: + +```json +{ + "bin": { + "ktx": "./dist/bin.js" + } +} +``` + +The npm package includes these assets: + +- Bundled JavaScript CLI output. +- Packaged demo assets. +- One KTX-owned Python wheel, for example + `python/kaelio_ktx-0.1.0-py3-none-any.whl`. +- A wheel checksum or runtime manifest that lets the CLI verify the bundled + Python payload before installation. + +The Python wheel contains the current `semantic_layer` and `ktx_daemon` +modules. It exposes at least the `ktx-daemon` console script. + +## Runtime installation + +KTX creates a managed Python runtime only when a command needs Python-backed +behavior. The runtime lives outside the npm cache so it survives `npx` runs. + +The runtime root is platform-specific: + +- macOS: `~/Library/Application Support/kaelio/ktx/runtime` +- Linux: `${XDG_DATA_HOME:-~/.local/share}/kaelio/ktx/runtime` +- Windows: `%LOCALAPPDATA%/Kaelio/KTX/runtime` + +The runtime is versioned by the npm package version. A versioned runtime avoids +mixing JavaScript and Python code from incompatible releases. + +The installer performs these steps: + +1. Locate `uv`. +2. Create a virtual environment under the versioned runtime directory. +3. Install the bundled KTX wheel into that environment. +4. Write a runtime manifest with the CLI version, wheel checksum, Python + executable, daemon executable, and installed feature set. + +For lightweight Python support, the install command uses the bundled wheel's +default dependency set. For local embeddings, the installer adds the embeddings +extra only when selected: + +```bash +uv pip install "/path/to/kaelio_ktx-0.1.0-py3-none-any.whl" +uv pip install "/path/to/kaelio_ktx-0.1.0-py3-none-any.whl[local-embeddings]" +``` + +## Feature installation levels + +KTX manages Python runtime features in levels so first use stays fast. + +`core` includes: + +- `sqlglot` +- `pydantic` +- `pyyaml` +- `fastapi` +- `uvicorn` +- lightweight daemon dependencies + +`local-embeddings` adds: + +- `sentence-transformers` +- `torch` +- model download support for `all-MiniLM-L6-v2` + +Commands that only need semantic-layer SQL generation require `core`. +Commands that need local embeddings require `local-embeddings`. + +## Command behavior + +Pure TypeScript commands run without the managed Python runtime. + +Python-backed one-shot operations use the managed `ktx-daemon` executable +directly. Examples include semantic query compilation, semantic validation, +semantic source generation, and sqlglot-backed table identifier parsing. + +Repeated or expensive operations use a managed HTTP daemon. Local embeddings use +the daemon because loading the model for every one-shot process is too slow. + +KTX provides runtime management commands: + +```bash +ktx runtime install +ktx runtime status +ktx runtime start +ktx runtime stop +ktx runtime doctor +ktx runtime prune +``` + +Normal commands can install the runtime lazily. Runtime commands make that +behavior inspectable and debuggable. + +## Daemon lifecycle + +The daemon binds to `127.0.0.1` on an available random port. KTX writes daemon +state to the runtime manifest or an adjacent state file: + +```json +{ + "pid": 12345, + "port": 58731, + "version": "0.1.0", + "features": ["core", "local-embeddings"], + "startedAt": "2026-05-11T00:00:00Z" +} +``` + +Before reusing a daemon, KTX checks that the process is alive, the port responds +to `/health`, and the daemon version matches the CLI version. If any check +fails, KTX treats the daemon as stale and starts a new one. + +KTX uses one-shot Python for short operations by default. It starts the daemon +only when a command benefits from process reuse. + +## Interactive and CI behavior + +In an interactive terminal, KTX prompts before installing the managed runtime +for the first time. The prompt states that Python dependencies will be +downloaded. + +With `--yes`, KTX installs the required runtime features without prompting. + +With `--no-input`, KTX fails if a required runtime feature is missing and no +explicit auto-install flag is present. The error prints the exact command to +prepare the runtime. + +For local embeddings, KTX prompts separately because the dependency and model +downloads are larger than the core runtime. + +## Error handling + +If `uv` is missing, KTX prints a focused error that explains how to install it +and how to retry. A later release can add a bundled or downloaded `uv` strategy. + +If Python runtime installation fails, KTX preserves install logs in the runtime +directory and prints the log path. + +If the daemon fails to start, KTX prints the captured daemon stdout and stderr +path. It falls back to one-shot mode only when the requested operation supports +one-shot execution. + +If JavaScript and Python versions don't match, KTX reinstalls the managed +runtime for the current npm package version. + +## Release flow + +The release builds the Python wheel before packing npm artifacts. The npm pack +step includes the wheel as an asset. + +Release checks must cover: + +1. Clean install of the packed npm package. +2. `npx` execution of the packed package. +3. First-run managed runtime install from the bundled wheel. +4. One-shot semantic-layer query through the managed runtime. +5. Runtime status and doctor output. +6. Daemon start, health check, reuse, and stop. +7. Optional local embeddings smoke in a separate job or opt-in check. + +## Open decisions + +KTX still needs a final decision on whether `uv` is a hard prerequisite or a +bootstrap dependency that KTX downloads automatically. + +KTX also needs the final Python distribution name. This spec uses +`kaelio-ktx` as the distribution name and `kaelio_ktx` in wheel filenames. + +## Success criteria + +Users can run `npx @kaelio/ktx ...` and complete Python-backed KTX operations +without manually installing a KTX Python package. + +Users who install `@kaelio/ktx` locally can run `npx ktx ...` through the local +project's npm binary resolution. + +The first Python-backed command installs only the core runtime. Local embedding +dependencies install only after the user selects local embeddings or explicitly +requests the `local-embeddings` runtime feature. + +KTX can diagnose and repair stale or mismatched managed runtimes without asking +users to delete directories manually. diff --git a/examples/package-artifacts/README.md b/examples/package-artifacts/README.md index 2db3817b..67c8d212 100644 --- a/examples/package-artifacts/README.md +++ b/examples/package-artifacts/README.md @@ -4,14 +4,21 @@ The package artifact smoke checks create temporary projects instead of storing sample projects in this directory. Run the checks from `ktx/`: ```bash -source .venv/bin/activate pnpm run artifacts:check ``` -The npm smoke project installs the generated `@ktx/context` and `@ktx/cli` -tarballs, imports public package entry points, and runs installed `ktx` -commands against a generated local project. +The npm smoke project installs the generated public `@kaelio/ktx` tarball, +imports the package entry point, and runs installed `ktx` commands against a +generated local project. -The Python smoke project installs `ktx-daemon` through the local artifact -directory, imports `semantic_layer` and `ktx_daemon`, and runs -`python -m ktx_daemon semantic-validate`. +The managed Python runtime smoke requires `uv` on `PATH`, isolates +`KTX_RUNTIME_ROOT`, verifies `ktx runtime status`, runs `ktx sl query --yes` to +install the core runtime from the bundled wheel, checks `ktx runtime doctor`, +starts and reuses the managed daemon, stops it, previews a stale runtime with +`ktx runtime prune --dry-run`, verifies confirmation is required, and removes +the stale runtime with `ktx runtime prune --yes`. + +The artifact manifest contains the public `@kaelio/ktx` npm tarball and the +bundled `kaelio-ktx` runtime wheel. The smoke does not install standalone +Python packages directly; Python-backed behavior is verified through the +managed runtime installed from the npm package. diff --git a/examples/postgres-historic/README.md b/examples/postgres-historic/README.md index 1a97cba2..f97d4b9b 100644 --- a/examples/postgres-historic/README.md +++ b/examples/postgres-historic/README.md @@ -13,8 +13,8 @@ generates query workload under separate users, runs `ktx setup` with - Docker with Compose v2 - Node and pnpm matching the KTX workspace -- `KTX_SQL_ANALYSIS_URL` or `KTX_DAEMON_URL` pointing at a running SQL-analysis - service that exposes `/api/sql/analyze-for-fingerprint` +- `uv` on `PATH` so the KTX-managed Python runtime can install the bundled + runtime wheel ## Run @@ -24,8 +24,9 @@ From the KTX repository root: examples/postgres-historic/scripts/smoke.sh ``` -The smoke creates a temporary KTX project, starts Postgres on -`127.0.0.1:55432`, and uses this connection URL: +The smoke creates a temporary KTX project, isolates the managed Python runtime +under the temporary project parent, starts Postgres on `127.0.0.1:55432`, and +uses this connection URL: ```bash postgresql://ktx_reader:ktx_reader@127.0.0.1:55432/analytics # pragma: allowlist secret @@ -83,10 +84,11 @@ Historic SQL (warehouse)` when `pg_stat_statements` is installed, Run local historic-SQL ingest: ```bash -node packages/cli/dist/bin.js --project-dir /tmp/ktx-postgres-historic dev ingest run \ +pnpm run ktx -- dev ingest run --project-dir /tmp/ktx-postgres-historic \ --connection-id warehouse \ --adapter historic-sql \ --plain \ + --yes \ --no-input ``` @@ -111,5 +113,6 @@ The manifest should have `dialect: "postgres"`, `degraded: true`, - Missing grants: confirm `GRANT pg_read_all_stats TO ktx_reader;`. - Empty templates: rerun `scripts/generate-workload.sh base` and keep `--historic-sql-min-calls 2` for the smoke. -- SQL-analysis failures: set `KTX_SQL_ANALYSIS_URL` or `KTX_DAEMON_URL` to a - running service URL before running `scripts/smoke.sh`. +- SQL-analysis failures: run `pnpm run ktx -- runtime doctor` from the KTX + repository root and confirm `uv`, the bundled Python wheel, and the managed + runtime all pass. diff --git a/examples/postgres-historic/scripts/smoke.sh b/examples/postgres-historic/scripts/smoke.sh index 5b1be929..488535a4 100755 --- a/examples/postgres-historic/scripts/smoke.sh +++ b/examples/postgres-historic/scripts/smoke.sh @@ -8,22 +8,20 @@ COMPOSE_FILE="$EXAMPLE_DIR/docker-compose.yml" PROJECT_PARENT="${KTX_POSTGRES_HISTORIC_PROJECT_PARENT:-$(mktemp -d)}" PROJECT_DIR="$PROJECT_PARENT/postgres-historic-ktx" KTX_BIN="$KTX_ROOT/packages/cli/dist/bin.js" +export KTX_RUNTIME_ROOT="$PROJECT_PARENT/managed-runtime" +unset KTX_DAEMON_URL +unset KTX_SQL_ANALYSIS_URL cleanup() { + if [[ -f "$KTX_BIN" ]]; then + node "$KTX_BIN" runtime stop >/dev/null 2>&1 || true + fi if [[ "${KTX_POSTGRES_HISTORIC_KEEP_DOCKER:-0}" != "1" ]]; then docker compose -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 || true fi } trap cleanup EXIT -require_sql_analysis_url() { - if [[ -n "${KTX_SQL_ANALYSIS_URL:-}" || -n "${KTX_DAEMON_URL:-}" ]]; then - return - fi - echo "Set KTX_SQL_ANALYSIS_URL or KTX_DAEMON_URL before running this smoke." >&2 - exit 1 -} - latest_manifest() { find "$PROJECT_DIR/raw-sources/warehouse/historic-sql" -name manifest.json | sort | tail -n 1 } @@ -60,9 +58,19 @@ const jobId = process.argv[4]; const { loadKtxProject } = await import(join(ktxRoot, 'packages/context/dist/project/index.js')); const { runLocalStageOnlyIngest } = await import(join(ktxRoot, 'packages/context/dist/ingest/index.js')); const { createKtxCliLocalIngestAdapters } = await import(join(ktxRoot, 'packages/cli/dist/local-adapters.js')); +const { getKtxCliPackageInfo } = await import(join(ktxRoot, 'packages/cli/dist/index.js')); const project = await loadKtxProject({ projectDir }); -const adapters = createKtxCliLocalIngestAdapters(project, { historicSqlConnectionId: 'warehouse' }); +const cliVersion = getKtxCliPackageInfo().version; +const managedRuntimeIo = { stdout: process.stdout, stderr: process.stderr }; +const adapters = createKtxCliLocalIngestAdapters(project, { + historicSqlConnectionId: 'warehouse', + managedDaemon: { + cliVersion, + installPolicy: 'auto', + io: managedRuntimeIo, + }, +}); const adapter = adapters.find((candidate) => candidate.source === 'historic-sql'); if (!adapter) throw new Error('historic-sql adapter was not registered for local run'); const record = await runLocalStageOnlyIngest({ @@ -88,7 +96,6 @@ NODE cd "$KTX_ROOT" pnpm --filter @ktx/context run build pnpm --filter @ktx/cli run build -require_sql_analysis_url docker compose -f "$COMPOSE_FILE" up -d --wait "$EXAMPLE_DIR/scripts/generate-workload.sh" base diff --git a/package.json b/package.json index bda78420..243a96eb 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,8 @@ "native:rebuild": "pnpm -r rebuild better-sqlite3", "setup:dev": "node scripts/setup-dev.mjs", "release:published-smoke": "node scripts/published-package-smoke.mjs --require-config", + "release:local-embeddings-smoke": "node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in", + "release:npm-publish": "node scripts/publish-public-npm-package.mjs", "release:readiness": "node scripts/release-readiness.mjs", "relationships:acquire-public-fixtures": "node scripts/acquire-public-benchmark-fixtures.mjs", "relationships:rebuild-public-snapshots": "node scripts/build-benchmark-snapshot.mjs --rebuild-all", diff --git a/packages/cli/src/agent-runtime.test.ts b/packages/cli/src/agent-runtime.test.ts index a7634103..808ddac3 100644 --- a/packages/cli/src/agent-runtime.test.ts +++ b/packages/cli/src/agent-runtime.test.ts @@ -105,4 +105,48 @@ describe('agent runtime helpers', () => { queryExecutor, }); }); + + it('creates managed semantic compute when no test override is injected', async () => { + const project = { + projectDir: tempDir, + configPath: join(tempDir, 'ktx.yaml'), + config: { project: 'revenue', connections: {} }, + coreConfig: {}, + git: {}, + fileStore: {}, + } as never; + const ports = { semanticLayer: {} } as never; + const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; + const loadProject = vi.fn(async () => project); + const createContextTools = vi.fn(() => ports); + const createManagedSemanticLayerCompute = vi.fn(async () => semanticLayerCompute); + const { io } = makeIo(); + + await expect( + createKtxAgentRuntime( + { + projectDir: tempDir, + enableSemanticCompute: true, + enableQueryExecution: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + io, + }, + { + loadProject, + createContextTools, + createManagedSemanticLayerCompute, + }, + ), + ).resolves.toMatchObject({ project, ports, semanticLayerCompute }); + + expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io, + }); + expect(createContextTools).toHaveBeenCalledWith(project, { + semanticLayerCompute, + }); + }); }); diff --git a/packages/cli/src/agent-runtime.ts b/packages/cli/src/agent-runtime.ts index 98ebcb3a..feccae7c 100644 --- a/packages/cli/src/agent-runtime.ts +++ b/packages/cli/src/agent-runtime.ts @@ -1,9 +1,13 @@ import { readFile } from 'node:fs/promises'; import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from '@ktx/context/connections'; -import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon'; +import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon'; import { createLocalProjectMcpContextPorts, type KtxMcpContextPorts } from '@ktx/context/mcp'; import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; +import { + createManagedPythonSemanticLayerComputePort, + type KtxManagedPythonInstallPolicy, +} from './managed-python-command.js'; export const KTX_AGENT_MAX_ROWS_CAP = 1000; @@ -11,6 +15,9 @@ export interface KtxAgentRuntimeOptions { projectDir: string; enableSemanticCompute: boolean; enableQueryExecution: boolean; + cliVersion?: string; + runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; + io?: KtxCliIo; } export interface KtxAgentRuntime { @@ -24,6 +31,7 @@ export interface KtxAgentRuntimeDeps { loadProject?: typeof loadKtxProject; createContextTools?: typeof createLocalProjectMcpContextPorts; createSemanticLayerCompute?: () => KtxSemanticLayerComputePort; + createManagedSemanticLayerCompute?: typeof createManagedPythonSemanticLayerComputePort; createQueryExecutor?: () => KtxSqlQueryExecutorPort; } @@ -57,14 +65,34 @@ export function parseAgentMaxRows(value: number | undefined): number { return value; } +async function createAgentSemanticLayerCompute( + options: KtxAgentRuntimeOptions, + deps: KtxAgentRuntimeDeps, +): Promise { + if (!options.enableSemanticCompute) { + return undefined; + } + if (deps.createSemanticLayerCompute) { + return deps.createSemanticLayerCompute(); + } + if (!options.cliVersion || !options.runtimeInstallPolicy || !options.io) { + throw new Error('Managed Python semantic compute requires cliVersion, runtimeInstallPolicy, and io.'); + } + const createManagedSemanticLayerCompute = + deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort; + return createManagedSemanticLayerCompute({ + cliVersion: options.cliVersion, + installPolicy: options.runtimeInstallPolicy, + io: options.io, + }); +} + export async function createKtxAgentRuntime( options: KtxAgentRuntimeOptions, deps: KtxAgentRuntimeDeps = {}, ): Promise { const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: options.projectDir }); - const semanticLayerCompute = options.enableSemanticCompute - ? (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)() - : undefined; + const semanticLayerCompute = await createAgentSemanticLayerCompute(options, deps); const queryExecutor = options.enableQueryExecution ? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)() : undefined; diff --git a/packages/cli/src/agent.test.ts b/packages/cli/src/agent.test.ts index a57e7d04..043bdddb 100644 --- a/packages/cli/src/agent.test.ts +++ b/packages/cli/src/agent.test.ts @@ -231,6 +231,8 @@ describe('runKtxAgent', () => { queryFile, execute: true, maxRows: 100, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'never', }, io.io, { createRuntime: async () => runtime() }, @@ -240,6 +242,39 @@ describe('runKtxAgent', () => { expect(JSON.parse(io.stdout())).toMatchObject({ sql: 'select 1', rows: [[1]] }); }); + it('passes managed runtime options into default SL query runtime creation', async () => { + const queryFile = join(tempDir, 'sl-query.json'); + const io = makeIo(); + const createRuntime = vi.fn(async () => runtime()); + await writeFile(queryFile, '{"measures":["total_revenue"],"dimensions":[]}', 'utf-8'); + + await expect( + runKtxAgent( + { + command: 'sl-query', + projectDir: tempDir, + json: true, + connectionId: 'warehouse', + queryFile, + execute: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + io.io, + { createRuntime }, + ), + ).resolves.toBe(0); + + expect(createRuntime).toHaveBeenCalledWith({ + projectDir: tempDir, + enableSemanticCompute: true, + enableQueryExecution: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + io: io.io, + }); + }); + it('executes read-only SQL from a SQL file with an explicit row limit', async () => { const sqlFile = join(tempDir, 'query.sql'); const fakeRuntime = runtime(); diff --git a/packages/cli/src/agent.ts b/packages/cli/src/agent.ts index ea2a224e..61d85b8c 100644 --- a/packages/cli/src/agent.ts +++ b/packages/cli/src/agent.ts @@ -17,6 +17,7 @@ import { noIndexedSourcesSlSearchReadiness, type KtxAgentSlSearchReadinessDetail, } from './agent-search-readiness.js'; +import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; import { readKtxSetupStatus, type KtxSetupStatus } from './setup.js'; export type KtxAgentArgs = @@ -32,6 +33,8 @@ export type KtxAgentArgs = queryFile: string; execute: boolean; maxRows?: number; + cliVersion: string; + runtimeInstallPolicy: KtxManagedPythonInstallPolicy; } | { command: 'wiki-search'; projectDir: string; json: true; query: string; limit: number } | { command: 'wiki-read'; projectDir: string; json: true; pageId: string } @@ -42,6 +45,9 @@ export interface KtxAgentDeps extends KtxAgentRuntimeDeps { projectDir: string; enableSemanticCompute: boolean; enableQueryExecution: boolean; + cliVersion?: string; + runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; + io?: KtxCliIo; }) => Promise; readSetupStatus?: ( projectDir: string, @@ -68,23 +74,22 @@ function writeAgentSlSearchReadinessError(io: KtxCliIo, detail: KtxAgentSlSearch writeAgentJsonError(io, detail.message, { code: detail.code, nextSteps: detail.nextSteps }); } -async function runtimeFor(args: KtxAgentArgs, deps: KtxAgentDeps): Promise { +async function runtimeFor(args: KtxAgentArgs, deps: KtxAgentDeps, io: KtxCliIo): Promise { const needsSemanticCompute = args.command === 'sl-query'; const needsQueryExecution = args.command === 'sql-execute' || (args.command === 'sl-query' && args.execute); - return deps.createRuntime - ? deps.createRuntime({ - projectDir: args.projectDir, - enableSemanticCompute: needsSemanticCompute, - enableQueryExecution: needsQueryExecution, - }) - : createKtxAgentRuntime( - { - projectDir: args.projectDir, - enableSemanticCompute: needsSemanticCompute, - enableQueryExecution: needsQueryExecution, - }, - deps, - ); + const runtimeOptions = { + projectDir: args.projectDir, + enableSemanticCompute: needsSemanticCompute, + enableQueryExecution: needsQueryExecution, + ...(args.command === 'sl-query' + ? { + cliVersion: args.cliVersion, + runtimeInstallPolicy: args.runtimeInstallPolicy, + io, + } + : {}), + }; + return deps.createRuntime ? deps.createRuntime(runtimeOptions) : createKtxAgentRuntime(runtimeOptions, deps); } function connectionIdForSource(runtime: KtxAgentRuntime, requested: string | undefined): string { @@ -101,7 +106,7 @@ export async function runKtxAgent(args: KtxAgentArgs, io: KtxCliIo, deps: KtxAge return 0; } - const runtime = await runtimeFor(args, deps); + const runtime = await runtimeFor(args, deps, io); if (args.command === 'context') { const [status, connections, semanticLayer] = await Promise.all([ diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index f96e6f1c..fd7a50ef 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -4,6 +4,7 @@ import { registerAgentCommands } from './commands/agent-commands.js'; import { registerConnectionCommands } from './commands/connection-commands.js'; import { registerWikiCommands } from './commands/knowledge-commands.js'; import { registerPublicIngestCommands } from './commands/public-ingest-commands.js'; +import { registerRuntimeCommands } from './commands/runtime-commands.js'; import { registerServeCommands } from './commands/serve-commands.js'; import { registerSetupCommands } from './commands/setup-commands.js'; import { registerSlCommands } from './commands/sl-commands.js'; @@ -17,6 +18,7 @@ profileMark('module:cli-program'); export interface KtxCliCommandContext { io: KtxCliIo; deps: KtxCliDeps; + packageInfo: KtxCliPackageInfo; setExitCode: (code: number) => void; runInit: (args: { projectDir: string; projectName?: string; force: boolean }, io: KtxCliIo) => Promise; writeDebug?: (command: string, commandContext: CommandWithGlobalOptions) => void; @@ -177,6 +179,7 @@ async function runBareInteractiveCommand( skipAgents: false, inputMode: 'auto', yes: false, + cliVersion: context.packageInfo.version, skipLlm: false, skipEmbeddings: false, databaseSchemas: [], @@ -205,6 +208,7 @@ export async function runCommanderKtxCli( const context: KtxCliCommandContext = { io, deps, + packageInfo: info, setExitCode: (code: number) => { exitCode = code; }, @@ -229,6 +233,9 @@ export async function runCommanderKtxCli( registerSlCommands(program, context); profileMark('commander:register-sl'); + registerRuntimeCommands(program, context); + profileMark('commander:register-runtime'); + registerServeCommands(program, context); profileMark('commander:register-serve'); diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index 124d132d..77b75eb8 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -1,3 +1,5 @@ +import { createRequire } from 'node:module'; + import type { KtxConnectionMetabaseSetupArgs } from './commands/connection-metabase-setup.js'; import type { KtxConnectionNotionArgs } from './commands/connection-notion.js'; import type { KtxAgentArgs } from './agent.js'; @@ -7,6 +9,7 @@ import type { KtxDoctorArgs } from './doctor.js'; import type { KtxIngestArgs } from './ingest.js'; import type { KtxKnowledgeArgs } from './knowledge.js'; import type { KtxPublicIngestArgs } from './public-ingest.js'; +import type { KtxRuntimeArgs } from './runtime.js'; import type { KtxScanArgs } from './scan.js'; import type { KtxServeArgs } from './serve.js'; import type { KtxSetupArgs } from './setup.js'; @@ -15,9 +18,11 @@ import { profileMark, profileSpan } from './startup-profile.js'; profileMark('module:cli-runtime'); +const requirePackageJson = createRequire(import.meta.url); + export interface KtxCliPackageInfo { - name: '@ktx/cli'; - version: '0.0.0-private'; + name: string; + version: string; contextPackageName: '@ktx/context'; } @@ -37,15 +42,31 @@ export interface KtxCliDeps { doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise; ingest?: (args: KtxIngestArgs, io: KtxCliIo) => Promise; publicIngest?: (args: KtxPublicIngestArgs, io: KtxCliIo) => Promise; + runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise; scan?: (args: KtxScanArgs, io: KtxCliIo) => Promise; knowledge?: (args: KtxKnowledgeArgs, io: KtxCliIo) => Promise; sl?: (args: KtxSlArgs, io: KtxCliIo) => Promise; } export function getKtxCliPackageInfo(): KtxCliPackageInfo { + return packageInfoFromJson(requirePackageJson('../package.json')); +} + +export function packageInfoFromJson(packageJson: unknown): KtxCliPackageInfo { + if ( + typeof packageJson !== 'object' || + packageJson === null || + !('name' in packageJson) || + !('version' in packageJson) || + typeof packageJson.name !== 'string' || + typeof packageJson.version !== 'string' + ) { + throw new Error('Invalid KTX CLI package metadata'); + } + return { - name: '@ktx/cli', - version: '0.0.0-private', + name: packageJson.name, + version: packageJson.version, contextPackageName: '@ktx/context', }; } diff --git a/packages/cli/src/command-schemas.ts b/packages/cli/src/command-schemas.ts index 0e251b96..1a442af7 100644 --- a/packages/cli/src/command-schemas.ts +++ b/packages/cli/src/command-schemas.ts @@ -64,6 +64,8 @@ export const slQueryCommandSchema = z.object({ }), format: z.enum(['json', 'sql']), execute: z.boolean(), + cliVersion: z.string().min(1), + runtimeInstallPolicy: z.enum(['prompt', 'auto', 'never']), maxRows: z.number().int().positive().optional(), }); diff --git a/packages/cli/src/commands/agent-commands.ts b/packages/cli/src/commands/agent-commands.ts index 57cc94c3..2593991a 100644 --- a/packages/cli/src/commands/agent-commands.ts +++ b/packages/cli/src/commands/agent-commands.ts @@ -2,6 +2,7 @@ import { Option, type Command } from '@commander-js/extra-typings'; import type { KtxAgentArgs } from '../agent.js'; import type { KtxCliCommandContext } from '../cli-program.js'; import { parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js'; +import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; async function runAgent(context: KtxCliCommandContext, args: KtxAgentArgs): Promise { const runner = context.deps.agent ?? (await import('../agent.js')).runKtxAgent; @@ -73,10 +74,19 @@ export function registerAgentCommands(program: Command, context: KtxCliCommandCo .requiredOption('--connection-id ', 'Connection id for execution') .requiredOption('--query-file ', 'JSON semantic-layer query file') .option('--execute', 'Execute the compiled query against the connection', false) + .option('--yes', 'Install the managed Python runtime without prompting when required', false) + .option('--no-input', 'Disable interactive managed runtime installation') .option('--max-rows ', 'Maximum rows to return when executing', parsePositiveIntegerOption) .action( async ( - options: { connectionId: string; queryFile: string; execute: boolean; maxRows?: number }, + options: { + connectionId: string; + queryFile: string; + execute: boolean; + maxRows?: number; + yes?: boolean; + input?: boolean; + }, command, ) => { await runAgent(context, { @@ -86,6 +96,8 @@ export function registerAgentCommands(program: Command, context: KtxCliCommandCo connectionId: options.connectionId, queryFile: options.queryFile, execute: options.execute, + cliVersion: context.packageInfo.version, + runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), ...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}), }); }, diff --git a/packages/cli/src/commands/ingest-commands.ts b/packages/cli/src/commands/ingest-commands.ts index e546c4c4..772c107d 100644 --- a/packages/cli/src/commands/ingest-commands.ts +++ b/packages/cli/src/commands/ingest-commands.ts @@ -3,6 +3,7 @@ import { type Command, Option } from '@commander-js/extra-typings'; import { type KtxCliCommandContext, type OutputModeOptions, resolveCommandProjectDir } from '../cli-program.js'; import type { KtxCliDeps, KtxCliIo } from '../index.js'; import type { KtxIngestArgs, KtxIngestOutputMode } from '../ingest.js'; +import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; import { profileMark } from '../startup-profile.js'; profileMark('module:commands/ingest-commands'); @@ -75,6 +76,7 @@ export function registerIngestCommands( .addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz'])) .addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz'])) .addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json'])) + .option('--yes', 'Install the managed Python runtime without prompting when required', false) .option('--no-input', 'Disable interactive terminal input for visualization') .action(async (options, command) => { if (options.reportFile) { @@ -89,6 +91,8 @@ export function registerIngestCommands( adapter: options.adapter, sourceDir: options.sourceDir ? resolve(options.sourceDir) : undefined, databaseIntrospectionUrl: options.databaseIntrospectionUrl || undefined, + cliVersion: context.packageInfo.version, + runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), ...(options.debugLlmRequestFile ? { debugLlmRequestFile: resolve(options.debugLlmRequestFile) } : {}), outputMode: outputMode(options), ...inputMode(options), diff --git a/packages/cli/src/commands/runtime-commands.ts b/packages/cli/src/commands/runtime-commands.ts new file mode 100644 index 00000000..8f478658 --- /dev/null +++ b/packages/cli/src/commands/runtime-commands.ts @@ -0,0 +1,100 @@ +import { type Command, Option } from '@commander-js/extra-typings'; +import type { KtxCliCommandContext } from '../cli-program.js'; +import type { KtxRuntimeArgs } from '../runtime.js'; + +type RuntimeFeature = Extract['feature']; + +function createRuntimeFeatureOption() { + return new Option('--feature ', 'Runtime feature level') + .choices(['core', 'local-embeddings']) + .default('core'); +} + +async function runRuntimeArgs(context: KtxCliCommandContext, args: KtxRuntimeArgs): Promise { + const runner = context.deps.runtime ?? (await import('../runtime.js')).runKtxRuntime; + context.setExitCode(await runner(args, context.io)); +} + +export function registerRuntimeCommands(program: Command, context: KtxCliCommandContext): void { + const runtime = program + .command('runtime') + .description('Install, inspect, and prune the KTX-managed Python runtime') + .showHelpAfterError(); + + runtime + .command('install') + .description('Install the bundled Python runtime wheel into the managed runtime') + .addOption(createRuntimeFeatureOption()) + .option('--yes', 'Accept runtime installation without prompting', false) + .option('--force', 'Reinstall even when the runtime already looks ready', false) + .action(async (options: { feature: RuntimeFeature; yes?: boolean; force?: boolean }) => { + await runRuntimeArgs(context, { + command: 'install', + cliVersion: context.packageInfo.version, + feature: options.feature, + force: options.force === true, + }); + }); + + runtime + .command('start') + .description('Start the KTX-managed Python HTTP daemon') + .addOption(createRuntimeFeatureOption()) + .option('--force', 'Restart even when a matching daemon is already running', false) + .action(async (options: { feature: RuntimeFeature; force?: boolean }) => { + await runRuntimeArgs(context, { + command: 'start', + cliVersion: context.packageInfo.version, + feature: options.feature, + force: options.force === true, + }); + }); + + runtime + .command('stop') + .description('Stop the KTX-managed Python HTTP daemon') + .action(async () => { + await runRuntimeArgs(context, { + command: 'stop', + cliVersion: context.packageInfo.version, + }); + }); + + runtime + .command('status') + .description('Show managed Python runtime status') + .option('--json', 'Print JSON output', false) + .action(async (options: { json?: boolean }) => { + await runRuntimeArgs(context, { + command: 'status', + cliVersion: context.packageInfo.version, + json: options.json === true, + }); + }); + + runtime + .command('doctor') + .description('Check managed Python runtime prerequisites and installation') + .option('--json', 'Print JSON output', false) + .action(async (options: { json?: boolean }) => { + await runRuntimeArgs(context, { + command: 'doctor', + cliVersion: context.packageInfo.version, + json: options.json === true, + }); + }); + + runtime + .command('prune') + .description('Remove stale managed Python runtimes for older CLI versions') + .option('--dry-run', 'List stale runtimes without deleting them', false) + .option('--yes', 'Confirm deletion of stale runtime directories', false) + .action(async (options: { dryRun?: boolean; yes?: boolean }) => { + await runRuntimeArgs(context, { + command: 'prune', + cliVersion: context.packageInfo.version, + dryRun: options.dryRun === true, + yes: options.yes === true, + }); + }); +} diff --git a/packages/cli/src/commands/scan-commands.ts b/packages/cli/src/commands/scan-commands.ts index 9f3d35f7..fc30fafa 100644 --- a/packages/cli/src/commands/scan-commands.ts +++ b/packages/cli/src/commands/scan-commands.ts @@ -1,5 +1,6 @@ import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings'; import { type KtxCliCommandContext, parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js'; +import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; import type { KtxScanArgs } from '../scan.js'; import { profileMark } from '../startup-profile.js'; @@ -102,6 +103,8 @@ export function registerScanCommands(program: Command, context: KtxCliCommandCon ) .option('--dry-run', 'Run without writing scan results', false) .option('--database-introspection-url ', 'Daemon URL for live-database introspection') + .option('--yes', 'Install the managed Python runtime without prompting when required', false) + .option('--no-input', 'Disable interactive managed runtime installation') .showHelpAfterError() .addHelpText( 'after', @@ -126,6 +129,8 @@ export function registerScanCommands(program: Command, context: KtxCliCommandCon detectRelationships: mode === 'relationships', dryRun: options.dryRun === true, databaseIntrospectionUrl: options.databaseIntrospectionUrl, + cliVersion: context.packageInfo.version, + runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), }); }); diff --git a/packages/cli/src/commands/serve-commands.ts b/packages/cli/src/commands/serve-commands.ts index b7e659fb..28acf32d 100644 --- a/packages/cli/src/commands/serve-commands.ts +++ b/packages/cli/src/commands/serve-commands.ts @@ -1,5 +1,6 @@ import { type Command, InvalidArgumentError } from '@commander-js/extra-typings'; import { type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js'; +import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; import type { KtxServeArgs } from '../serve.js'; import { profileMark } from '../startup-profile.js'; @@ -20,6 +21,8 @@ export function registerServeCommands(program: Command, context: KtxCliCommandCo .option('--user-id ', 'Local user id', 'local') .option('--semantic-compute', 'Enable semantic-layer compute', false) .option('--semantic-compute-url ', 'HTTP semantic-layer compute URL') + .option('--yes', 'Install the managed Python runtime without prompting when required', false) + .option('--no-input', 'Disable interactive managed runtime installation') .option('--database-introspection-url ', 'Daemon URL for live-database introspection') .option('--execute-queries', 'Allow semantic-layer query execution', false) .option('--memory-capture', 'Enable memory capture', false) @@ -40,6 +43,8 @@ export function registerServeCommands(program: Command, context: KtxCliCommandCo executeQueries: options.executeQueries === true, memoryCapture: options.memoryCapture === true, memoryModel: options.memoryModel, + cliVersion: context.packageInfo.version, + runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), }; const runner = context.deps.serveStdio ?? (await import('../serve.js')).runKtxServeStdio; context.setExitCode(await runner(args)); diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index 1d5a933a..16e3cc28 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -371,6 +371,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo skipAgents: options.skipAgents === true, inputMode: options.input === false ? 'disabled' : 'auto', yes: options.yes === true, + cliVersion: context.packageInfo.version, ...(options.anthropicApiKeyEnv ? { anthropicApiKeyEnv: options.anthropicApiKeyEnv } : {}), ...(options.anthropicApiKeyFile ? { anthropicApiKeyFile: options.anthropicApiKeyFile } : {}), ...(options.anthropicModel ? { anthropicModel: options.anthropicModel } : {}), diff --git a/packages/cli/src/commands/sl-commands.ts b/packages/cli/src/commands/sl-commands.ts index 24ab95aa..36d75fac 100644 --- a/packages/cli/src/commands/sl-commands.ts +++ b/packages/cli/src/commands/sl-commands.ts @@ -6,6 +6,7 @@ import { resolveCommandProjectDir, } from '../cli-program.js'; import { slQueryCommandSchema } from '../command-schemas.js'; +import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; import type { KtxSlArgs } from '../sl.js'; import { profileMark } from '../startup-profile.js'; @@ -121,6 +122,8 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte .option('--include-empty', 'Include empty rows', false) .addOption(new Option('--format ', 'json or sql').choices(['json', 'sql']).default('json')) .option('--execute', 'Execute the compiled query', false) + .option('--yes', 'Install the managed Python runtime without prompting when required', false) + .option('--no-input', 'Disable interactive managed runtime installation') .option('--max-rows ', 'Maximum rows to return when executing', parsePositiveIntegerOption) .action(async (options, command) => { if (options.measure.length === 0) { @@ -141,6 +144,8 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte }, format: options.format, execute: options.execute === true, + cliVersion: context.packageInfo.version, + runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), ...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}), }); await runSlArgs(context, args); diff --git a/packages/cli/src/dev.test.ts b/packages/cli/src/dev.test.ts index 639244a6..167513d5 100644 --- a/packages/cli/src/dev.test.ts +++ b/packages/cli/src/dev.test.ts @@ -234,6 +234,8 @@ describe('dev Commander tree', () => { detectRelationships: false, dryRun: true, databaseIntrospectionUrl: undefined, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', }, scanIo.io, ); @@ -259,6 +261,8 @@ describe('dev Commander tree', () => { detectRelationships: true, dryRun: false, databaseIntrospectionUrl: undefined, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', }, io.io, ); @@ -661,6 +665,8 @@ describe('dev Commander tree', () => { adapter: 'metabase', sourceDir: undefined, databaseIntrospectionUrl: undefined, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', outputMode: 'json', }, io.io, diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index aa751925..f26215d1 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { getKtxCliPackageInfo, + packageInfoFromJson, rendererUnavailableVizFallback, renderMemoryFlowTui, resolveVizFallback, @@ -56,6 +57,19 @@ describe('getKtxCliPackageInfo', () => { version: '0.0.0-private', }); }); + + it('normalizes public package metadata from package.json contents', () => { + expect( + packageInfoFromJson({ + name: '@kaelio/ktx', + version: '0.1.0', + }), + ).toEqual({ + name: '@kaelio/ktx', + version: '0.1.0', + contextPackageName: '@ktx/context', + }); + }); }); describe('memory-flow renderer exports', () => { @@ -108,7 +122,7 @@ describe('runKtxCli', () => { await expect(runKtxCli(['--help'], testIo.io)).resolves.toBe(0); expect(testIo.stdout()).toContain('Usage: ktx [options] [command]'); - for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'serve', 'status']) { + for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'runtime', 'serve', 'status']) { expect(testIo.stdout()).toContain(`${command}`); } for (const removed of ['demo', 'init', 'connect', 'scan', 'ask', 'knowledge', 'agent', 'completion']) { @@ -124,6 +138,151 @@ describe('runKtxCli', () => { expect(testIo.stderr()).toBe(''); }); + it('routes runtime management commands with the CLI package version', async () => { + const runtime = vi.fn(async () => 0); + const installIo = makeIo(); + const startIo = makeIo(); + const stopIo = makeIo(); + const statusIo = makeIo(); + const doctorIo = makeIo(); + const pruneIo = makeIo(); + + await expect( + runKtxCli(['runtime', 'install', '--feature', 'local-embeddings', '--force', '--yes'], installIo.io, { + runtime, + }), + ).resolves.toBe(0); + await expect( + runKtxCli(['runtime', 'start', '--feature', 'local-embeddings', '--force'], startIo.io, { runtime }), + ).resolves.toBe(0); + await expect(runKtxCli(['runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['runtime', 'doctor'], doctorIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(0); + + expect(runtime).toHaveBeenNthCalledWith( + 1, + { + command: 'install', + cliVersion: '0.0.0-private', + feature: 'local-embeddings', + force: true, + }, + installIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 2, + { + command: 'start', + cliVersion: '0.0.0-private', + feature: 'local-embeddings', + force: true, + }, + startIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 3, + { + command: 'stop', + cliVersion: '0.0.0-private', + }, + stopIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 4, + { + command: 'status', + cliVersion: '0.0.0-private', + json: true, + }, + statusIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 5, + { + command: 'doctor', + cliVersion: '0.0.0-private', + json: false, + }, + doctorIo.io, + ); + expect(runtime).toHaveBeenNthCalledWith( + 6, + { + command: 'prune', + cliVersion: '0.0.0-private', + dryRun: true, + yes: false, + }, + pruneIo.io, + ); + }); + + it('routes sl query managed runtime install policies', async () => { + const sl = vi.fn(async () => 0); + + const promptIo = makeIo(); + await expect( + runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count'], promptIo.io, { sl }), + ).resolves.toBe(0); + expect(sl).toHaveBeenLastCalledWith( + expect.objectContaining({ + command: 'query', + projectDir: tempDir, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', + query: expect.objectContaining({ measures: ['orders.order_count'], dimensions: [] }), + }), + promptIo.io, + ); + + const autoIo = makeIo(); + await expect( + runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes'], autoIo.io, { + sl, + }), + ).resolves.toBe(0); + expect(sl).toHaveBeenLastCalledWith( + expect.objectContaining({ + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'auto', + }), + autoIo.io, + ); + + const noInputIo = makeIo(); + await expect( + runKtxCli( + ['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--no-input'], + noInputIo.io, + { sl }, + ), + ).resolves.toBe(0); + expect(sl).toHaveBeenLastCalledWith( + expect.objectContaining({ + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'never', + }), + noInputIo.io, + ); + }); + + it('rejects conflicting sl query runtime install flags', async () => { + const io = makeIo(); + const sl = vi.fn(async () => 0); + + await expect( + runKtxCli( + ['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes', '--no-input'], + io.io, + { sl }, + ), + ).resolves.toBe(1); + + expect(sl).not.toHaveBeenCalled(); + expect(io.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); + }); + it('exposes demo under setup help instead of root help', async () => { const testIo = makeIo(); @@ -179,6 +338,7 @@ describe('runKtxCli', () => { skipAgents: false, inputMode: 'auto', yes: false, + cliVersion: '0.0.0-private', skipLlm: false, skipEmbeddings: false, databaseSchemas: [], @@ -439,6 +599,8 @@ describe('runKtxCli', () => { executeQueries: false, memoryCapture: false, memoryModel: undefined, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', }); }); @@ -757,6 +919,8 @@ describe('runKtxCli', () => { adapter: 'fake', sourceDir: tempDir, databaseIntrospectionUrl: undefined, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'never', debugLlmRequestFile: `${tempDir}/debug.jsonl`, outputMode: 'json', inputMode: 'disabled', @@ -770,6 +934,60 @@ describe('runKtxCli', () => { expect(ingestReplayHelpIo.stderr()).toBe(''); }); + it('routes ingest managed runtime install policies', async () => { + const autoIo = makeIo(); + const conflictIo = makeIo(); + const ingest = vi.fn(async () => 0); + + await expect( + runKtxCli( + [ + 'dev', + 'ingest', + 'run', + '--project-dir', + tempDir, + '--connection-id', + 'warehouse', + '--adapter', + 'looker', + '--yes', + ], + autoIo.io, + { ingest }, + ), + ).resolves.toBe(0); + await expect( + runKtxCli( + [ + 'dev', + 'ingest', + 'run', + '--project-dir', + tempDir, + '--connection-id', + 'warehouse', + '--adapter', + 'looker', + '--yes', + '--no-input', + ], + conflictIo.io, + { ingest }, + ), + ).resolves.toBe(1); + + expect(ingest).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'run', + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'auto', + }), + autoIo.io, + ); + expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); + }); + it('dispatches public connection through the existing connection implementation', async () => { const testIo = makeIo(); const connection = vi.fn(async () => 0); @@ -870,6 +1088,7 @@ describe('runKtxCli', () => { command: 'run', projectDir: tempDir, inputMode: 'disabled', + cliVersion: '0.0.0-private', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', anthropicModel: 'claude-sonnet-4-6', skipLlm: false, @@ -977,6 +1196,7 @@ describe('runKtxCli', () => { projectDir: '/tmp/project', inputMode: 'disabled', yes: true, + cliVersion: '0.0.0-private', skipLlm: true, skipEmbeddings: true, databaseDrivers: ['postgres'], @@ -1239,6 +1459,8 @@ describe('runKtxCli', () => { queryFile: '/tmp/query.json', execute: true, maxRows: 100, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', }, }, { @@ -1287,6 +1509,104 @@ describe('runKtxCli', () => { expect(helpIo.stdout()).not.toContain('agent '); }); + it('routes hidden agent SL query managed runtime policies', async () => { + const autoIo = makeIo(); + const neverIo = makeIo(); + const conflictIo = makeIo(); + const agent = vi.fn(async () => 0); + + await expect( + runKtxCli( + [ + '--project-dir', + tempDir, + 'agent', + 'sl', + 'query', + '--json', + '--connection-id', + 'warehouse', + '--query-file', + '/tmp/query.json', + '--yes', + ], + autoIo.io, + { agent }, + ), + ).resolves.toBe(0); + + await expect( + runKtxCli( + [ + '--project-dir', + tempDir, + 'agent', + 'sl', + 'query', + '--json', + '--connection-id', + 'warehouse', + '--query-file', + '/tmp/query.json', + '--no-input', + ], + neverIo.io, + { agent }, + ), + ).resolves.toBe(0); + + await expect( + runKtxCli( + [ + '--project-dir', + tempDir, + 'agent', + 'sl', + 'query', + '--json', + '--connection-id', + 'warehouse', + '--query-file', + '/tmp/query.json', + '--yes', + '--no-input', + ], + conflictIo.io, + { agent }, + ), + ).resolves.toBe(1); + + expect(agent).toHaveBeenNthCalledWith( + 1, + { + command: 'sl-query', + projectDir: tempDir, + json: true, + connectionId: 'warehouse', + queryFile: '/tmp/query.json', + execute: false, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'auto', + }, + autoIo.io, + ); + expect(agent).toHaveBeenNthCalledWith( + 2, + { + command: 'sl-query', + projectDir: tempDir, + json: true, + connectionId: 'warehouse', + queryFile: '/tmp/query.json', + execute: false, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'never', + }, + neverIo.io, + ); + expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); + }); + it('prints semantic-layer hybrid search metadata from the hidden agent sl list command', async () => { const agent = vi.fn(async (args, io) => { expect(args).toEqual({ @@ -1797,11 +2117,48 @@ describe('runKtxCli', () => { detectRelationships: false, dryRun: false, databaseIntrospectionUrl: undefined, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', }, testIo.io, ); }); + it('routes scan managed runtime install policies', async () => { + const autoIo = makeIo(); + const neverIo = makeIo(); + const conflictIo = makeIo(); + const scan = vi.fn().mockResolvedValue(0); + + await expect(runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--yes'], autoIo.io, { scan })) + .resolves.toBe(0); + await expect(runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--no-input'], neverIo.io, { scan })) + .resolves.toBe(0); + await expect( + runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--yes', '--no-input'], conflictIo.io, { + scan, + }), + ).resolves.toBe(1); + + expect(scan).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + command: 'run', + runtimeInstallPolicy: 'auto', + }), + autoIo.io, + ); + expect(scan).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + command: 'run', + runtimeInstallPolicy: 'never', + }), + neverIo.io, + ); + expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); + }); + it('dispatches serve public command options through Commander', async () => { const serveIo = makeIo(); const serveStdio = vi.fn(async () => 0); @@ -1836,10 +2193,65 @@ describe('runKtxCli', () => { executeQueries: true, memoryCapture: true, memoryModel: 'openai/gpt-5.2', + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'prompt', }); expect(serveIo.stderr()).toBe(''); }); + it('routes serve managed runtime install policies', async () => { + const autoIo = makeIo(); + const neverIo = makeIo(); + const conflictIo = makeIo(); + const serveStdio = vi.fn(async () => 0); + + await expect( + runKtxCli(['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--yes'], autoIo.io, { + serveStdio, + }), + ).resolves.toBe(0); + await expect( + runKtxCli(['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--no-input'], neverIo.io, { + serveStdio, + }), + ).resolves.toBe(0); + await expect( + runKtxCli( + ['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--yes', '--no-input'], + conflictIo.io, + { serveStdio }, + ), + ).resolves.toBe(1); + + expect(serveStdio).toHaveBeenNthCalledWith(1, { + mcp: 'stdio', + projectDir: tempDir, + userId: 'local', + semanticCompute: true, + semanticComputeUrl: undefined, + databaseIntrospectionUrl: undefined, + executeQueries: false, + memoryCapture: false, + memoryModel: undefined, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'auto', + }); + expect(serveStdio).toHaveBeenNthCalledWith(2, { + mcp: 'stdio', + projectDir: tempDir, + userId: 'local', + semanticCompute: true, + semanticComputeUrl: undefined, + databaseIntrospectionUrl: undefined, + executeQueries: false, + memoryCapture: false, + memoryModel: undefined, + cliVersion: '0.0.0-private', + runtimeInstallPolicy: 'never', + }); + expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); + }); + it('prints dev help for bare dev commands', async () => { const testIo = makeIo(); diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 6018c28c..96fbbeec 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -2,6 +2,7 @@ import { profileMark } from './startup-profile.js'; export { getKtxCliPackageInfo, + packageInfoFromJson, runInitForCommander, runKtxCli, type KtxCliDeps, @@ -42,6 +43,26 @@ export type { KtxSetupSourceType, } from './setup-sources.js'; export { runKtxSetupSourcesStep } from './setup-sources.js'; +export { runKtxRuntime, type KtxRuntimeArgs, type KtxRuntimeDeps } from './runtime.js'; +export { + allocateDaemonPort, + readManagedPythonDaemonStatus, + startManagedPythonDaemon, + stopManagedPythonDaemon, +} from './managed-python-daemon.js'; +export type { + ManagedPythonDaemonStartResult, + ManagedPythonDaemonState, + ManagedPythonDaemonStatus, + ManagedPythonDaemonStopResult, +} from './managed-python-daemon.js'; +export { + ensureManagedLocalEmbeddingsDaemon, + managedLocalEmbeddingHealthConfig, + managedLocalEmbeddingProjectConfig, + type ManagedLocalEmbeddingsDaemon, + type ManagedLocalEmbeddingsOptions, +} from './managed-local-embeddings.js'; export type { KtxMemoryFlowTuiIo, MemoryFlowTuiLiveSession } from './memory-flow-tui.js'; export { renderMemoryFlowTui, diff --git a/packages/cli/src/ingest.test.ts b/packages/cli/src/ingest.test.ts index d1299d8a..a2784266 100644 --- a/packages/cli/src/ingest.test.ts +++ b/packages/cli/src/ingest.test.ts @@ -11,7 +11,7 @@ import { type RunLocalIngestOptions, type SourceAdapter, } from '@ktx/context/ingest'; -import { ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project'; +import { initKtxProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxIngestArgs, runKtxIngest } from './ingest.js'; import { @@ -644,6 +644,59 @@ describe('runKtxIngest', () => { adapters: createdAdapters, adapter: 'fake', connectionId: 'warehouse', + pullConfigOptions: { + databaseIntrospectionUrl: 'http://127.0.0.1:8765', + }, + }), + ); + }); + + it('passes managed daemon options to adapters and pull-config options when no explicit daemon URL is set', async () => { + const projectDir = join(tempDir, 'managed-daemon-ingest-project'); + await initKtxProject({ projectDir, projectName: 'managed-daemon-ingest-project' }); + await writeWarehouseConfig(projectDir); + const createdAdapters: SourceAdapter[] = [ + { source: 'fake', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) }, + ]; + const createAdapters = vi.fn(() => createdAdapters as never); + const runLocal = vi.fn(async (input: RunLocalIngestOptions) => + completedLocalBundleRun(input, input.jobId ?? 'local-job-1'), + ); + const io = makeIo(); + + await expect( + runKtxIngest( + { + command: 'run', + projectDir, + connectionId: 'warehouse', + adapter: 'fake', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + outputMode: 'plain', + } satisfies KtxIngestArgs, + io.io, + { + createAdapters, + runLocalIngest: runLocal, + jobIdFactory: () => 'local-job-1', + }, + ), + ).resolves.toBe(0); + + const expectedManagedDaemon = { + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + }; + expect(createAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), { + managedDaemon: expectedManagedDaemon, + }); + expect(runLocal).toHaveBeenCalledWith( + expect.objectContaining({ + pullConfigOptions: { + managedDaemon: expectedManagedDaemon, + }, }), ); }); diff --git a/packages/cli/src/ingest.ts b/packages/cli/src/ingest.ts index 2e33372c..d9f4d434 100644 --- a/packages/cli/src/ingest.ts +++ b/packages/cli/src/ingest.ts @@ -17,6 +17,7 @@ import { import { loadKtxProject } from '@ktx/context/project'; import { readIngestReportSnapshotFile } from './ingest-report-file.js'; import { createKtxCliLocalIngestAdapters } from './local-adapters.js'; +import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; import { type KtxMemoryFlowStdin, renderMemoryFlowInteractively } from './memory-flow-interactive.js'; import { type KtxMemoryFlowTuiIo, @@ -40,6 +41,8 @@ export type KtxIngestArgs = adapter: string; sourceDir?: string; databaseIntrospectionUrl?: string; + cliVersion?: string; + runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; debugLlmRequestFile?: string; outputMode: KtxIngestOutputMode; inputMode?: KtxIngestInputMode; @@ -256,6 +259,20 @@ function initialRunMemoryFlowInput( }; } +function managedDaemonOptionsForIngestRun( + args: Extract, + io: KtxIngestIo, +) { + if (args.databaseIntrospectionUrl || !args.cliVersion || !args.runtimeInstallPolicy) { + return undefined; + } + return { + cliVersion: args.cliVersion, + installPolicy: args.runtimeInstallPolicy, + io, + }; +} + async function writeReportRecord( report: IngestReportSnapshot, outputMode: KtxIngestOutputMode, @@ -311,9 +328,11 @@ export async function runKtxIngest( const createAdapters = deps.createAdapters ?? createKtxCliLocalIngestAdapters; const executeLocalIngest = deps.runLocalIngest ?? runLocalIngest; const localIngestOptions = deps.localIngestOptions ?? {}; + const managedDaemon = managedDaemonOptionsForIngestRun(args, io); const adapterOptions = { ...(localIngestOptions.pullConfigOptions ?? {}), ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), + ...(managedDaemon ? { managedDaemon } : {}), ...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}), }; if (args.adapter === 'metabase' && args.sourceDir) { @@ -380,6 +399,7 @@ export async function runKtxIngest( trigger: 'manual_resync', jobId, ...localIngestOptions, + pullConfigOptions: adapterOptions, ...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}), ...(memoryFlow ? { memoryFlow } : {}), }); diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index 9c7669e8..90d17306 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -20,6 +20,12 @@ import { } from '@ktx/context/ingest'; import type { KtxLocalProject } from '@ktx/context/project'; import { createHttpSqlAnalysisPort } from '@ktx/context/sql-analysis'; +import { + createManagedDaemonLookerTableIdentifierParser, + createManagedDaemonSqlAnalysisPort, + managedDaemonDatabaseIntrospectionOptions, + type ManagedPythonCoreDaemonOptions, +} from './managed-python-http.js'; function hasSnowflakeDriver(connection: unknown): boolean { return ( @@ -29,13 +35,55 @@ function hasSnowflakeDriver(connection: unknown): boolean { ); } +function ktxCliDaemonDatabaseIntrospectionOptions( + options: KtxCliLocalIngestAdaptersOptions, +): DefaultLocalIngestAdaptersOptions['databaseIntrospection'] { + if (options.databaseIntrospectionUrl || options.databaseIntrospection?.requestJson || !options.managedDaemon) { + return options.databaseIntrospection; + } + return { + ...(options.databaseIntrospection ?? {}), + ...managedDaemonDatabaseIntrospectionOptions(options.managedDaemon), + }; +} + +function ktxCliLookerOptions( + options: KtxCliLocalIngestAdaptersOptions, +): DefaultLocalIngestAdaptersOptions['looker'] { + const looker = options.looker; + if (looker?.parser || looker?.daemonBaseUrl || process.env.KTX_DAEMON_URL || !options.managedDaemon) { + return looker; + } + return { + ...(looker ?? {}), + parser: createManagedDaemonLookerTableIdentifierParser(options.managedDaemon), + }; +} + +function ktxCliHistoricSqlAnalysis(options: KtxCliLocalIngestAdaptersOptions) { + if (options.sqlAnalysisUrl) { + return createHttpSqlAnalysisPort({ baseUrl: options.sqlAnalysisUrl }); + } + if (process.env.KTX_SQL_ANALYSIS_URL) { + return createHttpSqlAnalysisPort({ baseUrl: process.env.KTX_SQL_ANALYSIS_URL }); + } + if (process.env.KTX_DAEMON_URL) { + return createHttpSqlAnalysisPort({ baseUrl: process.env.KTX_DAEMON_URL }); + } + if (options.managedDaemon) { + return createManagedDaemonSqlAnalysisPort(options.managedDaemon); + } + return createHttpSqlAnalysisPort({ baseUrl: 'http://127.0.0.1:8765' }); +} + function createKtxCliLiveDatabaseIntrospection( project: KtxLocalProject, - options: DefaultLocalIngestAdaptersOptions = {}, + options: KtxCliLocalIngestAdaptersOptions = {}, ): LiveDatabaseIntrospectionPort { + const databaseIntrospection = ktxCliDaemonDatabaseIntrospectionOptions(options); const daemon = createDaemonLiveDatabaseIntrospection({ connections: project.config.connections, - ...options.databaseIntrospection, + ...databaseIntrospection, ...(options.databaseIntrospectionUrl ? { baseUrl: options.databaseIntrospectionUrl } : {}), }); const sqlite = createSqliteLiveDatabaseIntrospection({ @@ -95,9 +143,10 @@ function createKtxCliLiveDatabaseIntrospection( }; } -interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions { +export interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions { historicSqlConnectionId?: string; sqlAnalysisUrl?: string; + managedDaemon?: ManagedPythonCoreDaemonOptions; } function isEnabledPostgresHistoricSqlConnection(connection: KtxPostgresConnectionConfig | undefined): boolean { @@ -145,13 +194,7 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli return undefined; } return { - sqlAnalysis: createHttpSqlAnalysisPort({ - baseUrl: - options.sqlAnalysisUrl ?? - process.env.KTX_SQL_ANALYSIS_URL ?? - process.env.KTX_DAEMON_URL ?? - 'http://127.0.0.1:8765', - }), + sqlAnalysis: ktxCliHistoricSqlAnalysis(options), postgresQueryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId), postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'), }; @@ -164,6 +207,8 @@ export function createKtxCliLocalIngestAdapters( const historicSql = historicSqlOptionsForLocalRun(project, options); const base = createDefaultLocalIngestAdapters(project, { ...options, + databaseIntrospection: ktxCliDaemonDatabaseIntrospectionOptions(options), + looker: ktxCliLookerOptions(options), ...(historicSql ? { historicSql } : {}), }); const liveDatabase = new LiveDatabaseSourceAdapter({ diff --git a/packages/cli/src/managed-local-embeddings.test.ts b/packages/cli/src/managed-local-embeddings.test.ts new file mode 100644 index 00000000..f0cb5a2f --- /dev/null +++ b/packages/cli/src/managed-local-embeddings.test.ts @@ -0,0 +1,180 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, +} from '@ktx/context'; +import { + ensureManagedLocalEmbeddingsDaemon, + managedLocalEmbeddingHealthConfig, + managedLocalEmbeddingProjectConfig, +} from './managed-local-embeddings.js'; +import type { ManagedPythonCommandRuntime } from './managed-python-command.js'; +import type { ManagedPythonDaemonStartResult } from './managed-python-daemon.js'; + +function makeIo() { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +function runtime(): ManagedPythonCommandRuntime { + return { + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', + }, + manifest: { + schemaVersion: 1, + cliVersion: '0.2.0', + installedAt: '2026-05-11T00:00:00.000Z', + asset: { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.2.0', + wheel: { + file: 'kaelio_ktx-0.2.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 123, + }, + }, + features: ['core', 'local-embeddings'], + python: { + executable: '/runtime/0.2.0/.venv/bin/python', + daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon', + }, + installLog: '/runtime/0.2.0/install.log', + }, + }; +} + +function daemonResult(status: 'started' | 'reused' = 'reused'): ManagedPythonDaemonStartResult { + return { + status, + layout: runtime().layout, + baseUrl: 'http://127.0.0.1:61234', + state: { + schemaVersion: 1, + pid: 12345, + host: '127.0.0.1', + port: 61234, + version: '0.2.0', + features: ['core', 'local-embeddings'], + startedAt: '2026-05-11T00:00:00.000Z', + stdoutLog: '/runtime/0.2.0/daemon.stdout.log', + stderrLog: '/runtime/0.2.0/daemon.stderr.log', + }, + }; +} + +describe('managedLocalEmbeddingProjectConfig', () => { + it('uses a stable managed runtime marker instead of a random daemon port', () => { + expect( + managedLocalEmbeddingProjectConfig({ + model: 'all-MiniLM-L6-v2', + dimensions: 384, + }), + ).toEqual({ + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { + base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + pathPrefix: '', + }, + }); + }); +}); + +describe('managedLocalEmbeddingHealthConfig', () => { + it('uses the active managed daemon URL for the immediate health check', () => { + expect( + managedLocalEmbeddingHealthConfig({ + baseUrl: 'http://127.0.0.1:61234', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + }), + ).toEqual({ + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, + }); + }); +}); + +describe('ensureManagedLocalEmbeddingsDaemon', () => { + it('ensures the local-embeddings feature and starts the managed daemon', async () => { + const io = makeIo(); + const ensureRuntime = vi.fn(async () => runtime()); + const startDaemon = vi.fn(async () => daemonResult('started')); + + await expect( + ensureManagedLocalEmbeddingsDaemon({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + ensureRuntime, + startDaemon, + }), + ).resolves.toEqual({ + baseUrl: 'http://127.0.0.1:61234', + env: { + [MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: 'http://127.0.0.1:61234', + }, + }); + + expect(ensureRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + feature: 'local-embeddings', + }); + expect(startDaemon).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['local-embeddings'], + force: false, + }); + expect(io.stderr()).toContain('Started KTX local embeddings daemon: http://127.0.0.1:61234'); + }); + + it('reuses an already running daemon without reporting a new start', async () => { + const io = makeIo(); + + await ensureManagedLocalEmbeddingsDaemon({ + cliVersion: '0.2.0', + installPolicy: 'prompt', + io: io.io, + ensureRuntime: vi.fn(async () => runtime()), + startDaemon: vi.fn(async () => daemonResult('reused')), + }); + + expect(io.stderr()).toContain('Using KTX local embeddings daemon: http://127.0.0.1:61234'); + }); +}); diff --git a/packages/cli/src/managed-local-embeddings.ts b/packages/cli/src/managed-local-embeddings.ts new file mode 100644 index 00000000..e47d605c --- /dev/null +++ b/packages/cli/src/managed-local-embeddings.ts @@ -0,0 +1,95 @@ +import { + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, +} from '@ktx/context'; +import type { KtxProjectEmbeddingConfig } from '@ktx/context/project'; +import type { KtxEmbeddingConfig } from '@ktx/llm'; +import type { KtxCliIo } from './cli-runtime.js'; +import { + ensureManagedPythonCommandRuntime, + type KtxManagedPythonInstallPolicy, + type ManagedPythonCommandRuntime, +} from './managed-python-command.js'; +import { startManagedPythonDaemon, type ManagedPythonDaemonStartResult } from './managed-python-daemon.js'; + +export interface ManagedLocalEmbeddingsDaemon { + baseUrl: string; + env: Record; +} + +export interface ManagedLocalEmbeddingsOptions { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + ensureRuntime?: (options: { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + feature: 'local-embeddings'; + }) => Promise; + startDaemon?: (options: { + cliVersion: string; + features: ['local-embeddings']; + force: boolean; + }) => Promise; +} + +export function managedLocalEmbeddingProjectConfig(input: { + model: string; + dimensions: number; +}): KtxProjectEmbeddingConfig { + return { + backend: 'sentence-transformers', + model: input.model, + dimensions: input.dimensions, + sentenceTransformers: { + base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + pathPrefix: '', + }, + }; +} + +export function managedLocalEmbeddingHealthConfig(input: { + baseUrl: string; + model: string; + dimensions: number; +}): KtxEmbeddingConfig { + return { + backend: 'sentence-transformers', + model: input.model, + dimensions: input.dimensions, + sentenceTransformers: { + baseURL: input.baseUrl, + pathPrefix: '', + }, + }; +} + +export async function ensureManagedLocalEmbeddingsDaemon( + options: ManagedLocalEmbeddingsOptions, +): Promise { + const ensureRuntime = options.ensureRuntime ?? ensureManagedPythonCommandRuntime; + const startDaemon = options.startDaemon ?? startManagedPythonDaemon; + + await ensureRuntime({ + cliVersion: options.cliVersion, + installPolicy: options.installPolicy, + io: options.io, + feature: 'local-embeddings', + }); + const daemon = await startDaemon({ + cliVersion: options.cliVersion, + features: ['local-embeddings'], + force: false, + }); + + const verb = daemon.status === 'started' ? 'Started' : 'Using'; + options.io.stderr.write(`${verb} KTX local embeddings daemon: ${daemon.baseUrl}\n`); + + return { + baseUrl: daemon.baseUrl, + env: { + [MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: daemon.baseUrl, + }, + }; +} diff --git a/packages/cli/src/managed-python-command.test.ts b/packages/cli/src/managed-python-command.test.ts new file mode 100644 index 00000000..d081c320 --- /dev/null +++ b/packages/cli/src/managed-python-command.test.ts @@ -0,0 +1,224 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + createManagedPythonSemanticLayerComputePort, + managedRuntimeInstallCommand, + runtimeInstallPolicyFromFlags, +} from './managed-python-command.js'; +import type { + InstalledKtxRuntimeManifest, + KtxRuntimeFeature, + ManagedPythonRuntimeInstallResult, + ManagedPythonRuntimeLayout, + ManagedPythonRuntimeStatus, +} from './managed-python-runtime.js'; + +function makeIo() { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +function layout(): ManagedPythonRuntimeLayout { + return { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', + }; +} + +function manifest(features: KtxRuntimeFeature[] = ['core']): InstalledKtxRuntimeManifest { + return { + schemaVersion: 1, + cliVersion: '0.2.0', + installedAt: '2026-05-11T00:00:00.000Z', + asset: { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.2.0', + wheel: { + file: 'kaelio_ktx-0.2.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 123, + }, + }, + features, + python: { + executable: '/runtime/0.2.0/.venv/bin/python', + daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon', + }, + installLog: '/runtime/0.2.0/install.log', + }; +} + +function readyStatus(features: KtxRuntimeFeature[] = ['core']): ManagedPythonRuntimeStatus { + return { + kind: 'ready', + detail: 'Runtime ready at /runtime/0.2.0', + layout: layout(), + manifest: manifest(features), + }; +} + +function missingStatus(): ManagedPythonRuntimeStatus { + return { + kind: 'missing', + detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', + layout: layout(), + }; +} + +function installResult(features: KtxRuntimeFeature[] = ['core']): ManagedPythonRuntimeInstallResult { + const installedManifest = manifest(features); + return { + status: 'installed', + layout: layout(), + asset: { + manifest: installedManifest.asset, + wheelPath: '/assets/python/kaelio_ktx-0.2.0-py3-none-any.whl', + }, + manifest: installedManifest, + }; +} + +describe('managedRuntimeInstallCommand', () => { + it('prints the exact command for each managed runtime feature', () => { + expect(managedRuntimeInstallCommand('core')).toBe('ktx runtime install --yes'); + expect(managedRuntimeInstallCommand('local-embeddings')).toBe( + 'ktx runtime install --feature local-embeddings --yes', + ); + }); +}); + +describe('runtimeInstallPolicyFromFlags', () => { + it('maps command flags to managed runtime install policies', () => { + expect(runtimeInstallPolicyFromFlags({})).toBe('prompt'); + expect(runtimeInstallPolicyFromFlags({ yes: false })).toBe('prompt'); + expect(runtimeInstallPolicyFromFlags({ yes: true })).toBe('auto'); + expect(runtimeInstallPolicyFromFlags({ input: false })).toBe('never'); + }); + + it('rejects conflicting runtime install flags', () => { + expect(() => runtimeInstallPolicyFromFlags({ yes: true, input: false })).toThrow( + 'Choose only one runtime install mode: --yes or --no-input', + ); + }); +}); + +describe('createManagedPythonSemanticLayerComputePort', () => { + it('uses the managed ktx-daemon executable when the runtime is ready', async () => { + const io = makeIo(); + const compute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; + const createPythonCompute = vi.fn(() => compute); + + await expect( + createManagedPythonSemanticLayerComputePort({ + cliVersion: '0.2.0', + installPolicy: 'never', + io: io.io, + readStatus: vi.fn(async () => readyStatus()), + installRuntime: vi.fn(), + createPythonCompute, + }), + ).resolves.toBe(compute); + + expect(createPythonCompute).toHaveBeenCalledWith({ + command: '/runtime/0.2.0/.venv/bin/ktx-daemon', + args: [], + }); + expect(io.stderr()).toBe(''); + }); + + it('fails with a preparation command when input is disabled and the runtime is missing', async () => { + const io = makeIo(); + const installRuntime = vi.fn(); + + await expect( + createManagedPythonSemanticLayerComputePort({ + cliVersion: '0.2.0', + installPolicy: 'never', + io: io.io, + readStatus: vi.fn(async () => missingStatus()), + installRuntime, + }), + ).rejects.toThrow('KTX Python runtime is required for this command. Run: ktx runtime install --yes'); + + expect(installRuntime).not.toHaveBeenCalled(); + }); + + it('installs the core runtime without prompting when policy is auto', async () => { + const io = makeIo(); + const compute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; + const createPythonCompute = vi.fn(() => compute); + const installRuntime = vi.fn(async () => installResult()); + + await expect( + createManagedPythonSemanticLayerComputePort({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + readStatus: vi.fn(async () => missingStatus()), + installRuntime, + createPythonCompute, + }), + ).resolves.toBe(compute); + + expect(installRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['core'], + force: false, + }); + expect(io.stderr()).toContain('Installing KTX Python runtime (core) with uv'); + expect(io.stderr()).toContain('KTX Python runtime ready: /runtime/0.2.0'); + }); + + it('prompts before installing when policy is prompt', async () => { + const io = makeIo(); + const confirmInstall = vi.fn(async () => true); + const installRuntime = vi.fn(async () => installResult()); + + await createManagedPythonSemanticLayerComputePort({ + cliVersion: '0.2.0', + installPolicy: 'prompt', + io: io.io, + readStatus: vi.fn(async () => missingStatus()), + installRuntime, + createPythonCompute: vi.fn(() => ({ query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() })), + confirmInstall, + }); + + expect(confirmInstall).toHaveBeenCalledWith( + 'KTX needs to install the core Python runtime. This downloads Python dependencies with uv. Continue?', + ); + expect(installRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['core'], + force: false, + }); + }); +}); diff --git a/packages/cli/src/managed-python-command.ts b/packages/cli/src/managed-python-command.ts new file mode 100644 index 00000000..0a8a193c --- /dev/null +++ b/packages/cli/src/managed-python-command.ts @@ -0,0 +1,135 @@ +import { cancel, confirm, isCancel } from '@clack/prompts'; +import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon'; +import type { KtxCliIo } from './cli-runtime.js'; +import { + installManagedPythonRuntime, + readManagedPythonRuntimeStatus, + type InstalledKtxRuntimeManifest, + type KtxRuntimeFeature, + type ManagedPythonRuntimeInstallOptions, + type ManagedPythonRuntimeInstallResult, + type ManagedPythonRuntimeLayout, + type ManagedPythonRuntimeLayoutOptions, + type ManagedPythonRuntimeStatus, +} from './managed-python-runtime.js'; + +export type KtxManagedPythonInstallPolicy = 'prompt' | 'auto' | 'never'; + +export function runtimeInstallPolicyFromFlags(options: { + yes?: boolean; + input?: boolean; +}): KtxManagedPythonInstallPolicy { + if (options.yes === true && options.input === false) { + throw new Error('Choose only one runtime install mode: --yes or --no-input'); + } + if (options.yes === true) { + return 'auto'; + } + return options.input === false ? 'never' : 'prompt'; +} + +export interface ManagedPythonCommandRuntime { + layout: ManagedPythonRuntimeLayout; + manifest: InstalledKtxRuntimeManifest; +} + +export interface ManagedPythonCommandDeps { + readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; + installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise; + confirmInstall?: (message: string) => Promise; +} + +export interface ManagedPythonCommandOptions extends ManagedPythonCommandDeps { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + feature?: KtxRuntimeFeature; +} + +export interface ManagedPythonSemanticLayerComputeOptions extends ManagedPythonCommandOptions { + createPythonCompute?: typeof createPythonSemanticLayerComputePort; +} + +export function managedRuntimeInstallCommand(feature: KtxRuntimeFeature): string { + return feature === 'local-embeddings' + ? 'ktx runtime install --feature local-embeddings --yes' + : 'ktx runtime install --yes'; +} + +function installPrompt(feature: KtxRuntimeFeature): string { + const label = feature === 'local-embeddings' ? 'local embeddings Python runtime' : 'core Python runtime'; + return `KTX needs to install the ${label}. This downloads Python dependencies with uv. Continue?`; +} + +function runtimeRequiredMessage(feature: KtxRuntimeFeature): string { + return `KTX Python runtime is required for this command. Run: ${managedRuntimeInstallCommand(feature)}`; +} + +function hasFeature(manifest: InstalledKtxRuntimeManifest, feature: KtxRuntimeFeature): boolean { + return manifest.features.includes(feature); +} + +async function defaultConfirmInstall(message: string): Promise { + if (process.stdin.isTTY !== true || process.stdout.isTTY !== true) { + return false; + } + const response = await confirm({ message, initialValue: true }); + if (isCancel(response)) { + cancel('Runtime installation cancelled.'); + return false; + } + return response === true; +} + +export async function ensureManagedPythonCommandRuntime( + options: ManagedPythonCommandOptions, +): Promise { + const feature = options.feature ?? 'core'; + const readStatus = options.readStatus ?? readManagedPythonRuntimeStatus; + const installRuntime = options.installRuntime ?? installManagedPythonRuntime; + const status = await readStatus({ cliVersion: options.cliVersion }); + + if (status.kind === 'ready' && status.manifest && hasFeature(status.manifest, feature)) { + return { layout: status.layout, manifest: status.manifest }; + } + + if (options.installPolicy === 'never') { + throw new Error(runtimeRequiredMessage(feature)); + } + + if (options.installPolicy === 'prompt') { + const confirmInstall = options.confirmInstall ?? defaultConfirmInstall; + const confirmed = await confirmInstall(installPrompt(feature)); + if (!confirmed) { + throw new Error(`KTX Python runtime installation was cancelled. Run: ${managedRuntimeInstallCommand(feature)}`); + } + } + + options.io.stderr.write(`Installing KTX Python runtime (${feature}) with uv...\n`); + const installed = await installRuntime({ + cliVersion: options.cliVersion, + features: [feature], + force: false, + }); + options.io.stderr.write(`KTX Python runtime ready: ${installed.layout.versionDir}\n`); + return { layout: installed.layout, manifest: installed.manifest }; +} + +export async function createManagedPythonSemanticLayerComputePort( + options: ManagedPythonSemanticLayerComputeOptions, +): Promise { + const runtime = await ensureManagedPythonCommandRuntime({ + cliVersion: options.cliVersion, + installPolicy: options.installPolicy, + io: options.io, + feature: 'core', + ...(options.readStatus ? { readStatus: options.readStatus } : {}), + ...(options.installRuntime ? { installRuntime: options.installRuntime } : {}), + ...(options.confirmInstall ? { confirmInstall: options.confirmInstall } : {}), + }); + const createPythonCompute = options.createPythonCompute ?? createPythonSemanticLayerComputePort; + return createPythonCompute({ + command: runtime.manifest.python.daemonExecutable, + args: [], + }); +} diff --git a/packages/cli/src/managed-python-daemon.test.ts b/packages/cli/src/managed-python-daemon.test.ts new file mode 100644 index 00000000..24df2a78 --- /dev/null +++ b/packages/cli/src/managed-python-daemon.test.ts @@ -0,0 +1,239 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + readManagedPythonDaemonStatus, + startManagedPythonDaemon, + stopManagedPythonDaemon, + type ManagedPythonDaemonChild, + type ManagedPythonDaemonFetch, + type ManagedPythonDaemonSpawn, + type ManagedPythonDaemonState, +} from './managed-python-daemon.js'; +import type { + InstalledKtxRuntimeManifest, + ManagedPythonRuntimeInstallResult, + ManagedPythonRuntimeLayout, +} from './managed-python-runtime.js'; + +function layout(root: string): ManagedPythonRuntimeLayout { + return { + cliVersion: '0.2.0', + runtimeRoot: join(root, 'runtime'), + versionDir: join(root, 'runtime', '0.2.0'), + venvDir: join(root, 'runtime', '0.2.0', '.venv'), + manifestPath: join(root, 'runtime', '0.2.0', 'manifest.json'), + installLogPath: join(root, 'runtime', '0.2.0', 'install.log'), + assetDir: join(root, 'assets', 'python'), + assetManifestPath: join(root, 'assets', 'python', 'manifest.json'), + pythonPath: join(root, 'runtime', '0.2.0', '.venv', 'bin', 'python'), + daemonPath: join(root, 'runtime', '0.2.0', '.venv', 'bin', 'ktx-daemon'), + daemonStatePath: join(root, 'runtime', '0.2.0', 'daemon.json'), + daemonStdoutPath: join(root, 'runtime', '0.2.0', 'daemon.stdout.log'), + daemonStderrPath: join(root, 'runtime', '0.2.0', 'daemon.stderr.log'), + }; +} + +function manifest(root: string, features: Array<'core' | 'local-embeddings'> = ['core']): InstalledKtxRuntimeManifest { + const runtimeLayout = layout(root); + return { + schemaVersion: 1, + cliVersion: '0.2.0', + installedAt: '2026-05-11T00:00:00.000Z', + asset: { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.2.0', + wheel: { + file: 'kaelio_ktx-0.2.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 123, + }, + }, + features, + python: { + executable: runtimeLayout.pythonPath, + daemonExecutable: runtimeLayout.daemonPath, + }, + installLog: runtimeLayout.installLogPath, + }; +} + +function installResult(root: string, features: Array<'core' | 'local-embeddings'> = ['core']): ManagedPythonRuntimeInstallResult { + return { + status: 'ready', + layout: layout(root), + asset: { + manifest: manifest(root, features).asset, + wheelPath: join(root, 'assets', 'python', 'kaelio_ktx-0.2.0-py3-none-any.whl'), + }, + manifest: manifest(root, features), + }; +} + +function makeFetch(version = '0.2.0'): ManagedPythonDaemonFetch { + return vi.fn(async () => ({ + ok: true, + status: 200, + json: async () => ({ status: 'healthy', version }), + text: async () => '', + })); +} + +function makeSpawn(pid = 4242): ManagedPythonDaemonSpawn { + return vi.fn((_command, _args, _options): ManagedPythonDaemonChild => ({ + pid, + unref: vi.fn(), + })); +} + +function runningState(root: string, overrides: Partial = {}): ManagedPythonDaemonState { + const runtimeLayout = layout(root); + return { + schemaVersion: 1, + pid: 4242, + host: '127.0.0.1', + port: 58731, + version: '0.2.0', + features: ['core'], + startedAt: '2026-05-11T00:00:00.000Z', + stdoutLog: runtimeLayout.daemonStdoutPath, + stderrLog: runtimeLayout.daemonStderrPath, + ...overrides, + }; +} + +describe('managed Python daemon lifecycle', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-managed-daemon-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('reports stopped when no daemon state exists', async () => { + const status = await readManagedPythonDaemonStatus({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + processAlive: vi.fn(() => false), + fetch: makeFetch(), + }); + + expect(status.kind).toBe('stopped'); + expect(status.detail).toContain('No daemon state'); + }); + + it('starts ktx-daemon serve-http, waits for health, and writes state', async () => { + const spawnDaemon = makeSpawn(5555); + const installRuntime = vi.fn(async () => installResult(tempDir)); + + const result = await startManagedPythonDaemon({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + features: ['core'], + installRuntime, + spawnDaemon, + fetch: makeFetch(), + allocatePort: vi.fn(async () => 61234), + now: () => new Date('2026-05-11T00:00:00.000Z'), + pollIntervalMs: 1, + }); + + expect(result.status).toBe('started'); + expect(result.baseUrl).toBe('http://127.0.0.1:61234'); + expect(installRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + features: ['core'], + force: false, + }); + expect(spawnDaemon).toHaveBeenCalledWith( + layout(tempDir).daemonPath, + ['serve-http', '--host', '127.0.0.1', '--port', '61234'], + expect.objectContaining({ + detached: true, + env: expect.objectContaining({ KTX_DAEMON_VERSION: '0.2.0' }), + }), + ); + expect(JSON.parse(await readFile(layout(tempDir).daemonStatePath, 'utf8'))).toMatchObject({ + pid: 5555, + port: 61234, + version: '0.2.0', + features: ['core'], + stdoutLog: layout(tempDir).daemonStdoutPath, + stderrLog: layout(tempDir).daemonStderrPath, + }); + }); + + it('reuses a healthy daemon with the requested feature set', async () => { + await mkdir(layout(tempDir).versionDir, { recursive: true }); + await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`); + const spawnDaemon = makeSpawn(9999); + + const result = await startManagedPythonDaemon({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + features: ['core'], + installRuntime: vi.fn(async () => installResult(tempDir)), + spawnDaemon, + fetch: makeFetch(), + processAlive: vi.fn(() => true), + pollIntervalMs: 1, + }); + + expect(result.status).toBe('reused'); + expect(result.baseUrl).toBe('http://127.0.0.1:58731'); + expect(spawnDaemon).not.toHaveBeenCalled(); + }); + + it('starts a fresh daemon when the previous state is stale', async () => { + await mkdir(layout(tempDir).versionDir, { recursive: true }); + await writeFile( + layout(tempDir).daemonStatePath, + `${JSON.stringify(runningState(tempDir, { version: '0.1.0' }), null, 2)}\n`, + ); + + const result = await startManagedPythonDaemon({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + features: ['core'], + installRuntime: vi.fn(async () => installResult(tempDir)), + spawnDaemon: makeSpawn(6666), + fetch: makeFetch(), + processAlive: vi.fn(() => true), + killProcess: vi.fn(), + allocatePort: vi.fn(async () => 61235), + now: () => new Date('2026-05-11T00:00:00.000Z'), + pollIntervalMs: 1, + }); + + expect(result.status).toBe('started'); + expect(JSON.parse(await readFile(layout(tempDir).daemonStatePath, 'utf8'))).toMatchObject({ + pid: 6666, + port: 61235, + version: '0.2.0', + }); + }); + + it('stops a recorded daemon and removes the state file', async () => { + await mkdir(layout(tempDir).versionDir, { recursive: true }); + await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`); + const killProcess = vi.fn(); + + const result = await stopManagedPythonDaemon({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + processAlive: vi.fn(() => true), + killProcess, + }); + + expect(result.status).toBe('stopped'); + expect(killProcess).toHaveBeenCalledWith(4242); + await expect(readFile(layout(tempDir).daemonStatePath, 'utf8')).rejects.toThrow(); + }); +}); diff --git a/packages/cli/src/managed-python-daemon.ts b/packages/cli/src/managed-python-daemon.ts new file mode 100644 index 00000000..4b128c63 --- /dev/null +++ b/packages/cli/src/managed-python-daemon.ts @@ -0,0 +1,397 @@ +import { spawn } from 'node:child_process'; +import { mkdir, open, readFile, rm, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:net'; +import { setTimeout as delay } from 'node:timers/promises'; +import { z } from 'zod'; +import { + installManagedPythonRuntime, + managedPythonRuntimeLayout, + runtimeFeatureSchema, + type KtxRuntimeFeature, + type ManagedPythonRuntimeInstallOptions, + type ManagedPythonRuntimeInstallResult, + type ManagedPythonRuntimeLayout, + type ManagedPythonRuntimeLayoutOptions, +} from './managed-python-runtime.js'; + +export interface ManagedPythonDaemonState { + schemaVersion: 1; + pid: number; + host: '127.0.0.1'; + port: number; + version: string; + features: KtxRuntimeFeature[]; + startedAt: string; + stdoutLog: string; + stderrLog: string; +} + +export type ManagedPythonDaemonStatus = + | { kind: 'stopped'; detail: string; layout: ManagedPythonRuntimeLayout } + | { kind: 'running'; detail: string; layout: ManagedPythonRuntimeLayout; state: ManagedPythonDaemonState; baseUrl: string } + | { kind: 'stale'; detail: string; layout: ManagedPythonRuntimeLayout; state?: ManagedPythonDaemonState }; + +export interface ManagedPythonDaemonStartResult { + status: 'started' | 'reused'; + layout: ManagedPythonRuntimeLayout; + state: ManagedPythonDaemonState; + baseUrl: string; +} + +export interface ManagedPythonDaemonStopResult { + status: 'stopped' | 'already-stopped'; + layout: ManagedPythonRuntimeLayout; + state?: ManagedPythonDaemonState; +} + +export interface ManagedPythonDaemonChild { + pid?: number; + unref(): void; +} + +export type ManagedPythonDaemonSpawn = ( + command: string, + args: string[], + options: { + detached: boolean; + stdio: ['ignore', number, number]; + env: NodeJS.ProcessEnv; + }, +) => ManagedPythonDaemonChild; + +export type ManagedPythonDaemonFetch = ( + url: string, +) => Promise<{ + ok: boolean; + status: number; + json(): Promise; + text(): Promise; +}>; + +export interface ManagedPythonDaemonStartOptions extends ManagedPythonRuntimeLayoutOptions { + features: KtxRuntimeFeature[]; + force?: boolean; + installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise; + spawnDaemon?: ManagedPythonDaemonSpawn; + fetch?: ManagedPythonDaemonFetch; + allocatePort?: () => Promise; + processAlive?: (pid: number) => boolean; + killProcess?: (pid: number) => void; + now?: () => Date; + startupTimeoutMs?: number; + pollIntervalMs?: number; +} + +export interface ManagedPythonDaemonStatusOptions extends ManagedPythonRuntimeLayoutOptions { + fetch?: ManagedPythonDaemonFetch; + processAlive?: (pid: number) => boolean; +} + +export interface ManagedPythonDaemonStopOptions extends ManagedPythonRuntimeLayoutOptions { + processAlive?: (pid: number) => boolean; + killProcess?: (pid: number) => void; +} + +const daemonStateSchema = z.object({ + schemaVersion: z.literal(1), + pid: z.number().int().positive(), + host: z.literal('127.0.0.1'), + port: z.number().int().min(1).max(65535), + version: z.string().min(1), + features: z.array(runtimeFeatureSchema).min(1), + startedAt: z.string().min(1), + stdoutLog: z.string().min(1), + stderrLog: z.string().min(1), +}); + +function normalizeFeatures(features: KtxRuntimeFeature[]): KtxRuntimeFeature[] { + const requested = new Set(['core', ...features]); + return runtimeFeatureSchema.options.filter((feature) => requested.has(feature)); +} + +function hasFeatures(state: ManagedPythonDaemonState, features: KtxRuntimeFeature[]): boolean { + return normalizeFeatures(features).every((feature) => state.features.includes(feature)); +} + +function defaultFetch(url: string): ReturnType { + return fetch(url) as ReturnType; +} + +function defaultProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function defaultKillProcess(pid: number): void { + try { + process.kill(pid, 'SIGTERM'); + } catch (error) { + const code = (error as { code?: unknown }).code; + if (code !== 'ESRCH') { + throw error; + } + } +} + +function defaultSpawnDaemon( + command: string, + args: string[], + options: Parameters[2], +): ManagedPythonDaemonChild { + return spawn(command, args, options); +} + +function baseUrl(state: Pick): string { + return `http://${state.host}:${state.port}`; +} + +async function readState(path: string): Promise { + try { + return daemonStateSchema.parse(JSON.parse(await readFile(path, 'utf8')) as unknown); + } catch (error) { + const code = (error as { code?: unknown }).code; + if (code === 'ENOENT') { + return undefined; + } + throw error; + } +} + +async function writeState(path: string, state: ManagedPythonDaemonState): Promise { + await writeFile(path, `${JSON.stringify(state, null, 2)}\n`); +} + +async function healthOk(input: { + state: ManagedPythonDaemonState; + cliVersion: string; + fetch: ManagedPythonDaemonFetch; +}): Promise<{ ok: true } | { ok: false; detail: string }> { + try { + const response = await input.fetch(`${baseUrl(input.state)}/health`); + if (!response.ok) { + return { ok: false, detail: `Health check returned HTTP ${response.status}: ${await response.text()}` }; + } + const body = (await response.json()) as unknown; + if (!body || typeof body !== 'object' || Array.isArray(body)) { + return { ok: false, detail: 'Health check returned non-object JSON' }; + } + const record = body as Record; + if (record.status !== 'healthy') { + return { ok: false, detail: `Health check returned status ${String(record.status)}` }; + } + if (record.version !== input.cliVersion) { + return { + ok: false, + detail: `Daemon version ${String(record.version)} does not match CLI ${input.cliVersion}`, + }; + } + return { ok: true }; + } catch (error) { + return { ok: false, detail: error instanceof Error ? error.message : String(error) }; + } +} + +export async function readManagedPythonDaemonStatus( + options: ManagedPythonDaemonStatusOptions, +): Promise { + const layout = managedPythonRuntimeLayout(options); + let state: ManagedPythonDaemonState | undefined; + try { + state = await readState(layout.daemonStatePath); + } catch (error) { + return { + kind: 'stale', + detail: `Daemon state is invalid: ${error instanceof Error ? error.message : String(error)}`, + layout, + }; + } + if (!state) { + return { kind: 'stopped', detail: `No daemon state at ${layout.daemonStatePath}`, layout }; + } + if (state.version !== options.cliVersion) { + return { + kind: 'stale', + detail: `Daemon is for CLI ${state.version}, current CLI is ${options.cliVersion}`, + layout, + state, + }; + } + const processAlive = options.processAlive ?? defaultProcessAlive; + if (!processAlive(state.pid)) { + return { kind: 'stale', detail: `Daemon process ${state.pid} is not running`, layout, state }; + } + const health = await healthOk({ + state, + cliVersion: options.cliVersion, + fetch: options.fetch ?? defaultFetch, + }); + if (!health.ok) { + return { kind: 'stale', detail: health.detail, layout, state }; + } + return { kind: 'running', detail: `Daemon running at ${baseUrl(state)}`, layout, state, baseUrl: baseUrl(state) }; +} + +export async function allocateDaemonPort(): Promise { + return await new Promise((resolve, reject) => { + const server = createServer(); + server.on('error', reject); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + server.close(() => { + if (address && typeof address === 'object') { + resolve(address.port); + return; + } + reject(new Error('Failed to allocate a daemon port')); + }); + }); + }); +} + +async function waitForHealth(input: { + state: ManagedPythonDaemonState; + cliVersion: string; + fetch: ManagedPythonDaemonFetch; + timeoutMs: number; + pollIntervalMs: number; +}): Promise { + const deadline = Date.now() + input.timeoutMs; + let lastDetail = 'daemon did not answer health checks'; + while (Date.now() <= deadline) { + const health = await healthOk({ + state: input.state, + cliVersion: input.cliVersion, + fetch: input.fetch, + }); + if (health.ok) { + return; + } + lastDetail = health.detail; + await delay(input.pollIntervalMs); + } + throw new Error(`KTX Python daemon failed to start: ${lastDetail}. stderr: ${input.state.stderrLog}`); +} + +async function removeState(layout: ManagedPythonRuntimeLayout): Promise { + await rm(layout.daemonStatePath, { force: true }); +} + +async function stopRecordedDaemon(input: { + layout: ManagedPythonRuntimeLayout; + state: ManagedPythonDaemonState; + processAlive: (pid: number) => boolean; + killProcess: (pid: number) => void; +}): Promise { + if (input.processAlive(input.state.pid)) { + input.killProcess(input.state.pid); + } + await removeState(input.layout); +} + +export async function startManagedPythonDaemon( + options: ManagedPythonDaemonStartOptions, +): Promise { + const features = normalizeFeatures(options.features); + const installRuntime = options.installRuntime ?? installManagedPythonRuntime; + const layoutOverrides = { + ...(options.runtimeRoot !== undefined ? { runtimeRoot: options.runtimeRoot } : {}), + ...(options.assetDir !== undefined ? { assetDir: options.assetDir } : {}), + ...(options.platform !== undefined ? { platform: options.platform } : {}), + ...(options.env !== undefined ? { env: options.env } : {}), + ...(options.homeDir !== undefined ? { homeDir: options.homeDir } : {}), + }; + const layout = managedPythonRuntimeLayout({ cliVersion: options.cliVersion, ...layoutOverrides }); + const processAlive = options.processAlive ?? defaultProcessAlive; + const killProcess = options.killProcess ?? defaultKillProcess; + const fetchImpl = options.fetch ?? defaultFetch; + + const status = await readManagedPythonDaemonStatus({ + cliVersion: options.cliVersion, + ...layoutOverrides, + fetch: fetchImpl, + processAlive, + }); + if (options.force !== true && status.kind === 'running' && hasFeatures(status.state, features)) { + return { status: 'reused', layout, state: status.state, baseUrl: status.baseUrl }; + } + if ('state' in status && status.state) { + await stopRecordedDaemon({ layout, state: status.state, processAlive, killProcess }); + } else { + await removeState(layout); + } + + const installed = await installRuntime({ + cliVersion: options.cliVersion, + ...layoutOverrides, + features, + force: false, + }); + + await mkdir(layout.versionDir, { recursive: true }); + const stdout = await open(layout.daemonStdoutPath, 'a'); + const stderr = await open(layout.daemonStderrPath, 'a'); + try { + const port = await (options.allocatePort ?? allocateDaemonPort)(); + const spawnDaemon = options.spawnDaemon ?? defaultSpawnDaemon; + const child = spawnDaemon( + installed.manifest.python.daemonExecutable, + ['serve-http', '--host', '127.0.0.1', '--port', String(port)], + { + detached: true, + stdio: ['ignore', stdout.fd, stderr.fd], + env: { + ...process.env, + KTX_DAEMON_VERSION: options.cliVersion, + }, + }, + ); + child.unref(); + if (!child.pid) { + throw new Error(`KTX Python daemon did not report a pid. stderr: ${layout.daemonStderrPath}`); + } + const state: ManagedPythonDaemonState = { + schemaVersion: 1, + pid: child.pid, + host: '127.0.0.1', + port, + version: options.cliVersion, + features: installed.manifest.features, + startedAt: (options.now ?? (() => new Date()))().toISOString(), + stdoutLog: layout.daemonStdoutPath, + stderrLog: layout.daemonStderrPath, + }; + await waitForHealth({ + state, + cliVersion: options.cliVersion, + fetch: fetchImpl, + timeoutMs: options.startupTimeoutMs ?? 10_000, + pollIntervalMs: options.pollIntervalMs ?? 100, + }); + await writeState(layout.daemonStatePath, state); + return { status: 'started', layout, state, baseUrl: baseUrl(state) }; + } finally { + await stdout.close(); + await stderr.close(); + } +} + +export async function stopManagedPythonDaemon( + options: ManagedPythonDaemonStopOptions, +): Promise { + const layout = managedPythonRuntimeLayout(options); + const state = await readState(layout.daemonStatePath); + if (!state) { + return { status: 'already-stopped', layout }; + } + await stopRecordedDaemon({ + layout, + state, + processAlive: options.processAlive ?? defaultProcessAlive, + killProcess: options.killProcess ?? defaultKillProcess, + }); + return { status: 'stopped', layout, state }; +} diff --git a/packages/cli/src/managed-python-http.test.ts b/packages/cli/src/managed-python-http.test.ts new file mode 100644 index 00000000..c0153c45 --- /dev/null +++ b/packages/cli/src/managed-python-http.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + createManagedDaemonHttpJsonRunner, + createManagedDaemonLookerTableIdentifierParser, + createManagedDaemonSqlAnalysisPort, + createManagedPythonDaemonBaseUrlResolver, + managedDaemonDatabaseIntrospectionOptions, +} from './managed-python-http.js'; + +function io() { + let stderr = ''; + return { + io: { + stdout: { write: vi.fn() }, + stderr: { write: (chunk: string) => (stderr += chunk) }, + }, + stderr: () => stderr, + }; +} + +describe('createManagedPythonDaemonBaseUrlResolver', () => { + it('ensures the core runtime, starts the daemon, reports the URL, and caches the result', async () => { + const testIo = io(); + const ensureRuntime = vi.fn(async () => ({ + layout: {} as never, + manifest: {} as never, + })); + const startDaemon = vi.fn(async () => ({ + status: 'started' as const, + layout: {} as never, + state: { pid: 1234 } as never, + baseUrl: 'http://127.0.0.1:61234', + })); + const resolveBaseUrl = createManagedPythonDaemonBaseUrlResolver({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: testIo.io, + ensureRuntime, + startDaemon, + }); + + await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); + await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); + + expect(ensureRuntime).toHaveBeenCalledTimes(1); + expect(ensureRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: testIo.io, + feature: 'core', + }); + expect(startDaemon).toHaveBeenCalledTimes(1); + expect(startDaemon).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['core'], + force: false, + }); + expect(testIo.stderr()).toContain('Started KTX Python daemon: http://127.0.0.1:61234'); + }); + + it('reports daemon reuse without reinstalling after the first resolved URL', async () => { + const testIo = io(); + const ensureRuntime = vi.fn(async () => ({ + layout: {} as never, + manifest: {} as never, + })); + const startDaemon = vi.fn(async () => ({ + status: 'reused' as const, + layout: {} as never, + state: { pid: 1234 } as never, + baseUrl: 'http://127.0.0.1:61234', + })); + const resolveBaseUrl = createManagedPythonDaemonBaseUrlResolver({ + cliVersion: '0.2.0', + installPolicy: 'never', + io: testIo.io, + ensureRuntime, + startDaemon, + }); + + await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); + await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); + + expect(ensureRuntime).toHaveBeenCalledTimes(1); + expect(startDaemon).toHaveBeenCalledTimes(1); + expect(testIo.stderr()).toContain('Using existing KTX Python daemon: http://127.0.0.1:61234'); + }); +}); + +describe('createManagedDaemonHttpJsonRunner', () => { + it('resolves the managed base URL lazily for each HTTP JSON request', async () => { + const postJson = vi.fn(async () => ({ ok: true })); + const runner = createManagedDaemonHttpJsonRunner({ + resolveBaseUrl: async () => 'http://127.0.0.1:61234', + postJson, + }); + + await expect(runner('/sql/parse-table-identifier', { items: [] })).resolves.toEqual({ ok: true }); + + expect(postJson).toHaveBeenCalledWith('http://127.0.0.1:61234', '/sql/parse-table-identifier', { items: [] }); + }); +}); + +describe('managed daemon ingest ports', () => { + it('creates a Looker table parser backed by the managed daemon runner', async () => { + const requestJson = vi.fn(async () => ({ + results: { + 'model.explore': { + ok: true, + catalog: 'warehouse', + schema: 'public', + name: 'orders', + canonical_table: 'public.orders', + }, + }, + })); + const parser = createManagedDaemonLookerTableIdentifierParser({ requestJson }); + + await expect( + parser.parse([{ key: 'model.explore', sql_table_name: 'public.orders', dialect: 'postgres' }]), + ).resolves.toEqual({ + 'model.explore': { + ok: true, + catalog: 'warehouse', + schema: 'public', + name: 'orders', + canonical_table: 'public.orders', + }, + }); + expect(requestJson).toHaveBeenCalledWith('/sql/parse-table-identifier', { + items: [{ key: 'model.explore', sql_table_name: 'public.orders', dialect: 'postgres' }], + }); + }); + + it('creates a SQL analysis port backed by the managed daemon runner', async () => { + const requestJson = vi.fn(async () => ({ + fingerprint: 'select-orders', + normalized_sql: 'SELECT * FROM public.orders WHERE id = ?', + tables_touched: ['public.orders'], + literal_slots: [{ position: 1, type: 'number', example_value: '42' }], + })); + const sqlAnalysis = createManagedDaemonSqlAnalysisPort({ requestJson }); + + await expect(sqlAnalysis.analyzeForFingerprint('SELECT * FROM public.orders WHERE id = 42', 'postgres')).resolves + .toEqual({ + fingerprint: 'select-orders', + normalizedSql: 'SELECT * FROM public.orders WHERE id = ?', + tablesTouched: ['public.orders'], + literalSlots: [{ position: 1, type: 'number', exampleValue: '42' }], + }); + expect(requestJson).toHaveBeenCalledWith('/api/sql/analyze-for-fingerprint', { + sql: 'SELECT * FROM public.orders WHERE id = 42', + dialect: 'postgres', + }); + }); + + it('returns live-database daemon request options backed by the managed runner', async () => { + const requestJson = vi.fn(async () => ({ + connection_id: 'warehouse', + tables: [], + })); + const options = managedDaemonDatabaseIntrospectionOptions({ requestJson }); + expect(options.requestJson).toBeDefined(); + + await expect(options.requestJson?.('/database/introspect', { connection_id: 'warehouse' })).resolves.toEqual({ + connection_id: 'warehouse', + tables: [], + }); + expect(requestJson).toHaveBeenCalledWith('/database/introspect', { connection_id: 'warehouse' }); + }); +}); diff --git a/packages/cli/src/managed-python-http.ts b/packages/cli/src/managed-python-http.ts new file mode 100644 index 00000000..1cd1f7d1 --- /dev/null +++ b/packages/cli/src/managed-python-http.ts @@ -0,0 +1,194 @@ +import { request as httpRequest } from 'node:http'; +import { request as httpsRequest } from 'node:https'; +import { URL } from 'node:url'; +import { + createDaemonLookerTableIdentifierParser, + type DaemonLiveDatabaseIntrospectionOptions, + type KtxDaemonDatabaseHttpJsonRunner, + type KtxDaemonTableIdentifierHttpJsonRunner, + type LookerTableIdentifierParser, +} from '@ktx/context/ingest'; +import { + createHttpSqlAnalysisPort, + type KtxSqlAnalysisHttpJsonRunner, + type SqlAnalysisPort, +} from '@ktx/context/sql-analysis'; +import type { KtxCliIo } from './cli-runtime.js'; +import { + ensureManagedPythonCommandRuntime, + type KtxManagedPythonInstallPolicy, + type ManagedPythonCommandRuntime, +} from './managed-python-command.js'; +import { startManagedPythonDaemon, type ManagedPythonDaemonStartResult } from './managed-python-daemon.js'; + +export type ManagedPythonHttpJsonRunner = ( + path: string, + payload: Record, +) => Promise>; + +export type ManagedPythonHttpPostJson = ( + baseUrl: string, + path: string, + payload: Record, +) => Promise>; + +export interface ManagedPythonCoreDaemonOptions { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + ensureRuntime?: (options: { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + feature: 'core'; + }) => Promise; + startDaemon?: (options: { + cliVersion: string; + features: ['core']; + force: false; + }) => Promise; +} + +export type ManagedPythonDaemonHttpOptions = + | { + requestJson: ManagedPythonHttpJsonRunner; + } + | { + resolveBaseUrl: () => Promise; + postJson?: ManagedPythonHttpPostJson; + } + | (ManagedPythonCoreDaemonOptions & { + postJson?: ManagedPythonHttpPostJson; + }); + +function normalizedBaseUrl(baseUrl: string): string { + return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`; +} + +function parseJsonObject(raw: string, path: string): Record { + const parsed = JSON.parse(raw) as unknown; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`KTX managed daemon HTTP ${path} returned non-object JSON`); + } + return parsed as Record; +} + +export async function postManagedDaemonJson( + baseUrl: string, + path: string, + payload: Record, +): Promise> { + return await new Promise((resolve, reject) => { + const target = new URL(path.replace(/^\//, ''), normalizedBaseUrl(baseUrl)); + const body = JSON.stringify(payload); + const client = target.protocol === 'https:' ? httpsRequest : httpRequest; + const request = client( + target, + { + method: 'POST', + headers: { + accept: 'application/json', + 'content-type': 'application/json', + 'content-length': Buffer.byteLength(body), + }, + }, + (response) => { + const chunks: Buffer[] = []; + response.on('data', (chunk: Buffer) => chunks.push(chunk)); + response.on('end', () => { + const text = Buffer.concat(chunks).toString('utf8'); + const statusCode = response.statusCode ?? 0; + if (statusCode < 200 || statusCode >= 300) { + reject(new Error(`KTX managed daemon HTTP ${path} failed with ${statusCode}: ${text}`)); + return; + } + try { + resolve(parseJsonObject(text, path)); + } catch (error) { + reject(error); + } + }); + }, + ); + request.on('error', reject); + request.end(body); + }); +} + +export function createManagedPythonDaemonBaseUrlResolver( + options: ManagedPythonCoreDaemonOptions, +): () => Promise { + let cachedBaseUrl: string | undefined; + + return async () => { + if (cachedBaseUrl) { + return cachedBaseUrl; + } + + const ensureRuntime = options.ensureRuntime ?? ensureManagedPythonCommandRuntime; + const startDaemon = options.startDaemon ?? startManagedPythonDaemon; + await ensureRuntime({ + cliVersion: options.cliVersion, + installPolicy: options.installPolicy, + io: options.io, + feature: 'core', + }); + const daemon = await startDaemon({ + cliVersion: options.cliVersion, + features: ['core'], + force: false, + }); + const verb = daemon.status === 'started' ? 'Started' : 'Using existing'; + options.io.stderr.write(`${verb} KTX Python daemon: ${daemon.baseUrl}\n`); + cachedBaseUrl = daemon.baseUrl; + return cachedBaseUrl; + }; +} + +function isRequestJsonOnly(options: ManagedPythonDaemonHttpOptions): options is { requestJson: ManagedPythonHttpJsonRunner } { + return 'requestJson' in options; +} + +function isResolveBaseUrlOnly( + options: ManagedPythonDaemonHttpOptions, +): options is { resolveBaseUrl: () => Promise; postJson?: ManagedPythonHttpPostJson } { + return 'resolveBaseUrl' in options; +} + +export function createManagedDaemonHttpJsonRunner( + options: ManagedPythonDaemonHttpOptions, +): ManagedPythonHttpJsonRunner { + if (isRequestJsonOnly(options)) { + return options.requestJson; + } + const resolveBaseUrl = isResolveBaseUrlOnly(options) + ? options.resolveBaseUrl + : createManagedPythonDaemonBaseUrlResolver(options); + const postJson = options.postJson ?? postManagedDaemonJson; + + return async (path, payload) => postJson(await resolveBaseUrl(), path, payload); +} + +export function createManagedDaemonLookerTableIdentifierParser( + options: ManagedPythonDaemonHttpOptions, +): LookerTableIdentifierParser { + return createDaemonLookerTableIdentifierParser({ + baseUrl: 'http://127.0.0.1:0', + requestJson: createManagedDaemonHttpJsonRunner(options) as KtxDaemonTableIdentifierHttpJsonRunner, + }); +} + +export function createManagedDaemonSqlAnalysisPort(options: ManagedPythonDaemonHttpOptions): SqlAnalysisPort { + return createHttpSqlAnalysisPort({ + baseUrl: 'http://127.0.0.1:0', + requestJson: createManagedDaemonHttpJsonRunner(options) as KtxSqlAnalysisHttpJsonRunner, + }); +} + +export function managedDaemonDatabaseIntrospectionOptions( + options: ManagedPythonDaemonHttpOptions, +): Pick { + return { + requestJson: createManagedDaemonHttpJsonRunner(options) as KtxDaemonDatabaseHttpJsonRunner, + }; +} diff --git a/packages/cli/src/managed-python-runtime.test.ts b/packages/cli/src/managed-python-runtime.test.ts new file mode 100644 index 00000000..d100e409 --- /dev/null +++ b/packages/cli/src/managed-python-runtime.test.ts @@ -0,0 +1,479 @@ +import { createHash } from 'node:crypto'; +import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + MISSING_UV_RUNTIME_INSTALL_MESSAGE, + doctorManagedPythonRuntime, + installManagedPythonRuntime, + managedPythonRuntimeLayout, + pruneManagedPythonRuntimes, + readManagedPythonRuntimeStatus, + verifyRuntimeAsset, + type ManagedPythonRuntimeExec, +} from './managed-python-runtime.js'; + +async function writeAsset(root: string, contents = 'wheel-bytes') { + const assetDir = join(root, 'assets', 'python'); + await mkdir(assetDir, { recursive: true }); + const wheelPath = join(assetDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'); + await writeFile(wheelPath, contents); + await writeFile( + join(assetDir, 'manifest.json'), + `${JSON.stringify( + { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.1.0', + wheel: { + file: 'kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: createHash('sha256').update(contents).digest('hex'), + bytes: Buffer.byteLength(contents), + }, + }, + null, + 2, + )}\n`, + ); + return { assetDir, wheelPath }; +} + +describe('managedPythonRuntimeLayout', () => { + it('uses the macOS application-support runtime root', () => { + const layout = managedPythonRuntimeLayout({ + cliVersion: '0.2.0', + platform: 'darwin', + env: {}, + homeDir: '/Users/alex', + assetDir: '/repo/packages/cli/assets/python', + }); + + expect(layout.runtimeRoot).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime'); + expect(layout.versionDir).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0'); + expect(layout.venvDir).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv'); + expect(layout.pythonPath).toBe( + '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv/bin/python', + ); + expect(layout.daemonPath).toBe( + '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv/bin/ktx-daemon', + ); + expect(layout.daemonStatePath).toBe( + '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.json', + ); + expect(layout.daemonStdoutPath).toBe( + '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.stdout.log', + ); + expect(layout.daemonStderrPath).toBe( + '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.stderr.log', + ); + expect(layout.assetManifestPath).toBe('/repo/packages/cli/assets/python/manifest.json'); + }); + + it('honors KTX_RUNTIME_ROOT before platform defaults', () => { + const layout = managedPythonRuntimeLayout({ + cliVersion: '0.2.0', + platform: 'darwin', + env: { KTX_RUNTIME_ROOT: '/tmp/ktx-runtime' }, + homeDir: '/Users/alex', + assetDir: '/repo/packages/cli/assets/python', + }); + + expect(layout.runtimeRoot).toBe('/tmp/ktx-runtime'); + expect(layout.versionDir).toBe('/tmp/ktx-runtime/0.2.0'); + }); + + it('honors XDG_DATA_HOME on Linux', () => { + const layout = managedPythonRuntimeLayout({ + cliVersion: '0.2.0', + platform: 'linux', + env: { XDG_DATA_HOME: '/var/xdg' }, + homeDir: '/home/alex', + assetDir: '/repo/packages/cli/assets/python', + }); + + expect(layout.runtimeRoot).toBe('/var/xdg/kaelio/ktx/runtime'); + expect(layout.versionDir).toBe('/var/xdg/kaelio/ktx/runtime/0.2.0'); + }); + + it('uses LocalAppData on Windows', () => { + const layout = managedPythonRuntimeLayout({ + cliVersion: '0.2.0', + platform: 'win32', + env: { LOCALAPPDATA: 'C:\\Users\\Alex\\AppData\\Local' }, + homeDir: 'C:\\Users\\Alex', + assetDir: 'C:\\repo\\packages\\cli\\assets\\python', + }); + + expect(layout.runtimeRoot).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime'); + expect(layout.pythonPath).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime/0.2.0/.venv/Scripts/python.exe'); + expect(layout.daemonPath).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime/0.2.0/.venv/Scripts/ktx-daemon.exe'); + }); +}); + +describe('verifyRuntimeAsset', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-asset-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('reads the manifest and verifies the wheel checksum', async () => { + const { assetDir, wheelPath } = await writeAsset(tempDir, 'valid-wheel'); + + const asset = await verifyRuntimeAsset({ assetDir }); + + expect(asset.manifest.distributionName).toBe('kaelio-ktx'); + expect(asset.manifest.normalizedName).toBe('kaelio_ktx'); + expect(asset.wheelPath).toBe(wheelPath); + }); + + it('rejects a wheel whose checksum does not match the manifest', async () => { + const { assetDir, wheelPath } = await writeAsset(tempDir, 'original'); + await writeFile(wheelPath, 'tampered'); + + await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow( + /Bundled Python runtime wheel checksum mismatch/, + ); + }); + + it('rejects an unsafe wheel filename in the manifest', async () => { + const { assetDir } = await writeAsset(tempDir, 'valid-wheel'); + await writeFile( + join(assetDir, 'manifest.json'), + `${JSON.stringify({ + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.1.0', + wheel: { + file: '../kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 1, + }, + })}\n`, + ); + + await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow(/Unsafe runtime wheel filename/); + }); +}); + +describe('installManagedPythonRuntime', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-install-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('creates a venv, installs the core wheel, and writes a manifest', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const commands: Array<{ command: string; args: string[] }> = []; + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { + commands.push({ command, args }); + return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' }; + }); + + const result = await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }); + + expect(result.status).toBe('installed'); + expect(commands).toEqual([ + { command: 'uv', args: ['--version'] }, + { command: 'uv', args: ['venv', result.layout.venvDir] }, + { + command: 'uv', + args: ['pip', 'install', '--python', result.layout.pythonPath, result.asset.wheelPath], + }, + ]); + const manifest = JSON.parse(await readFile(result.layout.manifestPath, 'utf8')) as { + cliVersion: string; + features: string[]; + python: { executable: string; daemonExecutable: string }; + }; + expect(manifest.cliVersion).toBe('0.2.0'); + expect(manifest.features).toEqual(['core']); + expect(manifest.python.executable).toBe(result.layout.pythonPath); + expect(manifest.python.daemonExecutable).toBe(result.layout.daemonPath); + }); + + it('installs the local-embeddings extra when requested', async () => { + const { assetDir } = await writeAsset(tempDir, 'embedding-wheel'); + const commands: Array<{ command: string; args: string[] }> = []; + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { + commands.push({ command, args }); + return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' }; + }); + + const result = await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['local-embeddings'], + exec, + }); + + expect(commands.at(-1)).toEqual({ + command: 'uv', + args: ['pip', 'install', '--python', result.layout.pythonPath, `${result.asset.wheelPath}[local-embeddings]`], + }); + const manifest = JSON.parse(await readFile(result.layout.manifestPath, 'utf8')) as { features: string[] }; + expect(manifest.features).toEqual(['core', 'local-embeddings']); + }); + + it('fails with the hard-prerequisite message when uv is missing', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const commands: Array<{ command: string; args: string[] }> = []; + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { + commands.push({ command, args }); + throw new Error('spawn uv ENOENT'); + }); + + await expect( + installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }), + ).rejects.toThrow(MISSING_UV_RUNTIME_INSTALL_MESSAGE); + + expect(commands).toEqual([{ command: 'uv', args: ['--version'] }]); + }); + + it('reuses an existing compatible runtime when force is false', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ + stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', + stderr: '', + })); + + const first = await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }); + await mkdir(join(first.layout.venvDir, 'bin'), { recursive: true }); + await writeFile(first.layout.pythonPath, '#!/usr/bin/env python\n'); + await writeFile(first.layout.daemonPath, '#!/usr/bin/env python\n'); + + const second = await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }); + + expect(second.status).toBe('ready'); + expect(exec).toHaveBeenCalledTimes(3); + }); + + it('keeps failed install logs in the versioned runtime directory', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { + if (command === 'uv' && args[0] === 'venv') { + throw Object.assign(new Error('uv venv failed'), { stdout: 'creating\n', stderr: 'bad python\n' }); + } + return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' }; + }); + + await expect( + installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }), + ).rejects.toThrow(/Python runtime install failed/); + + const log = await readFile(join(tempDir, 'runtime', '0.2.0', 'install.log'), 'utf8'); + expect(log).toContain('$ uv venv'); + expect(log).toContain('bad python'); + }); +}); + +describe('readManagedPythonRuntimeStatus', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-status-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('reports missing before install', async () => { + const status = await readManagedPythonRuntimeStatus({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir: join(tempDir, 'assets', 'python'), + }); + + expect(status.kind).toBe('missing'); + expect(status.detail).toContain('No runtime manifest'); + }); + + it('reports ready when manifest and executables exist', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ + stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', + stderr: '', + })); + const install = await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }); + await mkdir(join(install.layout.venvDir, 'bin'), { recursive: true }); + await writeFile(install.layout.pythonPath, '#!/usr/bin/env python\n'); + await writeFile(install.layout.daemonPath, '#!/usr/bin/env python\n'); + + const status = await readManagedPythonRuntimeStatus({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + }); + + expect(status.kind).toBe('ready'); + expect(status.manifest?.features).toEqual(['core']); + }); + + it('reports broken when an executable is missing', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ + stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', + stderr: '', + })); + await installManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + features: ['core'], + exec, + }); + + const status = await readManagedPythonRuntimeStatus({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + }); + + expect(status.kind).toBe('broken'); + expect(status.detail).toContain('Missing Python executable'); + }); +}); + +describe('doctorManagedPythonRuntime', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-doctor-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('checks uv, bundled assets, and installed runtime status', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ + stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', + stderr: '', + })); + + const checks = await doctorManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + exec, + }); + + expect(checks.map((check) => [check.id, check.status])).toEqual([ + ['uv', 'pass'], + ['asset', 'pass'], + ['runtime', 'fail'], + ]); + expect(checks[2]?.fix).toBe('Run: ktx runtime install --yes'); + }); + + it('reports uv as a hard prerequisite when uv is missing', async () => { + const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const exec: ManagedPythonRuntimeExec = vi.fn(async () => { + throw new Error('spawn uv ENOENT'); + }); + + const checks = await doctorManagedPythonRuntime({ + cliVersion: '0.2.0', + runtimeRoot: join(tempDir, 'runtime'), + assetDir, + exec, + }); + + expect(checks[0]).toEqual({ + id: 'uv', + label: 'uv', + status: 'fail', + detail: MISSING_UV_RUNTIME_INSTALL_MESSAGE, + fix: 'Install uv, make sure it is on PATH, and run: ktx runtime install --yes', + }); + }); +}); + +describe('pruneManagedPythonRuntimes', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-prune-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('removes stale version directories and keeps the current version', async () => { + const runtimeRoot = join(tempDir, 'runtime'); + await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true }); + await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true }); + await writeFile(join(runtimeRoot, 'README.txt'), 'not a runtime directory\n'); + + const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot }); + + expect(result.removed).toEqual([join(runtimeRoot, '0.1.0')]); + expect(result.kept).toEqual([join(runtimeRoot, '0.2.0')]); + await expect(stat(join(runtimeRoot, '0.1.0'))).rejects.toThrow(); + expect(await readdir(runtimeRoot)).toEqual(['0.2.0', 'README.txt']); + }); + + it('supports dry-run without deleting stale directories', async () => { + const runtimeRoot = join(tempDir, 'runtime'); + await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true }); + await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true }); + + const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot, dryRun: true }); + + expect(result.removed).toEqual([]); + expect(result.stale).toEqual([join(runtimeRoot, '0.1.0')]); + expect(await readdir(runtimeRoot)).toEqual(['0.1.0', '0.2.0']); + }); +}); diff --git a/packages/cli/src/managed-python-runtime.ts b/packages/cli/src/managed-python-runtime.ts new file mode 100644 index 00000000..2b715b69 --- /dev/null +++ b/packages/cli/src/managed-python-runtime.ts @@ -0,0 +1,444 @@ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { access, appendFile, mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { basename, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; +import { z } from 'zod'; + +const execFileAsync = promisify(execFile); + +export const runtimeFeatureSchema = z.enum(['core', 'local-embeddings']); +export type KtxRuntimeFeature = z.infer; + +const runtimeAssetManifestSchema = z.object({ + schemaVersion: z.literal(1), + distributionName: z.literal('kaelio-ktx'), + normalizedName: z.literal('kaelio_ktx'), + version: z.string().min(1), + wheel: z.object({ + file: z.string().min(1), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + bytes: z.number().int().nonnegative(), + }), +}); + +export type KtxRuntimeAssetManifest = z.infer; + +const installedRuntimeManifestSchema = z.object({ + schemaVersion: z.literal(1), + cliVersion: z.string().min(1), + installedAt: z.string().min(1), + asset: runtimeAssetManifestSchema, + features: z.array(runtimeFeatureSchema).min(1), + python: z.object({ + executable: z.string().min(1), + daemonExecutable: z.string().min(1), + }), + installLog: z.string().min(1), +}); + +export type InstalledKtxRuntimeManifest = z.infer; + +export interface ManagedPythonRuntimeLayoutOptions { + cliVersion: string; + platform?: NodeJS.Platform; + env?: NodeJS.ProcessEnv; + homeDir?: string; + runtimeRoot?: string; + assetDir?: string; +} + +export interface ManagedPythonRuntimeLayout { + cliVersion: string; + runtimeRoot: string; + versionDir: string; + venvDir: string; + manifestPath: string; + installLogPath: string; + assetDir: string; + assetManifestPath: string; + pythonPath: string; + daemonPath: string; + daemonStatePath: string; + daemonStdoutPath: string; + daemonStderrPath: string; +} + +export interface ManagedRuntimeAsset { + manifest: KtxRuntimeAssetManifest; + wheelPath: string; +} + +export type ManagedPythonRuntimeExec = ( + command: string, + args: string[], + options?: { cwd?: string; env?: NodeJS.ProcessEnv }, +) => Promise<{ stdout: string; stderr: string }>; + +export interface ManagedPythonRuntimeInstallOptions extends ManagedPythonRuntimeLayoutOptions { + features: KtxRuntimeFeature[]; + force?: boolean; + exec?: ManagedPythonRuntimeExec; +} + +export interface ManagedPythonRuntimeInstallResult { + status: 'ready' | 'installed'; + layout: ManagedPythonRuntimeLayout; + asset: ManagedRuntimeAsset; + manifest: InstalledKtxRuntimeManifest; +} + +export type ManagedPythonRuntimeStatusKind = 'missing' | 'ready' | 'mismatched' | 'broken'; + +export interface ManagedPythonRuntimeStatus { + kind: ManagedPythonRuntimeStatusKind; + detail: string; + layout: ManagedPythonRuntimeLayout; + manifest?: InstalledKtxRuntimeManifest; +} + +export interface ManagedPythonRuntimeDoctorCheck { + id: 'uv' | 'asset' | 'runtime'; + label: string; + status: 'pass' | 'fail'; + detail: string; + fix?: string; +} + +export interface ManagedPythonRuntimePruneResult { + runtimeRoot: string; + stale: string[]; + kept: string[]; + removed: string[]; +} + +export const MISSING_UV_RUNTIME_INSTALL_MESSAGE = + 'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx runtime install --yes'; + +function defaultAssetDir(): string { + return fileURLToPath(new URL('../assets/python/', import.meta.url)); +} + +function runtimeRootFor(input: Required>): string { + if (input.env.KTX_RUNTIME_ROOT) { + return input.env.KTX_RUNTIME_ROOT; + } + if (input.platform === 'darwin') { + return join(input.homeDir, 'Library', 'Application Support', 'kaelio', 'ktx', 'runtime'); + } + if (input.platform === 'win32') { + return join(input.env.LOCALAPPDATA ?? join(input.homeDir, 'AppData', 'Local'), 'Kaelio', 'KTX', 'runtime'); + } + return join(input.env.XDG_DATA_HOME ?? join(input.homeDir, '.local', 'share'), 'kaelio', 'ktx', 'runtime'); +} + +function executablePath(venvDir: string, platform: NodeJS.Platform, name: string): string { + if (platform === 'win32') { + return join(venvDir, 'Scripts', `${name}.exe`); + } + return join(venvDir, 'bin', name); +} + +export function managedPythonRuntimeLayout(options: ManagedPythonRuntimeLayoutOptions): ManagedPythonRuntimeLayout { + const platform = options.platform ?? process.platform; + const env = options.env ?? process.env; + const homeDir = options.homeDir ?? homedir(); + const runtimeRoot = options.runtimeRoot ?? runtimeRootFor({ platform, env, homeDir }); + const versionDir = join(runtimeRoot, options.cliVersion); + const venvDir = join(versionDir, '.venv'); + const assetDir = options.assetDir ?? defaultAssetDir(); + + return { + cliVersion: options.cliVersion, + runtimeRoot, + versionDir, + venvDir, + manifestPath: join(versionDir, 'manifest.json'), + installLogPath: join(versionDir, 'install.log'), + assetDir, + assetManifestPath: join(assetDir, 'manifest.json'), + pythonPath: executablePath(venvDir, platform, 'python'), + daemonPath: executablePath(venvDir, platform, 'ktx-daemon'), + daemonStatePath: join(versionDir, 'daemon.json'), + daemonStdoutPath: join(versionDir, 'daemon.stdout.log'), + daemonStderrPath: join(versionDir, 'daemon.stderr.log'), + }; +} + +async function pathExists(path: string): Promise { + try { + await access(path); + return true; + } catch { + return false; + } +} + +function assertSafeWheelFilename(file: string): void { + if (file !== basename(file) || file.includes('/') || file.includes('\\')) { + throw new Error(`Unsafe runtime wheel filename in bundled manifest: ${file}`); + } +} + +async function readJsonFile(path: string): Promise { + return JSON.parse(await readFile(path, 'utf8')) as unknown; +} + +export async function verifyRuntimeAsset(input: { assetDir: string }): Promise { + const manifestPath = join(input.assetDir, 'manifest.json'); + const manifest = runtimeAssetManifestSchema.parse(await readJsonFile(manifestPath)); + assertSafeWheelFilename(manifest.wheel.file); + const wheelPath = join(input.assetDir, manifest.wheel.file); + const wheel = await readFile(wheelPath); + const sha256 = createHash('sha256').update(wheel).digest('hex'); + if (sha256 !== manifest.wheel.sha256 || wheel.byteLength !== manifest.wheel.bytes) { + throw new Error(`Bundled Python runtime wheel checksum mismatch: ${wheelPath}`); + } + return { manifest, wheelPath }; +} + +function normalizeFeatures(features: KtxRuntimeFeature[]): KtxRuntimeFeature[] { + const requested = new Set(['core', ...features]); + return runtimeFeatureSchema.options.filter((feature) => requested.has(feature)); +} + +async function readInstalledManifest(path: string): Promise { + if (!(await pathExists(path))) { + return undefined; + } + return installedRuntimeManifestSchema.parse(await readJsonFile(path)); +} + +function hasFeatures(manifest: InstalledKtxRuntimeManifest, features: KtxRuntimeFeature[]): boolean { + return normalizeFeatures(features).every((feature) => manifest.features.includes(feature)); +} + +async function defaultExec( + command: string, + args: string[], + options: { cwd?: string; env?: NodeJS.ProcessEnv } = {}, +): Promise<{ stdout: string; stderr: string }> { + const result = await execFileAsync(command, args, { + cwd: options.cwd, + env: options.env, + encoding: 'utf8', + maxBuffer: 1024 * 1024 * 20, + }); + return { stdout: result.stdout, stderr: result.stderr }; +} + +function errorOutput(error: unknown): { stdout: string; stderr: string } { + const value = error as { stdout?: unknown; stderr?: unknown }; + return { + stdout: typeof value.stdout === 'string' ? value.stdout : '', + stderr: typeof value.stderr === 'string' ? value.stderr : '', + }; +} + +async function runLogged(input: { + exec: ManagedPythonRuntimeExec; + logPath: string; + command: string; + args: string[]; + cwd?: string; +}): Promise<{ stdout: string; stderr: string }> { + await appendFile(input.logPath, `$ ${input.command} ${input.args.join(' ')}\n`); + try { + const result = await input.exec(input.command, input.args, { cwd: input.cwd }); + if (result.stdout) { + await appendFile(input.logPath, result.stdout.endsWith('\n') ? result.stdout : `${result.stdout}\n`); + } + if (result.stderr) { + await appendFile(input.logPath, result.stderr.endsWith('\n') ? result.stderr : `${result.stderr}\n`); + } + return result; + } catch (error) { + const output = errorOutput(error); + if (output.stdout) { + await appendFile(input.logPath, output.stdout.endsWith('\n') ? output.stdout : `${output.stdout}\n`); + } + if (output.stderr) { + await appendFile(input.logPath, output.stderr.endsWith('\n') ? output.stderr : `${output.stderr}\n`); + } + throw new Error(`Python runtime install failed. Install log: ${input.logPath}`); + } +} + +async function ensureUv(exec: ManagedPythonRuntimeExec): Promise { + try { + const result = await exec('uv', ['--version']); + return result.stdout.trim() || 'uv available'; + } catch { + throw new Error(MISSING_UV_RUNTIME_INSTALL_MESSAGE); + } +} + +export async function installManagedPythonRuntime( + options: ManagedPythonRuntimeInstallOptions, +): Promise { + const layout = managedPythonRuntimeLayout(options); + const exec = options.exec ?? defaultExec; + const features = normalizeFeatures(options.features); + const asset = await verifyRuntimeAsset({ assetDir: layout.assetDir }); + const existing = await readInstalledManifest(layout.manifestPath); + if ( + options.force !== true && + existing && + existing.cliVersion === options.cliVersion && + existing.asset.wheel.sha256 === asset.manifest.wheel.sha256 && + hasFeatures(existing, features) && + (await pathExists(existing.python.executable)) && + (await pathExists(existing.python.daemonExecutable)) + ) { + return { status: 'ready', layout, asset, manifest: existing }; + } + + await rm(layout.versionDir, { recursive: true, force: true }); + await mkdir(layout.versionDir, { recursive: true }); + await writeFile(layout.installLogPath, ''); + await ensureUv(exec); + await runLogged({ exec, logPath: layout.installLogPath, command: 'uv', args: ['venv', layout.venvDir] }); + const wheelSpec = features.includes('local-embeddings') ? `${asset.wheelPath}[local-embeddings]` : asset.wheelPath; + await runLogged({ + exec, + logPath: layout.installLogPath, + command: 'uv', + args: ['pip', 'install', '--python', layout.pythonPath, wheelSpec], + }); + + const manifest: InstalledKtxRuntimeManifest = { + schemaVersion: 1, + cliVersion: options.cliVersion, + installedAt: new Date().toISOString(), + asset: asset.manifest, + features, + python: { + executable: layout.pythonPath, + daemonExecutable: layout.daemonPath, + }, + installLog: layout.installLogPath, + }; + await writeFile(layout.manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + return { status: 'installed', layout, asset, manifest }; +} + +export async function readManagedPythonRuntimeStatus( + options: ManagedPythonRuntimeLayoutOptions, +): Promise { + const layout = managedPythonRuntimeLayout(options); + let manifest: InstalledKtxRuntimeManifest | undefined; + try { + manifest = await readInstalledManifest(layout.manifestPath); + } catch (error) { + return { + kind: 'broken', + detail: `Runtime manifest is invalid: ${error instanceof Error ? error.message : String(error)}`, + layout, + }; + } + if (!manifest) { + return { kind: 'missing', detail: `No runtime manifest at ${layout.manifestPath}`, layout }; + } + if (manifest.cliVersion !== options.cliVersion) { + return { + kind: 'mismatched', + detail: `Runtime is for CLI ${manifest.cliVersion}, current CLI is ${options.cliVersion}`, + layout, + manifest, + }; + } + if (!(await pathExists(manifest.python.executable))) { + return { kind: 'broken', detail: `Missing Python executable: ${manifest.python.executable}`, layout, manifest }; + } + if (!(await pathExists(manifest.python.daemonExecutable))) { + return { kind: 'broken', detail: `Missing ktx-daemon executable: ${manifest.python.daemonExecutable}`, layout, manifest }; + } + return { kind: 'ready', detail: `Runtime ready at ${layout.versionDir}`, layout, manifest }; +} + +function check( + status: ManagedPythonRuntimeDoctorCheck['status'], + input: Omit, +): ManagedPythonRuntimeDoctorCheck { + return { status, ...input }; +} + +export async function doctorManagedPythonRuntime( + options: ManagedPythonRuntimeLayoutOptions & { exec?: ManagedPythonRuntimeExec }, +): Promise { + const exec = options.exec ?? defaultExec; + const checks: ManagedPythonRuntimeDoctorCheck[] = []; + try { + const version = await ensureUv(exec); + checks.push(check('pass', { id: 'uv', label: 'uv', detail: version })); + } catch (error) { + checks.push( + check('fail', { + id: 'uv', + label: 'uv', + detail: error instanceof Error ? error.message : String(error), + fix: 'Install uv, make sure it is on PATH, and run: ktx runtime install --yes', + }), + ); + } + + try { + const asset = await verifyRuntimeAsset({ assetDir: managedPythonRuntimeLayout(options).assetDir }); + checks.push(check('pass', { id: 'asset', label: 'Bundled Python wheel', detail: asset.wheelPath })); + } catch (error) { + checks.push( + check('fail', { + id: 'asset', + label: 'Bundled Python wheel', + detail: error instanceof Error ? error.message : String(error), + fix: 'Run: pnpm run artifacts:check', + }), + ); + } + + const status = await readManagedPythonRuntimeStatus(options); + checks.push( + check(status.kind === 'ready' ? 'pass' : 'fail', { + id: 'runtime', + label: 'Managed Python runtime', + detail: status.detail, + ...(status.kind === 'ready' ? {} : { fix: 'Run: ktx runtime install --yes' }), + }), + ); + return checks; +} + +export async function pruneManagedPythonRuntimes(options: { + cliVersion: string; + runtimeRoot: string; + dryRun?: boolean; +}): Promise { + if (!(await pathExists(options.runtimeRoot))) { + return { runtimeRoot: options.runtimeRoot, stale: [], kept: [], removed: [] }; + } + const entries = await readdir(options.runtimeRoot); + const stale: string[] = []; + const kept: string[] = []; + for (const entry of entries) { + const path = join(options.runtimeRoot, entry); + const info = await stat(path); + if (!info.isDirectory()) { + continue; + } + if (entry === options.cliVersion) { + kept.push(path); + } else { + stale.push(path); + } + } + const removed: string[] = []; + if (options.dryRun !== true) { + for (const path of stale) { + await rm(path, { recursive: true, force: true }); + removed.push(path); + } + } + return { runtimeRoot: options.runtimeRoot, stale, kept, removed }; +} diff --git a/packages/cli/src/runtime.test.ts b/packages/cli/src/runtime.test.ts new file mode 100644 index 00000000..e367d339 --- /dev/null +++ b/packages/cli/src/runtime.test.ts @@ -0,0 +1,315 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { + ManagedPythonDaemonStartResult, + ManagedPythonDaemonStopResult, +} from './managed-python-daemon.js'; +import type { + ManagedPythonRuntimeDoctorCheck, + ManagedPythonRuntimeInstallResult, + ManagedPythonRuntimeStatus, +} from './managed-python-runtime.js'; +import { runKtxRuntime, type KtxRuntimeDeps } from './runtime.js'; + +function makeIo() { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +describe('runKtxRuntime', () => { + it('installs the requested runtime feature and prints the manifest path', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + installRuntime: vi.fn(async (): Promise => ({ + status: 'installed', + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', + }, + asset: { + wheelPath: '/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl', + manifest: { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.1.0', + wheel: { + file: 'kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 10, + }, + }, + }, + manifest: { + schemaVersion: 1, + cliVersion: '0.2.0', + installedAt: '2026-05-11T00:00:00.000Z', + asset: { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.1.0', + wheel: { + file: 'kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 10, + }, + }, + features: ['core', 'local-embeddings'], + python: { + executable: '/runtime/0.2.0/.venv/bin/python', + daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon', + }, + installLog: '/runtime/0.2.0/install.log', + }, + })), + }; + + await expect( + runKtxRuntime( + { command: 'install', cliVersion: '0.2.0', feature: 'local-embeddings', force: true }, + io.io, + deps, + ), + ).resolves.toBe(0); + + expect(deps.installRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['local-embeddings'], + force: true, + }); + expect(io.stdout()).toContain('Installed KTX Python runtime'); + expect(io.stdout()).toContain('features: core, local-embeddings'); + expect(io.stdout()).toContain('manifest: /runtime/0.2.0/manifest.json'); + expect(io.stderr()).toBe(''); + }); + + it('starts the managed Python daemon and prints the base URL', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + startDaemon: vi.fn(async (): Promise => ({ + status: 'started', + baseUrl: 'http://127.0.0.1:61234', + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', + }, + state: { + schemaVersion: 1, + pid: 4242, + host: '127.0.0.1', + port: 61234, + version: '0.2.0', + features: ['core', 'local-embeddings'], + startedAt: '2026-05-11T00:00:00.000Z', + stdoutLog: '/runtime/0.2.0/daemon.stdout.log', + stderrLog: '/runtime/0.2.0/daemon.stderr.log', + }, + })), + }; + + await expect( + runKtxRuntime( + { command: 'start', cliVersion: '0.2.0', feature: 'local-embeddings', force: true }, + io.io, + deps, + ), + ).resolves.toBe(0); + + expect(deps.startDaemon).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + features: ['local-embeddings'], + force: true, + }); + expect(io.stdout()).toContain('Started KTX Python daemon'); + expect(io.stdout()).toContain('url: http://127.0.0.1:61234'); + expect(io.stdout()).toContain('pid: 4242'); + expect(io.stdout()).toContain('features: core, local-embeddings'); + expect(io.stdout()).toContain('stderr: /runtime/0.2.0/daemon.stderr.log'); + }); + + it('stops the managed Python daemon', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + stopDaemon: vi.fn(async (): Promise => ({ + status: 'stopped', + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', + }, + state: { + schemaVersion: 1, + pid: 4242, + host: '127.0.0.1', + port: 61234, + version: '0.2.0', + features: ['core'], + startedAt: '2026-05-11T00:00:00.000Z', + stdoutLog: '/runtime/0.2.0/daemon.stdout.log', + stderrLog: '/runtime/0.2.0/daemon.stderr.log', + }, + })), + }; + + await expect(runKtxRuntime({ command: 'stop', cliVersion: '0.2.0' }, io.io, deps)).resolves.toBe(0); + + expect(deps.stopDaemon).toHaveBeenCalledWith({ cliVersion: '0.2.0' }); + expect(io.stdout()).toContain('Stopped KTX Python daemon'); + expect(io.stdout()).toContain('pid: 4242'); + }); + + it('prints runtime status as JSON', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + readStatus: vi.fn(async (): Promise => ({ + kind: 'missing', + detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', + }, + })), + }; + + await expect(runKtxRuntime({ command: 'status', cliVersion: '0.2.0', json: true }, io.io, deps)).resolves.toBe(0); + + expect(JSON.parse(io.stdout())).toMatchObject({ + kind: 'missing', + detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', + layout: { runtimeRoot: '/runtime' }, + }); + }); + + it('returns failure for doctor when any check fails', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + doctorRuntime: vi.fn(async (): Promise => [ + { id: 'uv', label: 'uv', status: 'pass', detail: 'uv 0.9.5' }, + { + id: 'runtime', + label: 'Managed Python runtime', + status: 'fail', + detail: 'No runtime manifest', + fix: 'Run: ktx runtime install --yes', + }, + ]), + }; + + await expect(runKtxRuntime({ command: 'doctor', cliVersion: '0.2.0', json: false }, io.io, deps)).resolves.toBe(1); + + expect(io.stdout()).toContain('PASS uv: uv 0.9.5'); + expect(io.stdout()).toContain('FAIL Managed Python runtime: No runtime manifest'); + expect(io.stdout()).toContain('Fix: Run: ktx runtime install --yes'); + }); + + it('requires --yes before pruning stale runtime directories', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + pruneRuntime: vi.fn(async () => { + throw new Error('should not prune without --yes'); + }), + }; + + await expect(runKtxRuntime({ command: 'prune', cliVersion: '0.2.0', dryRun: false, yes: false }, io.io, deps)) + .resolves.toBe(1); + + expect(io.stderr()).toContain('Refusing to prune without --yes'); + expect(deps.pruneRuntime).not.toHaveBeenCalled(); + }); + + it('prints stale directories during prune dry-run', async () => { + const io = makeIo(); + const deps: KtxRuntimeDeps = { + readStatus: vi.fn(async (): Promise => ({ + kind: 'missing', + detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', + layout: { + cliVersion: '0.2.0', + runtimeRoot: '/runtime', + versionDir: '/runtime/0.2.0', + venvDir: '/runtime/0.2.0/.venv', + manifestPath: '/runtime/0.2.0/manifest.json', + installLogPath: '/runtime/0.2.0/install.log', + assetDir: '/assets/python', + assetManifestPath: '/assets/python/manifest.json', + pythonPath: '/runtime/0.2.0/.venv/bin/python', + daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', + daemonStatePath: '/runtime/0.2.0/daemon.json', + daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', + daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', + }, + })), + pruneRuntime: vi.fn(async () => ({ + runtimeRoot: '/runtime', + stale: ['/runtime/0.1.0'], + kept: ['/runtime/0.2.0'], + removed: [], + })), + }; + + await expect(runKtxRuntime({ command: 'prune', cliVersion: '0.2.0', dryRun: true, yes: false }, io.io, deps)) + .resolves.toBe(0); + + expect(io.stdout()).toContain('Stale KTX Python runtimes'); + expect(io.stdout()).toContain('/runtime/0.1.0'); + }); +}); diff --git a/packages/cli/src/runtime.ts b/packages/cli/src/runtime.ts new file mode 100644 index 00000000..fe2b5f74 --- /dev/null +++ b/packages/cli/src/runtime.ts @@ -0,0 +1,187 @@ +import type { KtxCliIo } from './cli-runtime.js'; +import { + startManagedPythonDaemon, + stopManagedPythonDaemon, + type ManagedPythonDaemonStartResult, + type ManagedPythonDaemonStopResult, +} from './managed-python-daemon.js'; +import { + doctorManagedPythonRuntime, + installManagedPythonRuntime, + pruneManagedPythonRuntimes, + readManagedPythonRuntimeStatus, + type KtxRuntimeFeature, + type ManagedPythonRuntimeDoctorCheck, + type ManagedPythonRuntimeInstallOptions, + type ManagedPythonRuntimeInstallResult, + type ManagedPythonRuntimeLayoutOptions, + type ManagedPythonRuntimePruneResult, + type ManagedPythonRuntimeStatus, +} from './managed-python-runtime.js'; + +export type KtxRuntimeArgs = + | { command: 'install'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean } + | { command: 'start'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean } + | { command: 'stop'; cliVersion: string } + | { command: 'status'; cliVersion: string; json: boolean } + | { command: 'doctor'; cliVersion: string; json: boolean } + | { command: 'prune'; cliVersion: string; dryRun: boolean; yes: boolean }; + +export interface KtxRuntimeDeps { + installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise; + startDaemon?: (options: { + cliVersion: string; + features: KtxRuntimeFeature[]; + force?: boolean; + }) => Promise; + stopDaemon?: (options: { cliVersion: string }) => Promise; + readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; + doctorRuntime?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; + pruneRuntime?: (options: { + cliVersion: string; + runtimeRoot: string; + dryRun?: boolean; + }) => Promise; +} + +function writeJson(io: KtxCliIo, value: unknown): void { + io.stdout.write(`${JSON.stringify(value, null, 2)}\n`); +} + +function writeInstallResult(io: KtxCliIo, result: ManagedPythonRuntimeInstallResult): void { + const verb = result.status === 'ready' ? 'Using existing' : 'Installed'; + io.stdout.write(`${verb} KTX Python runtime\n`); + io.stdout.write(`version: ${result.manifest.cliVersion}\n`); + io.stdout.write(`features: ${result.manifest.features.join(', ')}\n`); + io.stdout.write(`python: ${result.manifest.python.executable}\n`); + io.stdout.write(`daemon: ${result.manifest.python.daemonExecutable}\n`); + io.stdout.write(`manifest: ${result.layout.manifestPath}\n`); + io.stdout.write(`install log: ${result.layout.installLogPath}\n`); +} + +function writeDaemonStart(io: KtxCliIo, result: ManagedPythonDaemonStartResult): void { + const verb = result.status === 'reused' ? 'Using existing' : 'Started'; + io.stdout.write(`${verb} KTX Python daemon\n`); + io.stdout.write(`url: ${result.baseUrl}\n`); + io.stdout.write(`pid: ${result.state.pid}\n`); + io.stdout.write(`version: ${result.state.version}\n`); + io.stdout.write(`features: ${result.state.features.join(', ')}\n`); + io.stdout.write(`state: ${result.layout.daemonStatePath}\n`); + io.stdout.write(`stdout: ${result.state.stdoutLog}\n`); + io.stdout.write(`stderr: ${result.state.stderrLog}\n`); +} + +function writeDaemonStop(io: KtxCliIo, result: ManagedPythonDaemonStopResult): void { + if (result.status === 'already-stopped') { + io.stdout.write('KTX Python daemon already stopped\n'); + return; + } + io.stdout.write('Stopped KTX Python daemon\n'); + io.stdout.write(`pid: ${result.state?.pid ?? 'unknown'}\n`); + io.stdout.write(`state: ${result.layout.daemonStatePath}\n`); +} + +function writeStatus(io: KtxCliIo, status: ManagedPythonRuntimeStatus): void { + io.stdout.write('KTX Python runtime\n'); + io.stdout.write(`status: ${status.kind}\n`); + io.stdout.write(`detail: ${status.detail}\n`); + io.stdout.write(`runtime root: ${status.layout.runtimeRoot}\n`); + io.stdout.write(`version dir: ${status.layout.versionDir}\n`); + if (status.manifest) { + io.stdout.write(`features: ${status.manifest.features.join(', ')}\n`); + io.stdout.write(`python: ${status.manifest.python.executable}\n`); + io.stdout.write(`daemon: ${status.manifest.python.daemonExecutable}\n`); + } +} + +function writeDoctor(io: KtxCliIo, checks: ManagedPythonRuntimeDoctorCheck[]): void { + io.stdout.write('KTX Python runtime doctor\n'); + for (const check of checks) { + io.stdout.write(`${check.status.toUpperCase()} ${check.label}: ${check.detail}\n`); + if (check.fix) { + io.stdout.write(` Fix: ${check.fix}\n`); + } + } +} + +function writePrune(io: KtxCliIo, result: ManagedPythonRuntimePruneResult, dryRun: boolean): void { + if (result.stale.length === 0) { + io.stdout.write(`No stale KTX Python runtimes found under ${result.runtimeRoot}\n`); + return; + } + io.stdout.write(dryRun ? 'Stale KTX Python runtimes\n' : 'Removed stale KTX Python runtimes\n'); + for (const path of dryRun ? result.stale : result.removed) { + io.stdout.write(`${path}\n`); + } +} + +export async function runKtxRuntime( + args: KtxRuntimeArgs, + io: KtxCliIo = process, + deps: KtxRuntimeDeps = {}, +): Promise { + try { + if (args.command === 'install') { + const installRuntime = deps.installRuntime ?? installManagedPythonRuntime; + const result = await installRuntime({ + cliVersion: args.cliVersion, + features: [args.feature], + force: args.force, + }); + writeInstallResult(io, result); + return 0; + } + if (args.command === 'start') { + const startDaemon = deps.startDaemon ?? startManagedPythonDaemon; + const result = await startDaemon({ + cliVersion: args.cliVersion, + features: [args.feature], + force: args.force, + }); + writeDaemonStart(io, result); + return 0; + } + if (args.command === 'stop') { + const stopDaemon = deps.stopDaemon ?? stopManagedPythonDaemon; + const result = await stopDaemon({ cliVersion: args.cliVersion }); + writeDaemonStop(io, result); + return 0; + } + if (args.command === 'status') { + const readStatus = deps.readStatus ?? readManagedPythonRuntimeStatus; + const status = await readStatus({ cliVersion: args.cliVersion }); + if (args.json) { + writeJson(io, status); + } else { + writeStatus(io, status); + } + return 0; + } + if (args.command === 'doctor') { + const doctorRuntime = deps.doctorRuntime ?? doctorManagedPythonRuntime; + const checks = await doctorRuntime({ cliVersion: args.cliVersion }); + if (args.json) { + writeJson(io, { checks }); + } else { + writeDoctor(io, checks); + } + return checks.some((check) => check.status === 'fail') ? 1 : 0; + } + if (!args.dryRun && !args.yes) { + io.stderr.write('Refusing to prune without --yes. Preview with: ktx runtime prune --dry-run\n'); + return 1; + } + const status = await (deps.readStatus ?? readManagedPythonRuntimeStatus)({ cliVersion: args.cliVersion }); + const pruneRuntime = deps.pruneRuntime ?? pruneManagedPythonRuntimes; + const result = await pruneRuntime({ + cliVersion: args.cliVersion, + runtimeRoot: status.layout.runtimeRoot, + dryRun: args.dryRun, + }); + writePrune(io, result, args.dryRun); + return 0; + } catch (error) { + io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } +} diff --git a/packages/cli/src/scan.test.ts b/packages/cli/src/scan.test.ts index 603a0091..525ae53d 100644 --- a/packages/cli/src/scan.test.ts +++ b/packages/cli/src/scan.test.ts @@ -356,6 +356,49 @@ describe('runKtxScan', () => { expect(io.stdout()).not.toContain('/~'); }); + it('passes managed daemon options to local ingest adapters when no explicit daemon URL is set', async () => { + await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const createLocalIngestAdapters = vi.fn(() => []); + const runLocalScan = vi.fn( + async (_input: RunLocalScanOptions): Promise => ({ + runId: 'scan-run-1', + status: 'done', + done: true, + connectionId: 'warehouse', + mode: 'structural', + dryRun: false, + syncId: 'sync-1', + report, + }), + ); + const io = makeIo(); + + await expect( + runKtxScan( + { + command: 'run', + projectDir: tempDir, + connectionId: 'warehouse', + mode: 'structural', + detectRelationships: false, + dryRun: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + io.io, + { runLocalScan, createLocalIngestAdapters }, + ), + ).resolves.toBe(0); + + expect(createLocalIngestAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir }), { + managedDaemon: { + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + }, + }); + }); + it('explains warnings, capability gaps, and relationships in human scan summaries', async () => { await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); const runLocalScan = vi.fn( diff --git a/packages/cli/src/scan.ts b/packages/cli/src/scan.ts index accccf57..f89a9d18 100644 --- a/packages/cli/src/scan.ts +++ b/packages/cli/src/scan.ts @@ -33,6 +33,7 @@ import { import type { KtxCliIo } from './index.js'; import { createKtxCliLocalIngestAdapters } from './local-adapters.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; +import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; import { profileMark } from './startup-profile.js'; profileMark('module:scan'); @@ -46,6 +47,8 @@ export type KtxScanArgs = detectRelationships: boolean; dryRun: boolean; databaseIntrospectionUrl?: string; + cliVersion?: string; + runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; } | { command: 'status'; projectDir: string; runId: string } | { command: 'report'; projectDir: string; runId: string; json: boolean } @@ -220,6 +223,17 @@ function warningLine(warning: KtxScanWarning): string { return `${warning.code}: ${location}${warning.message}`; } +function managedDaemonOptionsForScanRun(args: Extract, io: KtxCliIo) { + if (args.databaseIntrospectionUrl || !args.cliVersion || !args.runtimeInstallPolicy) { + return undefined; + } + return { + cliVersion: args.cliVersion, + installPolicy: args.runtimeInstallPolicy, + io, + }; +} + function writeNeedsAttention(report: KtxScanReport, io: KtxCliIo): void { io.stdout.write('\nNeeds attention\n'); if (report.warnings.length === 0 && report.capabilityGaps.length === 0) { @@ -704,6 +718,7 @@ export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps return 0; } + const managedDaemon = managedDaemonOptionsForScanRun(args, io); const connector = args.mode !== 'structural' || args.detectRelationships ? await createKtxCliScanConnector(project, args.connectionId) @@ -720,7 +735,8 @@ export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps databaseIntrospectionUrl: args.databaseIntrospectionUrl, connector, adapters: (deps.createLocalIngestAdapters ?? createKtxCliLocalIngestAdapters)(project, { - databaseIntrospectionUrl: args.databaseIntrospectionUrl, + ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), + ...(managedDaemon ? { managedDaemon } : {}), }), progress, }); diff --git a/packages/cli/src/serve.test.ts b/packages/cli/src/serve.test.ts index ca328d07..ccffa548 100644 --- a/packages/cli/src/serve.test.ts +++ b/packages/cli/src/serve.test.ts @@ -6,6 +6,19 @@ import { initKtxProject } from '@ktx/context/project'; import { describe, expect, it, vi } from 'vitest'; import { runKtxServeStdio } from './serve.js'; +function makeManagedRuntimeIo() { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { write: (chunk: string) => (stdout += chunk) }, + stderr: { write: (chunk: string) => (stderr += chunk) }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + describe('runKtxServeStdio', () => { it('loads the project, creates local ports, and connects the server to stdio', async () => { const connect = vi.fn().mockResolvedValue(undefined); @@ -149,6 +162,9 @@ describe('runKtxServeStdio', () => { expect.objectContaining({ localIngest: expect.objectContaining({ adapters: expect.any(Array), + pullConfigOptions: { + databaseIntrospectionUrl: 'http://127.0.0.1:8765', + }, }), localScan: expect.objectContaining({ adapters: createdAdapters, @@ -161,6 +177,63 @@ describe('runKtxServeStdio', () => { }); }); + it('passes managed daemon options to MCP local ingest adapters and pull-config options', async () => { + const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never; + const adapters: SourceAdapter[] = [ + { source: 'looker', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) }, + ]; + const createIngestAdapters = vi.fn(() => adapters); + const createContextTools = vi.fn(() => ({ connections: { list: async () => [] } })); + const managedRuntimeIo = makeManagedRuntimeIo(); + + await expect( + runKtxServeStdio( + { + mcp: 'stdio', + projectDir: '/tmp/ktx-project', + userId: 'agent', + semanticCompute: false, + semanticComputeUrl: undefined, + databaseIntrospectionUrl: undefined, + executeQueries: false, + memoryCapture: false, + memoryModel: undefined, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + { + loadProject: async () => project, + createContextTools, + createIngestAdapters, + managedRuntimeIo: managedRuntimeIo.io, + createServer: vi.fn(() => ({ connect: vi.fn(async () => undefined) }) as never), + createTransport: vi.fn(() => ({}) as never), + stderr: { write: vi.fn() }, + }, + ), + ).resolves.toBe(0); + + const expectedManagedDaemon = { + cliVersion: '0.2.0', + installPolicy: 'auto', + io: managedRuntimeIo.io, + }; + expect(createIngestAdapters).toHaveBeenCalledWith(project, { + managedDaemon: expectedManagedDaemon, + }); + expect(createContextTools).toHaveBeenCalledWith( + project, + expect.objectContaining({ + localIngest: expect.objectContaining({ + adapters, + pullConfigOptions: { + managedDaemon: expectedManagedDaemon, + }, + }), + }), + ); + }); + it('uses CLI-native local ingest adapters for standalone scan tools', async () => { const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never; const createContextTools = vi.fn(() => ({}) as never); @@ -241,6 +314,53 @@ describe('runKtxServeStdio', () => { } }); + it('uses managed semantic compute when MCP semantic compute has no explicit HTTP URL', async () => { + const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never; + const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; + const createManagedSemanticLayerCompute = vi.fn(async () => semanticLayerCompute); + const createContextTools = vi.fn(() => ({ connections: { list: async () => [] } })); + const managedRuntimeIo = makeManagedRuntimeIo(); + + await expect( + runKtxServeStdio( + { + mcp: 'stdio', + projectDir: '/tmp/ktx-project', + userId: 'agent', + semanticCompute: true, + semanticComputeUrl: undefined, + databaseIntrospectionUrl: undefined, + executeQueries: false, + memoryCapture: false, + memoryModel: undefined, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + { + loadProject: async () => project, + createContextTools, + createManagedSemanticLayerCompute, + managedRuntimeIo: managedRuntimeIo.io, + createServer: vi.fn(() => ({ connect: vi.fn(async () => undefined) }) as never), + createTransport: vi.fn(() => ({}) as never), + stderr: { write: vi.fn() }, + }, + ), + ).resolves.toBe(0); + + expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: managedRuntimeIo.io, + }); + expect(createContextTools).toHaveBeenCalledWith( + project, + expect.objectContaining({ + semanticLayerCompute, + }), + ); + }); + it('uses the HTTP semantic compute port when a daemon URL is provided', async () => { const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never; const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; diff --git a/packages/cli/src/serve.ts b/packages/cli/src/serve.ts index 7857e7d6..0121834e 100644 --- a/packages/cli/src/serve.ts +++ b/packages/cli/src/serve.ts @@ -2,7 +2,6 @@ import { createLocalKtxLlmProviderFromConfig } from '@ktx/context'; import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from '@ktx/context/connections'; import { createHttpSemanticLayerComputePort, - createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort, } from '@ktx/context/daemon'; import { createDefaultLocalIngestAdapters, type LocalIngestMcpOptions } from '@ktx/context/ingest'; @@ -15,8 +14,14 @@ import { createLocalProjectMemoryCapture, type MemoryCaptureService } from '@ktx import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project'; import type { LocalScanMcpOptions } from '@ktx/context/scan'; import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import type { KtxCliIo } from './cli-runtime.js'; import { createKtxCliLocalIngestAdapters } from './local-adapters.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; +import { + createManagedPythonSemanticLayerComputePort, + type KtxManagedPythonInstallPolicy, +} from './managed-python-command.js'; +import type { ManagedPythonCoreDaemonOptions } from './managed-python-http.js'; import { profileMark } from './startup-profile.js'; profileMark('module:serve'); @@ -31,6 +36,8 @@ export interface KtxServeArgs { executeQueries: boolean; memoryCapture: boolean; memoryModel?: string; + cliVersion?: string; + runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; } interface KtxServeIo { @@ -48,8 +55,10 @@ interface KtxServeDeps { loadProject?: typeof loadKtxProject; createContextTools?: (project: KtxLocalProject, options?: LocalProjectContextToolOptions) => KtxMcpContextPorts; createSemanticLayerCompute?: () => KtxSemanticLayerComputePort; + createManagedSemanticLayerCompute?: typeof createManagedPythonSemanticLayerComputePort; + managedRuntimeIo?: KtxCliIo; createHttpSemanticLayerCompute?: (baseUrl: string) => KtxSemanticLayerComputePort; - createIngestAdapters?: typeof createDefaultLocalIngestAdapters; + createIngestAdapters?: typeof createKtxCliLocalIngestAdapters; createQueryExecutor?: () => KtxSqlQueryExecutorPort; createMemoryCapture?: typeof createLocalProjectMemoryCapture; createServer?: typeof createDefaultKtxMcpServer; @@ -57,6 +66,51 @@ interface KtxServeDeps { stderr?: KtxServeIo['stderr']; } +function requiredManagedRuntimeCliVersion(args: KtxServeArgs): string { + if (!args.cliVersion) { + throw new Error('Managed Python semantic compute requires a CLI version.'); + } + return args.cliVersion; +} + +function managedDaemonOptionsForServe( + args: KtxServeArgs, + deps: KtxServeDeps, +): ManagedPythonCoreDaemonOptions | undefined { + if (args.databaseIntrospectionUrl || !args.cliVersion) { + return undefined; + } + return { + cliVersion: args.cliVersion, + installPolicy: args.runtimeInstallPolicy ?? 'prompt', + io: deps.managedRuntimeIo ?? process, + }; +} + +async function createServeSemanticLayerCompute( + args: KtxServeArgs, + deps: KtxServeDeps, +): Promise { + if (!args.semanticCompute) { + return undefined; + } + if (args.semanticComputeUrl) { + return (deps.createHttpSemanticLayerCompute ?? ((baseUrl) => createHttpSemanticLayerComputePort({ baseUrl })))( + args.semanticComputeUrl, + ); + } + if (deps.createSemanticLayerCompute) { + return deps.createSemanticLayerCompute(); + } + const createManagedSemanticLayerCompute = + deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort; + return createManagedSemanticLayerCompute({ + cliVersion: requiredManagedRuntimeCliVersion(args), + installPolicy: args.runtimeInstallPolicy ?? 'prompt', + io: deps.managedRuntimeIo ?? process, + }); +} + export async function runKtxServeStdio(args: KtxServeArgs, deps: KtxServeDeps = {}): Promise { const loadProjectFn = deps.loadProject ?? loadKtxProject; const createContextToolsFn = deps.createContextTools ?? createLocalProjectMcpContextPorts; @@ -65,20 +119,17 @@ export async function runKtxServeStdio(args: KtxServeArgs, deps: KtxServeDeps = const stderr = deps.stderr ?? process.stderr; const project = await loadProjectFn({ projectDir: args.projectDir }); - const semanticLayerCompute = args.semanticCompute - ? args.semanticComputeUrl - ? (deps.createHttpSemanticLayerCompute ?? ((baseUrl) => createHttpSemanticLayerComputePort({ baseUrl })))( - args.semanticComputeUrl, - ) - : (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)() - : undefined; + const semanticLayerCompute = await createServeSemanticLayerCompute(args, deps); const queryExecutor = args.executeQueries ? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)() : undefined; const createIngestAdapters = deps.createIngestAdapters ?? createKtxCliLocalIngestAdapters; - const localAdapters = createIngestAdapters(project, { - databaseIntrospectionUrl: args.databaseIntrospectionUrl, - }); + const managedDaemon = managedDaemonOptionsForServe(args, deps); + const localAdapterOptions = { + ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), + ...(managedDaemon ? { managedDaemon } : {}), + }; + const localAdapters = createIngestAdapters(project, localAdapterOptions); const llmProvider = args.memoryCapture ? (createLocalKtxLlmProviderFromConfig(project.config.llm) ?? undefined) : undefined; @@ -90,6 +141,7 @@ export async function runKtxServeStdio(args: KtxServeArgs, deps: KtxServeDeps = : undefined; const localIngest: LocalIngestMcpOptions = { adapters: localAdapters, + pullConfigOptions: localAdapterOptions, ...(semanticLayerCompute ? { semanticLayerCompute } : {}), ...(queryExecutor ? { queryExecutor } : {}), }; diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index a4a7b4c3..ce3618e7 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -46,6 +46,15 @@ function makePromptAdapter(options: { }; } +function managedDaemon(baseUrl = 'http://127.0.0.1:61234') { + return { + baseUrl, + env: { + KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL: baseUrl, + }, + }; +} + describe('setup embeddings step', () => { let tempDir: string; @@ -67,6 +76,8 @@ describe('setup embeddings step', () => { { projectDir: tempDir, inputMode: 'auto', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', skipEmbeddings: false, }, io.io, @@ -94,10 +105,12 @@ describe('setup embeddings step', () => { { projectDir: tempDir, inputMode: 'auto', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', skipEmbeddings: false, }, io.io, - { prompts, env: {}, healthCheck }, + { prompts, env: {}, healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()) }, ); expect(result.status).toBe('ready'); @@ -106,7 +119,7 @@ describe('setup embeddings step', () => { backend: 'sentence-transformers', model: 'all-MiniLM-L6-v2', dimensions: 384, - sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' }, + sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, }); expect(vi.mocked(prompts.select).mock.calls.map((call) => call[0].message)).toEqual([ EMBEDDING_OPTION_PROMPT_MESSAGE, @@ -119,30 +132,38 @@ describe('setup embeddings step', () => { const io = makeIo(); const healthCheck = vi.fn(async () => ({ ok: true as const })); const prompts = makePromptAdapter({ selectValues: ['sentence-transformers'] }); + const ensureLocalEmbeddings = vi.fn(async () => managedDaemon()); const result = await runKtxSetupEmbeddingsStep( { projectDir: tempDir, inputMode: 'auto', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', skipEmbeddings: false, }, io.io, - { prompts, env: {}, healthCheck }, + { prompts, env: {}, healthCheck, ensureLocalEmbeddings }, ); expect(result.status).toBe('ready'); + expect(ensureLocalEmbeddings).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: io.io, + }); expect(healthCheck).toHaveBeenCalledWith({ backend: 'sentence-transformers', model: 'all-MiniLM-L6-v2', dimensions: 384, - sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' }, + sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.ingest.embeddings).toMatchObject({ backend: 'sentence-transformers', model: 'all-MiniLM-L6-v2', dimensions: 384, - sentenceTransformers: { base_url: 'http://127.0.0.1:8765', pathPrefix: '' }, + sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' }, }); expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings); expect(config.setup?.completed_steps).toContain('embeddings'); @@ -167,10 +188,12 @@ describe('setup embeddings step', () => { { projectDir: tempDir, inputMode: 'auto', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', skipEmbeddings: false, }, io.io, - { prompts, env: {}, healthCheck }, + { prompts, env: {}, healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()) }, ); await vi.waitFor(() => { @@ -192,10 +215,12 @@ describe('setup embeddings step', () => { { projectDir: tempDir, inputMode: 'disabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', skipEmbeddings: false, }, io.io, - { env: {}, healthCheck }, + { env: {}, healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()) }, ); expect(result.status).toBe('ready'); @@ -203,30 +228,59 @@ describe('setup embeddings step', () => { backend: 'sentence-transformers', model: 'all-MiniLM-L6-v2', dimensions: 384, - sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' }, + sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.ingest.embeddings).toMatchObject({ backend: 'sentence-transformers', model: 'all-MiniLM-L6-v2', dimensions: 384, - sentenceTransformers: { base_url: 'http://127.0.0.1:8765', pathPrefix: '' }, + sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' }, }); expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings); expect(config.setup?.completed_steps).toContain('embeddings'); }); + it('fails non-interactive local setup when the managed local embeddings runtime is missing', async () => { + const io = makeIo(); + const ensureLocalEmbeddings = vi.fn(async () => { + throw new Error( + 'KTX Python runtime is required for this command. Run: ktx runtime install --feature local-embeddings --yes', + ); + }); + + const result = await runKtxSetupEmbeddingsStep( + { + projectDir: tempDir, + inputMode: 'disabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'never', + skipEmbeddings: false, + }, + io.io, + { env: {}, ensureLocalEmbeddings }, + ); + + expect(result.status).toBe('failed'); + expect(io.stderr()).toContain( + 'KTX Python runtime is required for this command. Run: ktx runtime install --feature local-embeddings --yes', + ); + }); + it('does not persist embedding completion when the health check fails', async () => { const io = makeIo(); const result = await runKtxSetupEmbeddingsStep( { projectDir: tempDir, inputMode: 'disabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', skipEmbeddings: false, }, io.io, { env: {}, + ensureLocalEmbeddings: vi.fn(async () => managedDaemon()), healthCheck: vi.fn(async () => ({ ok: false as const, message: '401 invalid api key [redacted]' })), }, ); @@ -236,7 +290,7 @@ describe('setup embeddings step', () => { expect(config.setup?.completed_steps ?? []).not.toContain('embeddings'); expect(config.ingest.embeddings.backend).toBe('deterministic'); expect(io.stderr()).toContain('Local embedding health check failed: 401 invalid api key [redacted]'); - expect(io.stderr()).toContain('ktx-daemon serve-http --host 127.0.0.1 --port 8765'); + expect(io.stderr()).toContain('Prepare the runtime with: ktx runtime start --feature local-embeddings'); expect(io.stderr()).not.toContain('skip for now'); }); @@ -250,6 +304,8 @@ describe('setup embeddings step', () => { inputMode: 'disabled', embeddingBackend: 'openai', embeddingApiKeyEnv: 'OPENAI_API_KEY', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', skipEmbeddings: false, }, io.io, @@ -285,9 +341,20 @@ describe('setup embeddings step', () => { .mockResolvedValueOnce({ ok: true as const }); const result = await runKtxSetupEmbeddingsStep( - { projectDir: tempDir, inputMode: 'auto', skipEmbeddings: false }, + { + projectDir: tempDir, + inputMode: 'auto', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + skipEmbeddings: false, + }, io.io, - { prompts, env: { OPENAI_API_KEY: 'sk-openai-test' }, healthCheck }, + { + prompts, + env: { OPENAI_API_KEY: 'sk-openai-test' }, + healthCheck, + ensureLocalEmbeddings: vi.fn(async () => managedDaemon()), + }, ); expect(result.status).toBe('ready'); @@ -295,7 +362,7 @@ describe('setup embeddings step', () => { backend: 'sentence-transformers', model: 'all-MiniLM-L6-v2', dimensions: 384, - sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' }, + sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, }); expect(healthCheck).toHaveBeenNthCalledWith(2, { backend: 'openai', @@ -320,7 +387,13 @@ describe('setup embeddings step', () => { it('leaves setup incomplete when skipped', async () => { const result = await runKtxSetupEmbeddingsStep( - { projectDir: tempDir, inputMode: 'disabled', skipEmbeddings: true }, + { + projectDir: tempDir, + inputMode: 'disabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + skipEmbeddings: true, + }, makeIo().io, ); @@ -333,9 +406,20 @@ describe('setup embeddings step', () => { it('returns back without writing config when the local health check fails and Back is selected', async () => { const prompts = makePromptAdapter({ selectValues: ['sentence-transformers', 'back'] }); const result = await runKtxSetupEmbeddingsStep( - { projectDir: tempDir, inputMode: 'auto', skipEmbeddings: false }, + { + projectDir: tempDir, + inputMode: 'auto', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + skipEmbeddings: false, + }, makeIo().io, - { prompts, env: {}, healthCheck: vi.fn(async () => ({ ok: false as const, message: 'daemon unavailable' })) }, + { + prompts, + env: {}, + ensureLocalEmbeddings: vi.fn(async () => managedDaemon()), + healthCheck: vi.fn(async () => ({ ok: false as const, message: 'daemon unavailable' })), + }, ); expect(result.status).toBe('back'); @@ -371,10 +455,20 @@ describe('setup embeddings step', () => { const healthCheck = vi.fn(async () => ({ ok: true as const })); await expect( - runKtxSetupEmbeddingsStep({ projectDir: tempDir, inputMode: 'disabled', skipEmbeddings: false }, makeIo().io, { - env: { OPENAI_API_KEY: 'sk-openai-test' }, - healthCheck, - }), + runKtxSetupEmbeddingsStep( + { + projectDir: tempDir, + inputMode: 'disabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + skipEmbeddings: false, + }, + makeIo().io, + { + env: { OPENAI_API_KEY: 'sk-openai-test' }, + healthCheck, + }, + ), ).resolves.toMatchObject({ status: 'ready' }); expect(healthCheck).not.toHaveBeenCalled(); }); diff --git a/packages/cli/src/setup-embeddings.ts b/packages/cli/src/setup-embeddings.ts index 61cdc68d..acb97eaa 100644 --- a/packages/cli/src/setup-embeddings.ts +++ b/packages/cli/src/setup-embeddings.ts @@ -10,6 +10,13 @@ import { } from '@ktx/context/project'; import { type KtxEmbeddingConfig, type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from '@ktx/llm'; import type { KtxCliIo } from './cli-runtime.js'; +import { + ensureManagedLocalEmbeddingsDaemon, + managedLocalEmbeddingHealthConfig, + managedLocalEmbeddingProjectConfig, + type ManagedLocalEmbeddingsDaemon, +} from './managed-local-embeddings.js'; +import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; import { withMenuOptionsSpacing, withTextInputNavigation } from './prompt-navigation.js'; import { withSetupInterruptConfirmation } from './setup-interrupt.js'; import { envCredentialReference, writeProjectLocalSecretReference } from './setup-secrets.js'; @@ -19,6 +26,8 @@ export type KtxSetupEmbeddingBackend = 'openai' | 'sentence-transformers'; export interface KtxSetupEmbeddingsArgs { projectDir: string; inputMode: 'auto' | 'disabled'; + cliVersion: string; + runtimeInstallPolicy: KtxManagedPythonInstallPolicy; embeddingBackend?: KtxSetupEmbeddingBackend; embeddingApiKeyEnv?: string; embeddingApiKeyFile?: string; @@ -44,6 +53,11 @@ export interface KtxSetupEmbeddingsDeps { env?: NodeJS.ProcessEnv; prompts?: KtxSetupEmbeddingsPromptAdapter; healthCheck?: (config: KtxEmbeddingConfig) => Promise; + ensureLocalEmbeddings?: (options: { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxCliIo; + }) => Promise; } type BackendChoice = KtxSetupEmbeddingBackend | 'back'; @@ -62,9 +76,6 @@ const DEFAULTS: Record< }; const LOCAL_EMBEDDING_BACKEND: KtxSetupEmbeddingBackend = 'sentence-transformers'; -const LOCAL_EMBEDDING_DAEMON_COMMAND = 'ktx-daemon serve-http --host 127.0.0.1 --port 8765'; -const LOCAL_EMBEDDING_DAEMON_DEV_COMMAND = - 'cd ktx && source .venv/bin/activate && uv run ktx-daemon serve-http --host 127.0.0.1 --port 8765'; const EMBEDDING_OPTION_PROMPT_CONTEXT = 'KTX uses embeddings for semantic search over semantic-layer sources, wiki context, schema metadata, ' + 'and relationship evidence.'; @@ -302,10 +313,10 @@ async function chooseEmbeddingBackend( function localEmbeddingSetupMessage(message: string): string { return [ `Local embedding health check failed: ${message}`, - 'Local embeddings use the KTX Python daemon. KTX can call ktx-daemon automatically when it is on PATH.', - `For repeated inference, start the HTTP daemon in another terminal with: ${LOCAL_EMBEDDING_DAEMON_COMMAND}`, - `From the KTX repo, use: ${LOCAL_EMBEDDING_DAEMON_DEV_COMMAND}`, - 'The first run may download the all-MiniLM-L6-v2 model, so it can take a minute.', + 'Local embeddings use the KTX-managed Python runtime.', + 'Prepare the runtime with: ktx runtime start --feature local-embeddings', + 'Use --yes with setup to install and start the runtime without prompting.', + 'The first run may download Python packages and the all-MiniLM-L6-v2 model.', ].join('\n'); } @@ -432,12 +443,34 @@ export async function runKtxSetupEmbeddingsStep( credentialValue = credential.value; } - const healthConfig = buildHealthConfig({ - backend: selectedBackend, - model, - dimensions, - credentialValue, - }); + let managedLocalEmbeddings: ManagedLocalEmbeddingsDaemon | undefined; + if (selectedBackend === LOCAL_EMBEDDING_BACKEND) { + const ensureLocalEmbeddings = deps.ensureLocalEmbeddings ?? ensureManagedLocalEmbeddingsDaemon; + try { + managedLocalEmbeddings = await ensureLocalEmbeddings({ + cliVersion: args.cliVersion, + installPolicy: args.runtimeInstallPolicy, + io, + }); + } catch (error) { + io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + return { status: 'failed', projectDir: args.projectDir }; + } + } + + const healthConfig = + selectedBackend === LOCAL_EMBEDDING_BACKEND && managedLocalEmbeddings + ? managedLocalEmbeddingHealthConfig({ + baseUrl: managedLocalEmbeddings.baseUrl, + model, + dimensions, + }) + : buildHealthConfig({ + backend: selectedBackend, + model, + dimensions, + credentialValue, + }); const progress = startHealthCheckProgress(io, healthCheckStartText(selectedBackend, model, dimensions)); let health: KtxEmbeddingHealthCheckResult; try { @@ -450,12 +483,14 @@ export async function runKtxSetupEmbeddingsStep( progress.succeed(`Embedding test passed (${model}, ${dimensions} dimensions)`); await persistEmbeddingConfig( args.projectDir, - buildProjectEmbeddingConfig({ - backend: selectedBackend, - model, - dimensions, - credentialRef, - }), + selectedBackend === LOCAL_EMBEDDING_BACKEND + ? managedLocalEmbeddingProjectConfig({ model, dimensions }) + : buildProjectEmbeddingConfig({ + backend: selectedBackend, + model, + dimensions, + credentialRef, + }), ); io.stdout.write(`Embeddings ready: yes (${model}, ${dimensions} dimensions)\n`); return { status: 'ready', projectDir: args.projectDir }; diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index 20f12e6e..7cb0d0df 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -318,6 +318,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'disabled', yes: true, + cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, databaseSchemas: [], @@ -364,6 +365,7 @@ describe('setup status', () => { skipAgents: false, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, databaseSchemas: [], @@ -410,6 +412,7 @@ describe('setup status', () => { skipAgents: false, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, databaseSchemas: [], @@ -434,6 +437,7 @@ describe('setup status', () => { skipAgents: false, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, databaseSchemas: [], @@ -472,6 +476,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, databaseSchemas: [], @@ -530,6 +535,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, databaseSchemas: [], @@ -597,6 +603,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, databaseSchemas: [], @@ -661,6 +668,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: true, databaseSchemas: [], @@ -697,6 +705,7 @@ describe('setup status', () => { skipAgents: false, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, databaseSchemas: [], @@ -733,6 +742,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'disabled', yes: false, + cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, databaseSchemas: [], @@ -764,6 +774,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'disabled', yes: false, + cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, databaseSchemas: [], @@ -791,6 +802,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'disabled', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: true, databaseSchemas: [], @@ -819,6 +831,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'disabled', yes: false, + cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', anthropicModel: 'claude-sonnet-4-6', skipLlm: false, @@ -858,7 +871,8 @@ describe('setup status', () => { agents: false, skipAgents: true, inputMode: 'disabled', - yes: false, + yes: true, + cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', anthropicModel: 'claude-sonnet-4-6', skipLlm: false, @@ -878,6 +892,8 @@ describe('setup status', () => { expect.objectContaining({ projectDir: tempDir, inputMode: 'disabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', embeddingBackend: 'openai', embeddingApiKeyEnv: 'OPENAI_API_KEY', skipEmbeddings: false, @@ -886,6 +902,43 @@ describe('setup status', () => { ); }); + it('passes no-input runtime policy to the embeddings step', async () => { + const io = makeIo(); + const embeddings = vi.fn(async () => ({ status: 'failed' as const, projectDir: tempDir })); + + await expect( + runKtxSetup( + { + command: 'run', + projectDir: tempDir, + mode: 'new', + agents: false, + agentScope: 'project', + agentInstallMode: 'cli', + skipAgents: true, + inputMode: 'disabled', + yes: false, + cliVersion: '0.2.0', + skipLlm: true, + skipEmbeddings: false, + databaseSchemas: [], + skipDatabases: true, + skipSources: true, + }, + io.io, + { embeddings }, + ), + ).resolves.toBe(1); + + expect(embeddings).toHaveBeenCalledWith( + expect.objectContaining({ + cliVersion: '0.2.0', + runtimeInstallPolicy: 'never', + }), + io.io, + ); + }); + it('lets Back from embedding setup return to the model step instead of exiting', async () => { const testIo = makeIo(); const modelResults = [ @@ -905,6 +958,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, databaseSchemas: [], @@ -952,6 +1006,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, databaseSchemas: [], @@ -997,6 +1052,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: true, databaseSchemas: [], @@ -1037,6 +1093,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'disabled', yes: false, + cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', anthropicModel: 'claude-sonnet-4-6', skipLlm: false, @@ -1084,6 +1141,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'disabled', yes: true, + cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, skipDatabases: true, @@ -1130,6 +1188,7 @@ describe('setup status', () => { agents: false, inputMode: 'disabled', yes: true, + cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, skipDatabases: true, @@ -1191,6 +1250,7 @@ describe('setup status', () => { agentInstallMode: 'cli', inputMode: 'disabled', yes: true, + cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, skipDatabases: true, @@ -1244,6 +1304,7 @@ describe('setup status', () => { agentInstallMode: 'cli', inputMode: 'disabled', yes: true, + cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, skipDatabases: true, @@ -1277,6 +1338,7 @@ describe('setup status', () => { agents: false, inputMode: 'disabled', yes: true, + cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, skipDatabases: true, @@ -1352,6 +1414,7 @@ describe('setup status', () => { agents: false, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, skipDatabases: false, @@ -1416,6 +1479,7 @@ describe('setup status', () => { agents: false, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, skipDatabases: false, @@ -1509,6 +1573,7 @@ describe('setup status', () => { agents: false, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, skipDatabases: false, @@ -1604,6 +1669,7 @@ describe('setup status', () => { agents: false, inputMode: 'auto', yes: false, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, skipDatabases: false, @@ -1667,6 +1733,7 @@ describe('setup status', () => { agentInstallMode: 'both', inputMode: 'disabled', yes: true, + cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, skipDatabases: false, @@ -1715,6 +1782,7 @@ describe('setup status', () => { skipAgents: true, inputMode: 'disabled', yes: false, + cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', anthropicModel: 'claude-sonnet-4-6', skipLlm: false, diff --git a/packages/cli/src/setup.ts b/packages/cli/src/setup.ts index b9b0b412..5eac2e27 100644 --- a/packages/cli/src/setup.ts +++ b/packages/cli/src/setup.ts @@ -65,6 +65,7 @@ export type KtxSetupArgs = skipAgents?: boolean; inputMode: 'auto' | 'disabled'; yes: boolean; + cliVersion: string; anthropicApiKeyEnv?: string; anthropicApiKeyFile?: string; anthropicModel?: string; @@ -406,6 +407,13 @@ function writeContextNotReadyForAgents(projectDir: string, io: KtxCliIo): void { io.stderr.write(`Then install agent integration:\n ktx setup --agents --project-dir ${resolve(projectDir)}\n`); } +function setupRuntimeInstallPolicy(args: Extract): 'prompt' | 'auto' | 'never' { + if (args.yes) { + return 'auto'; + } + return args.inputMode === 'disabled' ? 'never' : 'prompt'; +} + export async function runKtxSetup(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetupDeps = {}): Promise { try { return await runKtxSetupInner(args, io, deps); @@ -609,6 +617,8 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup { projectDir: projectResult.projectDir, inputMode: args.inputMode, + cliVersion: args.cliVersion, + runtimeInstallPolicy: setupRuntimeInstallPolicy(args), ...(args.embeddingBackend ? { embeddingBackend: args.embeddingBackend } : {}), ...(args.embeddingApiKeyEnv ? { embeddingApiKeyEnv: args.embeddingApiKeyEnv } : {}), ...(args.embeddingApiKeyFile ? { embeddingApiKeyFile: args.embeddingApiKeyFile } : {}), diff --git a/packages/cli/src/sl.test.ts b/packages/cli/src/sl.test.ts index d623e35b..bd746b0b 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/src/sl.test.ts @@ -129,6 +129,8 @@ joins: [] query: { measures: ['orders.order_count'], dimensions: [] }, format: 'sql', execute: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', }, { stdout, stderr }, { loadProject, createSemanticLayerCompute }, @@ -139,6 +141,67 @@ joins: [] expect(stderr.write).not.toHaveBeenCalled(); }); + it('creates default sl query compute through the managed runtime helper', async () => { + const projectDir = join(tempDir, 'project'); + const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); + project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + await project.fileStore.writeFile( + 'semantic-layer/warehouse/orders.yaml', + `name: orders +table: public.orders +grain: [id] +columns: + - name: id + type: number +measures: + - name: order_count + expr: count(*) +joins: [] +`, + 'ktx', + 'ktx@example.com', + 'Add orders source', + ); + + const stdout = { write: vi.fn() }; + const stderr = { write: vi.fn() }; + const compute = { + query: vi.fn(async () => ({ + sql: 'select count(*) as order_count from public.orders', + dialect: 'postgres', + columns: [{ name: 'orders.order_count' }], + plan: {}, + })), + validateSources: vi.fn(), + generateSources: vi.fn(), + }; + const createManagedSemanticLayerCompute = vi.fn(async () => compute); + + await expect( + runKtxSl( + { + command: 'query', + projectDir, + connectionId: 'warehouse', + query: { measures: ['orders.order_count'], dimensions: [] }, + format: 'sql', + execute: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + { stdout, stderr }, + { createManagedSemanticLayerCompute }, + ), + ).resolves.toBe(0); + + expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: { stdout, stderr }, + }); + expect(stdout.write).toHaveBeenCalledWith('select count(*) as order_count from public.orders\n'); + }); + it('executes sl query through the injected query executor', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); @@ -194,6 +257,8 @@ joins: [] format: 'json', execute: true, maxRows: 20, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', }, { stdout, stderr }, { @@ -292,6 +357,8 @@ joins: [] format: 'json', execute: true, maxRows: 20, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', }, { stdout, stderr }, { createSemanticLayerCompute }, diff --git a/packages/cli/src/sl.ts b/packages/cli/src/sl.ts index 17087328..a82f84d8 100644 --- a/packages/cli/src/sl.ts +++ b/packages/cli/src/sl.ts @@ -1,5 +1,5 @@ import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from '@ktx/context/connections'; -import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon'; +import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon'; import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; import { compileLocalSlQuery, @@ -9,6 +9,10 @@ import { writeLocalSlSource, type SemanticLayerQueryInput, } from '@ktx/context/sl'; +import { + createManagedPythonSemanticLayerComputePort, + type KtxManagedPythonInstallPolicy, +} from './managed-python-command.js'; import { profileMark } from './startup-profile.js'; profileMark('module:sl'); @@ -28,6 +32,8 @@ export type KtxSlArgs = format: SlQueryFormat; execute: boolean; maxRows?: number; + cliVersion: string; + runtimeInstallPolicy: KtxManagedPythonInstallPolicy; }; interface KtxSlIo { @@ -38,6 +44,11 @@ interface KtxSlIo { interface KtxSlDeps { loadProject?: typeof loadKtxProject; createSemanticLayerCompute?: () => KtxSemanticLayerComputePort; + createManagedSemanticLayerCompute?: (options: { + cliVersion: string; + installPolicy: KtxManagedPythonInstallPolicy; + io: KtxSlIo; + }) => Promise; createQueryExecutor?: () => KtxSqlQueryExecutorPort; } @@ -97,7 +108,13 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx return 0; } if (args.command === 'query') { - const compute = (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)(); + const compute = deps.createSemanticLayerCompute + ? deps.createSemanticLayerCompute() + : await (deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort)({ + cliVersion: args.cliVersion, + installPolicy: args.runtimeInstallPolicy, + io, + }); const queryExecutor = args.execute ? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)() : undefined; const result = await compileLocalSlQuery(project as KtxLocalProject, { connectionId: args.connectionId, diff --git a/packages/context/src/ingest/local-ingest.ts b/packages/context/src/ingest/local-ingest.ts index c4086f4c..bc6294c4 100644 --- a/packages/context/src/ingest/local-ingest.ts +++ b/packages/context/src/ingest/local-ingest.ts @@ -41,10 +41,17 @@ export interface RunLocalIngestOptions { export interface LocalIngestMcpOptions extends Pick< RunLocalIngestOptions, - 'agentRunner' | 'llmProvider' | 'memoryModel' | 'semanticLayerCompute' | 'queryExecutor' | 'logger' -> { + | 'agentRunner' + | 'llmProvider' + | 'memoryModel' + | 'semanticLayerCompute' + | 'queryExecutor' + | 'logger' + | 'pullConfigOptions' + > { adapters?: SourceAdapter[]; jobIdFactory?: () => string; + runLocalIngest?: (options: RunLocalIngestOptions) => Promise; runLocalMetabaseIngest?: (options: RunLocalMetabaseIngestOptions) => Promise; } diff --git a/packages/context/src/llm/index.ts b/packages/context/src/llm/index.ts index 67e94b93..c9f039b8 100644 --- a/packages/context/src/llm/index.ts +++ b/packages/context/src/llm/index.ts @@ -11,6 +11,8 @@ export { summarizeKtxLlmDebugRequest, } from './debug-request-recorder.js'; export { + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, createLocalKtxEmbeddingProviderFromConfig, createLocalKtxLlmProviderFromConfig, resolveLocalKtxEmbeddingConfig, diff --git a/packages/context/src/llm/local-config.test.ts b/packages/context/src/llm/local-config.test.ts index 96292b56..ffb00b36 100644 --- a/packages/context/src/llm/local-config.test.ts +++ b/packages/context/src/llm/local-config.test.ts @@ -5,6 +5,8 @@ import { type KtxProjectLlmConfig, } from '../project/config.js'; import { + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, createLocalKtxEmbeddingProviderFromConfig, createLocalKtxLlmProviderFromConfig, resolveLocalKtxEmbeddingConfig, @@ -104,6 +106,45 @@ describe('local KTX embedding config', () => { }); }); + it('resolves managed sentence-transformers config from the CLI-provided daemon URL', () => { + const config: KtxProjectEmbeddingConfig = { + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { + base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + pathPrefix: '', + }, + batchSize: 32, + }; + + expect( + resolveLocalKtxEmbeddingConfig(config, { + [MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: 'http://127.0.0.1:61234', + }), + ).toEqual({ + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, + batchSize: 32, + }); + }); + + it('returns null for managed sentence-transformers when no daemon URL is available', () => { + const config: KtxProjectEmbeddingConfig = { + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + sentenceTransformers: { + base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + pathPrefix: '', + }, + }; + + expect(resolveLocalKtxEmbeddingConfig(config, {})).toBeNull(); + }); + it('constructs deterministic embeddings from the default project config', () => { const createKtxEmbeddingProvider = vi.fn(() => ({}) as never); const provider = createLocalKtxEmbeddingProviderFromConfig( diff --git a/packages/context/src/llm/local-config.ts b/packages/context/src/llm/local-config.ts index f0654642..76f1905f 100644 --- a/packages/context/src/llm/local-config.ts +++ b/packages/context/src/llm/local-config.ts @@ -16,6 +16,9 @@ interface LocalConfigDeps { createKtxEmbeddingProvider?: typeof createKtxEmbeddingProvider; } +export const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL = 'managed:local-embeddings'; +export const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV = 'KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL'; + function resolveOptional(value: string | undefined, env: NodeJS.ProcessEnv): string | undefined { return resolveKtxConfigReference(value, env) || undefined; } @@ -89,6 +92,19 @@ export function createLocalKtxLlmProviderFromConfig( return resolved ? (deps.createKtxLlmProvider ?? createKtxLlmProvider)(resolved) : null; } +function resolveSentenceTransformersBaseUrl( + value: string | undefined, + env: NodeJS.ProcessEnv, +): string | undefined { + if (!value) { + return undefined; + } + if (value === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL) { + return resolveOptional(`env:${MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV}`, env); + } + return value; +} + export function resolveLocalKtxEmbeddingConfig( config: KtxProjectEmbeddingConfig, env: NodeJS.ProcessEnv, @@ -96,6 +112,22 @@ export function resolveLocalKtxEmbeddingConfig( if (config.backend === 'none') { return null; } + if (config.backend === 'sentence-transformers') { + const baseURL = resolveSentenceTransformersBaseUrl(config.sentenceTransformers?.base_url, env); + if (!baseURL) { + return null; + } + return { + backend: config.backend, + model: config.model ?? 'all-MiniLM-L6-v2', + dimensions: config.dimensions, + sentenceTransformers: { + baseURL, + pathPrefix: config.sentenceTransformers?.pathPrefix, + }, + batchSize: config.batchSize, + }; + } return { backend: config.backend, model: config.model ?? 'deterministic', diff --git a/packages/context/src/mcp/local-project-ports.test.ts b/packages/context/src/mcp/local-project-ports.test.ts index b4534f56..f5aa52c0 100644 --- a/packages/context/src/mcp/local-project-ports.test.ts +++ b/packages/context/src/mcp/local-project-ports.test.ts @@ -845,6 +845,65 @@ describe('createLocalProjectMcpContextPorts', () => { expect(agentRunner.runLoop).toHaveBeenCalledTimes(1); }); + it('passes local ingest pull-config options into runLocalIngest', async () => { + const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + project.config.connections.warehouse = { driver: 'postgres' }; + project.config.ingest.adapters = ['looker']; + const runLocalIngest = vi.fn(async () => ({ + result: { ok: true }, + report: { + id: 'report-1', + runId: 'run-1', + jobId: 'job-1', + sourceKey: 'looker', + connectionId: 'warehouse', + body: { + syncId: 'sync-1', + workUnits: [], + failedWorkUnits: [], + diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 0 }, + provenanceRows: [], + }, + }, + }) as never); + const ports = createLocalProjectMcpContextPorts(project, { + localIngest: { + adapters: [ + { source: 'looker', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) }, + ], + pullConfigOptions: { + looker: { + daemonBaseUrl: 'http://127.0.0.1:61234', + }, + }, + runLocalIngest, + }, + }); + + await expect( + ports.ingest?.trigger({ + adapter: 'looker', + connectionId: 'warehouse', + trigger: 'manual_resync', + config: {}, + }), + ).resolves.toMatchObject({ + runId: 'run-1', + jobId: 'job-1', + reportId: 'report-1', + }); + + expect(runLocalIngest).toHaveBeenCalledWith( + expect.objectContaining({ + pullConfigOptions: { + looker: { + daemonBaseUrl: 'http://127.0.0.1:61234', + }, + }, + }), + ); + }); + it('triggers fetch-capable local ingest without sourceDir config', async () => { const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); project.config.connections.warehouse = { diff --git a/packages/context/src/mcp/local-project-ports.ts b/packages/context/src/mcp/local-project-ports.ts index d2ad139f..331a14ae 100644 --- a/packages/context/src/mcp/local-project-ports.ts +++ b/packages/context/src/mcp/local-project-ports.ts @@ -586,6 +586,7 @@ export function createLocalProjectMcpContextPorts( metabaseConnectionId: input.connectionId, trigger: input.trigger, jobIdFactory: options.localIngest?.jobIdFactory, + pullConfigOptions: options.localIngest?.pullConfigOptions, agentRunner: options.localIngest?.agentRunner, llmProvider: options.localIngest?.llmProvider, memoryModel: options.localIngest?.memoryModel, @@ -610,12 +611,14 @@ export function createLocalProjectMcpContextPorts( }; } - const result = await runLocalIngest({ + const executeLocalIngest = options.localIngest?.runLocalIngest ?? runLocalIngest; + const result = await executeLocalIngest({ project, adapters: options.localIngest?.adapters ?? createDefaultLocalIngestAdapters(project), adapter: input.adapter, connectionId: input.connectionId, sourceDir, + pullConfigOptions: options.localIngest?.pullConfigOptions, trigger: input.trigger, jobId: options.localIngest?.jobIdFactory?.(), agentRunner: options.localIngest?.agentRunner, diff --git a/packages/context/src/package-exports.test.ts b/packages/context/src/package-exports.test.ts index fd84c9c5..e22d64fa 100644 --- a/packages/context/src/package-exports.test.ts +++ b/packages/context/src/package-exports.test.ts @@ -132,6 +132,10 @@ describe('@ktx/context package exports', () => { expect(root.assertSearchBackendConformanceCase).toBeTypeOf('function'); expect(root.assertSearchBackendCapabilities).toBeTypeOf('function'); expect(root.createLocalKtxEmbeddingProviderFromConfig).toBeTypeOf('function'); + expect(root.MANAGED_SENTENCE_TRANSFORMERS_BASE_URL).toBe('managed:local-embeddings'); + expect(root.MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV).toBe( + 'KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL', + ); expect(agent).toBeDefined(); expect(agent.AgentRunnerService).toBeTypeOf('function'); expect(root.AgentRunnerService).toBeTypeOf('function'); diff --git a/python/ktx-daemon/pyproject.toml b/python/ktx-daemon/pyproject.toml index 57a2760a..16437879 100644 --- a/python/ktx-daemon/pyproject.toml +++ b/python/ktx-daemon/pyproject.toml @@ -15,15 +15,19 @@ dependencies = [ "psycopg[binary]>=3.2.0", "pydantic>=2.9.0", "requests>=2.32.0", - "sentence-transformers>=5.1.1", "sqlglot>=26", - "torch>=2.2.0", "uvicorn[standard]>=0.32.0", ] [project.scripts] ktx-daemon = "ktx_daemon.__main__:main" +[project.optional-dependencies] +local-embeddings = [ + "sentence-transformers>=5.1.1", + "torch>=2.2.0", +] + [project.urls] Homepage = "https://github.com/kaelio/ktx" Repository = "https://github.com/kaelio/ktx" diff --git a/python/ktx-daemon/src/ktx_daemon/app.py b/python/ktx-daemon/src/ktx_daemon/app.py index 73220227..272b0c24 100644 --- a/python/ktx-daemon/src/ktx_daemon/app.py +++ b/python/ktx-daemon/src/ktx_daemon/app.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import os from collections.abc import Callable from typing import Any @@ -80,7 +81,11 @@ def create_app( @app.get("/health") async def health() -> dict[str, str]: - return {"status": "healthy"} + response = {"status": "healthy"} + version = os.environ.get("KTX_DAEMON_VERSION") + if version: + response["version"] = version + return response @app.post("/database/introspect", response_model=DatabaseIntrospectionResponse) async def database_introspect( diff --git a/python/ktx-daemon/tests/test_app.py b/python/ktx-daemon/tests/test_app.py index fe3c1e4d..cd5c4f16 100644 --- a/python/ktx-daemon/tests/test_app.py +++ b/python/ktx-daemon/tests/test_app.py @@ -69,6 +69,16 @@ def test_health_endpoint_returns_healthy() -> None: assert response.json() == {"status": "healthy"} +def test_health_endpoint_returns_managed_runtime_version(monkeypatch) -> None: + monkeypatch.setenv("KTX_DAEMON_VERSION", "0.2.0") + client = TestClient(create_app()) + + response = client.get("/health") + + assert response.status_code == 200 + assert response.json() == {"status": "healthy", "version": "0.2.0"} + + def test_database_introspect_endpoint_returns_snapshot() -> None: calls = [] diff --git a/release-policy.json b/release-policy.json index ce814787..2eba9eaf 100644 --- a/release-policy.json +++ b/release-policy.json @@ -1,37 +1,27 @@ { "schemaVersion": 1, - "releaseMode": "ci-artifact-only", + "releaseMode": "npm-public-release-ready", "npm": { - "publish": false, + "publish": true, "registry": null, - "packages": [ - "@ktx/cli", - "@ktx/connector-bigquery", - "@ktx/connector-clickhouse", - "@ktx/connector-mysql", - "@ktx/connector-postgres", - "@ktx/connector-snowflake", - "@ktx/connector-sqlite", - "@ktx/connector-sqlserver", - "@ktx/context", - "@ktx/llm" - ] + "access": "public", + "tag": "next", + "packages": ["@kaelio/ktx"] }, "python": { "publish": false, "repository": null, - "packages": ["ktx-sl", "ktx-daemon"] + "packages": ["kaelio-ktx"] }, "publishedPackageSmoke": { - "packageName": null, - "version": "latest", + "packageName": "@kaelio/ktx", + "version": "0.1.0-rc.0", "registry": null }, - "requiredBeforePublishing": [ - "Choose npm registry and package visibility.", - "Choose Python package repository.", - "Choose public release versions.", - "Configure registry credentials outside source control.", - "Choose release tag and provenance policy." - ] + "runtimeInstaller": { + "uvStrategy": "path-prerequisite", + "bootstrapUv": false, + "missingUvBehavior": "focused-error" + }, + "requiredBeforePublishing": [] } diff --git a/scripts/build-public-npm-package.mjs b/scripts/build-public-npm-package.mjs new file mode 100644 index 00000000..1a9ef8bc --- /dev/null +++ b/scripts/build-public-npm-package.mjs @@ -0,0 +1,263 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +export const PUBLIC_NPM_PACKAGE_NAME = '@kaelio/ktx'; +export const PUBLIC_NPM_PACKAGE_VERSION = '0.1.0-rc.0'; + +export function publicNpmPackageTarballName(version = PUBLIC_NPM_PACKAGE_VERSION) { + return `kaelio-ktx-${version}.tgz`; +} + +export const PUBLIC_BUNDLED_WORKSPACE_PACKAGES = [ + '@ktx/llm', + '@ktx/context', + '@ktx/connector-bigquery', + '@ktx/connector-clickhouse', + '@ktx/connector-mysql', + '@ktx/connector-postgres', + '@ktx/connector-snowflake', + '@ktx/connector-sqlite', + '@ktx/connector-sqlserver', +]; + +export const PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS = { + '@ktx/llm': 'packages/llm', + '@ktx/context': 'packages/context', + '@ktx/connector-bigquery': 'packages/connector-bigquery', + '@ktx/connector-clickhouse': 'packages/connector-clickhouse', + '@ktx/connector-mysql': 'packages/connector-mysql', + '@ktx/connector-postgres': 'packages/connector-postgres', + '@ktx/connector-snowflake': 'packages/connector-snowflake', + '@ktx/connector-sqlite': 'packages/connector-sqlite', + '@ktx/connector-sqlserver': 'packages/connector-sqlserver', +}; + +function scriptRootDir() { + return resolve(dirname(fileURLToPath(import.meta.url)), '..'); +} + +export function publicNpmPackageLayout(rootDir = scriptRootDir(), version = PUBLIC_NPM_PACKAGE_VERSION) { + return { + rootDir, + packageVersion: version, + cliPackageRoot: join(rootDir, 'packages', 'cli'), + packRoot: join(rootDir, 'dist', 'public-npm-package'), + npmDir: join(rootDir, 'dist', 'artifacts', 'npm'), + tarballPath: join(rootDir, 'dist', 'artifacts', 'npm', publicNpmPackageTarballName(version)), + }; +} + +async function readJson(path) { + return JSON.parse(await readFile(path, 'utf8')); +} + +async function writeJson(path, value) { + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); +} + +function sortedObject(entries) { + return Object.fromEntries([...entries].sort(([left], [right]) => left.localeCompare(right))); +} + +function isWorkspacePackageName(name) { + return name.startsWith('@ktx/'); +} + +function parseCaretVersion(value) { + const match = /^\^(\d+)\.(\d+)\.(\d+)$/.exec(value); + if (!match) { + return null; + } + return { + major: Number(match[1]), + minor: Number(match[2]), + patch: Number(match[3]), + }; +} + +function compareParsedVersions(left, right) { + return left.major - right.major || left.minor - right.minor || left.patch - right.patch; +} + +function mergeDependencyVersion(name, previous, next) { + if (previous === next) { + return previous; + } + + const previousCaret = parseCaretVersion(previous); + const nextCaret = parseCaretVersion(next); + if (previousCaret && nextCaret && previousCaret.major === nextCaret.major) { + return compareParsedVersions(previousCaret, nextCaret) >= 0 ? previous : next; + } + + throw new Error(`Incompatible dependency versions for ${name}: ${previous} and ${next}`); +} + +export function collectPublicDependencies(packageJsons) { + const dependencies = new Map(); + + for (const packageJson of packageJsons) { + for (const [name, version] of Object.entries(packageJson.dependencies ?? {})) { + if (isWorkspacePackageName(name)) { + continue; + } + const previous = dependencies.get(name); + dependencies.set(name, previous ? mergeDependencyVersion(name, previous, version) : version); + } + } + + return sortedObject(dependencies); +} + +export function publicNpmPackageJson(cliPackageJson, dependencies, version = PUBLIC_NPM_PACKAGE_VERSION) { + return { + name: PUBLIC_NPM_PACKAGE_NAME, + version, + description: 'Standalone KTX context layer for database agents', + private: false, + type: 'module', + engines: cliPackageJson.engines ?? { node: '>=22.0.0' }, + bin: { ktx: './dist/bin.js' }, + main: cliPackageJson.main ?? 'dist/index.js', + types: cliPackageJson.types ?? 'dist/index.d.ts', + exports: cliPackageJson.exports ?? { + '.': { + types: './dist/index.d.ts', + import: './dist/index.js', + default: './dist/index.js', + }, + './package.json': './package.json', + }, + files: ['dist', 'assets'], + dependencies, + bundledDependencies: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, + license: cliPackageJson.license ?? 'Apache-2.0', + repository: { + type: 'git', + url: 'git+https://github.com/kaelio/ktx.git', + }, + bugs: { + url: 'https://github.com/kaelio/ktx/issues', + }, + homepage: 'https://github.com/kaelio/ktx#readme', + }; +} + +function bundledWorkspacePackageJson(packageJson) { + const dependencies = Object.fromEntries( + Object.entries(packageJson.dependencies ?? {}).filter(([name]) => !isWorkspacePackageName(name)), + ); + + return { + name: packageJson.name, + version: packageJson.version ?? PUBLIC_NPM_PACKAGE_VERSION, + private: true, + type: packageJson.type ?? 'module', + main: packageJson.main, + types: packageJson.types, + exports: packageJson.exports, + files: packageJson.files, + dependencies: sortedObject(Object.entries(dependencies)), + license: packageJson.license ?? 'Apache-2.0', + }; +} + +async function copyPackageFileEntries(sourceRoot, targetRoot, packageJson) { + for (const entry of packageJson.files ?? ['dist']) { + await cp(join(sourceRoot, entry), join(targetRoot, entry), { + recursive: true, + force: true, + }); + } +} + +async function copyCliPackage(layout, cliPackageJson, dependencies) { + await copyPackageFileEntries(layout.cliPackageRoot, layout.packRoot, cliPackageJson); + await writeJson( + join(layout.packRoot, 'package.json'), + publicNpmPackageJson(cliPackageJson, dependencies, layout.packageVersion), + ); +} + +async function copyBundledWorkspacePackage(rootDir, packageName, packageJson) { + const packageRoot = PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS[packageName]; + if (!packageRoot) { + throw new Error(`Missing bundled workspace package root for ${packageName}`); + } + + const sourceRoot = join(rootDir, packageRoot); + const targetRoot = join(rootDir, 'dist', 'public-npm-package', 'node_modules', ...packageName.split('/')); + await mkdir(targetRoot, { recursive: true }); + await copyPackageFileEntries(sourceRoot, targetRoot, packageJson); + await writeJson(join(targetRoot, 'package.json'), bundledWorkspacePackageJson(packageJson)); +} + +export async function createPublicNpmPackageTree(layout = publicNpmPackageLayout()) { + const cliPackageJson = await readJson(join(layout.cliPackageRoot, 'package.json')); + const bundledPackageJsons = await Promise.all( + PUBLIC_BUNDLED_WORKSPACE_PACKAGES.map(async (packageName) => { + const packageRoot = PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS[packageName]; + const packageJson = await readJson(join(layout.rootDir, packageRoot, 'package.json')); + if (packageJson.name !== packageName) { + throw new Error(`Unexpected package name in ${packageRoot}/package.json: ${packageJson.name}`); + } + return packageJson; + }), + ); + const dependencies = collectPublicDependencies([cliPackageJson, ...bundledPackageJsons]); + + await rm(layout.packRoot, { recursive: true, force: true }); + await mkdir(layout.packRoot, { recursive: true }); + await mkdir(layout.npmDir, { recursive: true }); + await copyCliPackage(layout, cliPackageJson, dependencies); + + for (const packageJson of bundledPackageJsons) { + await copyBundledWorkspacePackage(layout.rootDir, packageJson.name, packageJson); + } + + return { + layout, + packageJson: publicNpmPackageJson(cliPackageJson, dependencies, layout.packageVersion), + bundledPackages: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, + }; +} + +export function publicNpmPackCommand(layout = publicNpmPackageLayout()) { + return { + command: 'pnpm', + args: ['--config.node-linker=hoisted', 'pack', '--out', layout.tarballPath], + cwd: layout.packRoot, + }; +} + +export async function buildPublicNpmPackage(layout = publicNpmPackageLayout()) { + await createPublicNpmPackageTree(layout); + const pack = publicNpmPackCommand(layout); + await execFileAsync(pack.command, pack.args, { + cwd: pack.cwd, + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024, + }); + return layout.tarballPath; +} + +async function main() { + const tarball = await buildPublicNpmPackage(); + process.stdout.write(`Built ${PUBLIC_NPM_PACKAGE_NAME} package: ${tarball}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + try { + await main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/build-public-npm-package.test.mjs b/scripts/build-public-npm-package.test.mjs new file mode 100644 index 00000000..d4461049 --- /dev/null +++ b/scripts/build-public-npm-package.test.mjs @@ -0,0 +1,275 @@ +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { + PUBLIC_BUNDLED_WORKSPACE_PACKAGES, + PUBLIC_NPM_PACKAGE_NAME, + PUBLIC_NPM_PACKAGE_VERSION, + collectPublicDependencies, + createPublicNpmPackageTree, + publicNpmPackageJson, + publicNpmPackageLayout, + publicNpmPackageTarballName, + publicNpmPackCommand, +} from './build-public-npm-package.mjs'; + +async function writeJson(path, value) { + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); +} + +async function writePackage(root, packageRoot, packageJson, files = {}) { + const absoluteRoot = join(root, packageRoot); + await mkdir(absoluteRoot, { recursive: true }); + await writeJson(join(absoluteRoot, 'package.json'), packageJson); + + for (const [relativePath, contents] of Object.entries(files)) { + const target = join(absoluteRoot, relativePath); + await mkdir(join(target, '..'), { recursive: true }); + await writeFile(target, contents); + } +} + +async function writeWorkspaceFixture(root) { + await writePackage( + root, + 'packages/cli', + { + name: '@ktx/cli', + version: '0.0.0-private', + description: 'CLI wrapper for KTX', + type: 'module', + engines: { node: '>=22.0.0' }, + bin: { ktx: './dist/bin.js' }, + main: 'dist/index.js', + types: 'dist/index.d.ts', + exports: { + '.': { + types: './dist/index.d.ts', + import: './dist/index.js', + default: './dist/index.js', + }, + './package.json': './package.json', + }, + files: ['dist', 'assets'], + dependencies: { + '@clack/prompts': '1.3.0', + '@ktx/context': 'workspace:*', + commander: '14.0.3', + }, + license: 'Apache-2.0', + repository: { + type: 'git', + url: 'git+https://github.com/kaelio/ktx.git', + directory: 'packages/cli', + }, + }, + { + 'dist/bin.js': '#!/usr/bin/env node\n', + 'dist/index.js': 'export const cli = true;\n', + 'dist/index.d.ts': 'export declare const cli: true;\n', + 'assets/python/manifest.json': '{"schemaVersion":1}\n', + }, + ); + + await writePackage( + root, + 'packages/context', + { + name: '@ktx/context', + version: '0.0.0-private', + type: 'module', + main: 'dist/index.js', + exports: { '.': './dist/index.js' }, + files: ['dist', 'prompts', 'skills'], + dependencies: { + '@ktx/llm': 'workspace:*', + yaml: '^2.8.2', + }, + }, + { + 'dist/index.js': 'export const context = true;\n', + 'prompts/system.md': 'prompt\n', + 'skills/sl/SKILL.md': 'skill\n', + }, + ); + + await writePackage( + root, + 'packages/llm', + { + name: '@ktx/llm', + version: '0.0.0-private', + type: 'module', + main: 'dist/index.js', + exports: { '.': './dist/index.js' }, + files: ['dist'], + dependencies: { + ai: '^6.0.168', + }, + }, + { + 'dist/index.js': 'export const llm = true;\n', + }, + ); + + for (const packageName of PUBLIC_BUNDLED_WORKSPACE_PACKAGES.filter((name) => name.startsWith('@ktx/connector-'))) { + const directory = packageName.replace('@ktx/', ''); + await writePackage( + root, + `packages/${directory}`, + { + name: packageName, + version: '0.0.0-private', + type: 'module', + main: 'dist/index.js', + exports: { '.': './dist/index.js' }, + files: ['dist'], + dependencies: { + '@ktx/context': 'workspace:*', + }, + }, + { + 'dist/index.js': `export const name = ${JSON.stringify(packageName)};\n`, + }, + ); + } +} + +describe('publicNpmPackageLayout', () => { + it('uses the first public npm release version for the tarball name', () => { + const layout = publicNpmPackageLayout('/repo/ktx'); + + assert.equal(PUBLIC_NPM_PACKAGE_VERSION, '0.1.0-rc.0'); + assert.equal(publicNpmPackageTarballName(), 'kaelio-ktx-0.1.0-rc.0.tgz'); + assert.equal(layout.tarballPath, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz'); + }); +}); + +describe('collectPublicDependencies', () => { + it('unions external runtime dependencies and omits workspace packages', () => { + assert.deepEqual( + collectPublicDependencies([ + { + name: '@ktx/cli', + dependencies: { + '@ktx/context': 'workspace:*', + commander: '14.0.3', + zod: '^4.4.3', + }, + }, + { + name: '@ktx/context', + dependencies: { + '@ktx/llm': 'workspace:*', + commander: '14.0.3', + yaml: '^2.8.2', + zod: '^4.1.13', + }, + }, + ]), + { + commander: '14.0.3', + yaml: '^2.8.2', + zod: '^4.4.3', + }, + ); + }); + + it('fails on incompatible external dependency ranges', () => { + assert.throws( + () => + collectPublicDependencies([ + { name: '@ktx/cli', dependencies: { zod: '^4.4.3' } }, + { name: '@ktx/context', dependencies: { zod: '^3.25.0' } }, + ]), + /Incompatible dependency versions for zod/, + ); + }); +}); + +describe('publicNpmPackageJson', () => { + it('does not bundle the removed PostHog connector package', () => { + assert.equal(PUBLIC_BUNDLED_WORKSPACE_PACKAGES.includes('@ktx/connector-posthog'), false); + }); + + it('describes the public @kaelio/ktx binary package', () => { + const packageJson = publicNpmPackageJson( + { + name: '@ktx/cli', + version: '0.0.0-private', + engines: { node: '>=22.0.0' }, + bin: { ktx: './dist/bin.js' }, + main: 'dist/index.js', + types: 'dist/index.d.ts', + exports: { '.': './dist/index.js', './package.json': './package.json' }, + license: 'Apache-2.0', + }, + { commander: '14.0.3' }, + ); + + assert.equal(packageJson.name, PUBLIC_NPM_PACKAGE_NAME); + assert.equal(packageJson.version, '0.1.0-rc.0'); + assert.equal(packageJson.private, false); + assert.deepEqual(packageJson.bin, { ktx: './dist/bin.js' }); + assert.deepEqual(packageJson.dependencies, { commander: '14.0.3' }); + assert.deepEqual(packageJson.bundledDependencies, PUBLIC_BUNDLED_WORKSPACE_PACKAGES); + assert.deepEqual(packageJson.files, ['dist', 'assets']); + }); +}); + +describe('createPublicNpmPackageTree', () => { + it('copies CLI files, assets, and bundled internal workspace packages', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-public-npm-test-')); + try { + await writeWorkspaceFixture(root); + const layout = publicNpmPackageLayout(root); + + const result = await createPublicNpmPackageTree(layout); + + assert.equal(result.packageJson.name, '@kaelio/ktx'); + assert.equal(result.packageJson.dependencies.commander, '14.0.3'); + assert.equal(result.packageJson.dependencies.yaml, '^2.8.2'); + assert.equal(result.packageJson.dependencies.ai, '^6.0.168'); + assert.equal( + await readFile(join(layout.packRoot, 'assets', 'python', 'manifest.json'), 'utf8'), + '{"schemaVersion":1}\n', + ); + assert.equal( + await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'dist', 'index.js'), 'utf8'), + 'export const context = true;\n', + ); + assert.equal( + await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'prompts', 'system.md'), 'utf8'), + 'prompt\n', + ); + + const bundledContextJson = JSON.parse( + await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'package.json'), 'utf8'), + ); + assert.equal(bundledContextJson.private, true); + assert.deepEqual(bundledContextJson.dependencies, { yaml: '^2.8.2' }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); + +describe('publicNpmPackCommand', () => { + it('packs the assembled public package with pnpm', () => { + const layout = publicNpmPackageLayout('/repo/ktx'); + + assert.deepEqual(publicNpmPackCommand(layout), { + command: 'pnpm', + args: [ + '--config.node-linker=hoisted', + 'pack', + '--out', + '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz', + ], + cwd: '/repo/ktx/dist/public-npm-package', + }); + }); +}); diff --git a/scripts/build-python-runtime-wheel.mjs b/scripts/build-python-runtime-wheel.mjs new file mode 100644 index 00000000..9623b48a --- /dev/null +++ b/scripts/build-python-runtime-wheel.mjs @@ -0,0 +1,144 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +export const RUNTIME_WHEEL_DISTRIBUTION_NAME = 'kaelio-ktx'; +export const RUNTIME_WHEEL_NORMALIZED_NAME = 'kaelio_ktx'; +export const RUNTIME_WHEEL_PACKAGE_VERSION = '0.1.0'; + +function scriptRootDir() { + return resolve(dirname(fileURLToPath(import.meta.url)), '..'); +} + +export function runtimeWheelLayout(rootDir = scriptRootDir()) { + return { + rootDir, + semanticLayerSourceDir: join(rootDir, 'python', 'ktx-sl', 'semantic_layer'), + daemonSourceDir: join(rootDir, 'python', 'ktx-daemon', 'src', 'ktx_daemon'), + buildRoot: join(rootDir, 'dist', 'runtime-wheel-src'), + outputDir: join(rootDir, 'dist', 'artifacts', 'python'), + }; +} + +export function runtimeWheelPyproject() { + return `[project] +name = "${RUNTIME_WHEEL_DISTRIBUTION_NAME}" +version = "${RUNTIME_WHEEL_PACKAGE_VERSION}" +description = "Bundled Python runtime payload for the KTX npm package" +readme = "README.md" +requires-python = ">=3.13" +license = "Apache-2.0" +dependencies = [ + "fastapi>=0.115.0", + "lkml>=1.3.7", + "numpy>=2.2.6", + "orjson>=3.11.4", + "pandas>=2.2.3", + "psycopg[binary]>=3.2.0", + "pydantic>=2.9.0", + "pyyaml>=6", + "requests>=2.32.0", + "sqlglot>=26", + "uvicorn[standard]>=0.32.0", +] + +[project.optional-dependencies] +local-embeddings = [ + "sentence-transformers>=5.1.1", + "torch>=2.2.0", +] + +[project.scripts] +ktx-daemon = "ktx_daemon.__main__:main" + +[project.urls] +Homepage = "https://github.com/kaelio/ktx" +Repository = "https://github.com/kaelio/ktx" +Issues = "https://github.com/kaelio/ktx/issues" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["semantic_layer", "ktx_daemon"] +`; +} + +export function runtimeWheelReadme() { + return `# kaelio-ktx Python runtime + +Bundled Python runtime wheel for KTX. + +This wheel is built from the repository's \`semantic_layer\` and +\`ktx_daemon\` source trees for inclusion in the npm package. It is not a +separate public PyPI release artifact. +`; +} + +export async function createRuntimeWheelBuildTree(layout = runtimeWheelLayout()) { + await rm(layout.buildRoot, { recursive: true, force: true }); + await mkdir(layout.buildRoot, { recursive: true }); + await cp(layout.semanticLayerSourceDir, join(layout.buildRoot, 'semantic_layer'), { + recursive: true, + }); + await cp(layout.daemonSourceDir, join(layout.buildRoot, 'ktx_daemon'), { + recursive: true, + }); + await writeFile(join(layout.buildRoot, 'pyproject.toml'), runtimeWheelPyproject()); + await writeFile(join(layout.buildRoot, 'README.md'), runtimeWheelReadme()); +} + +export function runtimeWheelBuildCommand(layout = runtimeWheelLayout()) { + return { + command: 'uv', + args: ['build', '--wheel', '--out-dir', layout.outputDir, layout.buildRoot], + cwd: layout.rootDir, + }; +} + +async function runCommand(command, args, options) { + const result = await execFileAsync(command, args, { + cwd: options.cwd, + encoding: 'utf8', + maxBuffer: 1024 * 1024 * 20, + }); + if (result.stdout) { + process.stdout.write(result.stdout); + } + if (result.stderr) { + process.stderr.write(result.stderr); + } +} + +export async function buildRuntimeWheel(layout = runtimeWheelLayout()) { + await mkdir(layout.outputDir, { recursive: true }); + await createRuntimeWheelBuildTree(layout); + const command = runtimeWheelBuildCommand(layout); + await runCommand(command.command, command.args, { cwd: command.cwd }); + const pyproject = await readFile(join(layout.buildRoot, 'pyproject.toml'), 'utf8'); + return { + buildRoot: layout.buildRoot, + outputDir: layout.outputDir, + pyproject, + }; +} + +async function main() { + await buildRuntimeWheel(runtimeWheelLayout()); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + try { + await main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/build-python-runtime-wheel.test.mjs b/scripts/build-python-runtime-wheel.test.mjs new file mode 100644 index 00000000..2fbb3fbc --- /dev/null +++ b/scripts/build-python-runtime-wheel.test.mjs @@ -0,0 +1,115 @@ +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, + createRuntimeWheelBuildTree, + runtimeWheelBuildCommand, + runtimeWheelLayout, + runtimeWheelPyproject, +} from './build-python-runtime-wheel.mjs'; + +async function writeRuntimeSourceFixture(root) { + await mkdir(join(root, 'python', 'ktx-sl', 'semantic_layer'), { + recursive: true, + }); + await mkdir(join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon'), { + recursive: true, + }); + + await writeFile( + join(root, 'python', 'ktx-sl', 'semantic_layer', '__init__.py'), + 'SEMANTIC_LAYER_FIXTURE = True\n', + ); + await writeFile( + join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon', '__init__.py'), + 'KTX_DAEMON_FIXTURE = True\n', + ); + await writeFile( + join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon', '__main__.py'), + 'def main():\n return 0\n', + ); +} + +describe('runtimeWheelLayout', () => { + it('uses stable source, build, and output paths', () => { + const layout = runtimeWheelLayout('/repo/ktx'); + + assert.equal(layout.rootDir, '/repo/ktx'); + assert.equal(layout.semanticLayerSourceDir, '/repo/ktx/python/ktx-sl/semantic_layer'); + assert.equal(layout.daemonSourceDir, '/repo/ktx/python/ktx-daemon/src/ktx_daemon'); + assert.equal(layout.buildRoot, '/repo/ktx/dist/runtime-wheel-src'); + assert.equal(layout.outputDir, '/repo/ktx/dist/artifacts/python'); + }); +}); + +describe('runtimeWheelPyproject', () => { + it('describes one kaelio-ktx wheel with lazy local embeddings', () => { + const pyproject = runtimeWheelPyproject(); + + assert.match(pyproject, /name = "kaelio-ktx"/); + assert.match(pyproject, /version = "0\.1\.0"/); + assert.match(pyproject, /ktx-daemon = "ktx_daemon\.__main__:main"/); + assert.match(pyproject, /packages = \["semantic_layer", "ktx_daemon"\]/); + assert.match(pyproject, /\[project\.optional-dependencies\]/); + assert.match(pyproject, /local-embeddings = \[/); + assert.match(pyproject, /"sentence-transformers>=5\.1\.1"/); + assert.match(pyproject, /"torch>=2\.2\.0"/); + assert.doesNotMatch( + pyproject.match(/dependencies = \[[\s\S]*?\]/)?.[0] ?? '', + /sentence-transformers|torch/, + ); + }); +}); + +describe('createRuntimeWheelBuildTree', () => { + it('copies KTX-owned Python packages into the build tree', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-wheel-test-')); + try { + await writeRuntimeSourceFixture(root); + const layout = runtimeWheelLayout(root); + + await createRuntimeWheelBuildTree(layout); + + assert.equal( + await readFile(join(layout.buildRoot, 'semantic_layer', '__init__.py'), 'utf8'), + 'SEMANTIC_LAYER_FIXTURE = True\n', + ); + assert.equal( + await readFile(join(layout.buildRoot, 'ktx_daemon', '__main__.py'), 'utf8'), + 'def main():\n return 0\n', + ); + const pyproject = await readFile(join(layout.buildRoot, 'pyproject.toml'), 'utf8'); + assert.match(pyproject, /name = "kaelio-ktx"/); + assert.match(pyproject, /local-embeddings = \[/); + const readme = await readFile(join(layout.buildRoot, 'README.md'), 'utf8'); + assert.match(readme, /Bundled Python runtime wheel for KTX/); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); + +describe('runtimeWheelBuildCommand', () => { + it('runs uv build against the generated build tree', () => { + const layout = runtimeWheelLayout('/repo/ktx'); + + assert.deepEqual(runtimeWheelBuildCommand(layout), { + command: 'uv', + args: [ + 'build', + '--wheel', + '--out-dir', + '/repo/ktx/dist/artifacts/python', + '/repo/ktx/dist/runtime-wheel-src', + ], + cwd: '/repo/ktx', + }); + assert.equal(RUNTIME_WHEEL_DISTRIBUTION_NAME, 'kaelio-ktx'); + assert.equal(RUNTIME_WHEEL_PACKAGE_VERSION, '0.1.0'); + }); +}); diff --git a/scripts/check-boundaries.mjs b/scripts/check-boundaries.mjs index 5c43c575..f0ec86f0 100644 --- a/scripts/check-boundaries.mjs +++ b/scripts/check-boundaries.mjs @@ -7,6 +7,10 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; const codeExtensions = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.py']); const runtimeAssetPatterns = [/^packages\/[^/]+\/prompts\/.+\.md$/, /^packages\/[^/]+\/skills\/.+\.md$/]; const identifierSkipPrefixes = ['docs/', 'examples/', 'python/ktx-sl/plans/', 'python/ktx-sl/openspec/']; +const identifierAllowPatterns = [ + /^packages\/cli\/src\/(?:index|managed-local-embeddings|managed-python-command|managed-python-daemon|managed-python-runtime|runtime)(?:\.test)?\.ts$/, + /^scripts\/(?:build-public-npm-package|build-python-runtime-wheel|local-embeddings-runtime-smoke|package-artifacts|publish-public-npm-package|published-package-smoke|release-readiness)(?:\.test)?\.mjs$/, +]; const forbiddenIdentifierTerms = ['kae' + 'lio', 'Kae' + 'lio', 'KAE' + 'LIO_']; const appImportPatterns = [ @@ -98,6 +102,10 @@ function skipsIdentifierScan(relativePath) { return identifierSkipPrefixes.some((prefix) => relativePath.startsWith(prefix)); } +function allowsForbiddenIdentifier(relativePath) { + return identifierAllowPatterns.some((pattern) => pattern.test(relativePath)); +} + export function scanFileContent(relativePath, content) { const normalizedPath = normalizePath(relativePath); const violations = []; @@ -138,7 +146,11 @@ export function scanFileContent(relativePath, content) { } } - if (scansForForbiddenIdentifiers(normalizedPath) && !skipsIdentifierScan(normalizedPath)) { + if ( + scansForForbiddenIdentifiers(normalizedPath) && + !skipsIdentifierScan(normalizedPath) && + !allowsForbiddenIdentifier(normalizedPath) + ) { for (const term of forbiddenIdentifierTerms) { if (content.includes(term)) { violations.push({ diff --git a/scripts/check-boundaries.test.mjs b/scripts/check-boundaries.test.mjs index 39946464..8d7fabdd 100644 --- a/scripts/check-boundaries.test.mjs +++ b/scripts/check-boundaries.test.mjs @@ -65,6 +65,15 @@ describe('scanFileContent', () => { assert.equal(scanFileContent('python/ktx-sl/openspec/specs/semantic-layer/spec.md', name).length, 0); }); + it('allows public package identifiers in release packaging and managed runtime source', () => { + const name = lowerProductName(); + + assert.equal(scanFileContent('scripts/local-embeddings-runtime-smoke.mjs', `@${name}/ktx`).length, 0); + assert.equal(scanFileContent('scripts/package-artifacts.test.mjs', `${name}-ktx`).length, 0); + assert.equal(scanFileContent('scripts/publish-public-npm-package.test.mjs', `@${name}/ktx`).length, 0); + assert.equal(scanFileContent('packages/cli/src/managed-python-runtime.ts', `${name}_ktx`).length, 0); + }); + it('allows clean source files and clean runtime prompt assets', () => { assert.deepEqual( scanFileContent('packages/context/src/index.ts', "export const packageName = '@ktx/context';"), diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index 6eeeb13c..b9f63d65 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -6,6 +6,26 @@ async function readText(relativePath) { return readFile(new URL(`../${relativePath}`, import.meta.url), 'utf8'); } +function publicNpmPackageName() { + return `@${['kae', 'lio'].join('')}/ktx`; +} + +function runtimeWheelPackageName() { + return `${['kae', 'lio'].join('')}-ktx`; +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function publicPackagePattern(text) { + return new RegExp(text.replaceAll('{package}', escapeRegExp(publicNpmPackageName()))); +} + +function runtimeWheelPackagePattern(text) { + return new RegExp(text.replaceAll('{package}', escapeRegExp(runtimeWheelPackageName()))); +} + describe('standalone example docs', () => { it('documents the local warehouse example from the examples index', async () => { const examples = await readText('examples/README.md'); @@ -63,6 +83,16 @@ describe('standalone example docs', () => { assert.match(workload, /app_user/); assert.match(workload, /etl_user/); assert.match(smoke, /pg_stat_statements_reset/); + assert.match(smoke, /KTX_RUNTIME_ROOT/); + assert.match(smoke, /managedDaemon/); + assert.match(smoke, /installPolicy: 'auto'/); + assert.match(smoke, /getKtxCliPackageInfo/); + assert.doesNotMatch(smoke, /python-service/); + assert.doesNotMatch(smoke, /PYTHON_SERVICE/); + assert.doesNotMatch(smoke, /uvicorn app\.main:app/); + assert.doesNotMatch(smoke, /export KTX_SQL_ANALYSIS_URL/); + assert.doesNotMatch(readme, /python-service/); + assert.doesNotMatch(readme, /KTX_SQL_ANALYSIS_URL/); assert.match(smoke, /assert_manifest "\$FIRST_MANIFEST" true/); assert.match(smoke, /assert_manifest "\$SECOND_MANIFEST" false/); assert.match(smoke, /assert_manifest "\$RESET_MANIFEST" true/); @@ -113,6 +143,60 @@ describe('standalone example docs', () => { assert.match(rootReadme, /Tables: 1/); }); + it('documents public npm and managed runtime usage in the README', async () => { + const rootReadme = await readText('README.md'); + + assert.match(rootReadme, publicPackagePattern('npx {package} setup demo --no-input')); + assert.match(rootReadme, publicPackagePattern('npx {package} sl query')); + assert.match(rootReadme, publicPackagePattern('npm install {package}')); + assert.match(rootReadme, publicPackagePattern('npm install -g {package}')); + assert.match(rootReadme, /ktx runtime install/); + assert.match(rootReadme, /ktx runtime status/); + assert.match(rootReadme, /ktx runtime doctor/); + assert.match(rootReadme, /ktx runtime start/); + assert.match(rootReadme, /ktx runtime stop/); + assert.match(rootReadme, /ktx runtime prune --dry-run/); + assert.match(rootReadme, /ktx runtime prune --yes/); + assert.match(rootReadme, /KTX requires `uv` on `PATH`/); + assert.match(rootReadme, /KTX doesn't download `uv` automatically/); + assert.match( + rootReadme, + runtimeWheelPackagePattern( + 'release\\s+artifact manifest contains the public npm tarball and the\\s+bundled `{package}`\\s+runtime wheel', + ), + ); + assert.match(rootReadme, /source packages for\s+development, not public release artifacts/); + assert.match(rootReadme, /ktx serve --mcp stdio/); + assert.doesNotMatch(rootReadme, /uv run ktx-daemon serve-http/); + assert.doesNotMatch(rootReadme, /--semantic-compute-url http:\/\/127\.0\.0\.1:8765/); + }); + + it('documents the public package artifact smoke shape', async () => { + const readme = await readText('examples/package-artifacts/README.md'); + + assert.match(readme, publicPackagePattern('{package}')); + assert.match(readme, /managed Python runtime/); + assert.match( + readme, + new RegExp( + `public \`${escapeRegExp(publicNpmPackageName())}\` npm tarball and the\\s+bundled \`${escapeRegExp( + runtimeWheelPackageName(), + )}\`\\s+runtime wheel`, + ), + ); + assert.match(readme, /does not install standalone\s+Python packages directly/); + assert.doesNotMatch(readme, /standalone Python distributions/); + assert.doesNotMatch(readme, /installs the Python artifacts directly/); + assert.match(readme, /requires `uv` on `PATH`/); + assert.match(readme, /ktx runtime status/); + assert.match(readme, /ktx runtime doctor/); + assert.match(readme, /ktx runtime prune --dry-run/); + assert.match(readme, /ktx runtime prune --yes/); + assert.doesNotMatch(readme, /@ktx\/context/); + assert.doesNotMatch(readme, /@ktx\/cli/); + assert.doesNotMatch(readme, /python -m ktx_daemon semantic-validate/); + }); + it('replaces the fake-ingest smoke with a ktx scan walkthrough in the README', async () => { const rootReadme = await readText('README.md'); diff --git a/scripts/installed-live-database-smoke.mjs b/scripts/installed-live-database-smoke.mjs index bad155dd..7fe061c8 100644 --- a/scripts/installed-live-database-smoke.mjs +++ b/scripts/installed-live-database-smoke.mjs @@ -1,19 +1,15 @@ #!/usr/bin/env node -import { execFile, spawn } from 'node:child_process'; +import { execFile } from 'node:child_process'; import { once } from 'node:events'; import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { request as httpRequest } from 'node:http'; import { createServer } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; import { - findPythonArtifacts, npmSmokePackageJson, - npmSmokePythonEnv, packageArtifactLayout, - pythonArtifactInstallArgs, } from './package-artifacts.mjs'; const POSTGRES_IMAGE = process.env.KTX_ARTIFACT_POSTGRES_IMAGE ?? 'postgres:16-alpine'; @@ -238,93 +234,37 @@ async function seedPostgres(containerName) { requireSuccess('seed postgres catalog', result); } -function httpGetOk(url) { - return new Promise((resolve, reject) => { - const request = httpRequest(url, { method: 'GET' }, (response) => { - response.resume(); - response.on('end', () => resolve((response.statusCode ?? 0) >= 200 && (response.statusCode ?? 0) < 300)); - }); - request.on('error', reject); - request.end(); - }); -} - -function spawnLogged(command, args, options = {}) { - const stdout = []; - const stderr = []; - let spawnError; - const child = spawn(command, args, { - cwd: options.cwd, - env: options.env ?? process.env, - stdio: ['ignore', 'pipe', 'pipe'], - }); - child.stdout.on('data', (chunk) => stdout.push(chunk)); - child.stderr.on('data', (chunk) => stderr.push(chunk)); - child.on('error', (error) => { - spawnError = error; - }); +function managedRuntimeEnv(cleanInstallDir) { return { - child, - error() { - return spawnError; - }, - output() { - return { - stdout: Buffer.concat(stdout).toString('utf8'), - stderr: Buffer.concat(stderr).toString('utf8'), - }; - }, + ...process.env, + KTX_RUNTIME_ROOT: join(cleanInstallDir, 'managed-runtime'), }; } -async function waitForHttpHealth(url, daemon) { - const deadline = Date.now() + 15_000; - while (Date.now() < deadline) { - if (daemon.error()) { - const output = daemon.output(); - throw new Error( - `Failed to start ktx-daemon: ${daemon.error().message}\nstdout:\n${output.stdout}\nstderr:\n${output.stderr}`, - ); - } - if (daemon.child.exitCode !== null || daemon.child.signalCode !== null) { - const output = daemon.output(); - throw new Error(`ktx-daemon exited before health check passed\nstdout:\n${output.stdout}\nstderr:\n${output.stderr}`); - } - try { - if (await httpGetOk(url)) { - return; - } - } catch { - await new Promise((resolve) => setTimeout(resolve, 100)); - continue; - } - await new Promise((resolve) => setTimeout(resolve, 100)); +function parseDaemonBaseUrl(stdout) { + const match = stdout.match(/^url: (http:\/\/127\.0\.0\.1:\d+)$/m); + if (!match) { + throw new Error(`Daemon URL was not printed by runtime start:\n${stdout}`); } - const output = daemon.output(); - throw new Error(`Timed out waiting for ${url}\nstdout:\n${output.stdout}\nstderr:\n${output.stderr}`); + return match[1]; } -async function startDaemon(port, cleanInstallDir) { - const daemon = spawnLogged( - 'ktx-daemon', - ['serve-http', '--host', '127.0.0.1', '--port', String(port), '--log-level', 'warning'], - { cwd: cleanInstallDir, env: npmSmokePythonEnv(cleanInstallDir) }, - ); - await waitForHttpHealth(`http://127.0.0.1:${port}/health`, daemon); - return daemon; +async function startDaemon(cleanInstallDir) { + const result = await run('pnpm', ['exec', 'ktx', 'runtime', 'start'], { + cwd: cleanInstallDir, + env: managedRuntimeEnv(cleanInstallDir), + timeout: 120_000, + }); + requireSuccess('ktx runtime start', result); + return parseDaemonBaseUrl(result.stdout); } -async function stopDaemon(daemon) { - if (daemon.child.exitCode !== null || daemon.child.signalCode !== null) { - return; - } - daemon.child.kill('SIGTERM'); - const closed = once(daemon.child, 'close').then(() => true); - const timedOut = new Promise((resolve) => setTimeout(() => resolve(false), 5_000)); - if (!(await Promise.race([closed, timedOut]))) { - daemon.child.kill('SIGKILL'); - await once(daemon.child, 'close'); - } +async function stopDaemon(cleanInstallDir) { + await run('pnpm', ['exec', 'ktx', 'runtime', 'stop'], { + cwd: cleanInstallDir, + env: managedRuntimeEnv(cleanInstallDir), + timeout: 30_000, + }); } async function assertPathExists(path, label) { @@ -336,7 +276,6 @@ async function assertPathExists(path, label) { } async function prepareCleanInstall(layout, cleanInstallDir) { - const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); await assertPathExists(layout.contextTarball, '@ktx/context tarball'); await assertPathExists(layout.cliTarball, '@ktx/cli tarball'); await mkdir(cleanInstallDir, { recursive: true }); @@ -344,34 +283,24 @@ async function prepareCleanInstall(layout, cleanInstallDir) { await run('pnpm', ['install'], { cwd: cleanInstallDir, timeout: 120_000 }).then((result) => requireSuccess('pnpm install clean artifact project', result), ); - await run('uv', ['venv', '.venv'], { cwd: cleanInstallDir, timeout: 120_000 }).then((result) => - requireSuccess('uv venv clean artifact project', result), - ); - await run( - 'uv', - pythonArtifactInstallArgs( - join(cleanInstallDir, '.venv', process.platform === 'win32' ? 'Scripts/python.exe' : 'bin/python'), - pythonArtifacts, - ), - { - cwd: cleanInstallDir, - timeout: 120_000, - }, - ).then((result) => requireSuccess('install Python artifacts', result)); + await run('pnpm', ['exec', 'ktx', 'runtime', 'install', '--yes'], { + cwd: cleanInstallDir, + env: managedRuntimeEnv(cleanInstallDir), + timeout: 120_000, + }).then((result) => requireSuccess('install managed runtime', result)); } async function main() { const layout = packageArtifactLayout(); const root = await mkdtemp(join(tmpdir(), 'ktx-live-db-artifact-smoke-')); const containerName = smokeContainerName(); - let daemon; + let cleanInstallDir; + let daemonStarted = false; try { const postgresPort = await getAvailablePort(); - const daemonPort = await getAvailablePort(); const postgresUrl = buildPostgresUrl(postgresPort); - const cleanInstallDir = join(root, 'npm-clean-install'); + cleanInstallDir = join(root, 'npm-clean-install'); const projectDir = join(root, 'project'); - const databaseIntrospectionUrl = `http://127.0.0.1:${daemonPort}`; await startPostgresContainer(containerName, postgresPort); await waitForPostgres(containerName); @@ -386,11 +315,12 @@ async function main() { requireSuccess('ktx init', init); await writeFile(join(projectDir, 'ktx.yaml'), buildKtxYaml(postgresUrl), 'utf8'); - daemon = await startDaemon(daemonPort, cleanInstallDir); + const databaseIntrospectionUrl = await startDaemon(cleanInstallDir); + daemonStarted = true; const ingestRun = await run('pnpm', buildLiveDatabaseIngestArgs(projectDir, databaseIntrospectionUrl), { cwd: cleanInstallDir, - env: npmSmokePythonEnv(cleanInstallDir), + env: managedRuntimeEnv(cleanInstallDir), timeout: 120_000, }); requireSuccess('ktx dev ingest run live-database', ingestRun); @@ -403,7 +333,7 @@ async function main() { const runId = getRunId(ingestRun.stdout); const ingestStatus = await run('pnpm', buildLiveDatabaseStatusArgs(projectDir, runId), { cwd: cleanInstallDir, - env: npmSmokePythonEnv(cleanInstallDir), + env: managedRuntimeEnv(cleanInstallDir), timeout: 30_000, }); requireSuccess('ktx ingest status live-database', ingestStatus); @@ -414,8 +344,8 @@ async function main() { await assertPathExists(join(projectDir, '.ktx', 'db.sqlite'), 'SQLite local ingest state'); process.stdout.write(`Installed live-database artifact smoke passed: ${runId}\n`); } finally { - if (daemon) { - await stopDaemon(daemon); + if (daemonStarted && cleanInstallDir) { + await stopDaemon(cleanInstallDir); } await stopPostgresContainer(containerName); await rm(root, { recursive: true, force: true }); diff --git a/scripts/local-embeddings-runtime-smoke.mjs b/scripts/local-embeddings-runtime-smoke.mjs new file mode 100644 index 00000000..06f483d5 --- /dev/null +++ b/scripts/local-embeddings-runtime-smoke.mjs @@ -0,0 +1,397 @@ +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +import { + PUBLIC_NPM_PACKAGE_NAME, + PUBLIC_NPM_PACKAGE_VERSION, +} from './build-public-npm-package.mjs'; + +const execFileAsync = promisify(execFile); +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); +const DEFAULT_ROOT_DIR = resolve(SCRIPT_DIR, '..'); +const PUBLIC_NPM_ARTIFACT_DIR = join('dist', 'artifacts', 'npm'); +const OPT_IN_MESSAGE = + 'Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke.'; + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +export function expectedPublicKtxVersionPattern() { + return new RegExp( + `${escapeRegExp(PUBLIC_NPM_PACKAGE_NAME)} ${escapeRegExp(PUBLIC_NPM_PACKAGE_VERSION)}`, + ); +} + +export function localEmbeddingsSmokeOptIn(env = process.env, args = process.argv.slice(2)) { + if (env.KTX_RUN_LOCAL_EMBEDDINGS_SMOKE === '1' || args.includes('--force')) { + return { run: true }; + } + return { run: false, message: OPT_IN_MESSAGE }; +} + +export function publicKtxTarballName(files) { + const matches = files.filter((file) => /^kaelio-ktx-.+\.tgz$/.test(file)).sort(); + if (matches.length !== 1) { + throw new Error( + `Expected exactly one @kaelio/ktx tarball in ${PUBLIC_NPM_ARTIFACT_DIR}, found ${matches.length}: ${ + matches.join(', ') || 'none' + }. Run pnpm run artifacts:build first.`, + ); + } + return matches[0]; +} + +export async function selectPublicKtxTarball(rootDir = DEFAULT_ROOT_DIR) { + const npmArtifactDir = join(rootDir, PUBLIC_NPM_ARTIFACT_DIR); + const files = await readdir(npmArtifactDir); + return join(npmArtifactDir, publicKtxTarballName(files)); +} + +export function buildLocalEmbeddingsSmokeEnv(root, baseEnv = process.env) { + return { + ...baseEnv, + KTX_RUN_LOCAL_EMBEDDINGS_SMOKE: '1', + KTX_RUNTIME_ROOT: join(root, 'managed-runtime'), + HF_HOME: join(root, 'hf-home'), + TRANSFORMERS_CACHE: join(root, 'transformers-cache'), + SENTENCE_TRANSFORMERS_HOME: join(root, 'sentence-transformers-home'), + TORCH_HOME: join(root, 'torch-home'), + }; +} + +export function localEmbeddingsSmokeCommands(input) { + return [ + { + label: 'ktx public package version', + command: 'pnpm', + args: ['exec', 'ktx', '--version'], + timeoutMs: 60_000, + }, + { + label: 'ktx runtime status missing', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'status', '--json'], + timeoutMs: 60_000, + }, + { + label: 'ktx runtime install local embeddings', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], + timeoutMs: 1_200_000, + }, + { + label: 'ktx runtime status local embeddings ready', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'status', '--json'], + timeoutMs: 60_000, + }, + { + label: 'ktx runtime start local embeddings', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'start', '--feature', 'local-embeddings'], + timeoutMs: 300_000, + }, + { + label: 'ktx setup local embeddings', + command: 'pnpm', + args: [ + 'exec', + 'ktx', + 'setup', + '--project-dir', + input.projectDir, + '--new', + '--no-input', + '--yes', + '--skip-llm', + '--embedding-backend', + 'sentence-transformers', + '--skip-databases', + '--skip-sources', + '--skip-agents', + ], + timeoutMs: 900_000, + }, + { + label: 'ktx runtime stop local embeddings', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'stop'], + timeoutMs: 60_000, + }, + ]; +} + +export function parseDaemonBaseUrl(stdout) { + const match = stdout.match(/^url: (http:\/\/127\.0\.0\.1:\d+)$/m); + if (!match) { + throw new Error(`Daemon URL was not printed by runtime start:\n${stdout}`); + } + return match[1]; +} + +export function validateEmbeddingResponse(raw, expectedDimensions) { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('Embedding response must be a JSON object'); + } + const embedding = raw.embedding; + if (!Array.isArray(embedding)) { + throw new Error('Embedding response must include an embedding array'); + } + if (embedding.length !== expectedDimensions) { + throw new Error(`Expected embedding dimension ${expectedDimensions}, got ${embedding.length}`); + } + for (const [index, value] of embedding.entries()) { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`Embedding value at index ${index} is not a finite number`); + } + } +} + +async function run(command, args, options = {}) { + process.stdout.write(`$ ${command} ${args.join(' ')}\n`); + try { + const result = await execFileAsync(command, args, { + cwd: options.cwd, + env: { ...process.env, ...options.env }, + encoding: 'utf8', + maxBuffer: 1024 * 1024 * 20, + timeout: options.timeoutMs ?? 120_000, + }); + if (result.stdout) { + process.stdout.write(result.stdout); + } + if (result.stderr) { + process.stderr.write(result.stderr); + } + return { code: 0, stdout: result.stdout, stderr: result.stderr }; + } catch (error) { + const stdout = typeof error.stdout === 'string' ? error.stdout : ''; + const stderr = typeof error.stderr === 'string' ? error.stderr : error.message; + if (stdout) { + process.stdout.write(stdout); + } + if (stderr) { + process.stderr.write(stderr); + } + return { + code: typeof error.code === 'number' ? error.code : 1, + stdout, + stderr, + }; + } +} + +function requireSuccess(label, result, options = {}) { + if (result.code !== 0) { + throw new Error(`${label} failed with code ${result.code}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); + } + if (options.stderrPattern && !options.stderrPattern.test(result.stderr)) { + throw new Error(`${label} stderr did not match ${options.stderrPattern}\nstderr:\n${result.stderr}`); + } +} + +function parseJsonStdout(label, result) { + requireSuccess(label, result); + try { + return JSON.parse(result.stdout); + } catch (error) { + throw new Error(`${label} did not write JSON stdout: ${error.message}\nstdout:\n${result.stdout}`); + } +} + +function requireOutput(label, result, pattern) { + if (!pattern.test(result.stdout)) { + throw new Error(`${label} stdout did not match ${pattern}\nstdout:\n${result.stdout}`); + } +} + +async function postJson(baseUrl, path, payload, timeoutMs) { + const response = await fetch(new URL(path, baseUrl), { + method: 'POST', + headers: { + accept: 'application/json', + 'content-type': 'application/json', + }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(timeoutMs), + }); + const text = await response.text(); + if (!response.ok) { + throw new Error(`POST ${path} failed with ${response.status}: ${text}`); + } + try { + return JSON.parse(text); + } catch (error) { + throw new Error(`POST ${path} returned non-JSON response: ${error.message}\n${text}`); + } +} + +async function writeSmokePackage(projectDir, tarballPath) { + await mkdir(projectDir, { recursive: true }); + await writeFile( + join(projectDir, 'package.json'), + `${JSON.stringify( + { + name: 'ktx-local-embeddings-runtime-smoke', + version: '0.0.0', + private: true, + type: 'module', + dependencies: { + '@kaelio/ktx': `file:${tarballPath}`, + }, + }, + null, + 2, + )}\n`, + ); +} + +export async function runLocalEmbeddingsRuntimeSmoke(options = {}) { + const rootDir = options.rootDir ?? DEFAULT_ROOT_DIR; + const tarballPath = options.tarballPath ?? (await selectPublicKtxTarball(rootDir)); + const root = await mkdtemp(join(tmpdir(), 'ktx-local-embeddings-smoke-')); + const keepTemp = options.keepTemp ?? process.env.KTX_KEEP_LOCAL_EMBEDDINGS_SMOKE === '1'; + const installDir = join(root, 'installed-package'); + const projectDir = join(root, 'project'); + const smokeEnv = buildLocalEmbeddingsSmokeEnv(root); + const commands = localEmbeddingsSmokeCommands({ projectDir }); + let daemonStarted = false; + + try { + await writeSmokePackage(installDir, tarballPath); + requireSuccess( + 'pnpm install public package', + await run('pnpm', ['install', '--ignore-scripts=false'], { + cwd: installDir, + env: smokeEnv, + timeoutMs: 300_000, + }), + ); + + const version = await run(commands[0].command, commands[0].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[0].timeoutMs, + }); + requireSuccess(commands[0].label, version); + requireOutput(commands[0].label, version, expectedPublicKtxVersionPattern()); + + const missingStatus = parseJsonStdout( + commands[1].label, + await run(commands[1].command, commands[1].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[1].timeoutMs, + }), + ); + if (missingStatus.kind !== 'missing') { + throw new Error(`Expected missing runtime before install, got ${JSON.stringify(missingStatus)}`); + } + + const install = await run(commands[2].command, commands[2].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[2].timeoutMs, + }); + requireSuccess(commands[2].label, install); + requireOutput(commands[2].label, install, /Installed KTX Python runtime/); + requireOutput(commands[2].label, install, /features: core, local-embeddings/); + + const readyStatus = parseJsonStdout( + commands[3].label, + await run(commands[3].command, commands[3].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[3].timeoutMs, + }), + ); + if (readyStatus.kind !== 'ready') { + throw new Error(`Expected ready runtime after install, got ${JSON.stringify(readyStatus)}`); + } + if (!readyStatus.manifest?.features?.includes('local-embeddings')) { + throw new Error(`Runtime manifest did not include local-embeddings: ${JSON.stringify(readyStatus.manifest)}`); + } + + const start = await run(commands[4].command, commands[4].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[4].timeoutMs, + }); + requireSuccess(commands[4].label, start); + daemonStarted = true; + const baseUrl = parseDaemonBaseUrl(start.stdout); + + const embeddingResponse = await postJson( + baseUrl, + '/embeddings/compute', + { text: 'KTX local embeddings release smoke' }, + 900_000, + ); + validateEmbeddingResponse(embeddingResponse, 384); + process.stdout.write('KTX local embeddings daemon computed a 384-dimensional embedding\n'); + + const setup = await run(commands[5].command, commands[5].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[5].timeoutMs, + }); + requireSuccess(commands[5].label, setup); + requireOutput(commands[5].label, setup, /Embeddings ready: yes \(all-MiniLM-L6-v2\)/); + + const config = await readFile(join(projectDir, 'ktx.yaml'), 'utf8'); + if (!config.includes('base_url: managed:local-embeddings')) { + throw new Error(`ktx.yaml did not contain managed local embeddings marker:\n${config}`); + } + process.stdout.write('KTX setup persisted managed local embeddings marker\n'); + + const stop = await run(commands[6].command, commands[6].args, { + cwd: installDir, + env: smokeEnv, + timeoutMs: commands[6].timeoutMs, + }); + requireSuccess(commands[6].label, stop); + daemonStarted = false; + requireOutput(commands[6].label, stop, /Stopped KTX Python daemon/); + + process.stdout.write('KTX local embeddings runtime smoke verified\n'); + } finally { + if (daemonStarted) { + await run('pnpm', ['exec', 'ktx', 'runtime', 'stop'], { + cwd: installDir, + env: smokeEnv, + timeoutMs: 60_000, + }); + } + if (!keepTemp) { + await rm(root, { recursive: true, force: true }); + } else { + process.stdout.write(`Kept local embeddings smoke root: ${root}\n`); + } + } +} + +async function main() { + const args = process.argv.slice(2); + const optIn = localEmbeddingsSmokeOptIn(process.env, args); + if (!optIn.run) { + process.stdout.write(`Skipping KTX local embeddings runtime smoke. ${optIn.message}\n`); + if (args.includes('--require-opt-in')) { + process.exitCode = 1; + } + return; + } + + await runLocalEmbeddingsRuntimeSmoke(); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/scripts/local-embeddings-runtime-smoke.test.mjs b/scripts/local-embeddings-runtime-smoke.test.mjs new file mode 100644 index 00000000..9c13c2a0 --- /dev/null +++ b/scripts/local-embeddings-runtime-smoke.test.mjs @@ -0,0 +1,172 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { describe, it } from 'node:test'; + +import { + buildLocalEmbeddingsSmokeEnv, + expectedPublicKtxVersionPattern, + localEmbeddingsSmokeCommands, + localEmbeddingsSmokeOptIn, + parseDaemonBaseUrl, + publicKtxTarballName, + validateEmbeddingResponse, +} from './local-embeddings-runtime-smoke.mjs'; + +describe('localEmbeddingsSmokeOptIn', () => { + it('skips unless the smoke is explicitly enabled', () => { + assert.deepEqual(localEmbeddingsSmokeOptIn({}, []), { + run: false, + message: 'Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke.', + }); + }); + + it('runs when the environment opt-in is set', () => { + assert.deepEqual(localEmbeddingsSmokeOptIn({ KTX_RUN_LOCAL_EMBEDDINGS_SMOKE: '1' }, []), { + run: true, + }); + }); + + it('runs when --force is present', () => { + assert.deepEqual(localEmbeddingsSmokeOptIn({}, ['--force']), { + run: true, + }); + }); +}); + +describe('publicKtxTarballName', () => { + it('selects the public @kaelio/ktx tarball name', () => { + assert.equal( + publicKtxTarballName(['kaelio-ktx-0.1.0-rc.0.tgz', 'ignore-me.tgz']), + 'kaelio-ktx-0.1.0-rc.0.tgz', + ); + }); + + it('fails when the public package tarball is missing', () => { + assert.throws( + () => publicKtxTarballName(['ktx-cli-0.0.0-private.tgz']), + /Expected exactly one @kaelio\/ktx tarball/, + ); + }); + + it('fails when multiple public package tarballs are present', () => { + assert.throws( + () => publicKtxTarballName(['kaelio-ktx-0.1.0-rc.0.tgz', 'kaelio-ktx-0.2.0.tgz']), + /Expected exactly one @kaelio\/ktx tarball/, + ); + }); +}); + +describe('expectedPublicKtxVersionPattern', () => { + it('matches the public package version and rejects the private workspace version', () => { + const pattern = expectedPublicKtxVersionPattern(); + + assert.match('@kaelio/ktx 0.1.0-rc.0\n', pattern); + assert.doesNotMatch('@kaelio/ktx 0.0.0-private\n', pattern); + }); +}); + +describe('buildLocalEmbeddingsSmokeEnv', () => { + it('isolates the runtime root and model caches inside the smoke root', () => { + const env = buildLocalEmbeddingsSmokeEnv('/tmp/ktx-local-embedding-smoke', { + PATH: '/usr/bin', + }); + + assert.equal(env.PATH, '/usr/bin'); + assert.equal(env.KTX_RUN_LOCAL_EMBEDDINGS_SMOKE, '1'); + assert.equal(env.KTX_RUNTIME_ROOT, '/tmp/ktx-local-embedding-smoke/managed-runtime'); + assert.equal(env.HF_HOME, '/tmp/ktx-local-embedding-smoke/hf-home'); + assert.equal(env.TRANSFORMERS_CACHE, '/tmp/ktx-local-embedding-smoke/transformers-cache'); + assert.equal(env.SENTENCE_TRANSFORMERS_HOME, '/tmp/ktx-local-embedding-smoke/sentence-transformers-home'); + assert.equal(env.TORCH_HOME, '/tmp/ktx-local-embedding-smoke/torch-home'); + }); +}); + +describe('localEmbeddingsSmokeCommands', () => { + it('describes the installed-package commands needed for the smoke', () => { + const commands = localEmbeddingsSmokeCommands({ + projectDir: '/tmp/ktx-local-embedding-smoke/project', + }); + + assert.deepEqual(commands.map((command) => command.label), [ + 'ktx public package version', + 'ktx runtime status missing', + 'ktx runtime install local embeddings', + 'ktx runtime status local embeddings ready', + 'ktx runtime start local embeddings', + 'ktx setup local embeddings', + 'ktx runtime stop local embeddings', + ]); + assert.deepEqual(commands[2], { + label: 'ktx runtime install local embeddings', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], + timeoutMs: 1_200_000, + }); + assert.deepEqual(commands[4], { + label: 'ktx runtime start local embeddings', + command: 'pnpm', + args: ['exec', 'ktx', 'runtime', 'start', '--feature', 'local-embeddings'], + timeoutMs: 300_000, + }); + assert.deepEqual(commands[5].args, [ + 'exec', + 'ktx', + 'setup', + '--project-dir', + '/tmp/ktx-local-embedding-smoke/project', + '--new', + '--no-input', + '--yes', + '--skip-llm', + '--embedding-backend', + 'sentence-transformers', + '--skip-databases', + '--skip-sources', + '--skip-agents', + ]); + }); +}); + +describe('parseDaemonBaseUrl', () => { + it('extracts the daemon URL from runtime start output', () => { + assert.equal( + parseDaemonBaseUrl('Started KTX Python daemon\nurl: http://127.0.0.1:61234\nfeatures: local-embeddings\n'), + 'http://127.0.0.1:61234', + ); + }); + + it('rejects output without a daemon URL', () => { + assert.throws(() => parseDaemonBaseUrl('Started KTX Python daemon\n'), /Daemon URL was not printed/); + }); +}); + +describe('validateEmbeddingResponse', () => { + it('accepts a finite embedding vector with the expected dimensions', () => { + validateEmbeddingResponse({ embedding: [0.1, -0.2, 0.3] }, 3); + }); + + it('rejects a vector with the wrong dimensions', () => { + assert.throws( + () => validateEmbeddingResponse({ embedding: [0.1, 0.2] }, 3), + /Expected embedding dimension 3, got 2/, + ); + }); + + it('rejects non-finite embedding values', () => { + assert.throws( + () => validateEmbeddingResponse({ embedding: [0.1, Number.NaN, 0.3] }, 3), + /Embedding value at index 1 is not a finite number/, + ); + }); +}); + +describe('package script', () => { + it('registers the opt-in local embeddings smoke command', async () => { + const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); + + assert.equal( + packageJson.scripts['release:local-embeddings-smoke'], + 'node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in', + ); + }); +}); diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index d05b30bf..032df825 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -4,13 +4,27 @@ import { createHash } from 'node:crypto'; import { execFile } from 'node:child_process'; import { access, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -const PACKAGE_VERSION = '0.0.0-private'; -const PYTHON_PACKAGE_VERSION = '0.1.0'; +import { + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_NORMALIZED_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, +} from './build-python-runtime-wheel.mjs'; +import { + PUBLIC_NPM_PACKAGE_NAME, + PUBLIC_NPM_PACKAGE_VERSION, + publicNpmPackageTarballName, +} from './build-public-npm-package.mjs'; -export const NPM_ARTIFACT_PACKAGES = [ +export { + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_NORMALIZED_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, +}; + +export const INTERNAL_NPM_WORKSPACE_PACKAGES = [ { name: '@ktx/context', packageRoot: 'packages/context' }, { name: '@ktx/llm', packageRoot: 'packages/llm' }, { name: '@ktx/connector-bigquery', packageRoot: 'packages/connector-bigquery' }, @@ -23,29 +37,25 @@ export const NPM_ARTIFACT_PACKAGES = [ { name: '@ktx/cli', packageRoot: 'packages/cli' }, ]; -const CONNECTOR_PACKAGE_NAMES = NPM_ARTIFACT_PACKAGES +export const NPM_ARTIFACT_PACKAGES = [{ name: PUBLIC_NPM_PACKAGE_NAME, packageRoot: 'packages/cli' }]; + +export const CLI_PYTHON_ASSET_MANIFEST = 'manifest.json'; + +const CONNECTOR_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES .map((packageInfo) => packageInfo.name) .filter((packageName) => packageName.startsWith('@ktx/connector-')); -const ordersSource = { - name: 'orders', - table: 'public.orders', - grain: ['id'], - columns: [ - { name: 'id', type: 'number' }, - { name: 'status', type: 'string' }, - { name: 'amount', type: 'number' }, - ], - measures: [{ name: 'order_count', expr: 'count(*)' }], - joins: [], -}; +const NPM_ARTIFACT_BUILD_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli']; function scriptRootDir() { return resolve(dirname(fileURLToPath(import.meta.url)), '..'); } function npmPackageTarballName(packageName) { - return `${packageName.replace('@ktx/', 'ktx-')}-${PACKAGE_VERSION}.tgz`; + if (packageName !== PUBLIC_NPM_PACKAGE_NAME) { + throw new Error(`Unsupported npm artifact package: ${packageName}`); + } + return publicNpmPackageTarballName(PUBLIC_NPM_PACKAGE_VERSION); } function npmPackageTarballs(npmDir) { @@ -66,40 +76,40 @@ export function packageArtifactLayout(rootDir = scriptRootDir()) { npmDir, pythonDir, npmTarballs, - contextTarball: npmTarballs['@ktx/context'], - cliTarball: npmTarballs['@ktx/cli'], - connectorTarballs: Object.fromEntries( - CONNECTOR_PACKAGE_NAMES.map((packageName) => [packageName, npmTarballs[packageName]]), - ), + contextTarball: npmTarballs[PUBLIC_NPM_PACKAGE_NAME], + cliTarball: npmTarballs[PUBLIC_NPM_PACKAGE_NAME], + connectorTarballs: {}, manifestPath: join(artifactDir, 'manifest.json'), }; } export function buildArtifactCommands(layout) { - const npmBuildCommands = NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({ - command: 'pnpm', - args: ['--filter', packageInfo.name, 'run', 'build'], + const packagesByName = new Map(INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => [packageInfo.name, packageInfo])); + const npmBuildCommands = NPM_ARTIFACT_BUILD_ORDER.map((packageName) => { + const packageInfo = packagesByName.get(packageName); + if (!packageInfo) { + throw new Error(`Unknown npm artifact build package: ${packageName}`); + } + return { + command: 'pnpm', + args: ['--filter', packageInfo.name, 'run', 'build'], + cwd: layout.rootDir, + }; + }); + const publicPackageCommand = { + command: process.execPath, + args: ['scripts/build-public-npm-package.mjs'], cwd: layout.rootDir, - })); - const npmPackCommands = NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({ - command: 'pnpm', - args: ['--filter', packageInfo.name, 'pack', '--out', layout.npmTarballs[packageInfo.name]], - cwd: layout.rootDir, - })); + }; return [ ...npmBuildCommands, - ...npmPackCommands, { - command: 'uv', - args: ['build', '--package', 'ktx-sl', '--out-dir', layout.pythonDir], - cwd: layout.rootDir, - }, - { - command: 'uv', - args: ['build', '--package', 'ktx-daemon', '--out-dir', layout.pythonDir], + command: process.execPath, + args: ['scripts/build-python-runtime-wheel.mjs'], cwd: layout.rootDir, }, + publicPackageCommand, ]; } @@ -122,9 +132,9 @@ function normalizePythonDistributionName(name) { return name.replaceAll('-', '_'); } -function findOne(files, distributionName, suffix, label, pythonDir) { +function findOne(files, distributionName, suffix, label, pythonDir, version) { const normalized = normalizePythonDistributionName(distributionName); - const found = files.find((file) => file.startsWith(`${normalized}-${PYTHON_PACKAGE_VERSION}`) && file.endsWith(suffix)); + const found = files.find((file) => file.startsWith(`${normalized}-${version}`) && file.endsWith(suffix)); if (!found) { throw new Error(`Missing Python artifact: ${label}`); } @@ -135,10 +145,14 @@ export async function findPythonArtifacts(pythonDir) { const files = await readdir(pythonDir); return { - ktxSlWheel: findOne(files, 'ktx-sl', '.whl', 'ktx-sl wheel', pythonDir), - ktxSlSdist: findOne(files, 'ktx-sl', '.tar.gz', 'ktx-sl source distribution', pythonDir), - ktxDaemonWheel: findOne(files, 'ktx-daemon', '.whl', 'ktx-daemon wheel', pythonDir), - ktxDaemonSdist: findOne(files, 'ktx-daemon', '.tar.gz', 'ktx-daemon source distribution', pythonDir), + runtimeWheel: findOne( + files, + RUNTIME_WHEEL_DISTRIBUTION_NAME, + '.whl', + 'kaelio-ktx runtime wheel', + pythonDir, + RUNTIME_WHEEL_PACKAGE_VERSION, + ), }; } @@ -150,47 +164,6 @@ async function readJson(path) { return JSON.parse(await readFile(path, 'utf-8')); } -function readProjectBlock(toml, sourcePath) { - const lines = toml.split(/\r?\n/); - const block = []; - let inProject = false; - - for (const line of lines) { - if (/^\[project\]\s*$/.test(line)) { - inProject = true; - continue; - } - if (inProject && /^\[.*\]\s*$/.test(line)) { - break; - } - if (inProject) { - block.push(line); - } - } - - if (!inProject) { - throw new Error(`Missing [project] table in ${sourcePath}`); - } - return block.join('\n'); -} - -function readTomlStringField(projectBlock, fieldName, sourcePath) { - const match = projectBlock.match(new RegExp(`^${fieldName}\\s*=\\s*"([^"]+)"\\s*$`, 'm')); - if (!match) { - throw new Error(`Missing project.${fieldName} in ${sourcePath}`); - } - return match[1]; -} - -async function readPyprojectMetadata(path) { - const toml = await readFile(path, 'utf-8'); - const projectBlock = readProjectBlock(toml, path); - return { - name: readTomlStringField(projectBlock, 'name', path), - version: readTomlStringField(projectBlock, 'version', path), - }; -} - function releaseMetadataEntry({ ecosystem, packageName, packageRoot, packageVersion, privatePackage }) { return { ecosystem, @@ -204,17 +177,19 @@ function releaseMetadataEntry({ ecosystem, packageName, packageRoot, packageVers async function readNpmPackageMetadata(rootDir, packageInfo) { const packageJson = await readJson(join(rootDir, packageInfo.packageRoot, 'package.json')); - if (packageJson.name !== packageInfo.name) { + const expectedSourceName = packageInfo.name === PUBLIC_NPM_PACKAGE_NAME ? '@ktx/cli' : packageInfo.name; + if (packageJson.name !== expectedSourceName) { throw new Error( - `Unexpected package name in ${packageInfo.packageRoot}/package.json: expected ${packageInfo.name}, got ${packageJson.name}`, + `Unexpected package name in ${packageInfo.packageRoot}/package.json: expected ${expectedSourceName}, got ${packageJson.name}`, ); } + const isPublicKtxPackage = packageInfo.name === PUBLIC_NPM_PACKAGE_NAME; return releaseMetadataEntry({ ecosystem: 'npm', - packageName: packageJson.name, + packageName: packageInfo.name, packageRoot: packageInfo.packageRoot, - packageVersion: packageJson.version, - privatePackage: packageJson.private === true, + packageVersion: isPublicKtxPackage ? PUBLIC_NPM_PACKAGE_VERSION : packageJson.version, + privatePackage: isPublicKtxPackage ? false : packageJson.private === true, }); } @@ -222,23 +197,14 @@ export async function packageReleaseMetadata(rootDir = scriptRootDir()) { const npmPackages = await Promise.all( NPM_ARTIFACT_PACKAGES.map((packageInfo) => readNpmPackageMetadata(rootDir, packageInfo)), ); - const ktxSlPackage = await readPyprojectMetadata(join(rootDir, 'python', 'ktx-sl', 'pyproject.toml')); - const ktxDaemonPackage = await readPyprojectMetadata(join(rootDir, 'python', 'ktx-daemon', 'pyproject.toml')); return [ ...npmPackages, releaseMetadataEntry({ ecosystem: 'python', - packageName: ktxSlPackage.name, - packageRoot: 'python/ktx-sl', - packageVersion: ktxSlPackage.version, - privatePackage: false, - }), - releaseMetadataEntry({ - ecosystem: 'python', - packageName: ktxDaemonPackage.name, - packageRoot: 'python/ktx-daemon', - packageVersion: ktxDaemonPackage.version, + packageName: RUNTIME_WHEEL_DISTRIBUTION_NAME, + packageRoot: 'python/runtime-wheel', + packageVersion: RUNTIME_WHEEL_PACKAGE_VERSION, privatePackage: false, }), ]; @@ -268,23 +234,8 @@ function artifactPackageRecords(layout, pythonArtifacts, packages) { ...npmRecords, { artifactKind: 'wheel', - artifactPath: pythonArtifacts.ktxSlWheel, - metadata: requirePackageMetadata(packagesByName, 'ktx-sl'), - }, - { - artifactKind: 'sdist', - artifactPath: pythonArtifacts.ktxSlSdist, - metadata: requirePackageMetadata(packagesByName, 'ktx-sl'), - }, - { - artifactKind: 'wheel', - artifactPath: pythonArtifacts.ktxDaemonWheel, - metadata: requirePackageMetadata(packagesByName, 'ktx-daemon'), - }, - { - artifactKind: 'sdist', - artifactPath: pythonArtifacts.ktxDaemonSdist, - metadata: requirePackageMetadata(packagesByName, 'ktx-daemon'), + artifactPath: pythonArtifacts.runtimeWheel, + metadata: requirePackageMetadata(packagesByName, RUNTIME_WHEEL_DISTRIBUTION_NAME), }, ]; } @@ -428,15 +379,41 @@ export async function verifyArtifactManifest(layout, options = {}) { return manifest; } -export function pythonArtifactInstallArgs(python, pythonArtifacts) { - return [ - 'pip', - 'install', - '--python', - python, - pythonArtifacts.ktxSlWheel, - pythonArtifacts.ktxDaemonWheel, - ]; +function runtimeWheelAssetName(runtimeWheelPath) { + return runtimeWheelPath.split(sep).at(-1); +} + +export async function copyRuntimeWheelAssets(layout, pythonArtifacts) { + const assetDir = join(layout.rootDir, 'packages', 'cli', 'assets', 'python'); + const wheelFile = runtimeWheelAssetName(pythonArtifacts.runtimeWheel); + if (!wheelFile) { + throw new Error(`Unable to determine runtime wheel filename: ${pythonArtifacts.runtimeWheel}`); + } + const wheelContents = await readFile(pythonArtifacts.runtimeWheel); + await rm(assetDir, { recursive: true, force: true }); + await mkdir(assetDir, { recursive: true }); + const wheelPath = join(assetDir, wheelFile); + const manifestPath = join(assetDir, CLI_PYTHON_ASSET_MANIFEST); + await writeFile(wheelPath, wheelContents); + await writeFile( + manifestPath, + `${JSON.stringify( + { + schemaVersion: 1, + distributionName: RUNTIME_WHEEL_DISTRIBUTION_NAME, + normalizedName: RUNTIME_WHEEL_NORMALIZED_NAME, + version: RUNTIME_WHEEL_PACKAGE_VERSION, + wheel: { + file: wheelFile, + sha256: createHash('sha256').update(wheelContents).digest('hex'), + bytes: wheelContents.byteLength, + }, + }, + null, + 2, + )}\n`, + ); + return { assetDir, wheelPath, manifestPath }; } function runCommand(command, args, options = {}) { @@ -473,28 +450,19 @@ function runCommand(command, args, options = {}) { }); } -function npmTarballDependencyEntries(layout) { - return Object.fromEntries( - NPM_ARTIFACT_PACKAGES.map((packageInfo) => [ - packageInfo.name, - `file:${layout.npmTarballs[packageInfo.name]}`, - ]), - ); -} - export function npmSmokePackageJson(layout) { - const npmTarballDependencies = npmTarballDependencyEntries(layout); return { name: 'ktx-artifact-npm-smoke', version: '0.0.0', private: true, type: 'module', dependencies: { - ...npmTarballDependencies, - '@modelcontextprotocol/sdk': '^1.27.1', + '@kaelio/ktx': `file:${layout.cliTarball}`, + }, + devDependencies: { + 'better-sqlite3': '^12.6.2', }, pnpm: { - overrides: npmTarballDependencies, onlyBuiltDependencies: ['better-sqlite3'], }, }; @@ -502,101 +470,13 @@ export function npmSmokePackageJson(layout) { export function npmVerifySource() { return ` -const context = await import('@ktx/context'); -const project = await import('@ktx/context/project'); -const mcp = await import('@ktx/context/mcp'); -const memory = await import('@ktx/context/memory'); -const daemon = await import('@ktx/context/daemon'); -const ingest = await import('@ktx/context/ingest'); -const search = await import('@ktx/context/search'); -const llm = await import('@ktx/llm'); -const cli = await import('@ktx/cli'); -const bigqueryConnector = await import('@ktx/connector-bigquery'); -const clickhouseConnector = await import('@ktx/connector-clickhouse'); -const mysqlConnector = await import('@ktx/connector-mysql'); -const postgresConnector = await import('@ktx/connector-postgres'); -const snowflakeConnector = await import('@ktx/connector-snowflake'); -const sqliteConnector = await import('@ktx/connector-sqlite'); -const sqlserverConnector = await import('@ktx/connector-sqlserver'); +const cli = await import('@kaelio/ktx'); -if (context.ktxContextPackageInfo.name !== '@ktx/context') { - throw new Error('Unexpected @ktx/context package info'); +if (cli.getKtxCliPackageInfo().name !== '@kaelio/ktx') { + throw new Error('Unexpected @kaelio/ktx package info'); } -if (typeof llm.createKtxLlmProvider !== 'function') { - throw new Error('Missing createKtxLlmProvider export'); -} -if (typeof llm.KtxMessageBuilder !== 'function') { - throw new Error('Missing KtxMessageBuilder export'); -} -if (typeof llm.createKtxEmbeddingProvider !== 'function') { - throw new Error('Missing createKtxEmbeddingProvider export'); -} -if (typeof project.initKtxProject !== 'function') { - throw new Error('Missing initKtxProject export'); -} -if (typeof mcp.createDefaultKtxMcpServer !== 'function') { - throw new Error('Missing createDefaultKtxMcpServer export'); -} -if (typeof memory.createLocalProjectMemoryCapture !== 'function') { - throw new Error('Missing createLocalProjectMemoryCapture export'); -} -if (typeof search.HybridSearchCore !== 'function') { - throw new Error('Missing HybridSearchCore export from @ktx/context/search'); -} -if (typeof search.assertSearchBackendConformanceCase !== 'function') { - throw new Error('Missing assertSearchBackendConformanceCase export from @ktx/context/search'); -} -if (typeof search.assertSearchBackendCapabilities !== 'function') { - throw new Error('Missing assertSearchBackendCapabilities export from @ktx/context/search'); -} -if (typeof daemon.createPythonSemanticLayerComputePort !== 'function') { - throw new Error('Missing createPythonSemanticLayerComputePort export'); -} -const dbtExtractionExports = [ - ['parseMetricflowFiles', ingest.parseMetricflowFiles], - ['parseMetricflowPullConfig', ingest.parseMetricflowPullConfig], - ['importMetricflowSemanticModels', ingest.importMetricflowSemanticModels], - ['parseDbtSchemaFiles', ingest.parseDbtSchemaFiles], - ['toDescriptionUpdates', ingest.toDescriptionUpdates], - ['toRelationshipUpdates', ingest.toRelationshipUpdates], - ['mergeSemanticModelTables', ingest.mergeSemanticModelTables], - ['loadProjectInfo', ingest.loadProjectInfo], - ['loadDbtSchemaFiles', ingest.loadDbtSchemaFiles], -]; - -for (const [exportName, exportValue] of dbtExtractionExports) { - if (typeof exportValue !== 'function') { - throw new Error('Missing dbt extraction export: ' + exportName); - } -} - -const metricflowConfig = ingest.parseMetricflowPullConfig({ - repoUrl: 'https://example.com/acme/analytics.git', -}); -if (metricflowConfig.branch !== 'main' || metricflowConfig.path !== null) { - throw new Error('Unexpected MetricFlow pull-config defaults from installed @ktx/context/ingest'); -} -if (cli.getKtxCliPackageInfo().name !== '@ktx/cli') { - throw new Error('Unexpected @ktx/cli package info'); -} - -const connectorExports = [ - ['@ktx/connector-bigquery', bigqueryConnector.KtxBigQueryScanConnector, bigqueryConnector.KtxBigQueryDialect], - ['@ktx/connector-clickhouse', clickhouseConnector.KtxClickHouseScanConnector, clickhouseConnector.KtxClickHouseDialect], - ['@ktx/connector-mysql', mysqlConnector.KtxMysqlScanConnector, mysqlConnector.KtxMysqlDialect], - ['@ktx/connector-postgres', postgresConnector.KtxPostgresScanConnector, postgresConnector.KtxPostgresDialect], - ['@ktx/connector-snowflake', snowflakeConnector.KtxSnowflakeScanConnector, snowflakeConnector.KtxSnowflakeDialect], - ['@ktx/connector-sqlite', sqliteConnector.KtxSqliteScanConnector, sqliteConnector.KtxSqliteDialect], - ['@ktx/connector-sqlserver', sqlserverConnector.KtxSqlServerScanConnector, sqlserverConnector.KtxSqlServerDialect], -]; - -for (const [packageName, ScanConnector, Dialect] of connectorExports) { - if (typeof ScanConnector !== 'function') { - throw new Error('Missing scan connector export from ' + packageName); - } - if (typeof Dialect !== 'function') { - throw new Error('Missing dialect export from ' + packageName); - } +if (typeof cli.runKtxCli !== 'function') { + throw new Error('Missing runKtxCli export'); } `; } @@ -604,29 +484,14 @@ for (const [packageName, ScanConnector, Dialect] of connectorExports) { export function npmRuntimeSmokeSource() { return ` import assert from 'node:assert/strict'; -import { spawn, execFile } from 'node:child_process'; -import { once } from 'node:events'; +import Database from 'better-sqlite3'; +import { execFile } from 'node:child_process'; import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { request as httpRequest } from 'node:http'; -import { createServer } from 'node:net'; -import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; +import { join } from 'node:path'; import { promisify } from 'node:util'; -import { Client } from '@modelcontextprotocol/sdk/client/index.js'; -import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; -import { - createDaemonLookerTableIdentifierParser, - LocalLookerRuntimeStore, -} from '@ktx/context/ingest'; const execFileAsync = promisify(execFile); -const require = createRequire(import.meta.url); -const contextPackageRoot = dirname(require.resolve('@ktx/context/package.json')); - -async function requireContextRuntimeAsset(relativePath) { - await access(join(contextPackageRoot, relativePath)); -} async function run(command, args, options = {}) { process.stdout.write('$ ' + command + ' ' + args.join(' ') + '\\n'); @@ -655,6 +520,15 @@ function requireSuccess(label, result) { assert.equal(result.stderr, '', label + ' wrote unexpected stderr'); } +function requireSuccessWithStderr(label, result, stderrPattern) { + assert.equal( + result.code, + 0, + label + ' failed with code ' + result.code + '\\nstdout:\\n' + result.stdout + '\\nstderr:\\n' + result.stderr, + ); + assert.match(result.stderr, stderrPattern, label + ' stderr did not match ' + stderrPattern); +} + function requireOutput(label, result, text) { assert.match(result.stdout, text, label + ' output did not match ' + text); } @@ -681,174 +555,43 @@ function getRunId(stdout) { return match[1]; } -function requireToolNames(tools, expectedNames) { - const names = tools.tools.map((tool) => tool.name).sort(); - for (const expectedName of expectedNames) { - assert.ok(names.includes(expectedName), 'MCP tool list did not include ' + expectedName + ': ' + names.join(', ')); - } -} - -function structuredContent(result) { - assert.ok(result.structuredContent, 'MCP result did not include structuredContent'); - return result.structuredContent; -} - -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function getAvailablePort() { - const server = createServer(); - server.listen(0, '127.0.0.1'); - await once(server, 'listening'); - const address = server.address(); - if (!address || typeof address === 'string') { - server.close(); - throw new Error('expected TCP server address for daemon smoke'); - } - const port = address.port; - server.close(); - await once(server, 'close'); - return port; -} - -function httpGetOk(url) { - return new Promise((resolve, reject) => { - const request = httpRequest(url, { method: 'GET' }, (response) => { - response.resume(); - response.on('end', () => resolve((response.statusCode ?? 0) >= 200 && (response.statusCode ?? 0) < 300)); - }); - request.on('error', reject); - request.end(); - }); -} - -function spawnLogged(command, args, options = {}) { - const stdout = []; - const stderr = []; - let spawnError; - const child = spawn(command, args, { - cwd: options.cwd, - env: options.env ?? process.env, - stdio: ['ignore', 'pipe', 'pipe'], - }); - child.stdout.on('data', (chunk) => stdout.push(chunk)); - child.stderr.on('data', (chunk) => stderr.push(chunk)); - child.on('error', (error) => { - spawnError = error; - }); - return { - child, - error() { - return spawnError; - }, - output() { - return { - stdout: Buffer.concat(stdout).toString('utf8'), - stderr: Buffer.concat(stderr).toString('utf8'), - }; - }, - }; -} - -async function waitForHttpHealth(url, daemon) { - const deadline = Date.now() + 15_000; - while (Date.now() < deadline) { - if (daemon.error()) { - const output = daemon.output(); - throw new Error( - 'Failed to start ktx-daemon serve-http: ' + - daemon.error().message + - '\\nstdout:\\n' + - output.stdout + - '\\nstderr:\\n' + - output.stderr, - ); - } - if (daemon.child.exitCode !== null || daemon.child.signalCode !== null) { - const output = daemon.output(); - throw new Error( - 'ktx-daemon serve-http exited before health check passed\\nstdout:\\n' + - output.stdout + - '\\nstderr:\\n' + - output.stderr, - ); - } - try { - if (await httpGetOk(url)) { - return; - } - } catch { - await sleep(100); - continue; - } - await sleep(100); - } - const output = daemon.output(); - throw new Error('Timed out waiting for ' + url + '\\nstdout:\\n' + output.stdout + '\\nstderr:\\n' + output.stderr); -} - -async function startSemanticDaemon(port) { - const daemon = spawnLogged('ktx-daemon', [ - 'serve-http', - '--host', - '127.0.0.1', - '--port', - String(port), - '--log-level', - 'warning', - ]); - await waitForHttpHealth('http://127.0.0.1:' + port + '/health', daemon); - return daemon; -} - -async function stopSemanticDaemon(daemon) { - if (daemon.child.exitCode !== null || daemon.child.signalCode !== null) { - return; - } - daemon.child.kill('SIGTERM'); - const closed = once(daemon.child, 'close').then(() => true); - const timedOut = sleep(5_000).then(() => false); - if (!(await Promise.race([closed, timedOut]))) { - daemon.child.kill('SIGKILL'); - await once(daemon.child, 'close'); - } -} - async function writeSqliteWarehouse(projectDir) { - const createDb = await run('python', [ - '-c', - [ - 'import sqlite3', - 'import sys', - 'db_path = sys.argv[1]', - 'conn = sqlite3.connect(db_path)', - 'conn.executescript("""', - 'DROP TABLE IF EXISTS orders;', - 'CREATE TABLE orders (', - ' id INTEGER PRIMARY KEY,', - ' status TEXT NOT NULL,', - ' amount INTEGER NOT NULL', - ');', - "INSERT INTO orders (status, amount) VALUES ('paid', 20), ('paid', 30), ('open', 10);", - '""")', - 'conn.close()', - ].join('\\n'), - join(projectDir, 'warehouse.db'), - ]); - requireSuccess('create sqlite warehouse', createDb); + const database = new Database(join(projectDir, 'warehouse.db')); + try { + database.exec(\` +DROP TABLE IF EXISTS orders; +CREATE TABLE orders ( + id INTEGER PRIMARY KEY, + status TEXT NOT NULL, + amount INTEGER NOT NULL +); +INSERT INTO orders (status, amount) VALUES ('paid', 20), ('paid', 30), ('open', 10); +\`); + } finally { + database.close(); + } } -await requireContextRuntimeAsset('skills/notion_synthesize/SKILL.md'); -await requireContextRuntimeAsset('prompts/skills/page_triage_classifier.md'); -await requireContextRuntimeAsset('prompts/skills/light_extraction.md'); -process.stdout.write('packaged ingest runtime assets verified\\n'); - const root = await mkdtemp(join(tmpdir(), 'ktx-installed-cli-smoke-')); +const previousRuntimeRoot = process.env.KTX_RUNTIME_ROOT; +process.env.KTX_RUNTIME_ROOT = join(root, 'managed-runtime'); +let daemonStarted = false; try { const projectDir = join(root, 'project'); const sourceDir = join(root, 'source'); + const version = await run('pnpm', ['exec', 'ktx', '--version']); + requireSuccess('ktx public package version', version); + requireOutput('ktx public package version', version, /@kaelio\\/ktx 0\\.1\\.0/); + + const runtimeStatusBefore = parseJsonResult( + 'ktx runtime status missing', + await run('pnpm', ['exec', 'ktx', 'runtime', 'status', '--json']), + ); + assert.equal(runtimeStatusBefore.kind, 'missing'); + assert.equal(runtimeStatusBefore.layout.runtimeRoot, process.env.KTX_RUNTIME_ROOT); + process.stdout.write('ktx managed runtime starts missing in isolated root\\n'); + const missingProjectDir = join(root, 'missing-project'); await mkdir(missingProjectDir, { recursive: true }); const missingProjectSearch = await run('pnpm', [ @@ -955,22 +698,6 @@ try { ); await writeSqliteWarehouse(projectDir); - const lookerStore = new LocalLookerRuntimeStore({ dbPath: join(projectDir, '.ktx', 'db.sqlite') }); - await lookerStore.setCursors('prod-looker', { - dashboardsLastSyncedAt: null, - looksLastSyncedAt: null, - }); - await lookerStore.upsertConnectionMapping({ - lookerConnectionId: 'prod-looker', - lookerConnectionName: 'analytics', - ktxConnectionId: 'warehouse', - source: 'cli', - }); - const lookerMappings = await lookerStore.readMappings('prod-looker'); - assert.equal(lookerMappings.length, 1); - assert.equal(lookerMappings[0].ktxConnectionId, 'warehouse'); - process.stdout.write('Looker local runtime store verified\\n'); - await mkdir(join(projectDir, 'knowledge', 'global'), { recursive: true }); await writeFile( join(projectDir, 'knowledge', 'global', 'revenue.md'), @@ -1079,40 +806,100 @@ try { requireIncludes(agentSlSearchJson.sources[0].matchReasons, 'lexical', 'agent sl search match reasons'); process.stdout.write('ktx agent sl list hybrid metadata verified\\n'); - const slQueryFile = join(projectDir, 'sl-query.json'); - await writeFile(slQueryFile, '{"measures":["orders.order_count"],"dimensions":[]}\\n', 'utf-8'); - - const slQuery = await run('pnpm', ['exec', 'ktx', 'agent', 'sl', 'query', - '--json', + const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', '--connection-id', 'warehouse', - '--query-file', - slQueryFile, + '--measure', + 'orders.order_count', + '--format', + 'json', + '--yes', '--project-dir', projectDir, ]); - requireSuccess('ktx agent sl query', slQuery); - requireOutput('ktx agent sl query', slQuery, /"mode": "compile_only"/); - requireOutput('ktx agent sl query', slQuery, /orders/); + requireSuccessWithStderr( + 'ktx sl query first managed runtime install', + slQuery, + /Installing KTX Python runtime \\(core\\) with uv[\\s\\S]*KTX Python runtime ready:/, + ); + requireOutput('ktx sl query first managed runtime install', slQuery, /"mode": "compile_only"/); + requireOutput('ktx sl query first managed runtime install', slQuery, /orders/); - const sqliteSlQuery = await run('pnpm', ['exec', 'ktx', 'agent', 'sl', 'query', - '--json', + const runtimeStatusAfter = parseJsonResult( + 'ktx runtime status ready', + await run('pnpm', ['exec', 'ktx', 'runtime', 'status', '--json']), + ); + assert.equal(runtimeStatusAfter.kind, 'ready'); + assert.deepEqual(runtimeStatusAfter.manifest.features, ['core']); + assert.equal(runtimeStatusAfter.layout.runtimeRoot, process.env.KTX_RUNTIME_ROOT); + process.stdout.write('ktx managed runtime lazy install verified\\n'); + + const sqliteSlQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', '--connection-id', 'warehouse', - '--query-file', - slQueryFile, + '--measure', + 'orders.order_count', + '--format', + 'json', '--execute', '--max-rows', '100', + '--yes', '--project-dir', projectDir, ]); - requireSuccess('ktx agent sl query sqlite execute', sqliteSlQuery); - requireOutput('ktx agent sl query sqlite execute', sqliteSlQuery, /"dialect": "sqlite"/); - requireOutput('ktx agent sl query sqlite execute', sqliteSlQuery, /"mode": "executed"/); - requireOutput('ktx agent sl query sqlite execute', sqliteSlQuery, /"driver": "sqlite"/); - requireOutput('ktx agent sl query sqlite execute', sqliteSlQuery, /"rows": \\[\\s*\\[\\s*3\\s*\\]\\s*\\]/); - process.stdout.write('ktx agent sl query sqlite execute verified\\n'); + requireSuccess('ktx sl query sqlite execute', sqliteSlQuery); + requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"dialect": "sqlite"/); + requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"mode": "executed"/); + requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"driver": "sqlite"/); + requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"rows": \\[\\s*\\[\\s*3\\s*\\]\\s*\\]/); + process.stdout.write('ktx sl query sqlite execute verified\\n'); + + const runtimeDoctor = await run('pnpm', ['exec', 'ktx', 'runtime', 'doctor']); + requireSuccess('ktx runtime doctor', runtimeDoctor); + requireOutput('ktx runtime doctor', runtimeDoctor, /PASS uv/); + requireOutput('ktx runtime doctor', runtimeDoctor, /PASS Bundled Python wheel/); + requireOutput('ktx runtime doctor', runtimeDoctor, /PASS Managed Python runtime/); + process.stdout.write('ktx runtime doctor verified\\n'); + + const runtimeStart = await run('pnpm', ['exec', 'ktx', 'runtime', 'start']); + requireSuccess('ktx runtime start', runtimeStart); + daemonStarted = true; + requireOutput('ktx runtime start', runtimeStart, /Started KTX Python daemon/); + requireOutput('ktx runtime start', runtimeStart, /url: http:\\/\\/127\\.0\\.0\\.1:\\d+/); + requireOutput('ktx runtime start', runtimeStart, /features: core/); + + const runtimeStartReuse = await run('pnpm', ['exec', 'ktx', 'runtime', 'start']); + requireSuccess('ktx runtime start reuse', runtimeStartReuse); + requireOutput('ktx runtime start reuse', runtimeStartReuse, /Using existing KTX Python daemon/); + requireOutput('ktx runtime start reuse', runtimeStartReuse, /features: core/); + + const runtimeStop = await run('pnpm', ['exec', 'ktx', 'runtime', 'stop']); + requireSuccess('ktx runtime stop', runtimeStop); + daemonStarted = false; + requireOutput('ktx runtime stop', runtimeStop, /Stopped KTX Python daemon/); + process.stdout.write('ktx runtime daemon lifecycle verified\\n'); + + const staleRuntimeDir = join(process.env.KTX_RUNTIME_ROOT, '0.0.0'); + await mkdir(staleRuntimeDir, { recursive: true }); + + const runtimePruneDryRun = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune', '--dry-run']); + requireSuccess('ktx runtime prune dry run', runtimePruneDryRun); + requireOutput('ktx runtime prune dry run', runtimePruneDryRun, /Stale KTX Python runtimes/); + requireOutput('ktx runtime prune dry run', runtimePruneDryRun, /0\\.0\\.0/); + await access(staleRuntimeDir); + + const runtimePruneNeedsConfirmation = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune']); + assert.equal(runtimePruneNeedsConfirmation.code, 1, 'ktx runtime prune needs confirmation'); + assert.equal(runtimePruneNeedsConfirmation.stdout, '', 'ktx runtime prune needs confirmation wrote stdout'); + assert.match(runtimePruneNeedsConfirmation.stderr, /Refusing to prune without --yes/); + + const runtimePruneConfirmed = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune', '--yes']); + requireSuccess('ktx runtime prune confirmed', runtimePruneConfirmed); + requireOutput('ktx runtime prune confirmed', runtimePruneConfirmed, /Removed stale KTX Python runtimes/); + requireOutput('ktx runtime prune confirmed', runtimePruneConfirmed, /0\\.0\\.0/); + await assert.rejects(() => access(staleRuntimeDir)); + process.stdout.write('ktx runtime prune verified\\n'); const structuralScan = await run('pnpm', ['exec', 'ktx', 'dev', 'scan', 'warehouse', '--project-dir', @@ -1194,196 +981,15 @@ try { await access(join(projectDir, '.ktx', 'db.sqlite')); process.stdout.write('ktx dev ingest provider guard verified\\n'); - - await writeFile( - join(projectDir, 'ktx.yaml'), - [ - 'project: warehouse', - 'connections:', - ' warehouse:', - ' driver: sqlite', - ' path: warehouse.db', - ' readonly: true', - 'storage:', - ' state: sqlite', - ' search: sqlite-fts5', - 'scan:', - ' enrichment:', - ' mode: deterministic', - 'llm:', - ' provider:', - ' backend: gateway', - ' gateway:', - ' api_key: env:AI_GATEWAY_API_KEY', - ' models:', - ' default: smoke/provider', - 'ingest:', - ' adapters:', - ' - fake', - ' - live-database', - '', - ].join('\\n'), - 'utf-8', - ); - - const daemonPort = await getAvailablePort(); - const semanticComputeUrl = 'http://127.0.0.1:' + daemonPort; - process.stdout.write('ktx-daemon serve-http --host 127.0.0.1 --port ' + daemonPort + '\\n'); - const daemon = await startSemanticDaemon(daemonPort); - const lookerParser = createDaemonLookerTableIdentifierParser({ baseUrl: semanticComputeUrl }); - const parsedLookerTables = await lookerParser.parse([ - { key: 'orders', sql_table_name: 'orders', dialect: 'sqlite' }, - ]); - assert.equal(parsedLookerTables.orders.ok, true); - assert.equal(parsedLookerTables.orders.name, 'orders'); - assert.equal(parsedLookerTables.orders.canonical_table, 'orders'); - process.stdout.write('Looker daemon table identifier parser verified\\n'); - const client = new Client({ name: 'ktx-artifact-smoke-client', version: '0.0.0' }); - process.stdout.write('ktx serve --mcp stdio --semantic-compute-url ' + semanticComputeUrl + ' --execute-queries\\n'); - const transport = new StdioClientTransport({ - command: 'pnpm', - args: [ - 'exec', - 'ktx', - 'serve', '--mcp', 'stdio', - '--project-dir', - projectDir, - '--user-id', - 'artifact-smoke-user', - '--semantic-compute-url', - semanticComputeUrl, - '--execute-queries', - '--memory-capture', '--memory-model', 'smoke/provider', - ], - cwd: process.cwd(), - stderr: 'pipe', - env: { - ...process.env, - AI_GATEWAY_API_KEY: process.env.AI_GATEWAY_API_KEY ?? 'artifact-smoke-token', - }, - }); - const mcpServerStderr = []; - transport.stderr?.on('data', (chunk) => mcpServerStderr.push(chunk)); - - try { - await client.connect(transport); - const tools = await client.listTools(); - requireToolNames(tools, [ - 'connection_list', - 'connection_test', - 'ingest_status', - 'ingest_trigger', - 'knowledge_read', - 'knowledge_search', - 'knowledge_write', - 'memory_capture', - 'memory_capture_status', - 'scan_list_artifacts', - 'scan_read_artifact', - 'scan_report', - 'scan_status', - 'scan_trigger', - 'sl_list_sources', - 'sl_query', - 'sl_read_source', - 'sl_validate', - 'sl_write_source', - ]); - const slValidateResult = structuredContent(await client.callTool({ - name: 'sl_validate', - arguments: { - connectionId: 'warehouse', - names: ['orders'], - }, - })); - assert.equal(slValidateResult.success, true); - assert.deepEqual(slValidateResult.errors, []); - const slQueryResult = structuredContent(await client.callTool({ - name: 'sl_query', - arguments: { - connectionId: 'warehouse', - measures: ['orders.order_count'], - limit: 5, - }, - })); - assert.equal(slQueryResult.connectionId, 'warehouse'); - assert.equal(slQueryResult.dialect, 'sqlite'); - assert.match(slQueryResult.sql, /orders/); - assert.deepEqual(slQueryResult.headers, ['order_count']); - assert.deepEqual(slQueryResult.rows, [[3]]); - assert.equal(slQueryResult.totalRows, 1); - assert.equal(slQueryResult.plan.execution.mode, 'executed'); - assert.equal(slQueryResult.plan.execution.driver, 'sqlite'); - - const connectionTest = structuredContent(await client.callTool({ - name: 'connection_test', - arguments: { - connectionId: 'warehouse', - }, - })); - assert.equal(connectionTest.id, 'warehouse'); - assert.equal(connectionTest.ok, true); - - const mcpScanTrigger = structuredContent(await client.callTool({ - name: 'scan_trigger', - arguments: { - connectionId: 'warehouse', - mode: 'structural', - }, - })); - assert.equal(mcpScanTrigger.connectionId, 'warehouse'); - assert.equal(mcpScanTrigger.report.mode, 'structural'); - assert.equal(mcpScanTrigger.report.manifestShardsWritten, 1); - - const mcpScanStatus = structuredContent(await client.callTool({ - name: 'scan_status', - arguments: { - runId: mcpScanTrigger.runId, - }, - })); - assert.equal(mcpScanStatus.runId, mcpScanTrigger.runId); - assert.equal(mcpScanStatus.status, 'done'); - - const mcpScanReport = structuredContent(await client.callTool({ - name: 'scan_report', - arguments: { - runId: mcpScanTrigger.runId, - }, - })); - assert.equal(mcpScanReport.runId, mcpScanTrigger.runId); - assert.deepEqual(mcpScanReport.artifactPaths.manifestShards, ['semantic-layer/warehouse/_schema/public.yaml']); - - const mcpScanArtifacts = structuredContent(await client.callTool({ - name: 'scan_list_artifacts', - arguments: { - runId: mcpScanTrigger.runId, - }, - })); - const manifestArtifact = mcpScanArtifacts.artifacts.find((artifact) => artifact.type === 'manifest_shard'); - assert.ok(manifestArtifact, 'scan_list_artifacts did not include a manifest shard'); - assert.equal(manifestArtifact.path, 'semantic-layer/warehouse/_schema/public.yaml'); - - const mcpManifestRead = structuredContent(await client.callTool({ - name: 'scan_read_artifact', - arguments: { - runId: mcpScanTrigger.runId, - path: manifestArtifact.path, - }, - })); - assert.equal(mcpManifestRead.path, 'semantic-layer/warehouse/_schema/public.yaml'); - assert.equal(mcpManifestRead.type, 'manifest_shard'); - assert.match(mcpManifestRead.content, /orders:/); - } catch (error) { - const stderr = Buffer.concat(mcpServerStderr).toString('utf8'); - if (stderr) { - error.message += '\\nktx serve stderr:\\n' + stderr; - } - throw error; - } finally { - await client.close(); - await stopSemanticDaemon(daemon); - } } finally { + if (daemonStarted) { + await run('pnpm', ['exec', 'ktx', 'runtime', 'stop']); + } + if (previousRuntimeRoot === undefined) { + delete process.env.KTX_RUNTIME_ROOT; + } else { + process.env.KTX_RUNTIME_ROOT = previousRuntimeRoot; + } await rm(root, { recursive: true, force: true }); } `; @@ -1393,7 +999,7 @@ export function npmDemoSmokeSource() { return ` import assert from 'node:assert/strict'; import { execFile } from 'node:child_process'; -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; @@ -1434,6 +1040,8 @@ function requireStdout(label, result, pattern) { const root = await mkdtemp(join(tmpdir(), 'ktx-packed-demo-smoke-')); try { const projectDir = join(root, 'demo-project'); + const packageJson = JSON.parse(await readFile(join(process.cwd(), 'package.json'), 'utf8')); + assert.deepEqual(Object.keys(packageJson.dependencies), ['@kaelio/ktx']); const help = await run('pnpm', ['exec', 'ktx', '--help']); requireSuccess('ktx --help', help); @@ -1507,48 +1115,30 @@ try { `; } -export function pythonVerifySource() { - return ` -import importlib.metadata -import ktx_daemon -import semantic_layer - -assert importlib.metadata.version("ktx-sl") == "0.1.0" -assert importlib.metadata.version("ktx-daemon") == "0.1.0" -assert semantic_layer is not None -assert ktx_daemon.PACKAGE_NAME == "ktx-daemon" -`; -} - -function pythonExecutable(projectDir) { - if (process.platform === 'win32') { - return join(projectDir, '.venv', 'Scripts', 'python.exe'); - } - return join(projectDir, '.venv', 'bin', 'python'); -} - -export function npmSmokePythonEnv(projectDir, baseEnv = process.env) { - const binDir = process.platform === 'win32' ? join(projectDir, '.venv', 'Scripts') : join(projectDir, '.venv', 'bin'); - const existingPath = baseEnv.PATH ?? ''; - - return Object.assign({}, baseEnv, { - PATH: existingPath ? `${binDir}${delimiter}${existingPath}` : binDir, - }); -} - async function buildArtifacts(layout) { await rm(layout.artifactDir, { recursive: true, force: true }); await mkdir(layout.npmDir, { recursive: true }); await mkdir(layout.pythonDir, { recursive: true }); - for (const command of buildArtifactCommands(layout)) { + const commands = buildArtifactCommands(layout); + const npmBuildCount = NPM_ARTIFACT_BUILD_ORDER.length; + const npmPackStart = commands.length - 1; + + for (const command of commands.slice(0, npmBuildCount)) { + await runCommand(command.command, command.args, { cwd: command.cwd }); + } + for (const command of commands.slice(npmBuildCount, npmPackStart)) { + await runCommand(command.command, command.args, { cwd: command.cwd }); + } + const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); + await copyRuntimeWheelAssets(layout, pythonArtifacts); + for (const command of commands.slice(npmPackStart)) { await runCommand(command.command, command.args, { cwd: command.cwd }); } for (const packageInfo of NPM_ARTIFACT_PACKAGES) { await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`); } - await findPythonArtifacts(layout.pythonDir); await writeArtifactManifest(layout); await assertPathExists(artifactManifestPath(layout), 'artifact manifest'); } @@ -1557,10 +1147,8 @@ async function verifyNpmArtifacts(layout, tmpRoot) { for (const packageInfo of NPM_ARTIFACT_PACKAGES) { await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`); } - const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); const projectDir = join(tmpRoot, 'npm-clean-install'); - const python = pythonExecutable(projectDir); await mkdir(projectDir, { recursive: true }); await writeFile( join(projectDir, 'package.json'), @@ -1572,20 +1160,10 @@ async function verifyNpmArtifacts(layout, tmpRoot) { await runCommand('pnpm', ['install'], { cwd: projectDir }); await runCommand('pnpm', ['rebuild', 'better-sqlite3'], { cwd: projectDir }); - await runCommand('uv', ['venv', '.venv'], { cwd: projectDir }); - await runCommand('uv', pythonArtifactInstallArgs(python, pythonArtifacts), { - cwd: projectDir, - }); await runCommand('node', ['verify-npm.mjs'], { cwd: projectDir }); await runCommand('pnpm', ['exec', 'ktx', '--version'], { cwd: projectDir }); - await runCommand('node', ['verify-installed-cli.mjs'], { - cwd: projectDir, - env: npmSmokePythonEnv(projectDir), - }); - await runCommand('node', ['verify-installed-demo.mjs'], { - cwd: projectDir, - env: npmSmokePythonEnv(projectDir), - }); + await runCommand('node', ['verify-installed-cli.mjs'], { cwd: projectDir }); + await runCommand('node', ['verify-installed-demo.mjs'], { cwd: projectDir }); } async function verifyNpmDemoArtifacts(layout, tmpRoot) { @@ -1602,32 +1180,12 @@ async function verifyNpmDemoArtifacts(layout, tmpRoot) { await runCommand('node', ['verify-installed-demo.mjs'], { cwd: projectDir }); } -async function verifyPythonArtifacts(layout, tmpRoot) { - const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); - - const projectDir = join(tmpRoot, 'python-clean-install'); - await mkdir(projectDir, { recursive: true }); - const python = pythonExecutable(projectDir); - await writeFile(join(projectDir, 'verify_python.py'), pythonVerifySource()); - - await runCommand('uv', ['venv', '.venv'], { cwd: projectDir }); - await runCommand('uv', pythonArtifactInstallArgs(python, pythonArtifacts), { - cwd: projectDir, - }); - await runCommand(python, ['verify_python.py'], { cwd: projectDir }); - await runCommand(python, ['-m', 'ktx_daemon', 'semantic-validate'], { - cwd: projectDir, - input: `${JSON.stringify({ sources: [ordersSource], dialect: 'postgres' })}\n`, - }); -} - async function verifyArtifacts(layout) { await verifyArtifactManifest(layout); const tmpRoot = await mkdtemp(join(tmpdir(), 'ktx-artifacts-')); try { await verifyNpmArtifacts(layout, tmpRoot); - await verifyPythonArtifacts(layout, tmpRoot); } finally { await rm(tmpRoot, { recursive: true, force: true }); } diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index 5b18a9ed..624248d0 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -6,19 +6,22 @@ import { join } from 'node:path'; import { describe, it } from 'node:test'; import { + CLI_PYTHON_ASSET_MANIFEST, + INTERNAL_NPM_WORKSPACE_PACKAGES, + RUNTIME_WHEEL_DISTRIBUTION_NAME, + RUNTIME_WHEEL_NORMALIZED_NAME, + RUNTIME_WHEEL_PACKAGE_VERSION, artifactManifestPath, buildArtifactCommands, + copyRuntimeWheelAssets, findPythonArtifacts, NPM_ARTIFACT_PACKAGES, npmDemoSmokeSource, npmRuntimeSmokeSource, npmSmokePackageJson, - npmSmokePythonEnv, npmVerifySource, packageArtifactLayout, packageReleaseMetadata, - pythonArtifactInstallArgs, - pythonVerifySource, verifyArtifactManifest, writeArtifactManifest, } from './package-artifacts.mjs'; @@ -29,47 +32,21 @@ async function writeJson(path, value) { await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); } -const CONNECTOR_PACKAGE_NAMES = [ - '@ktx/connector-bigquery', - '@ktx/connector-clickhouse', - '@ktx/connector-mysql', - '@ktx/connector-postgres', - '@ktx/connector-snowflake', - '@ktx/connector-sqlite', - '@ktx/connector-sqlserver', -]; - -function packageRootForName(packageName) { - return `packages/${packageName.replace('@ktx/', '')}`; -} - -function expectedNpmArtifactPath(packageName) { - return `npm/${packageName.replace('@ktx/', 'ktx-')}-0.0.0-private.tgz`; -} +const INTERNAL_BUILD_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => packageInfo.name); +const CONNECTOR_PACKAGE_NAMES = INTERNAL_BUILD_PACKAGE_NAMES.filter((packageName) => + packageName.startsWith('@ktx/connector-'), +); +const NPM_BUILD_PACKAGE_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli']; async function writeReleaseMetadataInputs(root) { - const npmPackages = ['@ktx/context', '@ktx/llm', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli']; - - for (const packageName of npmPackages) { - const packageRoot = packageName === '@ktx/context' ? 'packages/context' : packageRootForName(packageName); - await mkdir(join(root, packageRoot), { recursive: true }); - await writeJson(join(root, packageRoot, 'package.json'), { - name: packageName, + for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { + await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); + await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { + name: packageInfo.name, version: '0.0.0-private', private: true, }); } - - await mkdir(join(root, 'python', 'ktx-sl'), { recursive: true }); - await mkdir(join(root, 'python', 'ktx-daemon'), { recursive: true }); - await writeFile( - join(root, 'python', 'ktx-sl', 'pyproject.toml'), - ['[project]', 'name = "ktx-sl"', 'version = "0.1.0"', ''].join('\n'), - ); - await writeFile( - join(root, 'python', 'ktx-daemon', 'pyproject.toml'), - ['[project]', 'name = "ktx-daemon"', 'version = "0.1.0"', ''].join('\n'), - ); } async function writeUploadableArtifactFixtures(layout) { @@ -81,10 +58,10 @@ async function writeUploadableArtifactFixtures(layout) { layout.npmTarballs[packageInfo.name], `${packageInfo.name}-tarball`, ]), - [join(layout.pythonDir, 'ktx_sl-0.1.0-py3-none-any.whl'), 'ktx-sl-wheel'], - [join(layout.pythonDir, 'ktx_sl-0.1.0.tar.gz'), 'ktx-sl-sdist'], - [join(layout.pythonDir, 'ktx_daemon-0.1.0-py3-none-any.whl'), 'ktx-daemon-wheel'], - [join(layout.pythonDir, 'ktx_daemon-0.1.0.tar.gz'), 'ktx-daemon-sdist'], + [ + join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + 'kaelio-ktx-runtime-wheel', + ], ]); for (const [path, contents] of fileContents) { @@ -99,47 +76,30 @@ describe('packageArtifactLayout', () => { assert.equal(layout.artifactDir, '/repo/ktx/dist/artifacts'); assert.equal(layout.npmDir, '/repo/ktx/dist/artifacts/npm'); assert.equal(layout.pythonDir, '/repo/ktx/dist/artifacts/python'); - assert.equal(layout.contextTarball, '/repo/ktx/dist/artifacts/npm/ktx-context-0.0.0-private.tgz'); - assert.equal(layout.cliTarball, '/repo/ktx/dist/artifacts/npm/ktx-cli-0.0.0-private.tgz'); - assert.equal( - layout.connectorTarballs['@ktx/connector-sqlite'], - '/repo/ktx/dist/artifacts/npm/ktx-connector-sqlite-0.0.0-private.tgz', - ); - assert.equal( - layout.connectorTarballs['@ktx/connector-postgres'], - '/repo/ktx/dist/artifacts/npm/ktx-connector-postgres-0.0.0-private.tgz', - ); - assert.deepEqual( - Object.keys(layout.npmTarballs), - NPM_ARTIFACT_PACKAGES.map((packageInfo) => packageInfo.name), - ); + assert.equal(layout.cliTarball, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz'); + assert.deepEqual(Object.keys(layout.npmTarballs), ['@kaelio/ktx']); }); }); describe('buildArtifactCommands', () => { - it('builds all TypeScript packages before packing npm artifacts and builds both Python packages', () => { + it('builds TypeScript packages and the runtime wheel before packing npm artifacts', () => { const layout = packageArtifactLayout('/repo/ktx'); const commands = buildArtifactCommands(layout); assert.deepEqual( - commands.slice(0, NPM_ARTIFACT_PACKAGES.length).map((command) => [command.command, command.args]), - NPM_ARTIFACT_PACKAGES.map((packageInfo) => ['pnpm', ['--filter', packageInfo.name, 'run', 'build']]), + commands.slice(0, NPM_BUILD_PACKAGE_ORDER.length).map((command) => [command.command, command.args]), + NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]), ); assert.deepEqual( - commands - .slice(NPM_ARTIFACT_PACKAGES.length, NPM_ARTIFACT_PACKAGES.length * 2) - .map((command) => [command.command, command.args]), - NPM_ARTIFACT_PACKAGES.map((packageInfo) => [ - 'pnpm', - ['--filter', packageInfo.name, 'pack', '--out', layout.npmTarballs[packageInfo.name]], + commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [ + command.command, + command.args, ]), + [[process.execPath, ['scripts/build-python-runtime-wheel.mjs']]], ); assert.deepEqual( - commands.slice(NPM_ARTIFACT_PACKAGES.length * 2).map((command) => [command.command, command.args]), - [ - ['uv', ['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python']], - ['uv', ['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python']], - ], + commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [command.command, command.args]), + [[process.execPath, ['scripts/build-public-npm-package.mjs']]], ); }); }); @@ -151,26 +111,18 @@ describe('packageReleaseMetadata', () => { await writeReleaseMetadataInputs(root); assert.deepEqual(await packageReleaseMetadata(root), [ - ...NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({ - ecosystem: 'npm', - packageName: packageInfo.name, - packageRoot: packageInfo.packageRoot, - packageVersion: '0.0.0-private', - private: true, - releaseMode: 'ci-artifact-only', - })), { - ecosystem: 'python', - packageName: 'ktx-sl', - packageRoot: 'python/ktx-sl', - packageVersion: '0.1.0', + ecosystem: 'npm', + packageName: '@kaelio/ktx', + packageRoot: 'packages/cli', + packageVersion: '0.1.0-rc.0', private: false, releaseMode: 'ci-artifact-only', }, { ecosystem: 'python', - packageName: 'ktx-daemon', - packageRoot: 'python/ktx-daemon', + packageName: 'kaelio-ktx', + packageRoot: 'python/runtime-wheel', packageVersion: '0.1.0', private: false, releaseMode: 'ci-artifact-only', @@ -183,19 +135,13 @@ describe('packageReleaseMetadata', () => { }); describe('findPythonArtifacts', () => { - it('finds one wheel and one source distribution for each Python package', async () => { + it('finds the bundled runtime wheel only', async () => { const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-')); try { - await writeFile(join(root, 'ktx_sl-0.1.0-py3-none-any.whl'), ''); - await writeFile(join(root, 'ktx_sl-0.1.0.tar.gz'), ''); - await writeFile(join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'), ''); - await writeFile(join(root, 'ktx_daemon-0.1.0.tar.gz'), ''); + await writeFile(join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), ''); assert.deepEqual(await findPythonArtifacts(root), { - ktxSlWheel: join(root, 'ktx_sl-0.1.0-py3-none-any.whl'), - ktxSlSdist: join(root, 'ktx_sl-0.1.0.tar.gz'), - ktxDaemonWheel: join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'), - ktxDaemonSdist: join(root, 'ktx_daemon-0.1.0.tar.gz'), + runtimeWheel: join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), }); } finally { await rm(root, { recursive: true, force: true }); @@ -205,7 +151,7 @@ describe('findPythonArtifacts', () => { it('throws when a required Python artifact is missing', async () => { const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-')); try { - await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: ktx-sl wheel/); + await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: kaelio-ktx runtime wheel/); } finally { await rm(root, { recursive: true, force: true }); } @@ -230,30 +176,24 @@ describe('artifact manifest', () => { assert.equal(manifest.sourceRevision, 'abc123'); assert.deepEqual( manifest.packages.filter((entry) => entry.ecosystem === 'npm'), - NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({ - ecosystem: 'npm', - packageName: packageInfo.name, - packageRoot: packageInfo.packageRoot, - packageVersion: '0.0.0-private', - private: true, - releaseMode: 'ci-artifact-only', - })), + [ + { + ecosystem: 'npm', + packageName: '@kaelio/ktx', + packageRoot: 'packages/cli', + packageVersion: '0.1.0-rc.0', + private: false, + releaseMode: 'ci-artifact-only', + }, + ], ); assert.deepEqual( manifest.packages.filter((entry) => entry.ecosystem === 'python'), [ { ecosystem: 'python', - packageName: 'ktx-sl', - packageRoot: 'python/ktx-sl', - packageVersion: '0.1.0', - private: false, - releaseMode: 'ci-artifact-only', - }, - { - ecosystem: 'python', - packageName: 'ktx-daemon', - packageRoot: 'python/ktx-daemon', + packageName: 'kaelio-ktx', + packageRoot: 'python/runtime-wheel', packageVersion: '0.1.0', private: false, releaseMode: 'ci-artifact-only', @@ -271,13 +211,15 @@ describe('artifact manifest', () => { path: file.path, })) .sort((left, right) => left.packageName.localeCompare(right.packageName)), - NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({ - artifactKind: 'tarball', - ecosystem: 'npm', - packageName: packageInfo.name, - packageVersion: '0.0.0-private', - path: expectedNpmArtifactPath(packageInfo.name), - })).sort((left, right) => left.packageName.localeCompare(right.packageName)), + [ + { + artifactKind: 'tarball', + ecosystem: 'npm', + packageName: '@kaelio/ktx', + packageVersion: '0.1.0-rc.0', + path: 'npm/kaelio-ktx-0.1.0-rc.0.tgz', + }, + ], ); assert.deepEqual( manifest.files @@ -293,38 +235,17 @@ describe('artifact manifest', () => { { artifactKind: 'wheel', ecosystem: 'python', - packageName: 'ktx-daemon', + packageName: 'kaelio-ktx', packageVersion: '0.1.0', - path: 'python/ktx_daemon-0.1.0-py3-none-any.whl', - }, - { - artifactKind: 'sdist', - ecosystem: 'python', - packageName: 'ktx-daemon', - packageVersion: '0.1.0', - path: 'python/ktx_daemon-0.1.0.tar.gz', - }, - { - artifactKind: 'wheel', - ecosystem: 'python', - packageName: 'ktx-sl', - packageVersion: '0.1.0', - path: 'python/ktx_sl-0.1.0-py3-none-any.whl', - }, - { - artifactKind: 'sdist', - ecosystem: 'python', - packageName: 'ktx-sl', - packageVersion: '0.1.0', - path: 'python/ktx_sl-0.1.0.tar.gz', + path: 'python/kaelio_ktx-0.1.0-py3-none-any.whl', }, ], ); - const sqliteEntry = manifest.files.find((file) => file.path === 'npm/ktx-connector-sqlite-0.0.0-private.tgz'); - assert.ok(sqliteEntry); - assert.equal(sqliteEntry.bytes, Buffer.byteLength('@ktx/connector-sqlite-tarball')); - assert.equal(sqliteEntry.sha256, createHash('sha256').update('@ktx/connector-sqlite-tarball').digest('hex')); + const npmEntry = manifest.files.find((file) => file.path === 'npm/kaelio-ktx-0.1.0-rc.0.tgz'); + assert.ok(npmEntry); + assert.equal(npmEntry.bytes, Buffer.byteLength('@kaelio/ktx-tarball')); + assert.equal(npmEntry.sha256, createHash('sha256').update('@kaelio/ktx-tarball').digest('hex')); const writtenManifest = JSON.parse(await readFile(artifactManifestPath(layout), 'utf-8')); assert.deepEqual(writtenManifest, manifest); @@ -351,7 +272,7 @@ describe('verifyArtifactManifest', () => { assert.equal(manifest.schemaVersion, 2); assert.equal(manifest.sourceRevision, 'abc123'); - assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 4); + assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 1); } finally { await rm(root, { recursive: true, force: true }); } @@ -418,48 +339,89 @@ describe('verifyArtifactManifest', () => { }); }); -describe('pythonArtifactInstallArgs', () => { - it('installs the built Python wheels by artifact path', () => { - const args = pythonArtifactInstallArgs('/tmp/smoke/.venv/bin/python', { - ktxSlWheel: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0-py3-none-any.whl', - ktxSlSdist: '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0.tar.gz', - ktxDaemonWheel: '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0-py3-none-any.whl', - ktxDaemonSdist: '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0.tar.gz', - }); +describe('copyRuntimeWheelAssets', () => { + it('copies the runtime wheel and checksum manifest into CLI assets', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-assets-test-')); + const layout = packageArtifactLayout(root); + try { + await mkdir(layout.pythonDir, { recursive: true }); + await writeFile( + join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + 'kaelio-ktx-runtime-wheel', + ); - assert.deepEqual(args, [ - 'pip', - 'install', - '--python', - '/tmp/smoke/.venv/bin/python', - '/repo/ktx/dist/artifacts/python/ktx_sl-0.1.0-py3-none-any.whl', - '/repo/ktx/dist/artifacts/python/ktx_daemon-0.1.0-py3-none-any.whl', - ]); - assert.equal(args.includes('ktx-daemon'), false); - assert.equal(args.includes('--find-links'), false); + const assets = await copyRuntimeWheelAssets(layout, { + runtimeWheel: join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), + }); + + assert.equal( + assets.wheelPath, + join(root, 'packages', 'cli', 'assets', 'python', 'kaelio_ktx-0.1.0-py3-none-any.whl'), + ); + assert.equal( + assets.manifestPath, + join(root, 'packages', 'cli', 'assets', 'python', CLI_PYTHON_ASSET_MANIFEST), + ); + const manifest = JSON.parse(await readFile(assets.manifestPath, 'utf8')); + assert.deepEqual(manifest, { + schemaVersion: 1, + distributionName: RUNTIME_WHEEL_DISTRIBUTION_NAME, + normalizedName: RUNTIME_WHEEL_NORMALIZED_NAME, + version: RUNTIME_WHEEL_PACKAGE_VERSION, + wheel: { + file: 'kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: createHash('sha256') + .update('kaelio-ktx-runtime-wheel') + .digest('hex'), + bytes: Buffer.byteLength('kaelio-ktx-runtime-wheel'), + }, + }); + } finally { + await rm(root, { recursive: true, force: true }); + } }); }); -describe('npmSmokePythonEnv', () => { - it('prepends the npm smoke virtualenv bin directory to PATH', () => { - const env = npmSmokePythonEnv('/tmp/ktx-npm-smoke', { PATH: '/usr/bin' }); +describe('verifyNpmArtifacts', () => { + it('does not prepare an external Python environment for the npm smoke', async () => { + const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8'); + const start = source.indexOf('async function verifyNpmArtifacts'); + const end = source.indexOf('async function verifyNpmDemoArtifacts'); + assert.ok(start > 0, 'verifyNpmArtifacts function must exist'); + assert.ok(end > start, 'verifyNpmDemoArtifacts must follow verifyNpmArtifacts'); - assert.match(env.PATH, /^\/tmp\/ktx-npm-smoke\/\.venv\/(bin|Scripts)/); - assert.match(env.PATH, /\/usr\/bin$/); + const body = source.slice(start, end); + assert.doesNotMatch(body, /uv', \['venv', '\.venv'\]/); + assert.doesNotMatch(body, /pythonArtifactInstallArgs/); + assert.doesNotMatch(body, /npmSmokePythonEnv/); + }); +}); + +describe('standalone Python artifact cleanup', () => { + it('does not build or verify standalone Python package artifacts', async () => { + const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8'); + + assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-sl'/); + assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-daemon'/); + assert.doesNotMatch(source, /async function verifyPythonArtifacts/); + assert.doesNotMatch(source, /pythonArtifactInstallArgs/); + assert.doesNotMatch(source, /pythonVerifySource/); + assert.doesNotMatch(source, /ktx_sl-0\.1\.0/); + assert.doesNotMatch(source, /ktx_daemon-0\.1\.0/); }); }); describe('verification snippets', () => { - it('pins smoke dependencies and connector packages to clean-install-safe artifacts', () => { + it('pins the smoke project to the public package artifact', () => { const layout = packageArtifactLayout('/repo/ktx'); - const packageJson = npmSmokePackageJson(layout); - for (const packageInfo of NPM_ARTIFACT_PACKAGES) { - assert.equal(packageJson.dependencies[packageInfo.name], `file:${layout.npmTarballs[packageInfo.name]}`); - assert.equal(packageJson.pnpm.overrides[packageInfo.name], `file:${layout.npmTarballs[packageInfo.name]}`); - } - assert.equal(packageJson.dependencies['@modelcontextprotocol/sdk'], '^1.27.1'); - assert.deepEqual(packageJson.pnpm.onlyBuiltDependencies, ['better-sqlite3']); + const packageJson = npmSmokePackageJson(layout); + assert.deepEqual(packageJson.dependencies, { + '@kaelio/ktx': `file:${layout.cliTarball}`, + }); + assert.deepEqual(packageJson.devDependencies, { + 'better-sqlite3': '^12.6.2', + }); }); it('exposes manifest verification as a package artifact command', async () => { @@ -472,115 +434,64 @@ describe('verification snippets', () => { assert.equal(packageJson.scripts['artifacts:verify-manifest'], 'node scripts/package-artifacts.mjs verify-manifest'); }); - it('verifies installed dbt extraction exports from @ktx/context/ingest', () => { - const source = npmVerifySource(); - - assert.match(source, /const ingest = await import\('@ktx\/context\/ingest'\);/); - assert.match(source, /const dbtExtractionExports = \[/); - assert.match(source, /throw new Error\('Missing dbt extraction export: ' \+ exportName\);/); - - for (const exportName of [ - 'parseMetricflowFiles', - 'parseMetricflowPullConfig', - 'importMetricflowSemanticModels', - 'parseDbtSchemaFiles', - 'toDescriptionUpdates', - 'toRelationshipUpdates', - 'mergeSemanticModelTables', - 'loadProjectInfo', - 'loadDbtSchemaFiles', - ]) { - assert.match(source, new RegExp(`\\['${exportName}', ingest\\.${exportName}\\]`)); - } - }); - - it('asserts the public npm and connector entry points that clean installs must expose', () => { - const source = npmVerifySource(); - - assert.match(source, /@ktx\/context/); - assert.match(source, /@ktx\/context\/project/); - assert.match(source, /@ktx\/context\/mcp/); - assert.match(source, /@ktx\/context\/memory/); - assert.match(source, /@ktx\/context\/daemon/); - assert.match(source, /@ktx\/cli/); - assert.match(source, /@ktx\/llm/); - assert.match(source, /createKtxLlmProvider/); - assert.match(source, /KtxMessageBuilder/); - assert.match(source, /createKtxEmbeddingProvider/); - assert.doesNotMatch(source, /createGatewayLlmProvider/); - assert.match(source, /createLocalProjectMemoryCapture/); - for (const packageName of CONNECTOR_PACKAGE_NAMES) { - assert.match(source, new RegExp(packageName.replace('/', '\\/'))); - } - assert.match(source, /KtxSqliteScanConnector/); - assert.match(source, /KtxPostgresScanConnector/); - assert.match(source, /KtxBigQueryScanConnector/); - assert.match(source, /KtxSnowflakeScanConnector/); - }); - - it('asserts installed hybrid search exports and CLI smoke coverage', () => { + it('asserts the public npm entry point that clean installs must expose', () => { const verifySource = npmVerifySource(); - const runtimeSource = npmRuntimeSmokeSource(); - const demoSource = npmDemoSmokeSource(); - assert.match(verifySource, /const search = await import\('@ktx\/context\/search'\);/); - assert.match(verifySource, /HybridSearchCore/); - assert.match(verifySource, /assertSearchBackendConformanceCase/); - assert.match(verifySource, /assertSearchBackendCapabilities/); - - assert.match(runtimeSource, /ktx agent wiki search hybrid metadata verified/); - assert.match(runtimeSource, /ktx agent sl list hybrid metadata verified/); - assert.match(runtimeSource, /agent_sl_search_missing_project/); - assert.match(runtimeSource, /agent_sl_search_no_connections/); - assert.match(runtimeSource, /agent_sl_search_no_indexed_sources/); - - assert.match(demoSource, /ktx seeded demo agent wiki search verified/); - assert.match(demoSource, /ktx seeded demo agent sl search verified/); + assert.match(verifySource, /const cli = await import\('@kaelio\/ktx'\);/); + assert.match(verifySource, /getKtxCliPackageInfo/); + assert.match(verifySource, /runKtxCli/); + assert.doesNotMatch(verifySource, /@ktx\/context/); + assert.doesNotMatch(verifySource, /@ktx\/llm/); + assert.doesNotMatch(verifySource, /@ktx\/connector-/); }); - it('runs installed CLI commands and MCP through an installed daemon HTTP server', () => { + it('runs installed CLI commands through the public package runtime', () => { const source = npmRuntimeSmokeSource(); - assert.match(source, /@modelcontextprotocol\/sdk\/client\/index\.js/); - assert.match(source, /@modelcontextprotocol\/sdk\/client\/stdio\.js/); - assert.match(source, /spawn\(command, args/); - assert.match(source, /createServer/); - assert.match(source, /request as httpRequest/); - assert.match(source, /getAvailablePort/); - assert.match(source, /startSemanticDaemon/); - assert.match(source, /waitForHttpHealth/); - assert.match(source, /stopSemanticDaemon/); - assert.match(source, /'ktx-daemon'/); - assert.match(source, /'serve-http'/); - assert.match(source, /'--host'/); - assert.match(source, /'127\.0\.0\.1'/); - assert.match(source, /'--port'/); - assert.match(source, /\/health/); - assert.match(source, /--semantic-compute-url/); - assert.match(source, /createDaemonLookerTableIdentifierParser/); - assert.match(source, /LocalLookerRuntimeStore/); - assert.match(source, /Looker daemon table identifier parser verified/); - assert.match(source, /Looker local runtime store verified/); - assert.match(source, /semanticComputeUrl/); + assert.match(source, /ktx public package version/); + assert.match(source, /@kaelio\\\/ktx 0\\\.1\\\.0/); + assert.match(source, /'ktx', 'sl', 'query'/); + assert.doesNotMatch(source, /@ktx\/context/); + assert.doesNotMatch(source, /@modelcontextprotocol/); + assert.doesNotMatch(source, /startSemanticDaemon/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'setup'/); assert.match(source, /knowledge', 'global', 'revenue\.md'/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'agent',\s*'wiki',\s*'search'/); assert.match(source, /semantic-layer', 'warehouse', 'orders\.yaml'/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'agent',\s*'sl',\s*'list'/); - assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'agent',\s*'sl',\s*'query'/); assert.match(source, /orders\.order_count/); assert.match(source, /sqlite3/); assert.match(source, /driver: sqlite/); assert.match(source, /path: warehouse\.db/); assert.match(source, /live-database/); assert.match(source, /'--execute'/); - assert.match(source, /'--execute-queries'/); - assert.match(source, /slValidateResult\.success, true/); - assert.match(source, /slQueryResult\.dialect, 'sqlite'/); - assert.match(source, /slQueryResult\.plan\.execution\.driver, 'sqlite'/); assert.match(source, /"mode": "compile_only"/); assert.match(source, /"mode": "executed"/); - assert.match(source, /ktx agent sl query sqlite execute/); + assert.match(source, /ktx sl query sqlite execute/); + assert.match(source, /import Database from 'better-sqlite3'/); + assert.doesNotMatch(source, /run\('python'/); + assert.match(source, /KTX_RUNTIME_ROOT/); + assert.match(source, /managed-runtime/); + assert.match(source, /ktx runtime status missing/); + assert.match(source, /runtimeStatusBefore\.kind, 'missing'/); + assert.ok(source.includes(String.raw`Installing KTX Python runtime \(core\) with uv`)); + assert.match(source, /KTX Python runtime ready:/); + assert.match(source, /ktx runtime status ready/); + assert.match(source, /runtimeStatusAfter\.kind, 'ready'/); + assert.match(source, /runtimeStatusAfter\.manifest\.features/); + assert.match(source, /ktx runtime doctor/); + assert.match(source, /PASS Managed Python runtime/); + assert.match(source, /ktx runtime start/); + assert.match(source, /ktx runtime start reuse/); + assert.match(source, /Using existing KTX Python daemon/); + assert.match(source, /ktx runtime stop/); + assert.match(source, /ktx runtime prune dry run/); + assert.match(source, /0\.0\.0/); + assert.match(source, /ktx runtime prune needs confirmation/); + assert.match(source, /Refusing to prune without --yes/); + assert.match(source, /ktx runtime prune confirmed/); + assert.match(source, /Removed stale KTX Python runtimes/); + assert.match(source, /assert\.rejects\(\(\) => access\(staleRuntimeDir\)\)/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'dev',\s*'scan',\s*'warehouse'/); assert.match(source, /'--mode',\s*'enriched'/); assert.doesNotMatch(source, /'--enrich'/); @@ -590,28 +501,7 @@ describe('verification snippets', () => { assert.match(source, /scanReportJson\.artifactPaths\.enrichmentArtifacts/); assert.match(source, /enrichment:/); assert.match(source, /mode: deterministic/); - assert.match(source, /backend: gateway/); - assert.match(source, /models:/); - assert.match(source, /default: smoke\/provider/); - assert.match(source, /api_key: env:AI_GATEWAY_API_KEY/); assert.match(source, /run\('pnpm', \['exec', 'ktx', 'dev', 'ingest', 'run'/); - assert.match(source, /'serve', '--mcp', 'stdio'/); - assert.doesNotMatch(source, /'--semantic-compute',\n\s*'--execute-queries'/); - assert.match(source, /'--memory-capture', '--memory-model', 'smoke\/provider'/); - assert.match(source, /mcpServerStderr/); - assert.match(source, /ktx serve stderr/); - assert.match(source, /sl_validate/); - assert.match(source, /sl_query/); - assert.match(source, /memory_capture/); - assert.match(source, /memory_capture_status/); - assert.match(source, /connection_test/); - assert.match(source, /scan_trigger/); - assert.match(source, /scan_status/); - assert.match(source, /scan_report/); - assert.match(source, /scan_list_artifacts/); - assert.match(source, /scan_read_artifact/); - assert.match(source, /mcpScanArtifacts\.artifacts\.find/); - assert.match(source, /AI_GATEWAY_API_KEY/); assert.match(source, /access\(join\(projectDir, '\.ktx', 'db\.sqlite'\)\)/); assert.match(source, /SQLite knowledge index/); assert.match(source, /ktx dev ingest run requires llm\\.provider\\.backend: anthropic, vertex, or gateway/); @@ -632,22 +522,8 @@ describe('verification snippets', () => { assert.match(source, /'dev', 'doctor', 'setup', '--no-input'/); assert.match(source, /'--plain'/); assert.match(source, /ktx setup demo seeded wrote unexpected stderr/); + assert.match(source, /Object\.keys\(packageJson\.dependencies\)/); + assert.match(source, /'@kaelio\/ktx'/); }); }); - - it('checks packaged ingest runtime assets in the installed npm smoke', () => { - const source = npmRuntimeSmokeSource(); - - assert.match(source, /notion_synthesize\/SKILL\.md/); - assert.match(source, /skills\/page_triage_classifier\.md/); - assert.match(source, /skills\/light_extraction\.md/); - }); - - it('asserts the Python modules that clean installs must expose', () => { - const source = pythonVerifySource(); - - assert.match(source, /semantic_layer/); - assert.match(source, /ktx_daemon/); - assert.match(source, /importlib.metadata/); - }); }); diff --git a/scripts/publish-public-npm-package.mjs b/scripts/publish-public-npm-package.mjs new file mode 100644 index 00000000..d82a157d --- /dev/null +++ b/scripts/publish-public-npm-package.mjs @@ -0,0 +1,87 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { access } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; + +import { packageArtifactLayout } from './package-artifacts.mjs'; +import { releaseReadinessReport } from './release-readiness.mjs'; + +const execFileAsync = promisify(execFile); + +export function resolvePublishMode(args = process.argv.slice(2)) { + return { live: args.includes('--publish') }; +} + +export function requireNpmPublicReleaseReady(report) { + if (report.releaseMode !== 'npm-public-release-ready' || report.npmPublishEnabled !== true || !report.npmPublish) { + throw new Error('release-policy.json must use npm-public-release-ready before publishing'); + } + return report.npmPublish; +} + +export function buildNpmPublishCommand(tarballPath, publish, mode) { + return { + command: 'pnpm', + args: [ + 'publish', + tarballPath, + '--access', + publish.access, + '--tag', + publish.tag, + ...(mode.live ? [] : ['--dry-run', '--no-git-checks']), + ], + env: publish.registry ? { npm_config_registry: publish.registry } : {}, + }; +} + +async function assertFileExists(path) { + try { + await access(path); + } catch { + throw new Error(`Missing npm tarball: ${path}. Run pnpm run artifacts:check first.`); + } +} + +async function runPublishCommand(command) { + process.stdout.write(`$ ${command.command} ${command.args.join(' ')}\n`); + await execFileAsync(command.command, command.args, { + env: { ...process.env, ...command.env }, + encoding: 'utf8', + maxBuffer: 1024 * 1024 * 20, + }); +} + +export async function publishPublicNpmPackage(options = {}) { + const rootDir = options.rootDir; + const mode = options.mode ?? resolvePublishMode(options.args); + const report = await releaseReadinessReport(rootDir); + const publish = requireNpmPublicReleaseReady(report); + const layout = packageArtifactLayout(rootDir); + const tarballPath = layout.cliTarball; + + await assertFileExists(tarballPath); + const command = buildNpmPublishCommand(tarballPath, publish, mode); + await runPublishCommand(command); + + process.stdout.write( + mode.live + ? `Published ${publish.packageName}@${publish.version} with tag ${publish.tag}\n` + : `Dry-run verified ${publish.packageName}@${publish.version} with tag ${publish.tag}\n`, + ); +} + +async function main() { + await publishPublicNpmPackage({ args: process.argv.slice(2) }); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + try { + await main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/publish-public-npm-package.test.mjs b/scripts/publish-public-npm-package.test.mjs new file mode 100644 index 00000000..704a8b16 --- /dev/null +++ b/scripts/publish-public-npm-package.test.mjs @@ -0,0 +1,109 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { describe, it } from 'node:test'; + +import { + buildNpmPublishCommand, + requireNpmPublicReleaseReady, + resolvePublishMode, +} from './publish-public-npm-package.mjs'; + +const readyReport = { + releaseMode: 'npm-public-release-ready', + npmPublishEnabled: true, + npmPublish: { + packageName: '@kaelio/ktx', + version: '0.1.0-rc.0', + access: 'public', + tag: 'next', + registry: null, + }, +}; + +describe('resolvePublishMode', () => { + it('dry-runs by default', () => { + assert.deepEqual(resolvePublishMode([]), { live: false }); + }); + + it('requires an explicit flag for live publish', () => { + assert.deepEqual(resolvePublishMode(['--publish']), { live: true }); + }); +}); + +describe('requireNpmPublicReleaseReady', () => { + it('accepts the npm public release ready report', () => { + assert.equal(requireNpmPublicReleaseReady(readyReport), readyReport.npmPublish); + }); + + it('rejects artifact-only reports', () => { + assert.throws( + () => + requireNpmPublicReleaseReady({ + releaseMode: 'ci-artifact-only', + npmPublishEnabled: false, + npmPublish: null, + }), + /release-policy.json must use npm-public-release-ready before publishing/, + ); + }); +}); + +describe('buildNpmPublishCommand', () => { + it('builds a dry-run pnpm publish command by default', () => { + assert.deepEqual( + buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz', readyReport.npmPublish, { + live: false, + }), + { + command: 'pnpm', + args: [ + 'publish', + '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz', + '--access', + 'public', + '--tag', + 'next', + '--dry-run', + '--no-git-checks', + ], + env: {}, + }, + ); + }); + + it('omits dry-run only for explicit live publish', () => { + assert.deepEqual( + buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz', readyReport.npmPublish, { + live: true, + }).args, + [ + 'publish', + '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz', + '--access', + 'public', + '--tag', + 'next', + ], + ); + }); + + it('uses npm_config_registry when a registry is configured', () => { + const publish = { + ...readyReport.npmPublish, + registry: 'https://registry.npmjs.org/', + }; + + assert.deepEqual( + buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.0.tgz', publish, { live: false }).env, + { npm_config_registry: 'https://registry.npmjs.org/' }, + ); + }); +}); + +describe('package script', () => { + it('registers release:npm-publish', async () => { + const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); + + assert.equal(packageJson.scripts['release:npm-publish'], 'node scripts/publish-public-npm-package.mjs'); + }); +}); diff --git a/scripts/published-package-smoke-config.mjs b/scripts/published-package-smoke-config.mjs index 71bcd862..6aea8688 100644 --- a/scripts/published-package-smoke-config.mjs +++ b/scripts/published-package-smoke-config.mjs @@ -1,3 +1,4 @@ +import { dirname, join } from 'node:path'; import assert from 'node:assert/strict'; import { readFile } from 'node:fs/promises'; @@ -28,6 +29,30 @@ function assertHttpRegistry(registry, label) { } } +function registryEnv(config) { + return config.registry ? { npm_config_registry: config.registry } : {}; +} + +function runtimeCommandEnv(config, runtimeRoot) { + return { ...registryEnv(config), KTX_RUNTIME_ROOT: runtimeRoot }; +} + +function semanticQueryArgs(projectDir) { + return [ + 'sl', + 'query', + '--project-dir', + projectDir, + '--connection-id', + 'orbit_demo', + '--measure', + 'contracts.contract_count', + '--format', + 'sql', + '--yes', + ]; +} + function normalizePolicyConfig(policyConfig = {}) { if (policyConfig === null || policyConfig === undefined) { return { packageName: null, version: DEFAULT_VERSION_TAG, registry: null }; @@ -114,39 +139,70 @@ export function publishedPackageSpec(config) { return `${config.packageName}@${config.packageVersion}`; } -export function buildPublishedPackageNpxCommand(config, args, label = 'published package command') { - const env = config.registry ? { npm_config_registry: config.registry } : {}; - +export function buildPublishedPackageNpxCommand(config, args, label = 'published package command', extraEnv = {}) { return { label, command: 'npx', args: ['--yes', publishedPackageSpec(config), ...args], - env, + env: { ...registryEnv(config), ...extraEnv }, }; } -export function buildPublishedPackageSmokeCommands(config, projectDir, emptyProjectDir) { +export function buildPublishedPackageSmokeCommands( + config, + projectDir, + runtimeRoot = join(dirname(projectDir), 'managed-runtime'), +) { + const runtimeEnv = runtimeCommandEnv(config, runtimeRoot); + const packageEnv = registryEnv(config); + const queryArgs = semanticQueryArgs(projectDir); + return [ - buildPublishedPackageNpxCommand(config, ['--version'], 'published package version'), + buildPublishedPackageNpxCommand(config, ['--version'], 'published package npx version'), buildPublishedPackageNpxCommand( config, - ['demo', '--project-dir', projectDir, '--no-input', '--plain'], - 'published package demo', - ), - buildPublishedPackageNpxCommand( - config, - ['agent', 'wiki', 'search', 'ARR contract', '--json', '--limit', '5', '--project-dir', projectDir], - 'published package wiki hybrid search', - ), - buildPublishedPackageNpxCommand( - config, - ['agent', 'sl', 'list', '--json', '--query', 'ARR', '--project-dir', projectDir], - 'published package semantic-layer hybrid search', - ), - buildPublishedPackageNpxCommand( - config, - ['agent', 'sl', 'list', '--json', '--query', 'revenue', '--project-dir', emptyProjectDir], - 'published package missing-project readiness', + ['setup', 'demo', '--project-dir', projectDir, '--no-input', '--plain'], + 'published package setup demo', + { KTX_RUNTIME_ROOT: runtimeRoot }, ), + buildPublishedPackageNpxCommand(config, queryArgs, 'published package npx sl query', { + KTX_RUNTIME_ROOT: runtimeRoot, + }), + { + label: 'published package local install', + command: 'pnpm', + args: ['add', publishedPackageSpec(config)], + env: packageEnv, + }, + { + label: 'published package local version', + command: 'npx', + args: ['ktx', '--version'], + env: packageEnv, + }, + { + label: 'published package local sl query', + command: 'npx', + args: ['ktx', ...queryArgs], + env: runtimeEnv, + }, + { + label: 'published package global install', + command: 'pnpm', + args: ['add', '--global', publishedPackageSpec(config)], + env: packageEnv, + }, + { + label: 'published package global version', + command: 'ktx', + args: ['--version'], + env: packageEnv, + }, + { + label: 'published package global sl query', + command: 'ktx', + args: queryArgs, + env: runtimeEnv, + }, ]; } diff --git a/scripts/published-package-smoke.mjs b/scripts/published-package-smoke.mjs index d97cdda0..af1795bd 100644 --- a/scripts/published-package-smoke.mjs +++ b/scripts/published-package-smoke.mjs @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { execFile } from 'node:child_process'; -import { mkdir, mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -23,6 +23,26 @@ export { const execFileAsync = promisify(execFile); const SMOKE_TIMEOUT_MS = 180_000; +const VERSION_LABELS = new Set([ + 'published package npx version', + 'published package local version', + 'published package global version', +]); + +const SEMANTIC_QUERY_LABELS = new Set([ + 'published package npx sl query', + 'published package local sl query', + 'published package global sl query', +]); + +export function isPublishedPackageVersionLabel(label) { + return VERSION_LABELS.has(label); +} + +export function isPublishedPackageSemanticQueryLabel(label) { + return SEMANTIC_QUERY_LABELS.has(label); +} + function scriptRootDir() { return resolve(dirname(fileURLToPath(import.meta.url)), '..'); } @@ -59,78 +79,34 @@ function requireSuccess(label, result) { ); } -function parseJson(label, text) { - try { - return JSON.parse(text); - } catch (error) { - throw new Error(`${label} did not produce JSON: ${error instanceof Error ? error.message : String(error)}\n${text}`); - } -} - -function assertHybridWikiSearch(result) { - const payload = parseJson('published package wiki search', result.stdout); - assert.ok(payload.totalFound > 0, 'published package wiki search should return results'); - assert.ok( - payload.results.some((entry) => Array.isArray(entry.matchReasons) && entry.matchReasons.length > 0), - 'published package wiki search should expose match reasons', - ); -} - -function assertHybridSlSearch(result) { - const payload = parseJson('published package semantic-layer search', result.stdout); - assert.ok(payload.totalSources > 0, 'published package semantic-layer search should return sources'); - assert.ok( - payload.sources.some((entry) => Array.isArray(entry.matchReasons) && entry.matchReasons.length > 0), - 'published package semantic-layer search should expose match reasons', - ); -} - -function assertMissingProjectReadiness(result, emptyProjectDir) { - assert.equal(result.code, 1, 'missing-project semantic-layer search should exit 1'); - assert.equal(result.stdout, '', 'missing-project semantic-layer search should not write JSON errors to stdout'); - - const payload = parseJson('published package missing-project semantic-layer search', result.stderr); - assert.deepEqual(payload, { - ok: false, - error: { - code: 'agent_sl_search_missing_project', - message: `Semantic-layer search needs an initialized KTX project at ${emptyProjectDir}.`, - nextSteps: [ - 'ktx demo', - `ktx setup --project-dir ${emptyProjectDir}`, - 'ktx ingest ', - `ktx agent sl list --json --query "revenue" --project-dir ${emptyProjectDir}`, - ], - }, - }); -} - export async function runPublishedPackageSmoke(config) { const root = await mkdtemp(join(tmpdir(), 'ktx-published-package-smoke-')); try { const projectDir = join(root, 'demo-project'); - const emptyProjectDir = join(root, 'empty-project'); - await mkdir(emptyProjectDir, { recursive: true }); - const commands = buildPublishedPackageSmokeCommands(config, projectDir, emptyProjectDir); - for (const command of commands.slice(0, 4)) { - const result = await runCommand(command.command, command.args, { env: command.env }); + const commands = buildPublishedPackageSmokeCommands(config, projectDir); + const pnpmHome = join(root, 'pnpm-home'); + const globalEnv = { + PNPM_HOME: pnpmHome, + PATH: `${pnpmHome}${process.platform === 'win32' ? ';' : ':'}${process.env.PATH ?? ''}`, + }; + for (const command of commands) { + const isGlobalCommand = command.label.includes('global'); + const result = await runCommand(command.command, command.args, { + cwd: command.label.includes('local') || isGlobalCommand ? root : undefined, + env: isGlobalCommand ? { ...globalEnv, ...command.env } : command.env, + }); requireSuccess(command.label, result); - if (command.label === 'published package wiki hybrid search') { - assertHybridWikiSearch(result); + if (isPublishedPackageVersionLabel(command.label)) { + assert.match(result.stdout, /@kaelio\/ktx /); } - if (command.label === 'published package semantic-layer hybrid search') { - assertHybridSlSearch(result); + if (isPublishedPackageSemanticQueryLabel(command.label)) { + assert.match(result.stdout, /SELECT/i); + assert.match(result.stdout, /contracts/i); } } - const missingProjectCommand = commands[4]; - const missingProject = await runCommand(missingProjectCommand.command, missingProjectCommand.args, { - env: missingProjectCommand.env, - }); - assertMissingProjectReadiness(missingProject, emptyProjectDir); - - process.stdout.write('published package hybrid search smoke verified\n'); + process.stdout.write('published package invocation smoke verified\n'); } finally { await rm(root, { recursive: true, force: true }); } diff --git a/scripts/published-package-smoke.test.mjs b/scripts/published-package-smoke.test.mjs index 66fa6670..6852e237 100644 --- a/scripts/published-package-smoke.test.mjs +++ b/scripts/published-package-smoke.test.mjs @@ -5,6 +5,8 @@ import { describe, it } from 'node:test'; import { buildPublishedPackageNpxCommand, buildPublishedPackageSmokeCommands, + isPublishedPackageSemanticQueryLabel, + isPublishedPackageVersionLabel, publishedPackageSpec, readPublishedPackageSmokeConfig, } from './published-package-smoke.mjs'; @@ -32,7 +34,7 @@ describe('published package smoke config', () => { assert.deepEqual( readPublishedPackageSmokeConfig( { - KTX_PUBLISHED_KTX_PACKAGE: '@ktx/cli-public', + KTX_PUBLISHED_KTX_PACKAGE: '@kaelio/ktx', KTX_PUBLISHED_KTX_VERSION: 'latest', KTX_PUBLISHED_KTX_REGISTRY: 'https://registry.npmjs.org/', }, @@ -42,7 +44,7 @@ describe('published package smoke config', () => { enabled: true, requireConfig: false, configSource: 'environment', - packageName: '@ktx/cli-public', + packageName: '@kaelio/ktx', packageVersion: 'latest', registry: 'https://registry.npmjs.org/', }, @@ -55,7 +57,7 @@ describe('published package smoke config', () => { {}, [], { - packageName: '@ktx/cli-public', + packageName: '@kaelio/ktx', version: '2026.5.8', registry: 'https://registry.npmjs.org/', }, @@ -64,7 +66,7 @@ describe('published package smoke config', () => { enabled: true, requireConfig: false, configSource: 'release-policy', - packageName: '@ktx/cli-public', + packageName: '@kaelio/ktx', packageVersion: '2026.5.8', registry: 'https://registry.npmjs.org/', }, @@ -75,12 +77,12 @@ describe('published package smoke config', () => { assert.deepEqual( readPublishedPackageSmokeConfig( { - KTX_PUBLISHED_KTX_PACKAGE: '@ktx/cli-from-env', + KTX_PUBLISHED_KTX_PACKAGE: '@kaelio/ktx', KTX_PUBLISHED_KTX_VERSION: 'latest', }, [], { - packageName: '@ktx/cli-from-policy', + packageName: '@kaelio/ktx', version: '2026.5.8', registry: 'https://registry.npmjs.org/', }, @@ -89,7 +91,7 @@ describe('published package smoke config', () => { enabled: true, requireConfig: false, configSource: 'environment', - packageName: '@ktx/cli-from-env', + packageName: '@kaelio/ktx', packageVersion: 'latest', registry: 'https://registry.npmjs.org/', }, @@ -125,7 +127,7 @@ describe('published package smoke config', () => { () => readPublishedPackageSmokeConfig( { - KTX_PUBLISHED_KTX_PACKAGE: '@ktx/cli-public', + KTX_PUBLISHED_KTX_PACKAGE: '@kaelio/ktx', KTX_PUBLISHED_KTX_VERSION: '--tag latest', }, [], @@ -136,7 +138,7 @@ describe('published package smoke config', () => { () => readPublishedPackageSmokeConfig( { - KTX_PUBLISHED_KTX_PACKAGE: '@ktx/cli-public', + KTX_PUBLISHED_KTX_PACKAGE: '@kaelio/ktx', KTX_PUBLISHED_KTX_REGISTRY: 'file:///tmp/npm', }, [], @@ -146,103 +148,166 @@ describe('published package smoke config', () => { }); }); +describe('published package smoke output validation labels', () => { + it('classifies version and semantic query commands', () => { + assert.equal(isPublishedPackageVersionLabel('published package npx version'), true); + assert.equal(isPublishedPackageVersionLabel('published package local version'), true); + assert.equal(isPublishedPackageVersionLabel('published package global version'), true); + assert.equal(isPublishedPackageVersionLabel('published package setup demo'), false); + + assert.equal(isPublishedPackageSemanticQueryLabel('published package npx sl query'), true); + assert.equal(isPublishedPackageSemanticQueryLabel('published package local sl query'), true); + assert.equal(isPublishedPackageSemanticQueryLabel('published package global sl query'), true); + assert.equal(isPublishedPackageSemanticQueryLabel('published package local install'), false); + }); +}); + describe('published package smoke command construction', () => { const config = { enabled: true, requireConfig: false, - packageName: '@ktx/cli-public', + packageName: '@kaelio/ktx', packageVersion: 'latest', registry: 'https://registry.npmjs.org/', }; it('builds the npx package spec from package name and version tag', () => { - assert.equal(publishedPackageSpec(config), '@ktx/cli-public@latest'); + assert.equal(publishedPackageSpec(config), '@kaelio/ktx@latest'); }); it('builds npx commands with a registry env patch instead of shell interpolation', () => { assert.deepEqual(buildPublishedPackageNpxCommand(config, ['--version']), { label: 'published package command', command: 'npx', - args: ['--yes', '@ktx/cli-public@latest', '--version'], + args: ['--yes', '@kaelio/ktx@latest', '--version'], env: { npm_config_registry: 'https://registry.npmjs.org/' }, }); }); - it('builds the full hybrid-search smoke command list', () => { - assert.deepEqual(buildPublishedPackageSmokeCommands(config, '/tmp/ktx-smoke/demo', '/tmp/ktx-smoke/empty'), [ - { - label: 'published package version', - command: 'npx', - args: ['--yes', '@ktx/cli-public@latest', '--version'], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package demo', - command: 'npx', - args: [ - '--yes', - '@ktx/cli-public@latest', - 'demo', - '--project-dir', - '/tmp/ktx-smoke/demo', - '--no-input', - '--plain', - ], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package wiki hybrid search', - command: 'npx', - args: [ - '--yes', - '@ktx/cli-public@latest', - 'agent', - 'wiki', - 'search', - 'ARR contract', - '--json', - '--limit', - '5', - '--project-dir', - '/tmp/ktx-smoke/demo', - ], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package semantic-layer hybrid search', - command: 'npx', - args: [ - '--yes', - '@ktx/cli-public@latest', - 'agent', - 'sl', - 'list', - '--json', - '--query', - 'ARR', - '--project-dir', - '/tmp/ktx-smoke/demo', - ], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package missing-project readiness', - command: 'npx', - args: [ - '--yes', - '@ktx/cli-public@latest', - 'agent', - 'sl', - 'list', - '--json', - '--query', - 'revenue', - '--project-dir', - '/tmp/ktx-smoke/empty', - ], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - ]); + it('builds the full public package smoke command list', () => { + assert.deepEqual( + buildPublishedPackageSmokeCommands( + config, + '/tmp/ktx-smoke/demo', + '/tmp/ktx-smoke/managed-runtime', + ), + [ + { + label: 'published package npx version', + command: 'npx', + args: ['--yes', '@kaelio/ktx@latest', '--version'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package setup demo', + command: 'npx', + args: [ + '--yes', + '@kaelio/ktx@latest', + 'setup', + 'demo', + '--project-dir', + '/tmp/ktx-smoke/demo', + '--no-input', + '--plain', + ], + env: { + npm_config_registry: 'https://registry.npmjs.org/', + KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', + }, + }, + { + label: 'published package npx sl query', + command: 'npx', + args: [ + '--yes', + '@kaelio/ktx@latest', + 'sl', + 'query', + '--project-dir', + '/tmp/ktx-smoke/demo', + '--connection-id', + 'orbit_demo', + '--measure', + 'contracts.contract_count', + '--format', + 'sql', + '--yes', + ], + env: { + npm_config_registry: 'https://registry.npmjs.org/', + KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', + }, + }, + { + label: 'published package local install', + command: 'pnpm', + args: ['add', '@kaelio/ktx@latest'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package local version', + command: 'npx', + args: ['ktx', '--version'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package local sl query', + command: 'npx', + args: [ + 'ktx', + 'sl', + 'query', + '--project-dir', + '/tmp/ktx-smoke/demo', + '--connection-id', + 'orbit_demo', + '--measure', + 'contracts.contract_count', + '--format', + 'sql', + '--yes', + ], + env: { + npm_config_registry: 'https://registry.npmjs.org/', + KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', + }, + }, + { + label: 'published package global install', + command: 'pnpm', + args: ['add', '--global', '@kaelio/ktx@latest'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package global version', + command: 'ktx', + args: ['--version'], + env: { npm_config_registry: 'https://registry.npmjs.org/' }, + }, + { + label: 'published package global sl query', + command: 'ktx', + args: [ + 'sl', + 'query', + '--project-dir', + '/tmp/ktx-smoke/demo', + '--connection-id', + 'orbit_demo', + '--measure', + 'contracts.contract_count', + '--format', + 'sql', + '--yes', + ], + env: { + npm_config_registry: 'https://registry.npmjs.org/', + KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', + }, + }, + ], + ); }); it('exposes the smoke through the package release script', async () => { diff --git a/scripts/release-readiness.mjs b/scripts/release-readiness.mjs index 48a57e6c..6b15e83e 100644 --- a/scripts/release-readiness.mjs +++ b/scripts/release-readiness.mjs @@ -5,6 +5,7 @@ import { dirname, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { packageArtifactLayout, packageReleaseMetadata, verifyArtifactManifest } from './package-artifacts.mjs'; +import { PUBLIC_NPM_PACKAGE_VERSION } from './build-public-npm-package.mjs'; import { readPublishedPackageSmokeConfig } from './published-package-smoke-config.mjs'; function scriptRootDir() { @@ -21,9 +22,11 @@ async function readJson(path) { const CI_ARTIFACT_ONLY_RELEASE_MODE = 'ci-artifact-only'; const PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE = 'published-package-smoke-required'; +const NPM_PUBLIC_RELEASE_READY_MODE = 'npm-public-release-ready'; const SUPPORTED_RELEASE_MODES = new Set([ CI_ARTIFACT_ONLY_RELEASE_MODE, PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE, + NPM_PUBLIC_RELEASE_READY_MODE, ]); export async function readReleasePolicy(rootDir = scriptRootDir()) { @@ -64,6 +67,19 @@ function assertStringArray(value, label) { } } +function assertNpmAccess(value) { + if (value !== 'public') { + throw new Error('Release policy npm.access must be public'); + } +} + +function assertNpmTag(value) { + assertString(value, 'Release policy npm.tag'); + if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(value)) { + throw new Error(`Invalid Release policy npm.tag: ${value}`); + } +} + function assertSupportedReleaseMode(releaseMode) { assertString(releaseMode, 'Release policy releaseMode'); if (!SUPPORTED_RELEASE_MODES.has(releaseMode)) { @@ -79,10 +95,31 @@ function assertRequiredBeforePublishing(policy) { } if ( - policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE && + (policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE || + policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) && policy.requiredBeforePublishing.length > 0 ) { - throw new Error('published-package-smoke-required release mode requires requiredBeforePublishing to be empty'); + throw new Error(`${policy.releaseMode} release mode requires requiredBeforePublishing to be empty`); + } +} + +function assertRuntimeInstallerPolicy(policy) { + assertPlainObject(policy.runtimeInstaller, 'Release policy runtimeInstaller'); + assertString(policy.runtimeInstaller.uvStrategy, 'Release policy runtimeInstaller.uvStrategy'); + assertBoolean(policy.runtimeInstaller.bootstrapUv, 'Release policy runtimeInstaller.bootstrapUv'); + assertString( + policy.runtimeInstaller.missingUvBehavior, + 'Release policy runtimeInstaller.missingUvBehavior', + ); + + if (policy.runtimeInstaller.uvStrategy !== 'path-prerequisite') { + throw new Error('Release policy runtimeInstaller.uvStrategy must be path-prerequisite'); + } + if (policy.runtimeInstaller.bootstrapUv !== false) { + throw new Error('Release policy runtimeInstaller.bootstrapUv must be false'); + } + if (policy.runtimeInstaller.missingUvBehavior !== 'focused-error') { + throw new Error('Release policy runtimeInstaller.missingUvBehavior must be focused-error'); } } @@ -107,6 +144,8 @@ export function validateReleasePolicy(policy) { assertBoolean(policy.npm.publish, 'Release policy npm.publish'); assertNullableString(policy.npm.registry, 'Release policy npm.registry'); + assertNpmAccess(policy.npm.access); + assertNpmTag(policy.npm.tag); assertStringArray(policy.npm.packages, 'Release policy npm.packages'); assertBoolean(policy.python.publish, 'Release policy python.publish'); @@ -117,6 +156,7 @@ export function validateReleasePolicy(policy) { assertNullableString(policy.publishedPackageSmoke.registry, 'Release policy publishedPackageSmoke.registry'); readPublishedPackageSmokeConfig({}, [], policy.publishedPackageSmoke); assertRequiredBeforePublishing(policy); + assertRuntimeInstallerPolicy(policy); return policy; } @@ -128,10 +168,12 @@ function metadataNames(metadata, ecosystem) { function publishedPackageSmokeGate(policy) { const config = readPublishedPackageSmokeConfig({}, [], policy.publishedPackageSmoke); - if (policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE && !config.enabled) { - throw new Error( - 'published-package-smoke-required release mode requires release-policy.json publishedPackageSmoke.packageName', - ); + if ( + (policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE || + policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) && + !config.enabled + ) { + throw new Error(`${policy.releaseMode} release mode requires release-policy.json publishedPackageSmoke.packageName`); } const base = @@ -140,6 +182,11 @@ function publishedPackageSmokeGate(policy) { status: 'not_required', reason: 'Published package smoke remains pending until release-policy.json enables npm registry publishing.', } + : policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE + ? { + status: 'required', + reason: 'Run the published package smoke after the npm package is published.', + } : { status: 'required', reason: 'Run the published package smoke before accepting the hybrid-search release.', @@ -180,23 +227,68 @@ function assertNonPublishingArtifactPolicy(policy, metadata) { throw new Error(`Package ${entry.packageName} releaseMode must remain ci-artifact-only`); } if (entry.ecosystem === 'npm') { - if (entry.private !== true) { + const isPublicKtxPackage = entry.packageName === '@kaelio/ktx'; + if (isPublicKtxPackage) { + if (entry.private !== false) { + throw new Error(`${policyLabel} npm package @kaelio/ktx must be publishable when npm.publish is false`); + } + if (entry.packageVersion !== PUBLIC_NPM_PACKAGE_VERSION) { + throw new Error(`${policyLabel} npm package @kaelio/ktx must use public version ${PUBLIC_NPM_PACKAGE_VERSION}`); + } + } else if (entry.private !== true) { throw new Error(`${policyLabel} npm package ${entry.packageName} must remain private`); - } - if (!entry.packageVersion.endsWith('-private')) { + } else if (!entry.packageVersion.endsWith('-private')) { throw new Error(`${policyLabel} npm package ${entry.packageName} must use a private version suffix`); } } } } +function assertNpmPublicReleaseReadyPolicy(policy, metadata) { + if (policy.npm.publish !== true) { + throw new Error('npm-public-release-ready policy requires npm.publish true'); + } + if (policy.python.publish !== false) { + throw new Error('npm-public-release-ready policy keeps python.publish false'); + } + if (policy.python.repository !== null) { + throw new Error('npm-public-release-ready policy keeps python.repository null'); + } + + assertSameMembers(policy.npm.packages, ['@kaelio/ktx'], 'Release policy npm.packages'); + assertSameMembers(policy.python.packages, metadataNames(metadata, 'python'), 'Release policy python.packages'); + + const npmMetadata = metadata.find((entry) => entry.ecosystem === 'npm' && entry.packageName === '@kaelio/ktx'); + if (!npmMetadata) { + throw new Error('npm-public-release-ready policy requires @kaelio/ktx artifact metadata'); + } + if (npmMetadata.private !== false) { + throw new Error('npm-public-release-ready policy requires @kaelio/ktx to be publishable'); + } + if (npmMetadata.packageVersion !== PUBLIC_NPM_PACKAGE_VERSION) { + throw new Error( + `npm-public-release-ready policy expected @kaelio/ktx ${PUBLIC_NPM_PACKAGE_VERSION}, got ${npmMetadata.packageVersion}`, + ); + } + if (policy.publishedPackageSmoke.packageName !== '@kaelio/ktx') { + throw new Error('npm-public-release-ready policy requires publishedPackageSmoke.packageName @kaelio/ktx'); + } + if (policy.publishedPackageSmoke.version !== PUBLIC_NPM_PACKAGE_VERSION) { + throw new Error(`npm-public-release-ready policy requires publishedPackageSmoke.version ${PUBLIC_NPM_PACKAGE_VERSION}`); + } +} + export async function releaseReadinessReport(rootDir = scriptRootDir()) { const policy = validateReleasePolicy(await readReleasePolicy(rootDir)); const layout = packageArtifactLayout(rootDir); const manifest = await verifyArtifactManifest(layout); const metadata = await packageReleaseMetadata(rootDir); - assertNonPublishingArtifactPolicy(policy, metadata); + if (policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) { + assertNpmPublicReleaseReadyPolicy(policy, metadata); + } else { + assertNonPublishingArtifactPolicy(policy, metadata); + } return { schemaVersion: 1, @@ -206,6 +298,17 @@ export async function releaseReadinessReport(rootDir = scriptRootDir()) { pythonPublishEnabled: policy.python.publish, packageNames: metadata.map((entry) => entry.packageName), publishedPackageSmokeGate: publishedPackageSmokeGate(policy), + runtimeInstaller: policy.runtimeInstaller, + npmPublish: + policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE + ? { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + access: policy.npm.access, + tag: policy.npm.tag, + registry: policy.npm.registry, + } + : null, blockedPublishingDecisions: policy.requiredBeforePublishing, }; } @@ -229,7 +332,17 @@ async function main() { process.stdout.write( `Published package smoke registry: ${report.publishedPackageSmokeGate.registry ?? 'default npm registry'}\n`, ); - process.stdout.write('Registry publishing remains disabled by release-policy.json.\n'); + process.stdout.write(`Runtime uv strategy: ${report.runtimeInstaller.uvStrategy}\n`); + process.stdout.write( + `Runtime uv bootstrap: ${report.runtimeInstaller.bootstrapUv ? 'enabled' : 'disabled'}\n`, + ); + if (report.npmPublish) { + process.stdout.write( + `NPM publish target: ${report.npmPublish.packageName}@${report.npmPublish.version} (${report.npmPublish.tag})\n`, + ); + } else { + process.stdout.write('Registry publishing remains disabled by release-policy.json.\n'); + } process.stdout.write('Required decisions before publishing:\n'); for (const decision of report.blockedPublishingDecisions) { process.stdout.write(`- ${decision}\n`); diff --git a/scripts/release-readiness.test.mjs b/scripts/release-readiness.test.mjs index 6bfa2484..ee7113c0 100644 --- a/scripts/release-readiness.test.mjs +++ b/scripts/release-readiness.test.mjs @@ -4,39 +4,28 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it } from 'node:test'; -import { NPM_ARTIFACT_PACKAGES, packageArtifactLayout, writeArtifactManifest } from './package-artifacts.mjs'; +import { + INTERNAL_NPM_WORKSPACE_PACKAGES, + NPM_ARTIFACT_PACKAGES, + packageArtifactLayout, + writeArtifactManifest, +} from './package-artifacts.mjs'; +import { PUBLIC_NPM_PACKAGE_VERSION } from './build-public-npm-package.mjs'; import { readReleasePolicy, releasePolicyPath, releaseReadinessReport } from './release-readiness.mjs'; async function writeJson(path, value) { await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); } -async function writeReleaseMetadataInputs(root, options = {}) { - for (const packageInfo of NPM_ARTIFACT_PACKAGES) { +async function writeReleaseMetadataInputs(root) { + for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { name: packageInfo.name, version: '0.0.0-private', - private: - packageInfo.name === '@ktx/context' - ? (options.contextPrivate ?? true) - : packageInfo.name === '@ktx/cli' - ? (options.cliPrivate ?? true) - : true, + private: true, }); } - - await mkdir(join(root, 'python', 'ktx-sl'), { recursive: true }); - await mkdir(join(root, 'python', 'ktx-daemon'), { recursive: true }); - - await writeFile( - join(root, 'python', 'ktx-sl', 'pyproject.toml'), - ['[project]', 'name = "ktx-sl"', 'version = "0.1.0"', ''].join('\n'), - ); - await writeFile( - join(root, 'python', 'ktx-daemon', 'pyproject.toml'), - ['[project]', 'name = "ktx-daemon"', 'version = "0.1.0"', ''].join('\n'), - ); } async function writeUploadableArtifactFixtures(layout) { @@ -48,10 +37,7 @@ async function writeUploadableArtifactFixtures(layout) { layout.npmTarballs[packageInfo.name], `${packageInfo.name}-tarball`, ]), - [join(layout.pythonDir, 'ktx_sl-0.1.0-py3-none-any.whl'), 'ktx-sl-wheel'], - [join(layout.pythonDir, 'ktx_sl-0.1.0.tar.gz'), 'ktx-sl-sdist'], - [join(layout.pythonDir, 'ktx_daemon-0.1.0-py3-none-any.whl'), 'ktx-daemon-wheel'], - [join(layout.pythonDir, 'ktx_daemon-0.1.0.tar.gz'), 'ktx-daemon-sdist'], + [join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), 'kaelio-ktx-runtime-wheel'], ]); for (const [path, contents] of fileContents) { @@ -68,24 +54,29 @@ function releasePolicy(overrides = {}) { npm: { publish: false, registry: null, - packages: NPM_ARTIFACT_PACKAGES.map((packageInfo) => packageInfo.name), + access: 'public', + tag: 'latest', + packages: ['@kaelio/ktx'], ...npmOverrides, }, python: { publish: false, repository: null, - packages: ['ktx-sl', 'ktx-daemon'], + packages: ['kaelio-ktx'], ...pythonOverrides, }, publishedPackageSmoke: { - packageName: null, + packageName: '@kaelio/ktx', version: 'latest', registry: null, }, + runtimeInstaller: { + uvStrategy: 'path-prerequisite', + bootstrapUv: false, + missingUvBehavior: 'focused-error', + }, requiredBeforePublishing: [ - 'Choose npm registry and package visibility.', - 'Choose Python package repository.', - 'Choose public release versions.', + 'Choose public release version.', 'Configure registry credentials outside source control.', 'Choose release tag and provenance policy.', ], @@ -98,7 +89,7 @@ async function writePolicy(root, policy = releasePolicy()) { } async function writeReadyFixture(root, options = {}) { - await writeReleaseMetadataInputs(root, options); + await writeReleaseMetadataInputs(root); await writePolicy(root, options.policy ?? releasePolicy()); const layout = packageArtifactLayout(root); await writeUploadableArtifactFixtures(layout); @@ -135,20 +126,24 @@ describe('release readiness policy', () => { sourceRevision: 'abc123', npmPublishEnabled: false, pythonPublishEnabled: false, - packageNames: [...NPM_ARTIFACT_PACKAGES.map((packageInfo) => packageInfo.name), 'ktx-sl', 'ktx-daemon'], + packageNames: ['@kaelio/ktx', 'kaelio-ktx'], publishedPackageSmokeGate: { status: 'not_required', script: 'pnpm run release:published-smoke', reason: 'Published package smoke remains pending until release-policy.json enables npm registry publishing.', - configSource: null, - packageName: null, + configSource: 'release-policy', + packageName: '@kaelio/ktx', version: 'latest', registry: null, }, + runtimeInstaller: { + uvStrategy: 'path-prerequisite', + bootstrapUv: false, + missingUvBehavior: 'focused-error', + }, + npmPublish: null, blockedPublishingDecisions: [ - 'Choose npm registry and package visibility.', - 'Choose Python package repository.', - 'Choose public release versions.', + 'Choose public release version.', 'Configure registry credentials outside source control.', 'Choose release tag and provenance policy.', ], @@ -164,7 +159,7 @@ describe('release readiness policy', () => { await writeReadyFixture(root, { policy: releasePolicy({ publishedPackageSmoke: { - packageName: '@ktx/cli-public', + packageName: '@kaelio/ktx', version: '2026.5.8', registry: 'https://registry.npmjs.org/', }, @@ -178,7 +173,7 @@ describe('release readiness policy', () => { script: 'pnpm run release:published-smoke', reason: 'Published package smoke remains pending until release-policy.json enables npm registry publishing.', configSource: 'release-policy', - packageName: '@ktx/cli-public', + packageName: '@kaelio/ktx', version: '2026.5.8', registry: 'https://registry.npmjs.org/', }); @@ -194,7 +189,7 @@ describe('release readiness policy', () => { policy: releasePolicy({ releaseMode: 'published-package-smoke-required', publishedPackageSmoke: { - packageName: '@ktx/cli-public', + packageName: '@kaelio/ktx', version: '2026.5.8', registry: 'https://registry.npmjs.org/', }, @@ -210,16 +205,22 @@ describe('release readiness policy', () => { sourceRevision: 'abc123', npmPublishEnabled: false, pythonPublishEnabled: false, - packageNames: [...NPM_ARTIFACT_PACKAGES.map((packageInfo) => packageInfo.name), 'ktx-sl', 'ktx-daemon'], + packageNames: ['@kaelio/ktx', 'kaelio-ktx'], publishedPackageSmokeGate: { status: 'required', script: 'pnpm run release:published-smoke', reason: 'Run the published package smoke before accepting the hybrid-search release.', configSource: 'release-policy', - packageName: '@ktx/cli-public', + packageName: '@kaelio/ktx', version: '2026.5.8', registry: 'https://registry.npmjs.org/', }, + runtimeInstaller: { + uvStrategy: 'path-prerequisite', + bootstrapUv: false, + missingUvBehavior: 'focused-error', + }, + npmPublish: null, blockedPublishingDecisions: [], }); } finally { @@ -227,12 +228,205 @@ describe('release readiness policy', () => { } }); + it('accepts the npm public release ready policy', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-test-')); + try { + await writeReadyFixture(root, { + policy: releasePolicy({ + releaseMode: 'npm-public-release-ready', + npm: { + publish: true, + registry: null, + access: 'public', + tag: 'latest', + }, + publishedPackageSmoke: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + requiredBeforePublishing: [], + }), + }); + + const report = await releaseReadinessReport(root); + + assert.deepEqual(report, { + schemaVersion: 1, + releaseMode: 'npm-public-release-ready', + sourceRevision: 'abc123', + npmPublishEnabled: true, + pythonPublishEnabled: false, + packageNames: ['@kaelio/ktx', 'kaelio-ktx'], + publishedPackageSmokeGate: { + status: 'required', + script: 'pnpm run release:published-smoke', + reason: 'Run the published package smoke after the npm package is published.', + configSource: 'release-policy', + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + runtimeInstaller: { + uvStrategy: 'path-prerequisite', + bootstrapUv: false, + missingUvBehavior: 'focused-error', + }, + npmPublish: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + access: 'public', + tag: 'latest', + registry: null, + }, + blockedPublishingDecisions: [], + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it('rejects npm public release ready mode without a runtime installer policy', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-policy-missing-test-')); + try { + await writeReadyFixture(root, { + policy: releasePolicy({ + releaseMode: 'npm-public-release-ready', + npm: { + publish: true, + registry: null, + access: 'public', + tag: 'latest', + }, + publishedPackageSmoke: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + runtimeInstaller: undefined, + requiredBeforePublishing: [], + }), + }); + + await assert.rejects( + () => releaseReadinessReport(root), + /Release policy runtimeInstaller must be a JSON object/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it('rejects uv bootstrap download policy for the first public npm release', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-policy-bootstrap-test-')); + try { + await writeReadyFixture(root, { + policy: releasePolicy({ + releaseMode: 'npm-public-release-ready', + npm: { + publish: true, + registry: null, + access: 'public', + tag: 'latest', + }, + publishedPackageSmoke: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + runtimeInstaller: { + uvStrategy: 'bootstrap-download', + bootstrapUv: true, + missingUvBehavior: 'download', + }, + requiredBeforePublishing: [], + }), + }); + + await assert.rejects( + () => releaseReadinessReport(root), + /Release policy runtimeInstaller\.uvStrategy must be path-prerequisite/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it('rejects npm public release ready mode when npm publish is disabled', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-disabled-test-')); + try { + await writeReadyFixture(root, { + policy: releasePolicy({ + releaseMode: 'npm-public-release-ready', + npm: { + publish: false, + registry: null, + access: 'public', + tag: 'latest', + }, + publishedPackageSmoke: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + requiredBeforePublishing: [], + }), + }); + + await assert.rejects( + () => releaseReadinessReport(root), + /npm-public-release-ready policy requires npm.publish true/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it('rejects npm public release ready mode when Python publishing is enabled', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-python-test-')); + try { + await writeReadyFixture(root, { + policy: releasePolicy({ + releaseMode: 'npm-public-release-ready', + npm: { + publish: true, + registry: null, + access: 'public', + tag: 'latest', + }, + python: { + publish: true, + repository: 'pypi', + }, + publishedPackageSmoke: { + packageName: '@kaelio/ktx', + version: PUBLIC_NPM_PACKAGE_VERSION, + registry: null, + }, + requiredBeforePublishing: [], + }), + }); + + await assert.rejects( + () => releaseReadinessReport(root), + /npm-public-release-ready policy keeps python.publish false/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it('rejects required published smoke mode without a package name', async () => { const root = await mkdtemp(join(tmpdir(), 'ktx-release-smoke-required-missing-config-test-')); try { await writeReadyFixture(root, { policy: releasePolicy({ releaseMode: 'published-package-smoke-required', + publishedPackageSmoke: { + packageName: null, + version: 'latest', + registry: null, + }, requiredBeforePublishing: [], }), }); @@ -253,7 +447,7 @@ describe('release readiness policy', () => { policy: releasePolicy({ releaseMode: 'published-package-smoke-required', publishedPackageSmoke: { - packageName: '@ktx/cli-public', + packageName: '@kaelio/ktx', version: 'latest', registry: null, }, @@ -345,14 +539,20 @@ describe('release readiness policy', () => { } }); - it('rejects a public npm package while releaseMode is ci-artifact-only', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-release-public-npm-test-')); + it('rejects release policy that still lists internal npm packages', async () => { + const root = await mkdtemp(join(tmpdir(), 'ktx-release-stale-internal-npm-policy-test-')); try { - await writeReadyFixture(root, { contextPrivate: false }); + await writeReadyFixture(root, { + policy: releasePolicy({ + npm: { + packages: ['@kaelio/ktx', '@ktx/context'], + }, + }), + }); await assert.rejects( () => releaseReadinessReport(root), - /ci-artifact-only policy npm package @ktx\/context must remain private/, + /Release policy npm\.packages mismatch/, ); } finally { await rm(root, { recursive: true, force: true }); diff --git a/scripts/release-workflow.test.mjs b/scripts/release-workflow.test.mjs new file mode 100644 index 00000000..7e313c9c --- /dev/null +++ b/scripts/release-workflow.test.mjs @@ -0,0 +1,21 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { describe, it } from 'node:test'; + +describe('release workflow', () => { + it('publishes only from manual dispatch with an explicit live input', async () => { + const workflow = await readFile(new URL('../.github/workflows/release.yml', import.meta.url), 'utf8'); + + assert.match(workflow, /^name: KTX Release$/m); + assert.match(workflow, /^ workflow_dispatch:$/m); + assert.match(workflow, /publish_live:/); + assert.match(workflow, /default: false/); + assert.match(workflow, /pnpm run artifacts:check/); + assert.match(workflow, /pnpm run release:readiness/); + assert.match(workflow, /pnpm run release:npm-publish$/m); + assert.match(workflow, /pnpm run release:npm-publish -- --publish/); + assert.match(workflow, /NODE_AUTH_TOKEN: \$\{\{ secrets.NPM_TOKEN \}\}/); + assert.doesNotMatch(workflow, /^ push:/m); + assert.doesNotMatch(workflow, /^ pull_request:/m); + }); +}); diff --git a/uv.lock b/uv.lock index df70d473..5458900e 100644 --- a/uv.lock +++ b/uv.lock @@ -452,11 +452,15 @@ dependencies = [ { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, { name = "requests" }, - { name = "sentence-transformers" }, { name = "sqlglot" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[package.optional-dependencies] +local-embeddings = [ + { name = "sentence-transformers" }, { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" }, { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" }, - { name = "uvicorn", extra = ["standard"] }, ] [package.dev-dependencies] @@ -476,11 +480,12 @@ requires-dist = [ { name = "psycopg", extras = ["binary"], specifier = ">=3.2.0" }, { name = "pydantic", specifier = ">=2.9.0" }, { name = "requests", specifier = ">=2.32.0" }, - { name = "sentence-transformers", specifier = ">=5.1.1" }, + { name = "sentence-transformers", marker = "extra == 'local-embeddings'", specifier = ">=5.1.1" }, { name = "sqlglot", specifier = ">=26" }, - { name = "torch", specifier = ">=2.2.0", index = "https://download.pytorch.org/whl/cpu" }, + { name = "torch", marker = "extra == 'local-embeddings'", specifier = ">=2.2.0", index = "https://download.pytorch.org/whl/cpu" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" }, ] +provides-extras = ["local-embeddings"] [package.metadata.requires-dev] dev = [