diff --git a/README.md b/README.md index 014ac600..f29d393f 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,11 @@ artifacts. You can inspect them, commit them, and serve them to any MCP client. ## Quick start -Run the pre-seeded demo from the repository root: +Run the pre-seeded demo through the public npm package: ```bash -pnpm install -pnpm run setup:dev -pnpm run ktx -- setup demo --no-input -pnpm run ktx -- setup demo inspect +npx @kaelio/ktx setup demo --no-input +npx @kaelio/ktx setup demo inspect ``` The default demo uses packaged sample data and prebuilt context. It does not @@ -35,7 +33,7 @@ require API keys, network access, or an LLM provider. To replay the packaged ingest run, use: ```bash -pnpm run ktx -- setup demo --mode replay --no-input +npx @kaelio/ktx setup demo --mode replay --no-input ``` To run the full agentic demo with an LLM provider, set a provider key for the @@ -43,22 +41,29 @@ current process: ```bash ANTHROPIC_API_KEY=$YOUR_ANTHROPIC_API_KEY \ - pnpm run ktx -- setup demo --mode full --no-input + npx @kaelio/ktx setup demo --mode full --no-input ``` Interactive full-demo setup can prompt for a provider key without writing the key to `ktx.yaml`. -## Build a local project - -Create a project from the repository root: +You can also install the CLI in a project or globally: ```bash -uv sync --all-packages -source .venv/bin/activate +npm install @kaelio/ktx +npx ktx --help +npm install -g @kaelio/ktx +ktx --help +``` +## Build a local project + +Create a project from a local workspace: + +```bash +npm install @kaelio/ktx PROJECT_DIR="$(mktemp -d)/ktx-demo" -pnpm run ktx -- init "$PROJECT_DIR" --name ktx-demo +npx ktx init "$PROJECT_DIR" --name ktx-demo ``` Create a SQLite warehouse: @@ -112,7 +117,7 @@ YAML Write and validate a semantic-layer source: ```bash -pnpm run ktx -- sl write accounts --project-dir "$PROJECT_DIR" \ +npx ktx sl write accounts --project-dir "$PROJECT_DIR" \ --connection-id warehouse --yaml 'name: accounts table: accounts description: CRM accounts with segmentation attributes. @@ -133,14 +138,14 @@ measures: joins: [] ' -pnpm run ktx -- sl validate accounts --project-dir "$PROJECT_DIR" \ +npx ktx sl validate accounts --project-dir "$PROJECT_DIR" \ --connection-id warehouse ``` Generate SQL and execute the query: ```bash -pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \ +npx ktx sl query --project-dir "$PROJECT_DIR" \ --connection-id warehouse \ --measure accounts.account_count \ --dimension accounts.segment \ @@ -148,7 +153,7 @@ pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \ --limit 5 \ --format sql -pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \ +npx ktx sl query --project-dir "$PROJECT_DIR" \ --connection-id warehouse \ --measure accounts.account_count \ --dimension accounts.segment \ @@ -161,8 +166,8 @@ pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \ List and test the warehouse connection: ```bash -pnpm run ktx -- connection list --project-dir "$PROJECT_DIR" -pnpm run ktx -- connection test warehouse --project-dir "$PROJECT_DIR" +npx ktx connection list --project-dir "$PROJECT_DIR" +npx ktx connection test warehouse --project-dir "$PROJECT_DIR" ``` The connection test prints the configured driver and discovered table count: @@ -179,34 +184,55 @@ Scan artifacts are written under ```bash -SCAN_OUTPUT="$(pnpm run ktx -- scan warehouse --project-dir "$PROJECT_DIR")" +SCAN_OUTPUT="$(npx ktx scan warehouse --project-dir "$PROJECT_DIR")" printf '%s\n' "$SCAN_OUTPUT" SCAN_RUN_ID="$(printf '%s\n' "$SCAN_OUTPUT" | awk '/^Run: / { print $2 }')" -pnpm run ktx -- scan status --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID" -pnpm run ktx -- scan report --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID" +npx ktx scan status --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID" +npx ktx scan report --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID" ``` For non-SQLite drivers, prefer credential references such as `--url env:NAME` or `--url file:PATH` over literal credential URLs. +## Managed Python runtime + +KTX installs its Python runtime only when a Python-backed command needs it. +The runtime lives outside the npm cache, is versioned by the installed CLI +version, and is managed by `ktx runtime` commands: + +```bash +npx ktx runtime install --yes +npx ktx runtime status +npx ktx runtime doctor +npx ktx runtime start +npx ktx runtime stop +``` + +Commands such as `npx @kaelio/ktx sl query ... --yes` can install the core +runtime lazily from the bundled wheel. Local embeddings remain lazy; prepare +them only when you select local `sentence-transformers` embeddings: + +```bash +npx ktx runtime install --feature local-embeddings --yes +npx ktx runtime start --feature local-embeddings +``` + ## Serve MCP -Start the Python compute daemon in one terminal: +Start the stdio MCP server from the project directory: ```bash -source .venv/bin/activate -uv run ktx-daemon serve-http --host 127.0.0.1 --port 8765 -``` - -Start the stdio MCP server in another terminal: - -```bash -pnpm run ktx -- serve --mcp stdio --project-dir "$PROJECT_DIR" \ +npx ktx serve --mcp stdio --project-dir "$PROJECT_DIR" \ --user-id local \ - --semantic-compute-url http://127.0.0.1:8765 \ - --execute-queries + --semantic-compute \ + --execute-queries \ + --yes ``` +The `--semantic-compute` flag uses the managed Python runtime when no explicit +semantic compute URL is provided. KTX starts or reuses the managed runtime as +needed. + The MCP server exposes `connection_list`, `knowledge_search`, `knowledge_read`, `knowledge_write`, `sl_list_sources`, `sl_read_source`, `sl_write_source`, `sl_validate`, `sl_query`, `ingest_trigger`, @@ -252,10 +278,10 @@ packages. ## Release status -This repository is prepared for source publication. Package publishing is still -disabled by `release-policy.json`; registry names, public versions, package -visibility, and provenance policy must be chosen before publishing artifacts to -npm or Python package indexes. +This repository builds a single public npm artifact named `@kaelio/ktx`. +Package publishing is still disabled by `release-policy.json`; registry +credentials, public versions, release tags, and provenance policy must be +chosen before publishing artifacts to npm or Python package indexes. Build local package artifacts with: diff --git a/examples/package-artifacts/README.md b/examples/package-artifacts/README.md index 2db3817b..dc4cdcbe 100644 --- a/examples/package-artifacts/README.md +++ b/examples/package-artifacts/README.md @@ -4,14 +4,18 @@ The package artifact smoke checks create temporary projects instead of storing sample projects in this directory. Run the checks from `ktx/`: ```bash -source .venv/bin/activate pnpm run artifacts:check ``` -The npm smoke project installs the generated `@ktx/context` and `@ktx/cli` -tarballs, imports public package entry points, and runs installed `ktx` -commands against a generated local project. +The npm smoke project installs the generated public `@kaelio/ktx` tarball, +imports the package entry point, and runs installed `ktx` commands against a +generated local project. -The Python smoke project installs `ktx-daemon` through the local artifact -directory, imports `semantic_layer` and `ktx_daemon`, and runs -`python -m ktx_daemon semantic-validate`. +The managed Python runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies +`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from +the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed +daemon, and stops it. + +The Python smoke project still installs the Python artifacts directly because +it verifies the standalone Python distributions that feed the bundled runtime +wheel. diff --git a/examples/postgres-historic/README.md b/examples/postgres-historic/README.md index 3e27b462..f97d4b9b 100644 --- a/examples/postgres-historic/README.md +++ b/examples/postgres-historic/README.md @@ -13,8 +13,8 @@ generates query workload under separate users, runs `ktx setup` with - Docker with Compose v2 - Node and pnpm matching the KTX workspace -- `python-service/.venv` already created, or `KTX_SQL_ANALYSIS_URL` pointing at - a running service that exposes `/api/sql/analyze-for-fingerprint` +- `uv` on `PATH` so the KTX-managed Python runtime can install the bundled + runtime wheel ## Run @@ -24,8 +24,9 @@ From the KTX repository root: examples/postgres-historic/scripts/smoke.sh ``` -The smoke creates a temporary KTX project, starts Postgres on -`127.0.0.1:55432`, and uses this connection URL: +The smoke creates a temporary KTX project, isolates the managed Python runtime +under the temporary project parent, starts Postgres on `127.0.0.1:55432`, and +uses this connection URL: ```bash postgresql://ktx_reader:ktx_reader@127.0.0.1:55432/analytics # pragma: allowlist secret @@ -83,10 +84,11 @@ Historic SQL (warehouse)` when `pg_stat_statements` is installed, Run local historic-SQL ingest: ```bash -node packages/cli/dist/bin.js --project-dir /tmp/ktx-postgres-historic dev ingest run \ +pnpm run ktx -- dev ingest run --project-dir /tmp/ktx-postgres-historic \ --connection-id warehouse \ --adapter historic-sql \ --plain \ + --yes \ --no-input ``` @@ -111,5 +113,6 @@ The manifest should have `dialect: "postgres"`, `degraded: true`, - Missing grants: confirm `GRANT pg_read_all_stats TO ktx_reader;`. - Empty templates: rerun `scripts/generate-workload.sh base` and keep `--historic-sql-min-calls 2` for the smoke. -- SQL-analysis failures: set `KTX_SQL_ANALYSIS_URL` to the running service URL - or create `python-service/.venv` before running `scripts/smoke.sh`. +- SQL-analysis failures: run `pnpm run ktx -- runtime doctor` from the KTX + repository root and confirm `uv`, the bundled Python wheel, and the managed + runtime all pass. diff --git a/examples/postgres-historic/scripts/smoke.sh b/examples/postgres-historic/scripts/smoke.sh index d948cf8e..488535a4 100755 --- a/examples/postgres-historic/scripts/smoke.sh +++ b/examples/postgres-historic/scripts/smoke.sh @@ -4,17 +4,17 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EXAMPLE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" KTX_ROOT="$(cd "$EXAMPLE_DIR/../.." && pwd)" -REPO_ROOT="$(cd "$KTX_ROOT/.." && pwd)" COMPOSE_FILE="$EXAMPLE_DIR/docker-compose.yml" PROJECT_PARENT="${KTX_POSTGRES_HISTORIC_PROJECT_PARENT:-$(mktemp -d)}" PROJECT_DIR="$PROJECT_PARENT/postgres-historic-ktx" KTX_BIN="$KTX_ROOT/packages/cli/dist/bin.js" -PYTHON_SERVICE_LOG="$PROJECT_PARENT/python-service.log" -PYTHON_SERVICE_PID="" +export KTX_RUNTIME_ROOT="$PROJECT_PARENT/managed-runtime" +unset KTX_DAEMON_URL +unset KTX_SQL_ANALYSIS_URL cleanup() { - if [[ -n "$PYTHON_SERVICE_PID" ]]; then - kill "$PYTHON_SERVICE_PID" >/dev/null 2>&1 || true + if [[ -f "$KTX_BIN" ]]; then + node "$KTX_BIN" runtime stop >/dev/null 2>&1 || true fi if [[ "${KTX_POSTGRES_HISTORIC_KEEP_DOCKER:-0}" != "1" ]]; then docker compose -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 || true @@ -22,31 +22,6 @@ cleanup() { } trap cleanup EXIT -start_sql_analysis_if_needed() { - if [[ -n "${KTX_SQL_ANALYSIS_URL:-}" ]]; then - return - fi - if [[ ! -d "$REPO_ROOT/python-service/.venv" ]]; then - echo "Set KTX_SQL_ANALYSIS_URL or create python-service/.venv before running this smoke." >&2 - exit 1 - fi - ( - cd "$REPO_ROOT/python-service" - source .venv/bin/activate - uvicorn app.main:app --host 127.0.0.1 --port 18081 >"$PYTHON_SERVICE_LOG" 2>&1 - ) & - PYTHON_SERVICE_PID="$!" - export KTX_SQL_ANALYSIS_URL="http://127.0.0.1:18081" - for _ in $(seq 1 60); do - if curl -fsS "$KTX_SQL_ANALYSIS_URL/health" >/dev/null 2>&1; then - return - fi - sleep 1 - done - echo "SQL analysis service did not become healthy. Log: $PYTHON_SERVICE_LOG" >&2 - exit 1 -} - latest_manifest() { find "$PROJECT_DIR/raw-sources/warehouse/historic-sql" -name manifest.json | sort | tail -n 1 } @@ -83,9 +58,19 @@ const jobId = process.argv[4]; const { loadKtxProject } = await import(join(ktxRoot, 'packages/context/dist/project/index.js')); const { runLocalStageOnlyIngest } = await import(join(ktxRoot, 'packages/context/dist/ingest/index.js')); const { createKtxCliLocalIngestAdapters } = await import(join(ktxRoot, 'packages/cli/dist/local-adapters.js')); +const { getKtxCliPackageInfo } = await import(join(ktxRoot, 'packages/cli/dist/index.js')); const project = await loadKtxProject({ projectDir }); -const adapters = createKtxCliLocalIngestAdapters(project, { historicSqlConnectionId: 'warehouse' }); +const cliVersion = getKtxCliPackageInfo().version; +const managedRuntimeIo = { stdout: process.stdout, stderr: process.stderr }; +const adapters = createKtxCliLocalIngestAdapters(project, { + historicSqlConnectionId: 'warehouse', + managedDaemon: { + cliVersion, + installPolicy: 'auto', + io: managedRuntimeIo, + }, +}); const adapter = adapters.find((candidate) => candidate.source === 'historic-sql'); if (!adapter) throw new Error('historic-sql adapter was not registered for local run'); const record = await runLocalStageOnlyIngest({ @@ -111,7 +96,6 @@ NODE cd "$KTX_ROOT" pnpm --filter @ktx/context run build pnpm --filter @ktx/cli run build -start_sql_analysis_if_needed docker compose -f "$COMPOSE_FILE" up -d --wait "$EXAMPLE_DIR/scripts/generate-workload.sh" base diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index 3d3aa168..16b97ad1 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -6,6 +6,18 @@ async function readText(relativePath) { return readFile(new URL(`../${relativePath}`, import.meta.url), 'utf8'); } +function publicNpmPackageName() { + return `@${['kae', 'lio'].join('')}/ktx`; +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function publicPackagePattern(text) { + return new RegExp(text.replaceAll('{package}', escapeRegExp(publicNpmPackageName()))); +} + describe('standalone example docs', () => { it('documents the local warehouse example from the examples index', async () => { const examples = await readText('examples/README.md'); @@ -63,6 +75,16 @@ describe('standalone example docs', () => { assert.match(workload, /app_user/); assert.match(workload, /etl_user/); assert.match(smoke, /pg_stat_statements_reset/); + assert.match(smoke, /KTX_RUNTIME_ROOT/); + assert.match(smoke, /managedDaemon/); + assert.match(smoke, /installPolicy: 'auto'/); + assert.match(smoke, /getKtxCliPackageInfo/); + assert.doesNotMatch(smoke, /python-service/); + assert.doesNotMatch(smoke, /PYTHON_SERVICE/); + assert.doesNotMatch(smoke, /uvicorn app\.main:app/); + assert.doesNotMatch(smoke, /export KTX_SQL_ANALYSIS_URL/); + assert.doesNotMatch(readme, /python-service/); + assert.doesNotMatch(readme, /KTX_SQL_ANALYSIS_URL/); assert.match(smoke, /assert_manifest "\$FIRST_MANIFEST" true/); assert.match(smoke, /assert_manifest "\$SECOND_MANIFEST" false/); assert.match(smoke, /assert_manifest "\$RESET_MANIFEST" true/); @@ -112,6 +134,35 @@ describe('standalone example docs', () => { assert.match(rootReadme, /Tables: 1/); }); + it('documents public npm and managed runtime usage in the README', async () => { + const rootReadme = await readText('README.md'); + + assert.match(rootReadme, publicPackagePattern('npx {package} setup demo --no-input')); + assert.match(rootReadme, publicPackagePattern('npx {package} sl query')); + assert.match(rootReadme, publicPackagePattern('npm install {package}')); + assert.match(rootReadme, publicPackagePattern('npm install -g {package}')); + assert.match(rootReadme, /ktx runtime install/); + assert.match(rootReadme, /ktx runtime status/); + assert.match(rootReadme, /ktx runtime doctor/); + assert.match(rootReadme, /ktx runtime start/); + assert.match(rootReadme, /ktx runtime stop/); + assert.match(rootReadme, /ktx serve --mcp stdio/); + assert.doesNotMatch(rootReadme, /uv run ktx-daemon serve-http/); + assert.doesNotMatch(rootReadme, /--semantic-compute-url http:\/\/127\.0\.0\.1:8765/); + }); + + it('documents the public package artifact smoke shape', async () => { + const readme = await readText('examples/package-artifacts/README.md'); + + assert.match(readme, publicPackagePattern('{package}')); + assert.match(readme, /managed Python runtime/); + assert.match(readme, /ktx runtime status/); + assert.match(readme, /ktx runtime doctor/); + assert.doesNotMatch(readme, /@ktx\/context/); + assert.doesNotMatch(readme, /@ktx\/cli/); + assert.doesNotMatch(readme, /python -m ktx_daemon semantic-validate/); + }); + it('replaces the fake-ingest smoke with a ktx scan walkthrough in the README', async () => { const rootReadme = await readText('README.md');