apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-10 08:05:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/main' into luca-martial/schema-select-ux-text

Browse source 
# Conflicts:
#	packages/cli/src/demo.test.ts
#	packages/context/src/ingest/local-adapters.ts
This commit is contained in:
Andrey Avtomonov 2026-05-12 14:47:28 +02:00
commit d0f650f44a
123 changed files with 3739 additions and 933 deletions
Download patch file Download diff file Expand all files Collapse all files

133
.github/workflows/ci.yml vendored
View file

@ -15,19 +15,20 @@ concurrency:
cancel-in-progress: true
jobs:
check:
typescript-checks:
name: TypeScript checks
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: "24"
cache: "pnpm"
@ -39,19 +40,101 @@ jobs:
- name: Run TypeScript checks
run: pnpm run check
- name: Run slow TypeScript tests
run: pnpm run test:slow
slow-context-tests:
name: Slow context tests
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: "24"
cache: "pnpm"
cache-dependency-path: "pnpm-lock.yaml"
- name: Install TypeScript dependencies
run: pnpm install --frozen-lockfile
- name: Build TypeScript packages
run: pnpm run build
- name: Run slow context tests
run: pnpm --filter @ktx/context run test:slow
slow-cli-tests:
name: Slow CLI tests
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: "24"
cache: "pnpm"
cache-dependency-path: "pnpm-lock.yaml"
- name: Install TypeScript dependencies
run: pnpm install --frozen-lockfile
- name: Build TypeScript packages
run: pnpm run build
- name: Run slow CLI tests
run: pnpm --filter @ktx/cli run test:slow
cli-smoke-tests:
name: CLI smoke tests
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: "24"
cache: "pnpm"
cache-dependency-path: "pnpm-lock.yaml"
- name: Install TypeScript dependencies
run: pnpm install --frozen-lockfile
- name: Run CLI smoke tests
run: pnpm run smoke
python-checks:
name: Python checks
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.13"
- name: Setup uv
uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
cache-dependency-glob: "uv.lock"
@ -62,11 +145,47 @@ jobs:
- name: Run Python checks
run: uv run pytest
artifact-checks:
name: Artifact checks
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: "24"
cache: "pnpm"
cache-dependency-path: "pnpm-lock.yaml"
- name: Install TypeScript dependencies
run: pnpm install --frozen-lockfile
- name: Setup Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.13"
- name: Setup uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
cache-dependency-glob: "uv.lock"
- name: Install Python dependencies
run: uv sync --all-packages
- name: Build and verify package artifacts
run: pnpm run artifacts:check
- name: Upload package artifacts
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ktx-package-artifacts-${{ github.sha }}
path: |

6
.github/workflows/release.yml vendored
View file

@ -24,12 +24,12 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: "24"
cache: "pnpm"
@ -44,7 +44,7 @@ jobs:
python-version: "3.13"
- name: Setup uv
uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
cache-dependency-glob: "uv.lock"

2
.gitignore vendored
View file

@ -43,6 +43,7 @@ yarn-error.log*
# Local project runtime state
.ktx/
**/.devtools/
*.db
*.sqlite
*.sqlite3
@ -64,3 +65,4 @@ yarn-error.log*
*.swo
*~
.vercel
.devtools

70
.pre-commit-config.yaml Normal file
View file

@ -0,0 +1,70 @@
# See https://pre-commit.com for hook documentation.
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v6.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-json
- id: check-toml
- id: check-added-large-files
args: ["--maxkb=1000"]
- id: check-merge-conflict
- id: check-case-conflict
- id: mixed-line-ending
- repo: https://github.com/asottile/pyupgrade
rev: v3.21.2
hooks:
- id: pyupgrade
name: pyupgrade (python)
files: ^python/
args: [--py313-plus]
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.2
hooks:
- id: ruff
name: ruff (python)
files: ^python/
args: [--fix, --exit-non-zero-on-fix]
- id: ruff-format
name: ruff format (python)
files: ^python/
- repo: local
hooks:
- id: ktx-package-checks
name: ktx package checks
entry: node scripts/precommit-check.mjs
language: system
files: ^(packages/|scripts/|python/|package\.json$|pnpm-lock\.yaml$|pnpm-workspace\.yaml$|release-policy\.json$|tsconfig\.base\.json$|pyproject\.toml$|uv\.lock$|uv\.toml$)
- repo: https://github.com/Yelp/detect-secrets
rev: v1.5.0
hooks:
- id: detect-secrets
exclude: |
(?x)^(
.*\.lock$|
.*pnpm-lock\.yaml$|
.*package-lock\.json$|
.*yarn\.lock$|
.*\.log$|
.*\.dump$|
.*\.sql$|
.*\.csv$|
.*\.db$|
.*\.sqlite$|
.*\.sqlite3$|
.*/node_modules/.*|
.*/\.venv/.*|
.*/dist/.*|
.*/build/.*|
.*/coverage/.*|
.*/htmlcov/.*|
.*\.gen\.ts$|
.*\.gen\.py$|
.*\.generated\.ts$
)$

13
AGENTS.md
View file

@ -156,6 +156,19 @@ pnpm run test 2>&1 | tee /tmp/ktx-test-output.log
- Do not manually edit generated or built output under `dist/`; edit source and
rebuild.
### CLI Standards
- Use Commander for CLI command trees, arguments, options, help text, custom
parsers, and async action dispatch. Prefer `@commander-js/extra-typings` for
typed command definitions, use `InvalidArgumentError` for parse failures, and
call `parseAsync` when actions await asynchronous work.
- Use `@clack/prompts` for interactive flows. Always handle cancellation with
`isCancel` plus `cancel`, stop active spinners before exiting, and keep prompts
grouped or factored so multi-step setup flows share cancellation behavior.
- Keep command behavior scriptable: prefer flags and config over prompts when
values are supplied, and reserve prompts for interactive missing input or
explicit setup flows.
### Zod Naming Convention
```typescript

368
README.md
View file

@ -1,9 +1,9 @@
<h1 align="center">
<img src="assets/ktx-readme-header.png" alt="KTX" width="472" />
<img src="assets/ktx-lockup.svg" alt="KTX" width="500" />
</h1>
<p align="center">
<strong>Workspace-first context layer for database agents</strong>
<strong>The context layer for analytics agents</strong>
</p>
<p align="center">
@ -14,26 +14,64 @@
---
KTX stores warehouse memory in a project directory, generates and validates
semantic-layer YAML, indexes knowledge, scans database schemas, and exposes the
result through a CLI and MCP server.
KTX turns warehouse metadata, semantic definitions, and business knowledge into
reviewable project files that agents can use while planning, querying, and
updating analytics work.
KTX projects are plain files: YAML, Markdown, SQLite state, and generated
artifacts. You can inspect them, commit them, and serve them to any MCP client.
A KTX project is a directory of plain files — YAML semantic sources, Markdown
knowledge pages, and SQLite state — that you commit to git and review in PRs,
just like dbt models.
## What KTX provides
## Who KTX is for
- Durable warehouse memory with semantic-layer sources and knowledge pages.
- Native scan connectors for SQLite, Postgres, MySQL, ClickHouse, SQL Server,
BigQuery, and Snowflake.
- Agentic ingest with provenance links, tool transcripts, and replay metadata.
- Local semantic-layer query planning and optional query execution.
- A stdio MCP server with tools for connections, knowledge, semantic-layer
sources, ingest reports, and replay.
KTX is built for analytics engineers and data teams who want data agents to
work on real analytics systems — not just generate one-off SQL.
Use KTX when you want agents to:
- **Generate SQL** from approved measures and joins
- **Repair semantic definitions** through reviewable diffs
- **Explain metric provenance** with warehouse evidence
- **Work alongside** dbt, LookML, MetricFlow, Looker, Metabase, and modern BI
platforms
Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, and
SQLite.
## Quick start
Run the pre-seeded demo through the public npm package:
Install the CLI and run the setup wizard:
```bash
npm install @kaelio/ktx
npm install -g @kaelio/ktx
ktx setup
```
The wizard walks through six steps: configuring your LLM provider, setting up
embeddings, connecting your database, adding context sources (dbt, LookML,
Metabase, Looker, Notion), building context, and installing agent integration.
If it exits before completion, rerun `ktx setup` to resume where you left off.
Check your project status:
```bash
ktx status
```
```
KTX project: /home/user/analytics
Project ready: yes
LLM ready: yes (claude-sonnet-4-6)
Embeddings ready: yes (text-embedding-3-small)
Primary sources configured: yes (postgres-warehouse)
Context sources configured: yes (dbt-main)
KTX context built: yes
Agent integration ready: yes (claude-code:project)
```
Run the packaged demo without installing globally:
```bash
npx @kaelio/ktx setup demo --no-input
@ -43,144 +81,21 @@ npx @kaelio/ktx setup demo inspect
The default demo uses packaged sample data and prebuilt context. It does not
require API keys, network access, or an LLM provider.
To replay the packaged ingest run, use:
Generate SQL from a semantic-layer source:
```bash
npx @kaelio/ktx setup demo --mode replay --no-input
```
To run the full agentic demo with an LLM provider, set a provider key for the
current process:
```bash
ANTHROPIC_API_KEY=$YOUR_ANTHROPIC_API_KEY \
npx @kaelio/ktx setup demo --mode full --no-input
```
Interactive full-demo setup can prompt for a provider key without writing the
key to `ktx.yaml`.
You can also install the CLI in a project or globally:
```bash
npm install @kaelio/ktx
npx ktx --help
npm install -g @kaelio/ktx
ktx --help
```
## Build a local project
Create a project from a local workspace:
```bash
npm install @kaelio/ktx
PROJECT_DIR="$(mktemp -d)/ktx-demo"
npx ktx init "$PROJECT_DIR" --name ktx-demo
```
Create a SQLite warehouse:
```bash
python - "$PROJECT_DIR/demo.db" <<'PY'
import sqlite3
import sys
conn = sqlite3.connect(sys.argv[1])
conn.executescript("""
DROP TABLE IF EXISTS accounts;
CREATE TABLE accounts (
account_id INTEGER PRIMARY KEY,
account_name TEXT NOT NULL,
segment TEXT NOT NULL,
region TEXT NOT NULL
);
INSERT INTO accounts VALUES
(1, 'Acme Analytics', 'Mid-Market', 'NA'),
(2, 'Beacon Bank', 'Enterprise', 'EMEA'),
(3, 'Cobalt Coffee', 'SMB', 'NA'),
(4, 'Delta Devices', 'Mid-Market', 'APAC'),
(5, 'Evergreen Energy', 'Enterprise', 'NA');
""")
conn.close()
PY
```
Replace the generated `ktx.yaml`:
```bash
cat > "$PROJECT_DIR/ktx.yaml" <<YAML
project: ktx-demo
connections:
warehouse:
driver: sqlite
path: $PROJECT_DIR/demo.db
readonly: true
storage:
state: sqlite
search: sqlite-fts5
git:
auto_commit: true
author: "ktx <ktx@example.com>"
memory:
auto_commit: true
YAML
```
Write and validate a semantic-layer source:
```bash
npx ktx sl write accounts --project-dir "$PROJECT_DIR" \
--connection-id warehouse --yaml 'name: accounts
table: accounts
description: CRM accounts with segmentation attributes.
grain:
- account_id
columns:
- name: account_id
type: number
- name: account_name
type: string
- name: segment
type: string
- name: region
type: string
measures:
- name: account_count
expr: count(account_id)
joins: []
'
npx ktx sl validate accounts --project-dir "$PROJECT_DIR" \
--connection-id warehouse
```
Generate SQL and execute the query:
```bash
npx ktx sl query --project-dir "$PROJECT_DIR" \
npx @kaelio/ktx sl query --project-dir "$PROJECT_DIR" \
--connection-id warehouse \
--measure accounts.account_count \
--dimension accounts.segment \
--order-by accounts.account_count:desc \
--limit 5 \
--format sql
npx ktx sl query --project-dir "$PROJECT_DIR" \
--connection-id warehouse \
--measure accounts.account_count \
--dimension accounts.segment \
--order-by accounts.account_count:desc \
--limit 5 \
--execute \
--max-rows 5
```
List and test the warehouse connection:
List and test a configured warehouse connection:
```bash
npx ktx connection list --project-dir "$PROJECT_DIR"
npx ktx connection test warehouse --project-dir "$PROJECT_DIR"
ktx connection list --project-dir "$PROJECT_DIR"
ktx connection test warehouse --project-dir "$PROJECT_DIR"
```
The connection test prints the configured driver and discovered table count:
@ -190,18 +105,44 @@ Driver: sqlite
Tables: 1
```
## What's in a project
```
my-project/
├── ktx.yaml # Project configuration
├── semantic-layer/
│ └── warehouse/
│ ├── orders.yaml # Semantic source definitions
│ ├── customers.yaml
│ └── order_items.yaml
├── knowledge/
│ ├── global/
│ │ ├── revenue.md # Business definitions and rules
│ │ └── segment-classification.md
│ └── user/
│ └── local/
├── raw-sources/
│ └── warehouse/
│ └── live-database/ # Scan artifacts and reports
└── .ktx/
└── db.sqlite # Local state (git-ignored)
```
Semantic sources and knowledge pages are committed to git. The `.ktx/` directory
holds ephemeral state and is git-ignored — delete it and KTX rebuilds on the
next run.
### Scan the demo warehouse
Scan artifacts are written under
`raw-sources/warehouse/live-database/<syncId>/` in the project directory.
```bash
SCAN_OUTPUT="$(npx ktx scan warehouse --project-dir "$PROJECT_DIR")"
SCAN_OUTPUT="$(ktx scan warehouse --project-dir "$PROJECT_DIR")"
printf '%s\n' "$SCAN_OUTPUT"
SCAN_RUN_ID="$(printf '%s\n' "$SCAN_OUTPUT" | awk '/^Run: / { print $2 }')"
npx ktx scan status --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID"
npx ktx scan report --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID"
ktx scan status --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID"
ktx scan report --project-dir "$PROJECT_DIR" "$SCAN_RUN_ID"
```
For non-SQLite drivers, prefer credential references such as `--url env:NAME`
@ -219,107 +160,118 @@ backed KTX commands. KTX doesn't download `uv` automatically; run
`ktx runtime doctor` if runtime installation fails:
```bash
npx ktx runtime install --yes
npx ktx runtime status
npx ktx runtime doctor
npx ktx runtime start
npx ktx runtime stop
npx ktx runtime prune --dry-run
npx ktx runtime prune --yes
ktx runtime install --yes
ktx runtime status
ktx runtime doctor
ktx runtime start
ktx runtime stop
ktx runtime prune --dry-run
ktx runtime prune --yes
```
Use `runtime prune --dry-run` to preview stale runtime directories from older
CLI versions. Add `--yes` to remove those stale directories after daemon
processes are stopped.
The release artifact manifest contains the public npm tarball and the bundled `kaelio-ktx`
runtime wheel. The `python/ktx-sl` and `python/ktx-daemon` directories remain
source packages for development, not public release artifacts.
Commands such as `npx @kaelio/ktx sl query ... --yes` can install the core
runtime lazily from the bundled wheel. Local embeddings remain lazy; prepare
them only when you select local `sentence-transformers` embeddings:
## Serve agents
KTX integrates with coding agents through CLI skills, an MCP server, or both.
The setup wizard configures this automatically — here's what each mode looks
like.
**CLI skills** — the agent calls `ktx` commands directly through a skill file
installed in your agent's config (e.g., `.claude/skills/ktx/SKILL.md`):
```bash
npx ktx runtime install --feature local-embeddings --yes
npx ktx runtime start --feature local-embeddings
ktx sl query --measure orders.revenue --dimension orders.status --format sql
ktx wiki search "revenue definition"
ktx sl validate orders
```
## Serve MCP
Start the stdio MCP server from the project directory:
**MCP server** — the agent calls KTX tools over the Model Context Protocol:
```bash
npx ktx serve --mcp stdio --project-dir "$PROJECT_DIR" \
ktx serve --mcp stdio \
--user-id local \
--semantic-compute \
--execute-queries \
--yes
```
The `--semantic-compute` flag uses the managed Python runtime when no explicit
semantic compute URL is provided. KTX starts or reuses the managed runtime as
needed.
This exposes tools for connections, knowledge search, semantic-layer sources,
validation, queries, ingestion, and replay. The `--semantic-compute` flag starts
the managed Python runtime for query planning automatically.
The MCP server exposes `connection_list`, `knowledge_search`,
The standalone MCP server exposes `connection_list`, `knowledge_search`,
`knowledge_read`, `knowledge_write`, `sl_list_sources`, `sl_read_source`,
`sl_write_source`, `sl_validate`, `sl_query`, `ingest_trigger`,
`ingest_status`, `ingest_report`, and `ingest_replay`.
Supported agents: Claude Code, Codex, Cursor, OpenCode, and any agent that
reads `.agents/` skills or MCP configuration.
## Workspace packages
- `packages/context`: core TypeScript context library.
- `packages/cli`: CLI wrapper over the context package.
- `packages/llm`: LLM and embedding provider helpers.
- `packages/connector-bigquery`: BigQuery scan connector.
- `packages/connector-clickhouse`: ClickHouse scan connector.
- `packages/connector-mysql`: MySQL scan connector.
- `packages/connector-postgres`: Postgres scan connector.
- `packages/connector-snowflake`: Snowflake scan connector.
- `packages/connector-sqlite`: SQLite scan connector.
- `packages/connector-sqlserver`: SQL Server scan connector.
- `python/ktx-sl`: semantic-layer engine.
- `python/ktx-daemon`: portable compute service for semantic-layer operations.
| Package | Purpose |
|---------|---------|
| `packages/cli` | CLI entry point |
| `packages/context` | Core context engine |
| `packages/llm` | LLM and embedding providers |
| `packages/connector-bigquery` | BigQuery scan connector |
| `packages/connector-clickhouse` | ClickHouse scan connector |
| `packages/connector-mysql` | MySQL scan connector |
| `packages/connector-postgres` | Postgres scan connector |
| `packages/connector-snowflake` | Snowflake scan connector |
| `packages/connector-sqlite` | SQLite scan connector |
| `packages/connector-sqlserver` | SQL Server scan connector |
| `python/ktx-sl` | Semantic-layer query planning |
| `python/ktx-daemon` | Portable compute service |
## Development
Install dependencies and run checks:
```bash
git clone https://github.com/kaelio/ktx.git
cd ktx
pnpm install
uv sync --all-groups
pnpm run build
pnpm run check
uv sync --all-packages
source .venv/bin/activate
uv run pytest
```
Use the optional development binary when you want a local `ktx-dev` command:
Use the development CLI for local testing:
```bash
pnpm run setup:dev
pnpm run link:dev
ktx-dev --help
```
The repository uses `pnpm` for TypeScript packages and `uv` for Python
packages.
### Debug LLM traces
## Release status
This repository builds one public npm artifact named `@kaelio/ktx`. The release
artifact manifest contains the public npm tarball and the bundled `kaelio-ktx`
runtime wheel. The first public npm handoff is policy-gated through
`release-policy.json`, which keeps Python package publishing disabled because
KTX-owned Python code ships inside the npm package as a bundled wheel. The
`python/ktx-sl` and `python/ktx-daemon` directories remain source packages for
development, not public release artifacts.
Build local package artifacts and verify the guarded dry-run publish path with:
KTX can capture local AI SDK DevTools traces for LLM calls that run through the
KTX provider. Enable it with an environment flag when running an LLM-backed
command:
```bash
source .venv/bin/activate
pnpm run artifacts:check
pnpm run release:readiness
pnpm run release:npm-publish
KTX_AI_DEVTOOLS_ENABLED=true ktx dev ingest run \
--connection-id warehouse \
--adapter metabase
```
Run the live npm publish only from the manual `KTX Release` workflow with the
`publish_live` input enabled after the `NPM_TOKEN` secret is configured.
Traces are written to `.devtools/generations.json` under the current working
directory. To inspect them, run:
```bash
pnpm dlx @ai-sdk/devtools
```
Then open `http://localhost:4983`. These traces are local-development-only and
store prompts, model outputs, tool arguments/results, and raw provider payloads
in plain text. Do not enable this in production or for sensitive runs.
The repository uses `pnpm` for TypeScript packages and `uv` for Python
packages. See [Contributing](docs-site/content/docs/community/contributing.mdx)
for full development setup, testing, and PR guidelines.
## License

32
assets/ktx-lockup.svg Normal file
View file

@ -0,0 +1,32 @@
<svg viewBox="0 0 500 200" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="ktx">
<!-- mascot -->
<g fill="none" stroke="#1B3139" stroke-width="16" stroke-linecap="round">
<path d="M 62 110 Q 32 130 44 152" />
<path d="M 88 116 Q 80 152 70 174" />
<path d="M 112 116 Q 120 152 130 174" />
</g>
<path
d="M 134 108 C 162 116, 172 96, 162 78 C 154 64, 168 56, 178 60"
fill="none" stroke="#FF8A4C" stroke-width="16" stroke-linecap="round"
/>
<path
d="M 48 102 C 48 56, 78 30, 100 30 C 122 30, 152 56, 152 102 C 152 116, 132 120, 100 120 C 68 120, 48 116, 48 102 Z"
fill="#1B3139"
/>
<path d="M 80 84 Q 86 77 92 84" fill="none" stroke="#F5F1EA" stroke-width="3.5" stroke-linecap="round" />
<path d="M 108 84 Q 114 77 120 84" fill="none" stroke="#F5F1EA" stroke-width="3.5" stroke-linecap="round" />
<!-- wordmark: 'ktx', half the logo height, vertically centered -->
<text
x="225"
y="145"
font-family="'JetBrains Mono', 'Fira Code', ui-monospace, 'SF Mono', Menlo, monospace"
font-size="140"
font-weight="600"
fill="#1B3139"
letter-spacing="-0.04em"
>ktx</text>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 1.1 KiB

BIN
assets/ktx-readme-header.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 28 KiB

24
docs-site/app/docs/[[...slug]]/page.tsx
View file

@ -17,6 +17,10 @@ function isDocsIndex(slug: string[] | undefined) {
return slug === undefined || slug.length === 0 || slug.join("/") === "";
}
function isHeroPage(slug: string[] | undefined) {
return slug?.join("/") === "getting-started/introduction";
}
export default async function Page(props: {
params: Promise<{ slug?: string[] }>;
}) {
@ -30,14 +34,22 @@ export default async function Page(props: {
const MDX = page.data.body;
const hero = isHeroPage(params.slug);
return (
<DocsPage toc={page.data.toc}>
<DocsTitle>{page.data.title}</DocsTitle>
<DocsDescription>{page.data.description}</DocsDescription>
<DocsPageActions
markdownUrl={`${page.url}.md`}
mdxSource={page.data.content}
/>
{!hero && (
<>
<div className="flex items-start justify-between gap-4">
<DocsTitle>{page.data.title}</DocsTitle>
<DocsPageActions
markdownUrl={`${page.url}.md`}
mdxSource={page.data.content}
/>
</div>
<DocsDescription>{page.data.description}</DocsDescription>
</>
)}
<DocsBody>
<MDX components={{ ...defaultMdxComponents, pre: CodeBlock }} />
</DocsBody>

79
docs-site/app/global.css
View file

@ -262,6 +262,74 @@ figure[data-rehype-pretty-code-figure]:has(.ktx-code) {
color: #c8c3bc !important;
}
/* ── Mode D: Output preview (wizard prompts, status output) ── */
.ktx-code-output {
background: var(--color-fd-muted);
border: 1px solid var(--color-fd-border);
border-left: 3px solid color-mix(in oklch, var(--color-fd-primary) 50%, var(--color-fd-border));
position: relative;
box-shadow: 0 1px 2px rgba(27, 27, 24, 0.02);
}
.dark .ktx-code-output {
background: #111a1e;
border-color: rgba(255, 255, 255, 0.05);
border-left-color: rgba(34, 211, 238, 0.25);
}
.ktx-code-output:hover {
border-color: color-mix(in oklch, var(--color-fd-primary) 25%, var(--color-fd-border));
border-left-color: var(--color-fd-primary);
}
.dark .ktx-code-output:hover {
border-color: rgba(255, 255, 255, 0.08);
border-left-color: rgba(34, 211, 238, 0.45);
}
.ktx-code-output-label {
position: absolute;
top: 8px;
right: 14px;
font-size: 10px;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.05em;
color: var(--color-fd-muted-foreground);
font-family: var(--font-display), var(--font-sans), sans-serif;
opacity: 0.4;
pointer-events: none;
z-index: 1;
}
.ktx-code-output-copy {
position: absolute !important;
top: 6px !important;
right: 6px !important;
opacity: 0;
transform: translateY(-4px);
transition: opacity 0.2s var(--ktx-ease), transform 0.2s var(--ktx-ease);
z-index: 2;
}
.ktx-code-output:hover .ktx-code-output-copy {
opacity: 0.5;
transform: translateY(0);
}
.ktx-code-output:hover .ktx-code-output-label {
opacity: 0;
}
.ktx-code-body-output {
background: transparent !important;
color: var(--ktx-ink-soft) !important;
}
.dark .ktx-code-body-output {
color: #8a9da6 !important;
}
/* ── Mode B: VS Code tab (filename) ───────── */
.ktx-code-tab {
background: var(--color-fd-card);
@ -495,14 +563,20 @@ th {
opacity: 0.7;
}
/* Hide the vertical indicator lines in sidebar sections */
#nd-sidebar div[data-state]::before,
#nd-sidebar a[data-active]::before {
content: none !important;
display: none !important;
}
/* Page link items */
#nd-sidebar a[data-active] {
font-size: 14px;
padding: 6px 12px;
border-radius: 6px;
margin-left: 0;
border-left: 2px solid transparent;
transition: background 0.15s ease, color 0.15s ease, border-color 0.15s ease;
transition: background 0.15s ease, color 0.15s ease;
}
#nd-sidebar a[data-active="false"]:hover {
@ -512,7 +586,6 @@ th {
#nd-sidebar a[data-active="true"] {
background: color-mix(in oklch, var(--color-fd-primary) 8%, transparent) !important;
border-left-color: var(--color-fd-primary) !important;
color: var(--color-fd-primary) !important;
font-weight: 500;
}

20
docs-site/components/code-block.tsx
View file

@ -52,12 +52,11 @@ export function CodeBlock(props: Props) {
const language = detectLanguage(props, children);
const codeText = extractText(children);
const isTerminal =
(language !== null && TERMINAL_LANGS.has(language)) ||
WIZARD_GLYPHS.test(codeText);
const isTerminal = language !== null && TERMINAL_LANGS.has(language);
const isOutput = !isTerminal && WIZARD_GLYPHS.test(codeText);
const hasTitle = typeof title === "string" && title.length > 0;
// Mode A — Terminal
// Mode A — Terminal (commands the user types)
if (isTerminal) {
return (
<div className="not-prose ktx-code ktx-code-terminal group">
@ -80,6 +79,19 @@ export function CodeBlock(props: Props) {
);
}
// Mode D — Output preview (wizard prompts, terminal output)
if (isOutput) {
return (
<div className="not-prose ktx-code ktx-code-output group relative">
<span className="ktx-code-output-label">output</span>
<CopyButton text={codeText} className="ktx-code-output-copy" />
<pre {...rest} className="ktx-code-body ktx-code-body-output">
{children}
</pre>
</div>
);
}
// Mode B — VS Code tab (filename present)
if (hasTitle) {
return (

2
docs-site/components/docs-page-actions.tsx
View file

@ -11,7 +11,7 @@ type Props = {
export function DocsPageActions({ markdownUrl, mdxSource }: Props) {
return (
<div className="not-prose mt-4 mb-8 flex flex-wrap items-center gap-2 border-b border-fd-border pb-6 text-xs">
<div className="not-prose flex flex-wrap items-center gap-2 text-xs">
<CopyMarkdownButton markdownUrl={markdownUrl} />
<a
href={markdownUrl}

163
docs-site/content/docs/benchmarks/link-detection.mdx
View file

@ -1,163 +0,0 @@
---
title: Link Detection
description: How KTX's relationship detection performs on real-world schemas.
---
KTX infers foreign key relationships between tables even when the database declares no primary keys or foreign key constraints. This is critical for analytics warehouses, where constraints are rarely enforced. This page documents the methodology, scoring pipeline, and a reproducible benchmark you can run yourself.
## Agent usage notes
Use this page when an agent needs to explain, tune, or verify relationship detection.
| Agent task | Relevant section | Command |
|------------|------------------|---------|
| Explain why KTX inferred a join | Detection pipeline | `ktx dev scan relationships <run-id> --status all` |
| Decide whether to accept or reject a candidate | Scoring and threshold configuration | `ktx dev scan relationships <run-id> --accept <candidate-id>` |
| Tune thresholds from reviewed decisions | Broader benchmark suite and calibration | `ktx dev scan relationship-thresholds --connection <connection-id>` |
| Reproduce the bundled benchmark | Reproducing the benchmark | `pnpm run relationships:verify-orbit` |
## What this measures
Most analytics warehouses — Snowflake, BigQuery, Redshift — don't enforce referential integrity constraints. Tables like `fct_product_events` reference `dim_accounts` by convention (`account_id` → `id`), but nothing in the schema says so.
KTX's relationship detection discovers these links automatically. The benchmark measures how accurately it recovers known foreign key relationships from a schema with **all declared constraints removed** — the hardest operating mode.
Metrics tracked:
- **Accepted** — relationships scored above the accept threshold (default 0.85) and written to the project manifest
- **Review** — relationships scored between the review threshold (0.55) and accept threshold, flagged for human review
- **Rejected** — relationships scored below the review threshold
- **Skipped** — relationships not evaluated (e.g., filtered by candidate limits)
## Methodology
### Detection pipeline
Relationship detection runs as a multi-stage pipeline during `ktx dev scan`:
1. **Candidate generation** — scans the schema for potential FK relationships using multiple heuristics: exact column name matches, normalized table name matching, name inflection (singular/plural), column suffix patterns (`_id`, `_key`, `_code`, `_uuid`), self-references (`parent_id`, `manager_id`), and optionally embedding similarity and LLM proposals.
2. **Column profiling** — samples up to 10,000 rows per column (configurable via `profile_sample_rows`) to collect statistics: row counts, null rates, distinct value counts, uniqueness ratios, sample values, and text length ranges.
3. **Validation** — tests each candidate relationship against actual data by measuring target uniqueness, source coverage, violation ratio, and value overlap between child and parent columns.
4. **Scoring** — combines 7 weighted signals into a confidence score:
| Signal | Weight | What it captures |
|--------|--------|-----------------|
| Name similarity | 0.24 | How closely column/table names match FK conventions |
| Value overlap | 0.22 | What percentage of FK values exist in the PK column |
| Profile uniqueness | 0.22 | How unique the target column values are |
| Type compatibility | 0.10 | Whether data types are compatible (hard gate — score is 0 if incompatible) |
| Embedding similarity | 0.10 | Semantic similarity between column names |
| Profile null rate | 0.08 | Presence of non-null values |
| Structural prior | 0.04 | Baseline structural hints from schema conventions |
Each signal is normalized to \[0, 1\], multiplied by its weight, and summed. The final confidence is `0.56 + (weighted_sum × 0.65)`, clamped to \[0, 1\].
5. **Graph resolution** — resolves conflicts when multiple candidates target the same column, detects primary keys (by name pattern and validation), and classifies each relationship into `accepted`, `review`, or `rejected` based on thresholds.
### Threshold configuration
```yaml
scan:
relationships:
accept_threshold: 0.85
review_threshold: 0.55
```
Relationships scoring above `accept_threshold` are automatically accepted into the project manifest. Those between `review_threshold` and `accept_threshold` are flagged for analyst review. Below `review_threshold`, they're rejected.
### Test fixture
The benchmark uses the **Orbit-style product warehouse** — a synthetic schema modeled after a real SaaS analytics warehouse with all declared constraints removed. The fixture is a SQLite database with 6 tables:
| Table | Role | Estimated rows |
|-------|------|---------------|
| `dim_accounts` | Dimension | 3 |
| `dim_users` | Dimension | 4 |
| `dim_workspaces` | Dimension | 4 |
| `fct_product_events` | Fact | 5 |
| `fct_invoices` | Fact | 3 |
| `support_tickets` | Fact | 4 |
**Ground truth:** 6 primary keys (one `id` column per table) and 9 foreign key relationships, all `many_to_one`:
| Source column | Target |
|--------------|--------|
| `dim_users.account_id` | `dim_accounts.id` |
| `dim_workspaces.account_id` | `dim_accounts.id` |
| `dim_workspaces.user_id` | `dim_users.id` |
| `fct_product_events.account_id` | `dim_accounts.id` |
| `fct_product_events.user_id` | `dim_users.id` |
| `fct_product_events.workspace_id` | `dim_workspaces.id` |
| `fct_invoices.account_id` | `dim_accounts.id` |
| `support_tickets.account_id` | `dim_accounts.id` |
| `support_tickets.user_id` | `dim_users.id` |
The fixture runs in multiple modes to isolate the contribution of each pipeline stage: with LLM disabled, profiling disabled, validation disabled, and embeddings disabled.
## Results
Results for the default configuration will be added after the benchmark run is finalized.
## Reproducing the benchmark
### Prerequisites
- Node.js 22+
- pnpm
- The KTX repository cloned and dependencies installed (`pnpm install`)
### Running
From the repository root:
```bash
pnpm run relationships:verify-orbit
```
This runs `ktx dev scan` against the bundled SQLite fixture with enrichment disabled, then generates a verification report at:
```text
examples/orbit-relationship-verification/reports/orbit-verification.md
```
The report includes the full relationship summary, enrichment details, artifact paths, and any warnings.
### Custom project
To run verification against your own database (e.g., a local Orbit project):
```bash
KTX_ORBIT_PROJECT_DIR=/path/to/your-project pnpm run relationships:verify-orbit
```
### Configuration
The benchmark project configuration lives at `examples/orbit-relationship-verification/ktx.yaml`:
```yaml
scan:
enrichment:
backend: none
relationships:
enabled: true
llm_proposals: false
accept_threshold: 0.85
review_threshold: 0.55
profile_sample_rows: 10000
validation_concurrency: 4
```
Adjust `accept_threshold` and `review_threshold` to see how threshold changes affect the accepted/review/rejected distribution. Lower thresholds accept more relationships (higher recall, lower precision); higher thresholds are more conservative.
## Broader benchmark suite
Beyond the Orbit fixture, KTX includes a full benchmark corpus at `packages/context/test/fixtures/relationship-benchmarks/` with fixtures across multiple tiers:
- **Unit** — minimal schemas testing individual heuristics
- **Row-bearing** — small schemas with data for validation testing
- **Product** — full warehouse schemas like the Orbit fixture
Fixtures from public datasets (Chinook, Sakila, AdventureWorks, Northwind) supplement the synthetic fixtures. The benchmark runner measures precision, recall, and F1 for both primary key and foreign key detection across all fixtures and modes.

5
docs-site/content/docs/benchmarks/meta.json
View file

@ -1,5 +0,0 @@
{
"title": "Benchmarks",
"defaultOpen": true,
"pages": ["link-detection"]
}

2
docs-site/content/docs/cli-reference/ktx-serve.mdx
View file

@ -68,7 +68,7 @@ The MCP server is typically configured through `ktx setup --agents` rather than
| Error | Cause | Recovery |
|-------|-------|----------|
| Agent cannot start server | The agent config cannot find the `ktx` binary | Run `pnpm run link:dev` or use an absolute command path in the agent config |
| Agent cannot start server | The agent config cannot find the `ktx` binary | Install `@kaelio/ktx` globally with `npm install -g @kaelio/ktx` or use an absolute command path in the agent config |
| Semantic tools are unavailable | Server was started without `--semantic-compute` | Add `--semantic-compute` or `--semantic-compute-url` to the server args |
| Query execution is denied | Server was started without `--execute-queries` | Add `--execute-queries` only for trusted projects where read-only execution is intended |
| Context resolves to wrong project | `KTX_PROJECT_DIR` is missing or points elsewhere | Set `KTX_PROJECT_DIR` to the project containing the intended `ktx.yaml` |

9
docs-site/content/docs/community/contributing.mdx
View file

@ -7,6 +7,11 @@ KTX is an open-source project and welcomes contributions — bug fixes, new conn
## Development setup
This page is for contributors working on the KTX repository. To install KTX for
an analytics project, use the published
[`@kaelio/ktx`](https://www.npmjs.com/package/@kaelio/ktx) package in the
[Quickstart](/docs/getting-started/quickstart).
### Prerequisites
- **Node.js 22+** and **pnpm** — for the TypeScript workspace
@ -44,7 +49,9 @@ pnpm run setup:dev
pnpm run link:dev
```
This makes the `ktx` command available globally, pointing at your local build.
This makes the `ktx-dev` command available globally, pointing at your local
build. Use this development binary when you need to test unpublished repository
changes.
## Repository structure

56
docs-site/content/docs/concepts/context-as-code.mdx
View file

@ -29,43 +29,51 @@ This reconciliation step is what separates auto-ingestion from a simple sync. A
Auto-ingestion is designed to plug into a PR-based workflow. Run ingestion on a branch, review the changed YAML and Markdown files, and merge them the same way you merge dbt models or application code.
```
dbt / Looker / Metabase KTX project repo
┌──────────────┐ ┌──────────────────────┐
│ Metadata │───ingestion──▶│ Branch: ingest/... │
│ changes │ │ │
└──────────────┘ │ + 3 new sources │
│ ~ 2 updated joins │
│ + 1 knowledge page │
│ │
│ ──── Open PR ──── │
│ │
│ Review semantic diff │
│ Approve & merge │
└──────────────────────┘
│
▼
Agents see updated
context immediately
```text
dbt / Looker / Metabase / Notion
|
v
metadata changes
|
v
nightly cron or CI ingest
|
v
branch: ingest/nightly
|
| + 3 new sources
| ~ 2 updated joins
| + 1 knowledge page
v
open PR
|
v
review semantic diff
|
v
approve & merge
|
v
agents see updated context
```
A typical branch shows a semantic diff: "this ingest added 3 new sources from dbt, updated 2 join definitions based on schema changes, and created 1 knowledge page from a Notion doc." Analytics engineers review the diff, verify that the new sources look correct, and merge.
Teams usually run this on demand while setting up a source, then schedule it once the source is stable. A cron job or CI schedule can run `ktx ingest --all --no-input` overnight on an ingest branch so the latest dbt manifests, BI metadata, and documentation updates are ready for review each morning.
Once merged, agents querying through KTX's MCP server or CLI see the updated context immediately. No deployment step, no cache invalidation, no restart. The files are the source of truth, and agents read them on every request.
This workflow gives you the same review guarantees you have for dbt models. No semantic source reaches production without a human approving it. But unlike maintaining context manually, the heavy lifting — discovering new tables, drafting source definitions, extracting business rules from documentation — is done by the ingestion agent. You review and approve. You don't write from scratch.
## Feedback loops
Context improves over time through three feedback channels.
Context improves over time through two feedback channels.
**Analyst corrections.** When an analytics engineer spots something wrong — a measure formula that doesn't match the business definition, a join that should be `many_to_one` instead of `one_to_many`, a knowledge page that's out of date — they edit the YAML or Markdown directly and commit. These corrections become part of the project's git history, and the next ingestion run respects them. If you manually fix a measure definition, KTX won't overwrite it on the next ingest.
**Agent feedback.** When an agent queries the semantic layer and gets unexpected results — a query that returns no rows because of a bad filter, a join path that produces duplicated results — it can flag the issue. These signals feed back into the context: knowledge pages can note known data quality issues, source definitions can be tightened with better filters or grain declarations, and relationship thresholds can be adjusted.
**Agent feedback.** When an agent queries the semantic layer and gets unexpected results — a query that returns no rows because of a bad filter, a join path that produces duplicated results — it can flag the issue. These signals feed back into the context: knowledge pages can note known data quality issues, and source definitions can be tightened with better filters, join paths, or grain declarations.
**Relationship calibration.** KTX infers foreign key relationships between tables automatically, even when the database has no declared constraints. It does this by analyzing column names, types, value distributions, and asking the LLM for proposals. Each inferred relationship gets a confidence score. You control two thresholds: `acceptThreshold` (relationships above this score are accepted automatically, default 0.85) and `reviewThreshold` (relationships between review and accept are flagged for human review, default 0.55). As you accept or reject proposals, the system learns which patterns match your schema conventions.
Each of these channels makes the next ingestion cycle better. Analyst corrections teach the system what your team considers authoritative. Agent feedback surfaces gaps in coverage. Relationship calibration tunes the discovery process to your warehouse's conventions. Context is not a static artifact — it's a living system that converges toward accuracy with every iteration.
Each of these channels makes the next ingestion cycle better. Analyst corrections teach the system what your team considers authoritative. Agent feedback surfaces gaps in coverage. Context is not a static artifact — it's a living system that converges toward accuracy with every iteration.
## Deterministic replay
@ -89,5 +97,5 @@ Use this page when an agent needs to explain review workflows, ingestion diffs,
|------------|------------------|-----------|
| Explain how generated context should be reviewed | The git workflow | [Building Context](/docs/guides/building-context) |
| Diagnose why ingestion changed a semantic source | Auto-ingestion and Deterministic replay | [ktx ingest](/docs/cli-reference/ktx-ingest) |
| Explain how context improves over time | Feedback loops | [Link Detection](/docs/benchmarks/link-detection) |
| Explain how context improves over time | Feedback loops | [Building Context](/docs/guides/building-context) |
| Tell a user what to commit | The git workflow | [Writing Context](/docs/guides/writing-context) |

30
docs-site/content/docs/concepts/the-context-layer.mdx
View file

@ -9,7 +9,7 @@ Give an agent access to your database and it will generate SQL. It might even pr
The agent doesn't know that `orders.amount` includes refunds and needs a status filter. It doesn't know that `customers` should join to `orders` on `customer_id`, not `id`. It doesn't know that your team stopped using `legacy_segments` six months ago, or that "enterprise" means contracts over $100k, not just big logos. It sees column names and types. It doesn't see your business.
This isn't a model capability problem. GPT-4, Claude, and Gemini can all write correct SQL — when they know what correct means. The gap is context: which tables matter, which joins are valid, which metrics are canonical, what the business terms actually refer to. Without that, agents produce plausible-looking artifacts that are subtly, dangerously wrong. Wrong enough to pass a glance, wrong enough to drive a decision.
This isn't a model capability problem. Claude Code, Codex, and your BI agents can write correct SQL when they know what correct means. The gap is context: which tables matter, which joins are valid, which metrics are canonical, what the business terms actually refer to. Without that, agents produce plausible-looking artifacts that are subtly, dangerously wrong. Wrong enough to pass a glance, wrong enough to drive a decision.
Analytics engineers already know this pain. It's the same reason you write dbt tests, maintain a data dictionary, and spend half of standup explaining why someone's dashboard number doesn't match the board deck. The difference is that agents make decisions at machine speed, so the wrong context propagates faster than a human can catch it.
@ -19,9 +19,9 @@ The industry has moved through three distinct approaches to getting AI and data
**Wave one: database access.** Connect an LLM to a database, let it generate SQL. This works for simple lookups — "how many orders last week?" — but breaks on anything that requires business knowledge. The agent guesses at joins, invents metrics, and hallucinates table relationships. Every query is a coin flip.
**Wave two: semantic layers and text-to-SQL.** Add structure. Define metrics in MetricFlow or Cube, expose schemas, build text-to-SQL pipelines. This is better — the agent knows that `revenue` means `sum(amount) where status != 'refunded'` — but it's still limited. Semantic layers define what to calculate, not why, when, or how to interpret the result. The agent can compute net revenue but doesn't know about the February refund anomaly, the segment reclassification, or the fact that `enterprise` changed definition last quarter.
**Wave two: semantic layers and text-to-SQL.** Add structure. Define metrics in MetricFlow or Cube, expose schemas, build text-to-SQL pipelines. This is better — the agent knows that `revenue` means `sum(amount) where status != 'refunded'` — but building and maintaining that structure by hand is manual, time-consuming, and still limited. Semantic layers define what to calculate, not why, when, or how to interpret the result. The agent can compute net revenue but doesn't know about the February refund anomaly, the segment reclassification, or the fact that `enterprise` changed definition last quarter.
**Wave three: agentic context.** AI is no longer just answering questions — it's generating dashboards, writing semantic definitions, proposing dbt models, creating tests and documentation. For that to work, agents need more than metric definitions. They need the full picture: business rules, data quality gotchas, relationship maps, historical context, and the institutional knowledge that lives in your team's heads. They need a context layer.
**Wave three: agentic context.** AI is no longer just answering questions — it's generating dashboards, writing semantic definitions, proposing dbt models, creating tests and documentation. For that to work, agents need more than metric definitions. They need the full picture: business rules, known data quality issues, relationship maps, historical context, and the institutional knowledge that lives in your team's heads. They need a context layer.
## What a context layer is
@ -29,6 +29,13 @@ A context layer is the infrastructure that gives agents the business knowledge t
KTX organizes context into four pillars:
- Semantic sources
- Knowledge pages
- Scan artifacts
- Provenance
Each pillar covers a different kind of context agents need before they can safely write SQL, update semantic definitions, or explain an analytics result.
**Semantic sources** are YAML definitions that describe your data in terms agents can reason about. Each source maps to a table or SQL query, declares its grain, defines typed columns, specifies valid joins, and exposes named measures with optional filters. This is where "revenue means `sum(amount)` excluding refunds" lives.
```yaml
@ -60,7 +67,7 @@ measures:
expr: count(id)
```
**Knowledge pages** are Markdown documents that capture business definitions, rules, and gotchas — the kind of context that doesn't fit in a schema definition. Pages have structured frontmatter (summary, tags, semantic layer references) and free-form content. Agents search them when they need to understand why a metric works a certain way, not just how to compute it.
**Knowledge pages** are Markdown documents that capture business definitions, rules, and operating context — the kind of context that doesn't fit in a schema definition. Pages have structured frontmatter (summary, tags, semantic layer references) and free-form content. Agents search them when they need to understand why a metric works a certain way, not just how to compute it.
```markdown
---
@ -90,13 +97,12 @@ Together, these four pillars give agents enough context to produce analytics art
## How KTX compares
KTX is a context layer, and its structured core is an agent-native semantic layer. That matters. MetricFlow, Cube, and Malloy all give teams ways to model metrics, dimensions, joins, and generated SQL. KTX covers that same semantic-layer job, then adds the surrounding context agents need to use it well: knowledge pages, schema scans, provenance, ingestion, validation, and MCP tools.
KTX is a context layer with an agent-native semantic layer at its core. MetricFlow, Cube, and Malloy model metrics, dimensions, joins, and generated SQL. KTX covers that semantic-layer work, then adds the context agents need to use and maintain it: knowledge pages, schema scans, provenance, ingestion, validation, and MCP tools.
The primary user is different. MetricFlow is centered on dbt-style metric definitions. Cube is centered on a governed semantic runtime for BI, applications, and agents. Malloy is centered on an expressive modeling and query language. KTX is centered on agents that need to read a semantic model, change it, validate it, inspect the generated SQL, and leave a reviewable git diff.
The workflow is the difference. Traditional semantic layers are powerful, but they are usually built and maintained through manual modeling work, product-specific runtimes, or language-specific workflows. They are not agent-native by default, which makes them harder for agents to inspect, edit, validate, and review in a tight loop. KTX is designed for agents that need to read context, change semantic files, inspect generated SQL, and leave a reviewable git diff.
| | KTX semantic layer | MetricFlow | Cube | Malloy |
|---|---|---|---|---|
| **Design center** | Agent-native semantic modeling inside a broader context layer | Metric definitions and dbt semantic models | Governed serving layer for BI, embedded analytics, APIs, and agents | Semantic modeling and analytical query language |
| **Model surface** | Plain YAML sources plus Markdown knowledge pages | YAML semantic models and metrics in a dbt project | YAML or JavaScript cubes, views, access policies, and pre-aggregations | `.malloy` models, query pipelines, notebooks, and annotations |
| **What it models** | Sources, columns, measures, segments, joins, grain, filters, default time dimensions, and context references | Semantic models, entities, dimensions, measures, metrics, time grains, and metric types | Cubes, views, measures, dimensions, segments, joins, hierarchies, policies, and rollups | Sources, joins, dimensions, measures, calculations, nested results, and query pipelines |
| **Agent edit loop** | First-class. Agents can patch small files, save imperfect drafts, run validation, query through MCP, inspect SQL, and refine in the same workflow | Possible, but the interface is a dbt/metric workflow rather than an agent context workflow | Possible through code-first models and platform APIs, but changes are tied to runtime deployment and governance concerns | Possible, but agents must operate in Malloy's language and compiler model |
@ -105,15 +111,7 @@ The primary user is different. MetricFlow is centered on dbt-style metric defini
| **Context around semantics** | Built in: wiki pages, scan artifacts, relationship inference, ingest transcripts, replay, and agent-facing MCP tools | Primarily metric and dbt project context | Descriptions and `meta.ai_context` inside the semantic model, plus platform agent features | Annotations/tags can carry metadata; surrounding context depends on the application |
| **Best fit** | Agents maintaining analytics code, metrics, joins, SQL, docs, and semantic definitions | Teams standardizing metrics inside dbt workflows | Production semantic APIs, BI integrations, access control, caching, and concurrency | Expressive modeling and exploratory analysis above SQL |
**Agent-native by design.** KTX's advantage is not just that the files are YAML. The whole loop is shaped for agents: sources are small, overlays can add measures or computed columns without copying entire generated schemas, writes are permissive so an agent can save a draft, and validation/query tools give immediate feedback. An agent can move from "this metric is wrong" to "here is the semantic diff, generated SQL, and supporting context" without leaving the project.
**A semantic layer plus the context to use it.** Traditional semantic layers define what to calculate. KTX also stores why the definition exists, where it came from, what schema evidence supports it, and what an agent did when it changed. A measure can live next to a knowledge page about exclusions, a scan artifact that proves the join path, and an ingest transcript that explains the source of the definition. That is the difference between giving an agent a metric catalog and giving it operational memory.
**Fan-out handling is explicit and reviewable.** KTX asks model authors and agents to declare grain and relationship direction. The planner uses that metadata to avoid silent row multiplication: it detects `one_to_many` fan-out paths, separates independent fact measures into aggregate-locality CTEs, and refuses filters that would be unsafe to apply after pre-aggregation. Cube, MetricFlow, and Malloy all have strong approaches to this class of problem, but KTX's approach is deliberately inspectable in the files and in the generated plan.
**Where other systems are stronger.** KTX draws a clear product boundary around agent-native context and semantic modeling. Cube is stronger when you need a production semantic API with access policies, pre-aggregations, refresh workers, and high-concurrency serving. MetricFlow is stronger when your primary workflow is dbt-native metric standardization. Malloy is stronger when you want a full analytical language with nested query shapes. KTX is strongest when the semantic layer is the substrate agents will read, edit, validate, and extend as part of day-to-day analytics engineering.
**When KTX replaces your semantic layer vs. works beside it.** If you do not have a semantic layer, KTX can build an agent-native one from your database schema and enrich it with generated descriptions and knowledge pages. If you already use MetricFlow, LookML, Looker, Metabase, dbt, or Notion, KTX ingests from those tools and merges their context into KTX's files. You can keep your existing BI or metric-serving system while using KTX as the semantic and contextual surface agents work against.
If you do not have a semantic layer, KTX can build an agent-native one from your database schema and enrich it with generated descriptions and knowledge pages. If you already use MetricFlow or LookML, KTX ingests from those tools and merges their context into KTX's files. You can keep your existing BI or metric-serving system while using KTX as the semantic and contextual surface agents work against.
## The plain-files philosophy

97
docs-site/content/docs/getting-started/introduction.mdx
View file

@ -1,71 +1,86 @@
---
title: Introduction
description: What KTX is and who it's for.
description: How KTX gives analytics agents trusted context for warehouse work.
---
Data agents can write SQL. The hard part is making sure they write the SQL your analytics team would have written.
KTX is the agent-native context layer for analytics engineering. At its core is a semantic layer: YAML sources that define tables, columns, measures, joins, grain, filters, segments, and computed fields. Around that core, KTX adds the context analytics agents need to work safely: warehouse scans, knowledge pages, ingestion from existing tools, provenance, validation, and MCP access.
KTX projects are plain files — YAML, Markdown, and SQLite — that you commit to git and review in PRs, just like dbt models. Agents can read them, edit them, validate them, query through them, and leave behind a diff your team can review.
<div className="not-prose mb-14">
<div className="mb-8">
<h1
className="text-4xl font-extrabold tracking-tight lg:text-5xl"
style={{
fontFamily: 'var(--font-display)',
background: 'linear-gradient(180deg, var(--color-fd-foreground) 0%, color-mix(in oklch, var(--color-fd-foreground) 75%, var(--color-fd-primary)) 100%)',
WebkitBackgroundClip: 'text',
backgroundClip: 'text',
color: 'transparent',
WebkitTextFillColor: 'transparent',
lineHeight: '1.1',
letterSpacing: '0',
}}
>
Make analytics context{'\n'}usable by agents
</h1>
<p className="mt-4 text-lg text-fd-muted-foreground max-w-2xl" style={{ lineHeight: '1.7' }}>
KTX turns warehouse metadata, semantic definitions, and business knowledge
into reviewable project files that agents can use while planning, querying,
and updating analytics work.
</p>
</div>
<div className="flex flex-wrap gap-3">
<a
href="/docs/getting-started/quickstart"
className="inline-flex h-10 items-center rounded-lg bg-fd-primary px-5 text-sm font-medium text-fd-primary-foreground transition-colors hover:opacity-90"
>
Get Started
</a>
<a
href="/docs/concepts/the-context-layer"
className="inline-flex h-10 items-center rounded-lg border border-fd-border bg-fd-background px-5 text-sm font-medium text-fd-foreground transition-colors hover:bg-fd-muted"
>
The Context Layer
</a>
<a
href="/docs/guides/building-context"
className="inline-flex h-10 items-center rounded-lg border border-fd-border bg-fd-background px-5 text-sm font-medium text-fd-foreground transition-colors hover:bg-fd-muted"
>
Building Context
</a>
</div>
</div>
## Who KTX is for
KTX is built for analytics engineers and data teams who want data agents to work on real analytics systems, not just generate one-off SQL.
KTX is built for analytics engineers and data teams who want data agents to
work on real analytics systems — not just generate one-off SQL.
Use KTX when you want agents to:
- Generate SQL from approved measures, dimensions, and joins
- Repair or extend semantic definitions through reviewable git diffs
- Explain where a metric definition came from and what business rules shape it
- Use warehouse scans and relationship evidence instead of guessing join paths
- Work alongside **dbt**, **LookML**, **MetricFlow**, **Looker**, **Metabase**, **Notion**, and BI platforms
- Work with warehouses like **PostgreSQL**, **Snowflake**, **BigQuery**, **ClickHouse**, **MySQL**, or **SQL Server**
- **Generate SQL** from approved measures and joins
- **Repair semantic definitions** through reviewable diffs
- **Explain metric provenance** with warehouse evidence
- **Work alongside** dbt, LookML, MetricFlow, Looker, Metabase, and modern BI platforms
If you've ever watched an agent confidently generate a query that joins on the wrong key or invents a metric that doesn't exist, KTX is the fix.
Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, and SQL Server.
## What KTX gives agents
- **A semantic layer they can edit** — plain YAML sources with measures, dimensions, joins, grain, segments, filters, and computed columns
- **Safe query planning** — grain-aware SQL generation, fan-out detection, chasm-trap handling, and dialect transpilation
- **Business context** — Markdown knowledge pages for definitions, rules, exceptions, and data quality notes
- **Schema evidence** — warehouse scans with table metadata, column stats, constraints, and inferred relationships
- **Provenance** — ingest transcripts and replay metadata that explain where context came from and why it changed
- **An agent-facing API** — MCP and CLI tools for reading, writing, validating, searching, and querying context
## How these docs are organized
## Explore the docs
<Cards>
<Card title="Quickstart" href="/docs/getting-started/quickstart">
Set up KTX and build your first context in under 10 minutes.
</Card>
<Card title="AI Resources" href="/docs/ai-resources">
Machine-readable docs and prompt recipes for coding assistants.
</Card>
<Card title="Concepts" href="/docs/concepts/the-context-layer">
Understand what a context layer is, why agents need one, and how KTX compares to other semantic layers.
Understand what a context layer is and why agents need one.
</Card>
<Card title="Guides" href="/docs/guides/building-context">
Hands-on workflows for scanning, ingesting, writing semantic sources, and serving agents.
</Card>
<Card title="Integrations" href="/docs/integrations/primary-sources">
Setup details for every supported database, context source, and agent client.
Hands-on workflows for scanning, ingesting, writing, and serving.
</Card>
<Card title="CLI Reference" href="/docs/cli-reference/ktx-setup">
Exhaustive flag and subcommand reference for every KTX command.
Complete flag and subcommand reference for every KTX command.
</Card>
</Cards>
## Next steps
- **Get hands-on** — follow the [Quickstart](/docs/getting-started/quickstart) to set up KTX with your own database in under 10 minutes.
- **Help a coding agent use the docs** — start with [AI Resources](/docs/ai-resources) or fetch [`/llms.txt`](/llms.txt).
- **Understand the theory** — read [The Context Layer](/docs/concepts/the-context-layer) to learn why schema access alone breaks on real analytics and how KTX addresses it.
## Agent usage notes
Use this page as the high-level routing document for KTX docs.
| Agent task | Read next |
|------------|-----------|
| Discover machine-readable docs | [AI Resources](/docs/ai-resources) |

74
docs-site/content/docs/getting-started/quickstart.mdx
View file

@ -9,44 +9,30 @@ If you are a coding assistant trying to decide which KTX docs page to read, star
## Workflow summary
Use this sequence when an agent needs to set up KTX from a fresh checkout:
Use this sequence when you are setting up KTX in an analytics project:
1. `pnpm install` — install workspace dependencies.
2. `pnpm run setup:dev` — build local packages and prepare the development CLI.
3. `pnpm run link:dev` — link the `ktx` command for local use.
4. `ktx setup` — create or resume a KTX project.
5. `ktx status` — verify project readiness.
6. `ktx sl list` — confirm semantic-layer sources are available.
7. `ktx sl query ... --format sql` — compile a semantic query without executing it.
1. `npm install -g @kaelio/ktx` — install the published KTX CLI from npm.
2. `ktx setup` — create or resume a KTX project.
The setup wizard is stateful. If it exits before completion, rerun `ktx setup` in the same project directory to resume from the first incomplete step.
## Prerequisites
- **Node.js 22+** and **pnpm**
- An **Anthropic API key** for LLM-powered enrichment and ingestion
- A **database connection** — PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, or SQLite
- Optionally, a **dbt project**, **LookML repo**, **Metabase instance**, or other context source
## Install and run setup
KTX is currently used from a local checkout or linked workspace CLI. Build and link the CLI first:
Install the published [`@kaelio/ktx`](https://www.npmjs.com/package/@kaelio/ktx) CLI:
```bash
git clone https://github.com/kaelio/ktx.git
cd ktx
pnpm install
pnpm run setup:dev
pnpm run link:dev
npm install -g @kaelio/ktx
```
Then run the setup wizard in the directory where you want your KTX project:
Then run the setup wizard:
```bash
ktx setup
```
The wizard walks through six steps. You can go back at any point, and if you exit early, running `ktx setup` again resumes where you left off.
The local checkout flow is only for contributors working on KTX itself. See [Contributing](/docs/community/contributing) for that setup.
The wizard walks through six steps. You can go back at any point, and if you exit early, rerunning `ktx setup` resumes where you left off.
## Step 1: Configure LLM
@ -86,10 +72,11 @@ KTX uses embeddings for semantic search over sources, wiki content, schema metad
**OpenAI embeddings** use `text-embedding-3-small` (1536 dimensions) and require an `OPENAI_API_KEY`.
**Local embeddings** use `all-MiniLM-L6-v2` (384 dimensions) via the KTX Python daemon. No API key is needed. If you run the daemon as a long-lived HTTP service, start it with:
**Local embeddings** use `all-MiniLM-L6-v2` (384 dimensions) via the KTX managed Python runtime. No API key is needed. KTX can install and start the runtime during setup; to prepare it ahead of time, run:
```bash
ktx-daemon serve-http --host 127.0.0.1 --port 8765
ktx runtime install --feature local-embeddings --yes
ktx runtime start --feature local-embeddings
```
## Step 3: Connect a database
@ -208,12 +195,15 @@ Then select which agents to install for:
│ ◻ Codex
│ ◻ Cursor
│ ◻ OpenCode
│ ◻ Custom agent (.agents)
```
**CLI mode** writes a skill file (e.g., `.claude/skills/ktx/SKILL.md`) that teaches the agent to call KTX commands directly.
**MCP mode** writes an MCP server configuration (e.g., `.mcp.json`) that lets the agent call KTX tools like `sl_query`, `knowledge_search`, and `sl_write_source` over the Model Context Protocol.
**Custom agent** uses the universal `.agents` target for agents that can read project-local skills or MCP configuration.
## Generated files
KTX writes project state as plain files so agents can inspect and edit changes in git.
@ -247,44 +237,14 @@ KTX context built: yes
Agent integration ready: yes (claude-code:project)
```
List your semantic sources:
```bash
ktx sl list
```
Query through the semantic layer:
```bash
ktx sl query \
--connection-id postgres-warehouse \
--measure orders.total_revenue \
--dimension orders.status \
--order-by orders.total_revenue:desc \
--limit 5 \
--format sql
```
This outputs the generated SQL. Add `--execute` to run it against your warehouse:
```bash
ktx sl query \
--connection-id postgres-warehouse \
--measure orders.total_revenue \
--dimension orders.status \
--order-by orders.total_revenue:desc \
--limit 5 \
--execute --max-rows 10
```
## Common errors
| Error or symptom | Likely cause | Recovery |
|------------------|--------------|----------|
| `ktx: command not found` | The local CLI has not been linked | Run `pnpm run setup:dev` and `pnpm run link:dev` from the KTX checkout, then open a new shell |
| `ktx: command not found` | The KTX package is not installed globally, or the shell cannot find the global binary | Run `npm install -g @kaelio/ktx` and open a new shell |
| LLM health check fails | Missing, invalid, or unauthorized Anthropic API key | Export `ANTHROPIC_API_KEY` or rerun `ktx setup` and choose the file-backed secret option |
| OpenAI embedding check fails | `OPENAI_API_KEY` is missing when OpenAI embeddings are selected | Export `OPENAI_API_KEY`, or rerun setup and choose local sentence-transformers embeddings |
| Local embeddings hang or fail | The Python daemon cannot start or the local model runtime is unavailable | Run `uv sync --all-groups`, then start `ktx-daemon serve-http --host 127.0.0.1 --port 8765` and rerun setup |
| Local embeddings hang or fail | The managed Python runtime cannot start or the local model runtime is unavailable | Install `uv`, run `ktx runtime doctor`, then run `ktx runtime install --feature local-embeddings --yes` and rerun setup |
| Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx connection add ... --force` or rerun setup |
| `KTX context built: no` in `ktx status` | Setup saved configuration but did not build context | Run `ktx setup context build` or rerun `ktx setup` and choose to build context now |
| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex --agent-install-mode both --project` using the target you need |

2
docs-site/content/docs/integrations/primary-sources.mdx
View file

@ -213,7 +213,7 @@ For multiple datasets:
| Method | Config |
|--------|--------|
| Service account JSON | `credentials_json: file:/path/to/key.json` |
| Environment variable | `credentials_json: env:GCP_CREDENTIALS_JSON` |
| Environment variable | `credentials_json: env:BIGQUERY_CREDENTIALS_JSON` |
The project ID is extracted automatically from the service account JSON file.

1
docs-site/content/docs/meta.json
View file

@ -6,7 +6,6 @@
"concepts",
"guides",
"integrations",
"benchmarks",
"cli-reference",
"ai-resources",
"community"

2
examples/orbit-relationship-verification/README.md
View file

@ -29,5 +29,5 @@ examples/orbit-relationship-verification/reports/orbit-verification.md
Use a real local Orbit project by overriding the project directory:
```bash
KTX_ORBIT_PROJECT_DIR=/path/to/orbit-project pnpm run relationships:verify-orbit
KTX_PROJECT_DIR=/path/to/orbit-project pnpm run relationships:verify-orbit
```

2
packages/cli/assets/demo/orbit/knowledge/global/new-hire-onboarding-policy.md
View file

@ -12,7 +12,7 @@ refs:
## New Hire Week-One Onboarding Policy
**Source:** Notion — People & Operating Norms, last edited 2026-05-07
**Source:** Notion — People & Operating Norms, last edited 2026-05-07
**Owner:** Manager (not People Ops)
---

3
packages/cli/assets/demo/orbit/knowledge/global/orbit-activation-kpi-glossary.md
View file

@ -20,7 +20,7 @@ tables:
# Activation KPI Glossary
**Owner team:** Growth
**Owner team:** Growth
**Source:** Notion — Orbit Demo Home / Data Team - Onboarding / Activation KPI Glossary, last edited 2026-05-07
Use this when a question is about signup-to-habit behavior. Orbit uses activation language across Growth, Product, and CS conversations.
@ -62,4 +62,3 @@ Growth conversations typically use D7 and D14 Activation Rate. Product and CS ma
## Relationship to Account-Level Activation
This glossary defines **customer-level** activation (signup-to-habit). The **account-level** activation workflow (requester login → first approved purchase request → account activated) is a separate concept tracked in `mart_account_activity` and governed by the January 2026 policy change. See `orbit-activation-policy-change-jan-2026` for that definition.

6
packages/cli/assets/demo/orbit/knowledge/global/orbit-activation-policy-change-jan-2026.md
View file

@ -14,9 +14,9 @@ sl_refs:
# Activation Policy Change — January 2026
**Governed metric key:** `activated_accounts`
**Owner team:** growth
**Notion:** `notion://notion_page_activation_policy_decision#policy-change`
**Governed metric key:** `activated_accounts`
**Owner team:** growth
**Notion:** `notion://notion_page_activation_policy_decision#policy-change`
**Sources:** `mart_account_activity`, `int_activation_policy_windows`, `stg_activation_events`
## Policy Boundary

6
packages/cli/assets/demo/orbit/knowledge/global/orbit-arr-contract-first-definition.md
View file

@ -15,9 +15,9 @@ sl_refs:
# ARR — Contract-First Definition
**Governed metric key:** `arr`
**Owner team:** finance
**Notion:** `notion://notion_page_arr_contract_reporting#arr-contract-first`
**Governed metric key:** `arr`
**Owner team:** finance
**Notion:** `notion://notion_page_arr_contract_reporting#arr-contract-first`
**Source:** `mart_arr_daily` (grain: `metric_date`)
## Rule

3
packages/cli/assets/demo/orbit/knowledge/global/orbit-company-overview.md
View file

@ -21,7 +21,7 @@ refs:
Orbit sells procurement workflow and spend-control software. The core value proposition: route purchase requests, collect approvals, onboard suppliers, and issue purchase orders without turning every exception into a status hunt.
**Primary buyers:** Finance, Procurement, Business Operations.
**Primary buyers:** Finance, Procurement, Business Operations.
**Daily users:** department admins, office managers, IT leads, legal ops partners — anyone who has to get a vendor through the building.
## Product Workflow
@ -69,4 +69,3 @@ Orbit sells procurement workflow and spend-control software. The core value prop
- "Supplier onboarding is split across three teams."
- "Renewals are visible too late."
- "People keep asking Finance for status because there is nowhere better to look."

6
packages/cli/assets/demo/orbit/knowledge/global/orbit-customer-health-risk-definition.md
View file

@ -14,9 +14,9 @@ sl_refs:
# Customer Health Risk Definition
**Governed metric key:** `active_customers`
**Owner team:** customer_success
**Notion:** `notion://notion_page_customer_health_playbook#risk-definition`
**Governed metric key:** `active_customers`
**Owner team:** customer_success
**Notion:** `notion://notion_page_customer_health_playbook#risk-definition`
**Sources:** `mart_customer_health`, `int_customer_health_signals`
## Risk Levels

5
packages/cli/assets/demo/orbit/knowledge/global/orbit-customers-source.md
View file

@ -18,8 +18,8 @@ tables:
# Orbit Customers Source
**Table:** `orbit_analytics.customer`
**Grain:** one row per signed-up customer
**Table:** `orbit_analytics.customer`
**Grain:** one row per signed-up customer
**Source:** Notion — Orbit Demo Home / Data Team - Onboarding / Orbit Customers Source, last edited 2026-05-07
Use this when a question needs customer identity, plan tier, signup timing, recent activity, or the standard customer joins.
@ -58,4 +58,3 @@ Always join through `customer.id`. Do not join on `email`.
- **Timezone:** `created_at` and `last_seen_at` are UTC. Confirm whether a question expects UTC or a local business day before filtering.
- **Paying vs. all:** `free` customers must be excluded from paying-customer follow-ups. Use `paying_customer_count`, not `customer_count`.
- **plan_tier values:** `free`, `pro`, `enterprise`. Note: `pro_plus` is a legacy alias for `growth` in the account/contract layer (see `orbit-plan-segment-normalization`), but `plan_tier` on this table uses `pro` not `pro_plus`.

1
packages/cli/assets/demo/orbit/knowledge/global/orbit-dbt-exposures.md
View file

@ -42,4 +42,3 @@ Declared in `models/exposures.yml`. All exposures are type `dashboard` with matu
- **Owner:** Growth (growth@orbit-demo.example.com)
- **Depends on:** `mart_account_activity`
- **Description:** Activation policy comparison around the January 2026 workflow update.

9
packages/cli/assets/demo/orbit/knowledge/global/orbit-dbt-project-overview.md
View file

@ -22,10 +22,10 @@ sl_refs:
# Orbit dbt Project Overview
**Project name:** `kaelio_demo`
**dbt version:** 1.0.0
**Profile target:** Postgres (`orbit_analytics` schema, `kaelio_demo` database)
**Raw source schema:** `orbit_raw`
**Project name:** `kaelio_demo`
**dbt version:** 1.0.0
**Profile target:** Postgres (`orbit_analytics` schema, `kaelio_demo` database)
**Raw source schema:** `orbit_raw`
**Analytics schema:** `orbit_analytics` (all models materialised as views by default)
## Model Layers
@ -52,4 +52,3 @@ sl_refs:
## Raw Source Tables (`orbit_raw` schema)
accounts, account_hierarchy, plans, contracts, subscriptions, contract_discount_terms, arr_movements, invoices, invoice_line_items, refunds, plan_segment_mapping, users, activation_events, sessions, purchase_requests, approval_events, suppliers, supplier_onboarding_events, purchase_orders, support_tickets, account_owners.

3
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-account-activity.md
View file

@ -20,7 +20,7 @@ tables:
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/106.json -->
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/107.json -->
**Table:** `orbit_analytics.mart_account_activity`
**Table:** `orbit_analytics.mart_account_activity`
**Grain:** one row per `policy_change_date`
## Columns
@ -47,4 +47,3 @@ tables:
- The January 2026 activation policy change (`policy_change_date = 2026-01-15`) is the primary boundary. `policy_version` in upstream events splits into `pre_2026_01_15` and `post_2026_01_15` cohorts.
- Rates are ratios (0–1); multiply by 100 for percentage display.
- See [orbit-activation-policy-change-jan-2026](orbit-activation-policy-change-jan-2026) for full policy context.

3
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-account-segments.md
View file

@ -19,7 +19,7 @@ tables:
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/69.json -->
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/100.json -->
**Table:** `orbit_analytics.mart_account_segments`
**Table:** `orbit_analytics.mart_account_segments`
**Grain:** one row per `account_id`
## Columns
@ -53,4 +53,3 @@ tables:
- `normalized_plan_code` maps `pro_plus` → `growth`. Always use `normalized_plan_code` for plan-based reporting. See [orbit-plan-segment-normalization](orbit-plan-segment-normalization).
- `segment` is derived from `canonical_plan_code × size_band` via `stg_plan_segment_mapping`.
- `contract_arr_cents` is the contract-first ARR value. See [orbit-arr-contract-first-definition](orbit-arr-contract-first-definition).

3
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-arr-daily.md
View file

@ -18,7 +18,7 @@ tables:
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/56.json -->
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/96.json -->
**Table:** `orbit_analytics.mart_arr_daily`
**Table:** `orbit_analytics.mart_arr_daily`
**Grain:** one row per `metric_date`
## Columns
@ -44,4 +44,3 @@ tables:
- ARR is calculated contract-first: active contract ARR takes precedence over subscription ARR for any covered period. See [orbit-arr-contract-first-definition](orbit-arr-contract-first-definition).
- `display` is a formatted label for UI rendering; use `arr_cents` for all arithmetic.

3
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-nrr-quarterly.md
View file

@ -20,7 +20,7 @@ tables:
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/98.json -->
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/103.json -->
**Table:** `orbit_analytics.mart_nrr_quarterly`
**Table:** `orbit_analytics.mart_nrr_quarterly`
**Grain:** one row per `quarter_label` × `segment`
## Columns
@ -53,4 +53,3 @@ tables:
- `net_revenue_retention` is a ratio, not a percentage. Multiply by 100 for display.
- Contraction includes discount expirations (classified as contraction, not churn). See [orbit-nrr-discount-expiration-treatment](orbit-nrr-discount-expiration-treatment).
- Enterprise is the primary executive reporting segment.

3
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-procurement-activity.md
View file

@ -18,7 +18,7 @@ tables:
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/88.json -->
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/108.json -->
**Table:** `orbit_analytics.mart_procurement_activity`
**Table:** `orbit_analytics.mart_procurement_activity`
**Grain:** one row per `week_start_date` × `contract_arr_threshold_cents`
## Columns
@ -45,4 +45,3 @@ tables:
- `active_requesters` counts non-internal, non-test requesters on large active contracts. See [orbit-procurement-qualifying-actions](orbit-procurement-qualifying-actions).
- The standard threshold is `contract_arr_threshold_cents = 20000000` ($200k ARR).
- Always filter by `contract_arr_threshold_cents` — the table contains rows for multiple threshold values.

3
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-retention-movement-breakout.md
View file

@ -19,7 +19,7 @@ tables:
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/105.json -->
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/115.json -->
**Table:** `orbit_analytics.mart_retention_movement_breakout`
**Table:** `orbit_analytics.mart_retention_movement_breakout`
**Grain:** one row per `quarter_label` × `segment` × `movement_type` × `movement_reason`
## Columns
@ -53,4 +53,3 @@ tables:
- Contraction includes discount expirations, classified as contraction (not churn), tracked via `movement_reason`. See [orbit-nrr-discount-expiration-treatment](orbit-nrr-discount-expiration-treatment).
- This table is the row-level source for `mart_nrr_quarterly` aggregations.
- Only one of `expansion_arr_cents`, `contraction_arr_cents`, `churned_arr_cents` is non-zero per row.

3
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-revenue-daily.md
View file

@ -20,7 +20,7 @@ tables:
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/102.json -->
<!-- from: raw-sources/postgres-warehouse/metabase/2026-05-12-035303-local-metabase-3-114d957b-f564-4f46-8d4c-2770720a95be/cards/104.json -->
**Table:** `orbit_analytics.mart_revenue_daily`
**Table:** `orbit_analytics.mart_revenue_daily`
**Grain:** one row per `revenue_date`
## Columns
@ -54,4 +54,3 @@ tables:
- `reconciliation_check` must be `true` on every row. Any `false` row indicates a data quality issue.
- Gross-to-net reconciliation: gross revenue − credits − refunds = net revenue. See [orbit-revenue-gross-to-net-reconciliation](orbit-revenue-gross-to-net-reconciliation).
- All amounts are in cents; divide by 100 for USD, by 100,000,000 for $M.

1
packages/cli/assets/demo/orbit/knowledge/global/orbit-metabase-sql-library-patterns.md
View file

@ -69,4 +69,3 @@ Card 48 is the canonical reference; card 55 is a filtered variant for large-cont
| 53 | Enterprise NRR quarter breakout | mart_nrr_quarterly | 0 |
| 54 | February credits drilldown | mart_revenue_daily | 0 |
| 55 | Large contract requesters | mart_account_segments | 0 |

6
packages/cli/assets/demo/orbit/knowledge/global/orbit-nrr-discount-expiration-treatment.md
View file

@ -15,9 +15,9 @@ sl_refs:
# NRR — Discount Expiration Treatment
**Governed metric key:** `net_revenue_retention`
**Owner team:** analytics
**Notion:** `notion://notion_page_retention_policy_current#nrr-definition` and `#discount-expiration-treatment`
**Governed metric key:** `net_revenue_retention`
**Owner team:** analytics
**Notion:** `notion://notion_page_retention_policy_current#nrr-definition` and `#discount-expiration-treatment`
**Sources:** `mart_nrr_quarterly`, `mart_retention_movement_breakout`
## NRR Definition

6
packages/cli/assets/demo/orbit/knowledge/global/orbit-plan-segment-normalization.md
View file

@ -14,9 +14,9 @@ sl_refs:
# Plan & Segment Normalization
**Governed metric key:** `segment`
**Owner team:** sales_ops
**Notion:** `notion://notion_page_sales_ops_segmentation#growth-plan-normalization`
**Governed metric key:** `segment`
**Owner team:** sales_ops
**Notion:** `notion://notion_page_sales_ops_segmentation#growth-plan-normalization`
**Sources:** `mart_account_segments`, `stg_plan_segment_mapping`, `stg_plans`
## Canonical Plan Codes

6
packages/cli/assets/demo/orbit/knowledge/global/orbit-procurement-qualifying-actions.md
View file

@ -14,9 +14,9 @@ sl_refs:
# Procurement — Qualifying Actions & Weekly Active Requesters
**Governed metric key:** `weekly_active_requesters`
**Owner team:** product
**Notion:** `notion://notion_page_procurement_instrumentation#qualifying-procurement-actions`
**Governed metric key:** `weekly_active_requesters`
**Owner team:** product
**Notion:** `notion://notion_page_procurement_instrumentation#qualifying-procurement-actions`
**Sources:** `mart_procurement_activity`, `int_procurement_qualifying_actions`
## Qualifying Action Definition

6
packages/cli/assets/demo/orbit/knowledge/global/orbit-revenue-gross-to-net-reconciliation.md
View file

@ -14,9 +14,9 @@ sl_refs:
# Revenue — Gross-to-Net Reconciliation
**Governed metric key:** `net_revenue`
**Owner team:** finance
**Notion:** `notion://notion_page_revenue_reporting_policy#gross-to-net-reconciliation`
**Governed metric key:** `net_revenue`
**Owner team:** finance
**Notion:** `notion://notion_page_revenue_reporting_policy#gross-to-net-reconciliation`
**Source:** `mart_revenue_daily` (grain: `revenue_date`)
## Formula

2
packages/cli/assets/demo/orbit/knowledge/global/sales-ops-cs-handoff-process.md
View file

@ -14,7 +14,7 @@ refs:
## Sales Ops → Customer Success Implementation Handoff
**Source:** Notion — People & Operating Norms, last edited 2026-05-07
**Source:** Notion — People & Operating Norms, last edited 2026-05-07
**Owner:** Sales Ops (sender), Customer Success (receiver)
---

58
packages/cli/src/clack.ts
View file

@ -1,4 +1,4 @@
import { spinner } from '@clack/prompts';
import { cancel, confirm, isCancel, log, spinner } from '@clack/prompts';
export interface KtxCliSpinner {
start(message: string): void;
@ -6,6 +6,62 @@ export interface KtxCliSpinner {
error(message: string): void;
}
export interface KtxCliPromptAdapter {
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
cancel(message: string): void;
log: {
info(message: string): void;
warn(message: string): void;
error(message: string): void;
success(message: string): void;
step(message: string): void;
};
spinner(): KtxCliSpinner;
}
export class KtxCliPromptCancelledError extends Error {
constructor(message = 'Operation cancelled.') {
super(message);
this.name = 'KtxCliPromptCancelledError';
}
}
export function createClackSpinner(): KtxCliSpinner {
return spinner();
}
export function createClackPromptAdapter(): KtxCliPromptAdapter {
return {
async confirm(options) {
const value = await confirm(options);
if (isCancel(value)) {
cancel('Operation cancelled.');
throw new KtxCliPromptCancelledError();
}
return value;
},
cancel(message) {
cancel(message);
},
log: {
info(message) {
log.info(message);
},
warn(message) {
log.warn(message);
},
error(message) {
log.error(message);
},
success(message) {
log.success(message);
},
step(message) {
log.step(message);
},
},
spinner() {
return createClackSpinner();
},
};
}

2
packages/cli/src/commands/connection-metabase-setup.test.ts
View file

@ -138,7 +138,7 @@ function makeIo(options: { isTTY?: boolean; stdinIsTTY?: boolean } = {}) {
describe('runKtxConnectionMetabaseSetup', () => {
const fakeMetabaseCredential = 'mb_example';
const existingMetabaseCredential = 'mb_existing';
const fakeAdminCredential = 'pw';
const fakeAdminCredential = 'admin-secret-value-123';
let tempDir: string;
let projectDir: string;

4
packages/cli/src/commands/runtime-commands.ts
View file

@ -53,10 +53,12 @@ export function registerRuntimeCommands(program: Command, context: KtxCliCommand
runtime
.command('stop')
.description('Stop the KTX-managed Python HTTP daemon')
.action(async () => {
.option('--all', 'Stop all KTX daemon processes recorded or discoverable on this machine', false)
.action(async (options: { all?: boolean }) => {
await runRuntimeArgs(context, {
command: 'stop',
cliVersion: context.packageInfo.version,
all: options.all === true,
});
});

64
packages/cli/src/connection.test.ts
View file

@ -1,7 +1,8 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKtxProject, parseKtxProjectConfig } from '@ktx/context/project';
import type { MetabaseRuntimeClient } from '@ktx/context/ingest';
import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from '@ktx/context/project';
import type { KtxConnectionDriver, KtxScanConnector, KtxSchemaSnapshot } from '@ktx/context/scan';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxConnection } from './connection.js';
@ -476,7 +477,7 @@ describe('runKtxConnection', () => {
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_AUTH_TOKEN',
authTokenRef: 'env:NOTION_TOKEN',
crawlMode: 'all_accessible',
rootPageIds: [],
rootDatabaseIds: [],
@ -492,7 +493,7 @@ describe('runKtxConnection', () => {
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('driver: notion');
expect(yaml).toContain('auth_token_ref: env:NOTION_AUTH_TOKEN');
expect(yaml).toContain('auth_token_ref: env:NOTION_TOKEN');
expect(yaml).toContain('crawl_mode: all_accessible');
expect(yaml).toContain('max_pages_per_run: 50');
expect(yaml).not.toContain('ntn_');
@ -515,7 +516,7 @@ describe('runKtxConnection', () => {
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_AUTH_TOKEN',
authTokenRef: 'env:NOTION_TOKEN',
crawlMode: 'all_accessible',
rootPageIds: [],
rootDatabaseIds: ['database-1'],
@ -598,6 +599,61 @@ describe('runKtxConnection', () => {
expect(io.stdout()).toContain('Tables: 2');
});
it('tests a configured Metabase connection through the Metabase runtime client', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const projectConfig = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
await writeFile(
join(projectDir, 'ktx.yaml'),
serializeKtxProjectConfig({
...projectConfig,
connections: {
...projectConfig.connections,
prod_metabase: {
driver: 'metabase',
api_url: 'http://metabase.example.test',
api_key: 'mb_test',
},
},
}),
'utf-8',
);
const testConnection = vi.fn(async () => ({ success: true as const }));
const getDatabases = vi.fn(async () => [
{ id: 1, name: 'Analytics', engine: 'postgres', details: {}, is_sample: false },
{ id: 2, name: 'Sample Database', engine: 'h2', details: {}, is_sample: true },
]);
const cleanup = vi.fn(async () => undefined);
const createMetabaseClient = vi.fn(
async (): Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>> => ({
testConnection,
getDatabases,
cleanup,
}),
);
const createScanConnector = vi.fn(async () => {
throw new Error('native scanner should not be used for Metabase');
});
const io = makeIo();
await expect(
runKtxConnection({ command: 'test', projectDir, connectionId: 'prod_metabase' }, io.io, {
createScanConnector,
createMetabaseClient,
}),
).resolves.toBe(0);
expect(createScanConnector).not.toHaveBeenCalled();
expect(createMetabaseClient).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), 'prod_metabase');
expect(testConnection).toHaveBeenCalledTimes(1);
expect(getDatabases).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledTimes(1);
expect(io.stdout()).toContain('Connection test passed: prod_metabase');
expect(io.stdout()).toContain('Driver: metabase');
expect(io.stdout()).toContain('Databases: 1');
expect(io.stderr()).toBe('');
});
it('cleans up the native scan connector when connection testing fails', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });

67
packages/cli/src/connection.ts
View file

@ -1,4 +1,10 @@
import { cancel, confirm, isCancel } from '@clack/prompts';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultMetabaseConnectionClientFactory,
type MetabaseRuntimeClient,
metabaseRuntimeConfigFromLocalConnection,
} from '@ktx/context/ingest';
import { type KtxLocalProject, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
import type { KtxScanConnector } from '@ktx/context/scan';
import type { KtxConnectionMappingArgs } from './commands/connection-mapping.js';
@ -61,6 +67,7 @@ interface KtxConnectionIo extends KtxCliIo {
interface KtxConnectionDeps {
createScanConnector?: typeof createKtxCliScanConnector;
createMetabaseClient?: typeof createDefaultMetabaseClient;
runMapping?: (argv: string[], io: KtxCliIo) => Promise<number>;
prompts?: KtxConnectionPromptAdapter;
}
@ -104,6 +111,12 @@ async function cleanupConnector(connector: KtxScanConnector | null): Promise<voi
}
}
function normalizedConnectionDriver(project: KtxLocalProject, connectionId: string): string {
return String(project.config.connections[connectionId]?.driver ?? '')
.trim()
.toLowerCase();
}
async function testNativeConnection(
project: KtxLocalProject,
connectionId: string,
@ -131,6 +144,48 @@ async function testNativeConnection(
}
}
async function createDefaultMetabaseClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>> {
const factory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(
metabaseConnectionId,
project.config.connections[metabaseConnectionId],
),
DEFAULT_METABASE_CLIENT_CONFIG,
);
return factory.createClient(connectionId);
}
async function testMetabaseConnection(
project: KtxLocalProject,
connectionId: string,
createMetabaseClient: typeof createDefaultMetabaseClient,
): Promise<{ driver: 'metabase'; databaseCount: number }> {
let client: Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'> | null = null;
try {
client = await createMetabaseClient(project, connectionId);
const testResult = await client.testConnection();
if (!testResult.success) {
throw new Error(
`Metabase connection test failed: ${testResult.error ?? testResult.message ?? 'unknown error'}`,
);
}
const databases = await client.getDatabases();
const databaseCount = databases.filter((database) => database.is_sample !== true).length;
if (databaseCount === 0) {
throw new Error('Metabase auth worked but no usable databases were returned');
}
return { driver: 'metabase', databaseCount };
} finally {
await client?.cleanup();
}
}
interface BufferedIo extends KtxCliIo {
stdoutText(): string;
stderrText(): string;
@ -399,6 +454,18 @@ export async function runKtxConnection(
return 0;
}
if (normalizedConnectionDriver(project, args.connectionId) === 'metabase') {
const result = await testMetabaseConnection(
project,
args.connectionId,
deps.createMetabaseClient ?? createDefaultMetabaseClient,
);
io.stdout.write(`Connection test passed: ${args.connectionId}\n`);
io.stdout.write(`Driver: ${result.driver}\n`);
io.stdout.write(`Databases: ${result.databaseCount}\n`);
return 0;
}
const result = await testNativeConnection(
project,
args.connectionId,

25
packages/cli/src/demo.test.ts
View file

@ -11,6 +11,9 @@ import type { renderMemoryFlowTui } from './memory-flow-tui.js';
import { KTX_NEXT_STEP_DIRECT_COMMANDS } from './next-steps.js';
import { resetVizFallbackWarningsForTest } from './viz-fallback.js';
const SEEDED_DEMO_SEMANTIC_SOURCE_COUNT = 46;
const SEEDED_DEMO_KNOWLEDGE_PAGE_COUNT = 28;
function makeIo(options: { isTTY?: boolean; columns?: number; rawMode?: boolean } = {}) {
let stdout = '';
let stderr = '';
@ -336,8 +339,14 @@ describe('runKtxDemo', () => {
notion: { pageCount: 8 },
},
generatedOutputs: {
semanticLayer: { manifestSourceCount: 46, fileCount: 46 },
knowledge: { manifestPageCount: 28, fileCount: 28 },
semanticLayer: {
manifestSourceCount: SEEDED_DEMO_SEMANTIC_SOURCE_COUNT,
fileCount: SEEDED_DEMO_SEMANTIC_SOURCE_COUNT,
},
knowledge: {
manifestPageCount: SEEDED_DEMO_KNOWLEDGE_PAGE_COUNT,
fileCount: SEEDED_DEMO_KNOWLEDGE_PAGE_COUNT,
},
links: { manifestLinkCount: 23, linkCount: 23 },
reports: { primaryPath: 'reports/seeded-demo-report.json', fileCount: 1 },
},
@ -636,10 +645,16 @@ describe('runKtxDemo', () => {
).resolves.toBe(0);
expect(seededIo.stdout()).toContain('Status: ready');
expect(seededIo.stdout()).toContain('Semantic-layer sources: 46 manifest, 46 files');
expect(seededIo.stdout()).toContain('Knowledge pages: 28 manifest, 28 files');
expect(seededIo.stdout()).toContain(
`Semantic-layer sources: ${SEEDED_DEMO_SEMANTIC_SOURCE_COUNT} manifest, ${SEEDED_DEMO_SEMANTIC_SOURCE_COUNT} files`,
);
expect(seededIo.stdout()).toContain(
`Knowledge pages: ${SEEDED_DEMO_KNOWLEDGE_PAGE_COUNT} manifest, ${SEEDED_DEMO_KNOWLEDGE_PAGE_COUNT} files`,
);
expect(seededIo.stdout()).not.toContain('Status: corrupt');
expect(seededIo.stdout()).not.toContain('Semantic-layer sources: 6 manifest, 0 files');
expect(seededIo.stdout()).not.toContain(
`Semantic-layer sources: ${SEEDED_DEMO_SEMANTIC_SOURCE_COUNT} manifest, 0 files`,
);
});
it('fails corrupted demo projects in no-input mode with reset guidance', async () => {

49
packages/cli/src/index.test.ts
View file

@ -2,6 +2,7 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { createRequire } from 'node:module';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKtxProject } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
@ -143,6 +144,7 @@ describe('runKtxCli', () => {
const installIo = makeIo();
const startIo = makeIo();
const stopIo = makeIo();
const stopAllIo = makeIo();
const statusIo = makeIo();
const doctorIo = makeIo();
const pruneIo = makeIo();
@ -156,6 +158,7 @@ describe('runKtxCli', () => {
runKtxCli(['runtime', 'start', '--feature', 'local-embeddings', '--force'], startIo.io, { runtime }),
).resolves.toBe(0);
await expect(runKtxCli(['runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0);
await expect(runKtxCli(['runtime', 'stop', '--all'], stopAllIo.io, { runtime })).resolves.toBe(0);
await expect(runKtxCli(['runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0);
await expect(runKtxCli(['runtime', 'doctor'], doctorIo.io, { runtime })).resolves.toBe(0);
await expect(runKtxCli(['runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(0);
@ -185,11 +188,21 @@ describe('runKtxCli', () => {
{
command: 'stop',
cliVersion: '0.0.0-private',
all: false,
},
stopIo.io,
);
expect(runtime).toHaveBeenNthCalledWith(
4,
{
command: 'stop',
cliVersion: '0.0.0-private',
all: true,
},
stopAllIo.io,
);
expect(runtime).toHaveBeenNthCalledWith(
5,
{
command: 'status',
cliVersion: '0.0.0-private',
@ -198,7 +211,7 @@ describe('runKtxCli', () => {
statusIo.io,
);
expect(runtime).toHaveBeenNthCalledWith(
5,
6,
{
command: 'doctor',
cliVersion: '0.0.0-private',
@ -207,7 +220,7 @@ describe('runKtxCli', () => {
doctorIo.io,
);
expect(runtime).toHaveBeenNthCalledWith(
6,
7,
{
command: 'prune',
cliVersion: '0.0.0-private',
@ -218,6 +231,17 @@ describe('runKtxCli', () => {
);
});
it('documents runtime stop all in command help', async () => {
const testIo = makeIo();
await expect(runKtxCli(['runtime', 'stop', '--help'], testIo.io)).resolves.toBe(0);
expect(testIo.stdout()).toContain('--all');
expect(testIo.stdout()).toContain('Stop all KTX daemon processes recorded or discoverable');
expect(testIo.stdout()).toContain('on this machine');
expect(testIo.stderr()).toBe('');
});
it('routes sl query managed runtime install policies', async () => {
const sl = vi.fn(async () => 0);
@ -310,6 +334,23 @@ describe('runKtxCli', () => {
expect(testIo.stderr()).toBe('');
});
it('keeps representative JSON command stdout parseable', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const commands = [
['--project-dir', projectDir, 'setup', 'status', '--json'],
['--project-dir', projectDir, 'sl', 'list', '--json'],
];
for (const argv of commands) {
const testIo = makeIo();
await expect(runKtxCli(argv, testIo.io)).resolves.toBe(0);
expect(() => JSON.parse(testIo.stdout())).not.toThrow();
expect(testIo.stderr()).toBe('');
}
});
it('starts setup for bare ktx in a TTY when no project is discoverable', async () => {
const { mkdtemp, realpath, rm } = await import('node:fs/promises');
const { tmpdir } = await import('node:os');
@ -1964,7 +2005,7 @@ describe('runKtxCli', () => {
'--project-dir',
tempDir,
'--token-env',
'NOTION_AUTH_TOKEN',
'NOTION_TOKEN',
'--crawl-mode',
'selected_roots',
'--root-page-id',
@ -1991,7 +2032,7 @@ describe('runKtxCli', () => {
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_AUTH_TOKEN',
authTokenRef: 'env:NOTION_TOKEN',
crawlMode: 'selected_roots',
rootPageIds: ['page-1'],
rootDatabaseIds: ['database-1'],

5
packages/cli/src/index.ts
View file

@ -47,13 +47,18 @@ export { runKtxRuntime, type KtxRuntimeArgs, type KtxRuntimeDeps } from './runti
export {
allocateDaemonPort,
readManagedPythonDaemonStatus,
stopAllManagedPythonDaemons,
startManagedPythonDaemon,
stopManagedPythonDaemon,
} from './managed-python-daemon.js';
export type {
ManagedPythonDaemonProcessInfo,
ManagedPythonDaemonStartResult,
ManagedPythonDaemonState,
ManagedPythonDaemonStatus,
ManagedPythonDaemonStopAllEntry,
ManagedPythonDaemonStopAllFailure,
ManagedPythonDaemonStopAllResult,
ManagedPythonDaemonStopResult,
} from './managed-python-daemon.js';
export {

6
packages/cli/src/ingest-viz.test.ts
View file

@ -331,8 +331,9 @@ describe('runKtxIngest viz and replay', () => {
).resolves.toBe(0);
expect(runLocal).toHaveBeenCalledWith(expect.objectContaining({ memoryFlow: expect.anything() }));
expect(io.stdout()).toContain('[5%] Fetching source files for warehouse/fake');
expect(io.stderr()).toContain('[5%] Fetching source files for warehouse/fake');
expect(io.stdout()).toContain('Job: plain-run');
expect(io.stdout()).not.toContain('[5%]');
expect(io.stdout()).not.toContain('KTX memory flow');
});
@ -407,8 +408,9 @@ describe('runKtxIngest viz and replay', () => {
expect(startLiveMemoryFlow).not.toHaveBeenCalled();
expect(runLocal).toHaveBeenCalledWith(expect.objectContaining({ memoryFlow: expect.anything() }));
expect(io.stdout()).toContain('[5%] Fetching source files for warehouse/fake');
expect(io.stderr()).toContain('[5%] Fetching source files for warehouse/fake');
expect(io.stdout()).toContain('Job: raw-missing-viz-run');
expect(io.stdout()).not.toContain('[5%]');
expect(io.stdout()).not.toContain('KTX memory flow');
expect(io.stderr()).toContain(
'Visualization requested but stdin raw mode is unavailable; printing plain output.',

2
packages/cli/src/ingest.test-utils.ts
View file

@ -546,7 +546,7 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync
),
).resolves.toBe(0);
expect(io.stderr()).toBe('');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).toContain(`target=warehouse_a database=1 status=done job=${jobId}`);

450
packages/cli/src/ingest.test.ts
View file

@ -14,6 +14,7 @@ import {
import { initKtxProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { type KtxIngestArgs, runKtxIngest } from './ingest.js';
import type { KtxCliLocalIngestAdaptersOptions } from './local-adapters.js';
import {
CliLookerSlWritingAgentRunner,
CliMetabaseAgentRunner,
@ -32,6 +33,7 @@ import {
writeWarehouseConfig,
} from './ingest.test-utils.js';
import { resetVizFallbackWarningsForTest } from './viz-fallback.js';
import { runKtxSetup } from './setup.js';
describe('runKtxIngest', () => {
let tempDir: string;
@ -105,6 +107,75 @@ describe('runKtxIngest', () => {
expect(statusIo.stderr()).toBe('');
});
it('prints provider setup guidance when a skip-llm setup project runs dev ingest', async () => {
const projectDir = join(tempDir, 'project');
const setupIo = makeIo();
await expect(
runKtxSetup(
{
command: 'run',
projectDir,
mode: 'new',
agents: false,
agentScope: 'project',
agentInstallMode: 'cli',
skipAgents: true,
inputMode: 'disabled',
yes: true,
cliVersion: '0.0.0-test',
skipLlm: true,
skipEmbeddings: true,
databaseDrivers: ['postgres'],
databaseConnectionId: 'warehouse',
databaseUrl: 'env:WAREHOUSE_URL',
databaseSchemas: [],
enableHistoricSql: true,
skipDatabases: false,
skipSources: true,
},
setupIo.io,
{
databasesDeps: {
testConnection: async (_projectDir, _connectionId, io) => {
io.stdout.write('Driver: postgres\nTables: 1\n');
return 0;
},
scanConnection: async () => 0,
historicSqlProbe: async () => ({ ok: true, lines: ['PASS Historic SQL probe skipped in test'] }),
},
context: async () => ({ status: 'skipped', projectDir }),
},
),
).resolves.toBe(0);
const sourceDir = join(tempDir, 'source');
await mkdir(join(sourceDir, 'orders'), { recursive: true });
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
const runIo = makeIo();
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'warehouse',
adapter: 'historic-sql',
sourceDir,
outputMode: 'plain',
},
runIo.io,
),
).resolves.toBe(1);
expect(runIo.stdout()).toBe('');
expect(runIo.stderr()).toContain(
'ktx dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner.',
);
expect(runIo.stderr()).toContain(
`ktx setup --project-dir ${projectDir} --anthropic-api-key-env ANTHROPIC_API_KEY --anthropic-model claude-sonnet-4-6 --no-input`,
);
});
it('routes metabase scheduled pulls to the fan-out runner and prints child summaries', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
@ -159,7 +230,7 @@ describe('runKtxIngest', () => {
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).toContain('warehouse_a');
expect(io.stdout()).toContain('metabase-child-1');
expect(io.stderr()).toBe('');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
});
it('returns a non-zero code when Metabase fan-out has failed children', async () => {
@ -229,7 +300,7 @@ describe('runKtxIngest', () => {
expect(io.stdout()).toContain('Metabase fan-out: partial_failure');
expect(io.stdout()).toContain('Failed work units: 1');
expect(io.stdout()).toContain('status=error');
expect(io.stderr()).toBe('');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
});
it('prints Metabase fan-out progress before the final summary', async () => {
@ -303,12 +374,56 @@ describe('runKtxIngest', () => {
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Metabase ingest: prod-metabase');
expect(io.stdout()).toContain('Targets: 1 mapped database');
expect(io.stdout()).toContain('- database=1 target=warehouse_a status=running job=metabase-child-1');
expect(io.stdout()).toContain('- database=1 target=warehouse_a status=done job=metabase-child-1');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
expect(io.stderr()).toContain('Targets: 1 mapped database');
expect(io.stderr()).toContain('- database=1 target=warehouse_a status=running job=metabase-child-1');
expect(io.stderr()).toContain('- database=1 target=warehouse_a status=done job=metabase-child-1');
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stderr()).toBe('');
expect(io.stdout()).not.toContain('status=running job=metabase-child-1');
});
it('writes metabase fan-out progress to stderr and final result to stdout', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo({ isTTY: true });
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'prod-metabase',
adapter: 'metabase',
outputMode: 'plain',
},
io.io,
{
runLocalMetabaseIngest: async (input) => {
input.progress?.onMetabaseFanoutPlanned?.({
metabaseConnectionId: 'prod-metabase',
children: [{ metabaseDatabaseId: 1, targetConnectionId: 'warehouse_a' }],
});
input.progress?.onMetabaseChildStarted?.({
metabaseConnectionId: 'prod-metabase',
metabaseDatabaseId: 1,
targetConnectionId: 'warehouse_a',
jobId: 'metabase-child-1',
});
return {
metabaseConnectionId: 'prod-metabase',
status: 'all_succeeded',
totals: { workUnits: 0, failedWorkUnits: 0 },
children: [],
};
},
},
),
).resolves.toBe(0);
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
expect(io.stderr()).toContain('status=running job=metabase-child-1');
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).not.toContain('status=running job=metabase-child-1');
});
it('runs Metabase scheduled ingest through the public CLI command path with real fan-out', async () => {
@ -393,7 +508,8 @@ describe('runKtxIngest', () => {
),
).resolves.toBe(0);
expect(io.stderr()).toBe('');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
expect(io.stderr()).toContain('Targets: 2 mapped databases');
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).toContain('Source: prod-metabase');
expect(io.stdout()).toContain('Children: 2');
@ -483,6 +599,46 @@ describe('runKtxIngest', () => {
expect(io.stderr()).toBe('');
});
it('keeps metabase JSON stdout free of operational adapter logs', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo();
let adapterOptions: KtxCliLocalIngestAdaptersOptions | undefined;
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'prod-metabase',
adapter: 'metabase',
outputMode: 'json',
},
io.io,
{
createAdapters: (_project, options) => {
adapterOptions = options;
options?.logger?.warn('adapter warning');
return [];
},
runLocalMetabaseIngest: async (input) => {
input.adapters.find((adapter) => adapter.source === 'metabase');
return {
metabaseConnectionId: 'prod-metabase',
status: 'all_succeeded',
totals: { workUnits: 0, failedWorkUnits: 0 },
children: [],
};
},
},
),
).resolves.toBe(0);
expect(adapterOptions?.logger).toEqual(expect.objectContaining({ warn: expect.any(Function) }));
expect(() => JSON.parse(io.stdout())).not.toThrow();
expect(io.stderr()).toBe('');
});
it('rejects source-dir uploads through the metabase fan-out route', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
@ -694,17 +850,22 @@ describe('runKtxIngest', () => {
),
).resolves.toBe(0);
expect(createAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), {
databaseIntrospectionUrl: 'http://127.0.0.1:8765',
});
expect(createAdapters).toHaveBeenCalledWith(
expect.objectContaining({ projectDir }),
expect.objectContaining({
databaseIntrospectionUrl: 'http://127.0.0.1:8765',
logger: expect.any(Object),
}),
);
expect(runLocal).toHaveBeenCalledWith(
expect.objectContaining({
adapters: createdAdapters,
adapter: 'fake',
connectionId: 'warehouse',
pullConfigOptions: {
pullConfigOptions: expect.objectContaining({
databaseIntrospectionUrl: 'http://127.0.0.1:8765',
},
logger: expect.any(Object),
}),
}),
);
});
@ -747,14 +908,19 @@ describe('runKtxIngest', () => {
installPolicy: 'auto',
io: io.io,
};
expect(createAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), {
managedDaemon: expectedManagedDaemon,
});
expect(createAdapters).toHaveBeenCalledWith(
expect.objectContaining({ projectDir }),
expect.objectContaining({
managedDaemon: expectedManagedDaemon,
logger: expect.any(Object),
}),
);
expect(runLocal).toHaveBeenCalledWith(
expect.objectContaining({
pullConfigOptions: {
pullConfigOptions: expect.objectContaining({
managedDaemon: expectedManagedDaemon,
},
logger: expect.any(Object),
}),
}),
);
});
@ -808,9 +974,13 @@ describe('runKtxIngest', () => {
),
).resolves.toBe(0);
expect(createAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), {
historicSqlConnectionId: 'warehouse',
});
expect(createAdapters).toHaveBeenCalledWith(
expect.objectContaining({ projectDir }),
expect.objectContaining({
historicSqlConnectionId: 'warehouse',
logger: expect.any(Object),
}),
);
expect(runLocal).toHaveBeenCalledWith(
expect.objectContaining({
adapters: createdAdapters,
@ -912,12 +1082,228 @@ describe('runKtxIngest', () => {
expect(stdout).toContain('[45%] Planned 1 work unit');
expect(stdout).toContain('[80%] Processed 1/1 work units');
expect(stdout).toContain('[100%] Ingest completed');
expect(stdout.indexOf('[5%] Fetching source files for warehouse/historic-sql')).toBeLessThan(
stdout.indexOf('Report: report-live-1'),
);
expect(stdout).toContain('Report: report-live-1');
expect(io.stderr()).toBe('');
});
it('writes plain TTY ingest progress and final report to stdout', async () => {
const projectDir = join(tempDir, 'project');
await writeWarehouseConfig(projectDir);
const sourceDir = join(tempDir, 'source');
await mkdir(join(sourceDir, 'orders'), { recursive: true });
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
const runLocal = vi.fn(async (input: RunLocalIngestOptions) => completedLocalBundleRun(input, 'local-job-1'));
const io = makeIo({ isTTY: true });
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'warehouse',
adapter: 'fake',
sourceDir,
outputMode: 'plain',
},
io.io,
{
env: interactiveEnv(),
runLocalIngest: runLocal,
},
),
).resolves.toBe(0);
expect(io.stdout()).toContain('[5%] Fetching source files for warehouse/fake');
expect(io.stdout()).toContain('Report: report-live-1');
expect(io.stderr()).toBe('');
});
it('prints plain WorkUnit step progress during long-running local ingest', async () => {
const projectDir = join(tempDir, 'historic-sql-step-progress-project');
await mkdir(projectDir, { recursive: true });
await writeFile(
join(projectDir, 'ktx.yaml'),
[
'project: historic-sql-step-progress-project',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:WAREHOUSE_DATABASE_URL',
' historicSql:',
' enabled: true',
' dialect: postgres',
' minExecutions: 2',
'ingest:',
' adapters:',
' - historic-sql',
'',
].join('\n'),
'utf-8',
);
const createdAdapters: SourceAdapter[] = [
{ source: 'historic-sql', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) },
];
const runLocal = vi.fn(async (input: RunLocalIngestOptions) => {
input.memoryFlow?.update({
plannedWorkUnits: [
{
unitKey: 'historic-sql-table-public-orders',
rawFiles: ['tables/public/orders.json'],
peerFileCount: 0,
dependencyCount: 0,
},
{
unitKey: 'historic-sql-table-public-customers',
rawFiles: ['tables/public/customers.json'],
peerFileCount: 0,
dependencyCount: 0,
},
],
});
input.memoryFlow?.emit({ type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 });
input.memoryFlow?.emit({
type: 'work_unit_started',
unitKey: 'historic-sql-table-public-orders',
skills: ['historic_sql_table_digest'],
stepBudget: 40,
});
input.memoryFlow?.emit({
type: 'work_unit_step',
unitKey: 'historic-sql-table-public-orders',
stepIndex: 7,
stepBudget: 40,
});
input.memoryFlow?.emit({
type: 'work_unit_finished',
unitKey: 'historic-sql-table-public-orders',
status: 'success',
});
input.memoryFlow?.finish('done');
return completedLocalBundleRun(input, input.jobId ?? 'historic-step-progress-job');
});
const io = makeIo({ isTTY: true });
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'warehouse',
adapter: 'historic-sql',
outputMode: 'plain',
},
io.io,
{
env: interactiveEnv(),
createAdapters: vi.fn(() => createdAdapters as never),
runLocalIngest: runLocal,
jobIdFactory: () => 'historic-step-progress-job',
},
),
).resolves.toBe(0);
const stdout = io.stdout();
expect(stdout).toContain('[45%] Planned 2 work units');
expect(stdout).toContain('[55%] Processing 1/2 work units: historic-sql-table-public-orders');
expect(stdout).toContain(
'\r[58%] Processing work units: 0/2 complete, 1 active; latest historic-sql-table-public-orders step 7/40\u001b[K',
);
expect(stdout).toContain('[68%] Processed 1/2 work units');
});
it('renders concurrent WorkUnit step progress as transient aggregate status', async () => {
const projectDir = join(tempDir, 'historic-sql-concurrent-progress-project');
await mkdir(projectDir, { recursive: true });
await writeFile(
join(projectDir, 'ktx.yaml'),
[
'project: historic-sql-concurrent-progress-project',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:WAREHOUSE_DATABASE_URL',
' historicSql:',
' enabled: true',
' dialect: postgres',
' minExecutions: 2',
'ingest:',
' adapters:',
' - historic-sql',
'',
].join('\n'),
'utf-8',
);
const createdAdapters: SourceAdapter[] = [
{ source: 'historic-sql', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) },
];
const workUnitKeys = [
'historic-sql-table-public-orders',
'historic-sql-table-public-customers',
'historic-sql-table-public-line-items',
'historic-sql-table-public-payments',
'historic-sql-table-public-products',
'historic-sql-table-public-suppliers',
];
const runLocal = vi.fn(async (input: RunLocalIngestOptions) => {
input.memoryFlow?.update({
plannedWorkUnits: workUnitKeys.map((unitKey) => ({
unitKey,
rawFiles: [`tables/${unitKey}.json`],
peerFileCount: 0,
dependencyCount: 0,
})),
});
input.memoryFlow?.emit({
type: 'chunks_planned',
chunkCount: workUnitKeys.length,
workUnitCount: workUnitKeys.length,
evictionCount: 0,
});
for (const unitKey of workUnitKeys) {
input.memoryFlow?.emit({
type: 'work_unit_started',
unitKey,
skills: ['historic_sql_table_digest'],
stepBudget: 40,
});
}
for (const unitKey of workUnitKeys) {
input.memoryFlow?.emit({ type: 'work_unit_step', unitKey, stepIndex: 1, stepBudget: 40 });
}
input.memoryFlow?.finish('done');
return completedLocalBundleRun(input, input.jobId ?? 'historic-concurrent-progress-job');
});
const io = makeIo({ isTTY: true });
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'warehouse',
adapter: 'historic-sql',
outputMode: 'plain',
},
io.io,
{
env: interactiveEnv(),
createAdapters: vi.fn(() => createdAdapters as never),
runLocalIngest: runLocal,
jobIdFactory: () => 'historic-concurrent-progress-job',
},
),
).resolves.toBe(0);
const stdout = io.stdout();
expect(stdout).toContain(
'\r[56%] Processing work units: 0/6 complete, 6 active; latest historic-sql-table-public-suppliers step 1/40\u001b[K',
);
expect(stdout).not.toContain(
'\n[56%] Processing 6/6 work units: historic-sql-table-public-suppliers step 1/40\n',
);
expect(stdout).toContain('\n[100%] Ingest completed\n');
});
it('passes local Looker pull-config options and agent runner into scheduled ingest for Looker scheduled ingest', async () => {
const projectDir = join(tempDir, 'project');
await writeWarehouseConfig(projectDir);
@ -958,15 +1344,19 @@ describe('runKtxIngest', () => {
),
).resolves.toBe(0);
expect(createAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), {
looker: {
parser: pullConfigOptions.looker.parser,
},
});
expect(createAdapters).toHaveBeenCalledWith(
expect.objectContaining({ projectDir }),
expect.objectContaining({
logger: expect.any(Object),
looker: {
parser: pullConfigOptions.looker.parser,
},
}),
);
expect(runLocal).toHaveBeenCalledWith(
expect.objectContaining({
agentRunner,
pullConfigOptions,
pullConfigOptions: expect.objectContaining(pullConfigOptions),
}),
);
});

114
packages/cli/src/ingest.ts
View file

@ -18,6 +18,7 @@ import {
} from '@ktx/context/ingest';
import { loadKtxProject } from '@ktx/context/project';
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
import { createCliOperationalLogger } from './io/logger.js';
import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
import { type KtxMemoryFlowStdin, renderMemoryFlowInteractively } from './memory-flow-interactive.js';
@ -142,22 +143,22 @@ function createMetabaseFanoutProgress(
connectionId: string,
io: KtxIngestIo,
): LocalMetabaseFanoutProgress {
io.stdout.write(`Metabase ingest: ${connectionId}\n`);
io.stdout.write('Checking mappings and scheduled-pull targets...\n');
io.stderr.write(`Metabase ingest: ${connectionId}\n`);
io.stderr.write('Checking mappings and scheduled-pull targets...\n');
return {
onMetabaseFanoutPlanned(event) {
io.stdout.write(`Targets: ${pluralize(event.children.length, 'mapped database')}\n`);
io.stderr.write(`Targets: ${pluralize(event.children.length, 'mapped database')}\n`);
for (const child of event.children) {
io.stdout.write(`- database=${child.metabaseDatabaseId} target=${child.targetConnectionId} status=queued\n`);
io.stderr.write(`- database=${child.metabaseDatabaseId} target=${child.targetConnectionId} status=queued\n`);
}
},
onMetabaseChildStarted(event) {
io.stdout.write(
io.stderr.write(
`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=running job=${event.jobId}\n`,
);
},
onMetabaseChildCompleted(event) {
io.stdout.write(
io.stderr.write(
`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=${event.status} job=${event.jobId}\n`,
);
},
@ -168,14 +169,51 @@ function formatDiffProgress(event: Extract<MemoryFlowEvent, { type: 'diff_comput
return `+${event.added}/~${event.modified}/-${event.deleted}/=${event.unchanged}`;
}
function completedWorkUnitCount(snapshot: MemoryFlowReplayInput): number {
return snapshot.events.filter((event) => event.type === 'work_unit_finished').length;
function workUnitEventsThrough(snapshot: MemoryFlowReplayInput, eventIndex: number): MemoryFlowEvent[] {
return snapshot.events.slice(0, eventIndex + 1);
}
function completedWorkUnitCountThrough(snapshot: MemoryFlowReplayInput, eventIndex: number): number {
return workUnitEventsThrough(snapshot, eventIndex).filter((event) => event.type === 'work_unit_finished').length;
}
function activeWorkUnitCountThrough(snapshot: MemoryFlowReplayInput, eventIndex: number): number {
const active = new Set<string>();
for (const event of workUnitEventsThrough(snapshot, eventIndex)) {
if (event.type === 'work_unit_started') {
active.add(event.unitKey);
}
if (event.type === 'work_unit_finished') {
active.delete(event.unitKey);
}
}
return active.size;
}
function plannedWorkUnitCountThrough(snapshot: MemoryFlowReplayInput, eventIndex: number): number {
if (snapshot.plannedWorkUnits.length > 0) {
return snapshot.plannedWorkUnits.length;
}
const planEvent = workUnitEventsThrough(snapshot, eventIndex)
.filter((event) => event.type === 'chunks_planned')
.at(-1);
return planEvent?.workUnitCount ?? completedWorkUnitCountThrough(snapshot, eventIndex);
}
function workUnitOrdinalThrough(snapshot: MemoryFlowReplayInput, eventIndex: number, unitKey: string): number {
const events = workUnitEventsThrough(snapshot, eventIndex);
const startedIndex = events.findIndex((event) => event.type === 'work_unit_started' && event.unitKey === unitKey);
if (startedIndex === -1) {
return completedWorkUnitCountThrough(snapshot, eventIndex) + 1;
}
return events.slice(0, startedIndex + 1).filter((event) => event.type === 'work_unit_started').length;
}
function plainIngestEventProgress(
event: MemoryFlowEvent,
snapshot: MemoryFlowReplayInput,
): { percent: number; message: string } | null {
eventIndex: number,
): { percent: number; message: string; transient?: boolean } | null {
switch (event.type) {
case 'source_acquired':
return {
@ -196,11 +234,28 @@ function plainIngestEventProgress(
};
case 'stage_skipped':
return { percent: 45, message: `Skipped ${event.stage}: ${event.reason}` };
case 'work_unit_started':
return { percent: 55, message: `Processing ${event.unitKey}` };
case 'work_unit_started': {
const total = plannedWorkUnitCountThrough(snapshot, eventIndex);
const ordinal = workUnitOrdinalThrough(snapshot, eventIndex, event.unitKey);
const progress = total > 0 ? `${ordinal}/${total} work units: ` : '';
return { percent: 55, message: `Processing ${progress}${event.unitKey}` };
}
case 'work_unit_step': {
const total = plannedWorkUnitCountThrough(snapshot, eventIndex);
const completed = completedWorkUnitCountThrough(snapshot, eventIndex);
const active = activeWorkUnitCountThrough(snapshot, eventIndex);
const stepFraction = event.stepBudget > 0 ? Math.min(1, event.stepIndex / event.stepBudget) : 0;
const percent = total > 0 ? 55 + Math.ceil(((completed + stepFraction) / total) * 25) : 55;
const latest = `${event.unitKey} step ${event.stepIndex}/${event.stepBudget}`;
return {
percent,
message: `Processing work units: ${completed}/${total} complete, ${active} active; latest ${latest}`,
transient: true,
};
}
case 'work_unit_finished': {
const total = snapshot.plannedWorkUnits.length || completedWorkUnitCount(snapshot);
const completed = completedWorkUnitCount(snapshot);
const total = plannedWorkUnitCountThrough(snapshot, eventIndex);
const completed = completedWorkUnitCountThrough(snapshot, eventIndex);
const percent = total > 0 ? 55 + Math.round((completed / total) * 25) : 80;
return {
percent,
@ -225,7 +280,6 @@ function plainIngestEventProgress(
case 'report_created':
return { percent: 98, message: `Created ingest report ${event.reportPath ?? event.runId}` };
case 'scope_detected':
case 'work_unit_step':
case 'candidate_action':
return null;
}
@ -242,15 +296,31 @@ function shouldWritePlainIngestProgress(
function createPlainIngestProgressRenderer(
args: Extract<KtxIngestArgs, { command: 'run' }>,
io: KtxIngestIo,
): { start(): void; update(snapshot: MemoryFlowReplayInput): void } {
): { start(): void; update(snapshot: MemoryFlowReplayInput): void; flush(): void } {
let printedEvents = 0;
let lastPercent = 0;
let printedCompletion = false;
let hasPendingTransient = false;
const write = (percent: number, message: string) => {
const flush = () => {
if (!hasPendingTransient) {
return;
}
io.stdout.write('\n');
hasPendingTransient = false;
};
const write = (percent: number, message: string, options?: { transient?: boolean }) => {
const nextPercent = Math.max(lastPercent, Math.max(0, Math.min(100, percent)));
lastPercent = nextPercent;
io.stdout.write(`[${nextPercent}%] ${message}\n`);
const line = `[${nextPercent}%] ${message}`;
if (options?.transient === true) {
io.stdout.write(`\r${line}\u001b[K`);
hasPendingTransient = true;
return;
}
flush();
io.stdout.write(`${line}\n`);
};
return {
@ -259,13 +329,14 @@ function createPlainIngestProgressRenderer(
},
update(snapshot) {
while (printedEvents < snapshot.events.length) {
const eventIndex = printedEvents;
const event = snapshot.events[printedEvents++];
if (!event) {
continue;
}
const progress = plainIngestEventProgress(event, snapshot);
const progress = plainIngestEventProgress(event, snapshot, eventIndex);
if (progress) {
write(progress.percent, progress.message);
write(progress.percent, progress.message, progress.transient === true ? { transient: true } : undefined);
}
}
if (!printedCompletion && snapshot.status !== 'running') {
@ -273,6 +344,7 @@ function createPlainIngestProgressRenderer(
write(100, snapshot.status === 'done' ? 'Ingest completed' : 'Ingest failed');
}
},
flush,
};
}
@ -435,11 +507,13 @@ export async function runKtxIngest(
const executeLocalIngest = deps.runLocalIngest ?? runLocalIngest;
const localIngestOptions = deps.localIngestOptions ?? {};
const managedDaemon = managedDaemonOptionsForIngestRun(args, io);
const operationalLogger = createCliOperationalLogger(io, args.outputMode);
const adapterOptions = {
...(localIngestOptions.pullConfigOptions ?? {}),
...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}),
...(managedDaemon ? { managedDaemon } : {}),
...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}),
logger: operationalLogger,
};
if (args.adapter === 'metabase' && args.sourceDir) {
throw new Error('source-dir uploads are not supported for the Metabase fan-out adapter');
@ -524,6 +598,7 @@ export async function runKtxIngest(
io.stdout.write(formatMemoryFlowFinalSummary(latestMemoryFlowSnapshot));
return reportStatus(result.report) === 'done' ? 0 : 1;
}
plainProgress?.flush();
await writeReportRecord(result.report, runOutputMode, io, {
interactive: (args.inputMode ?? 'auto') === 'auto',
renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
@ -531,6 +606,7 @@ export async function runKtxIngest(
});
return reportStatus(result.report) === 'done' ? 0 : 1;
} finally {
plainProgress?.flush();
liveTui?.close();
}
}

65
packages/cli/src/io/logger.test.ts Normal file
View file

@ -0,0 +1,65 @@
import { describe, expect, it, vi } from 'vitest';
import { createCliOperationalLogger, createNoopOperationalLogger } from './logger.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('createCliOperationalLogger', () => {
it('routes operational messages to stderr outside JSON mode', () => {
const io = makeIo();
const logger = createCliOperationalLogger(io.io, 'plain');
logger.log('progress');
logger.warn('warning');
logger.error('failure');
logger.debug?.('debug');
expect(io.stdout()).toBe('');
expect(io.stderr()).toBe('progress\nwarning\nfailure\ndebug\n');
});
it('suppresses operational messages in JSON mode by default', () => {
const io = makeIo();
const logger = createCliOperationalLogger(io.io, 'json');
logger.log('progress');
logger.warn('warning');
logger.error('failure');
logger.debug?.('debug');
expect(io.stdout()).toBe('');
expect(io.stderr()).toBe('');
});
});
describe('createNoopOperationalLogger', () => {
it('never writes', () => {
const logger = createNoopOperationalLogger();
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
logger.log('progress');
logger.warn('warning');
logger.error('failure');
logger.debug?.('debug');
expect(warn).not.toHaveBeenCalled();
});
});

40
packages/cli/src/io/logger.ts Normal file
View file

@ -0,0 +1,40 @@
import type { KtxCliIo } from '../cli-runtime.js';
import type { KtxOutputMode } from './mode.js';
export interface KtxOperationalLogger {
log(message: string): void;
warn(message: string): void;
error(message: string): void;
debug?(message: string): void;
}
export type KtxOperationalOutputMode = KtxOutputMode | 'viz';
function writeLine(io: KtxCliIo, message: string): void {
io.stderr.write(message.endsWith('\n') ? message : `${message}\n`);
}
export function createNoopOperationalLogger(): KtxOperationalLogger {
return {
log: () => undefined,
warn: () => undefined,
error: () => undefined,
debug: () => undefined,
};
}
export function createCliOperationalLogger(
io: KtxCliIo,
mode: KtxOperationalOutputMode,
): KtxOperationalLogger {
if (mode === 'json') {
return createNoopOperationalLogger();
}
return {
log: (message) => writeLine(io, message),
warn: (message) => writeLine(io, message),
error: (message) => writeLine(io, message),
debug: (message) => writeLine(io, message),
};
}

15
packages/cli/src/io/print-list.ts
View file

@ -28,6 +28,16 @@ export interface PrintListArgs<Row> {
io: KtxCliIo;
}
export interface KtxJsonResultEnvelope<T> {
kind: string;
data: T;
meta?: Record<string, unknown>;
}
export function writeJsonResult<T>(io: KtxCliIo, envelope: KtxJsonResultEnvelope<T>): void {
io.stdout.write(`${JSON.stringify(envelope, null, 2)}\n`);
}
export function printList<Row extends object>(args: PrintListArgs<Row>): void {
switch (args.mode) {
case 'json':
@ -61,12 +71,11 @@ function printListPlain<Row extends object>(args: PrintListArgs<Row>): void {
}
function printListJson<Row extends object>(args: PrintListArgs<Row>): void {
const envelope = {
writeJsonResult(args.io, {
kind: 'list',
data: { items: args.rows },
meta: { command: args.command },
};
args.io.stdout.write(`${JSON.stringify(envelope, null, 2)}\n`);
});
}
function pluralize(count: number, singular: string): string {

2
packages/cli/src/local-adapters.ts
View file

@ -35,6 +35,7 @@ import {
managedDaemonDatabaseIntrospectionOptions,
type ManagedPythonCoreDaemonOptions,
} from './managed-python-http.js';
import type { KtxOperationalLogger } from './io/logger.js';
function hasSnowflakeDriver(connection: unknown): boolean {
return (
@ -162,6 +163,7 @@ export interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdap
sqlAnalysis?: SqlAnalysisPort;
sqlAnalysisUrl?: string;
managedDaemon?: ManagedPythonCoreDaemonOptions;
logger?: KtxOperationalLogger;
}
function historicSqlRecord(connection: unknown): Record<string, unknown> | null {

42
packages/cli/src/managed-python-command.test.ts
View file

@ -214,6 +214,7 @@ describe('createManagedPythonSemanticLayerComputePort', () => {
expect(confirmInstall).toHaveBeenCalledWith(
'KTX needs to install the core Python runtime. This downloads Python dependencies with uv. Continue?',
io.io,
);
expect(installRuntime).toHaveBeenCalledWith({
cliVersion: '0.2.0',
@ -221,4 +222,45 @@ describe('createManagedPythonSemanticLayerComputePort', () => {
force: false,
});
});
it('uses injected runtime confirmation instead of reading process TTY directly', async () => {
const io = makeIo();
const compute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };
const installRuntime = vi.fn(async (): Promise<ManagedPythonRuntimeInstallResult> => installResult());
const confirmInstall = vi.fn(async () => true);
await expect(
createManagedPythonSemanticLayerComputePort({
cliVersion: '0.2.0',
installPolicy: 'prompt',
io: io.io,
readStatus: async () => missingStatus(),
installRuntime,
confirmInstall,
createPythonCompute: () => compute,
}),
).resolves.toBe(compute);
expect(confirmInstall).toHaveBeenCalledWith(
'KTX needs to install the core Python runtime. This downloads Python dependencies with uv. Continue?',
io.io,
);
expect(io.stderr()).toContain('Installing KTX Python runtime (core) with uv...');
});
it('can decide default runtime prompting from injected io capabilities', async () => {
const io = makeIo();
Object.assign(io.io.stdout, { isTTY: false });
await expect(
createManagedPythonSemanticLayerComputePort({
cliVersion: '0.2.0',
installPolicy: 'prompt',
io: io.io,
readStatus: async () => missingStatus(),
installRuntime: vi.fn(),
createPythonCompute: () => ({ query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }),
}),
).rejects.toThrow('KTX Python runtime installation was cancelled');
});
});

18
packages/cli/src/managed-python-command.ts
View file

@ -1,6 +1,6 @@
import { cancel, confirm, isCancel } from '@clack/prompts';
import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon';
import type { KtxCliIo } from './cli-runtime.js';
import { createClackPromptAdapter } from './clack.js';
import {
installManagedPythonRuntime,
readManagedPythonRuntimeStatus,
@ -36,7 +36,7 @@ export interface ManagedPythonCommandRuntime {
export interface ManagedPythonCommandDeps {
readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise<ManagedPythonRuntimeStatus>;
installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise<ManagedPythonRuntimeInstallResult>;
confirmInstall?: (message: string) => Promise<boolean>;
confirmInstall?: (message: string, io: KtxCliIo) => Promise<boolean>;
}
export interface ManagedPythonCommandOptions extends ManagedPythonCommandDeps {
@ -69,16 +69,12 @@ function hasFeature(manifest: InstalledKtxRuntimeManifest, feature: KtxRuntimeFe
return manifest.features.includes(feature);
}
async function defaultConfirmInstall(message: string): Promise<boolean> {
if (process.stdin.isTTY !== true || process.stdout.isTTY !== true) {
async function defaultConfirmInstall(message: string, io: KtxCliIo): Promise<boolean> {
if (io.stdout.isTTY !== true) {
return false;
}
const response = await confirm({ message, initialValue: true });
if (isCancel(response)) {
cancel('Runtime installation cancelled.');
return false;
}
return response === true;
const prompts = createClackPromptAdapter();
return await prompts.confirm({ message, initialValue: true });
}
export async function ensureManagedPythonCommandRuntime(
@ -99,7 +95,7 @@ export async function ensureManagedPythonCommandRuntime(
if (options.installPolicy === 'prompt') {
const confirmInstall = options.confirmInstall ?? defaultConfirmInstall;
const confirmed = await confirmInstall(installPrompt(feature));
const confirmed = await confirmInstall(installPrompt(feature), options.io);
if (!confirmed) {
throw new Error(`KTX Python runtime installation was cancelled. Run: ${managedRuntimeInstallCommand(feature)}`);
}

189
packages/cli/src/managed-python-daemon.test.ts
View file

@ -5,9 +5,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
readManagedPythonDaemonStatus,
startManagedPythonDaemon,
stopAllManagedPythonDaemons,
stopManagedPythonDaemon,
type ManagedPythonDaemonChild,
type ManagedPythonDaemonFetch,
type ManagedPythonDaemonProcessInfo,
type ManagedPythonDaemonSpawn,
type ManagedPythonDaemonState,
} from './managed-python-daemon.js';
@ -105,6 +107,24 @@ function runningState(root: string, overrides: Partial<ManagedPythonDaemonState>
};
}
function daemonStatePath(root: string, version: string): string {
return join(root, 'runtime', version, 'daemon.json');
}
function runningStateForVersion(
root: string,
version: string,
overrides: Partial<ManagedPythonDaemonState> = {},
): ManagedPythonDaemonState {
return {
...runningState(root),
version,
stdoutLog: join(root, 'runtime', version, 'daemon.stdout.log'),
stderrLog: join(root, 'runtime', version, 'daemon.stderr.log'),
...overrides,
};
}
describe('managed Python daemon lifecycle', () => {
let tempDir: string;
@ -170,6 +190,41 @@ describe('managed Python daemon lifecycle', () => {
});
});
it('makes a final health probe before reporting startup failure', async () => {
const spawnDaemon = makeSpawn(5556);
const installRuntime = vi.fn(async () => installResult(tempDir));
const fetch = vi
.fn<ManagedPythonDaemonFetch>()
.mockRejectedValueOnce(new Error('fetch failed'))
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => ({ status: 'healthy', version: '0.2.0' }),
text: async () => '',
});
const result = await startManagedPythonDaemon({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
features: ['core'],
installRuntime,
spawnDaemon,
fetch,
allocatePort: vi.fn(async () => 61234),
now: () => new Date('2026-05-11T00:00:00.000Z'),
startupTimeoutMs: 5,
pollIntervalMs: 20,
});
expect(result.status).toBe('started');
expect(fetch).toHaveBeenCalledTimes(2);
expect(JSON.parse(await readFile(layout(tempDir).daemonStatePath, 'utf8'))).toMatchObject({
pid: 5556,
port: 61234,
version: '0.2.0',
});
});
it('reuses a healthy daemon with the requested feature set', async () => {
await mkdir(layout(tempDir).versionDir, { recursive: true });
await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`);
@ -236,4 +291,138 @@ describe('managed Python daemon lifecycle', () => {
expect(killProcess).toHaveBeenCalledWith(4242);
await expect(readFile(layout(tempDir).daemonStatePath, 'utf8')).rejects.toThrow();
});
it('stops all recorded daemon states across runtime versions and removes state files', async () => {
await mkdir(join(tempDir, 'runtime', '0.1.0'), { recursive: true });
await mkdir(join(tempDir, 'runtime', '0.2.0'), { recursive: true });
await writeFile(
daemonStatePath(tempDir, '0.1.0'),
`${JSON.stringify(runningStateForVersion(tempDir, '0.1.0', { pid: 1111, port: 61111 }), null, 2)}\n`,
);
await writeFile(
daemonStatePath(tempDir, '0.2.0'),
`${JSON.stringify(runningStateForVersion(tempDir, '0.2.0', { pid: 2222, port: 62222 }), null, 2)}\n`,
);
const alive = new Set([1111, 2222]);
const killProcess = vi.fn((pid: number) => {
alive.delete(pid);
});
const result = await stopAllManagedPythonDaemons({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
listProcesses: vi.fn(async () => []),
processAlive: vi.fn((pid) => alive.has(pid)),
killProcess,
stopGraceMs: 0,
});
expect(result.failed).toHaveLength(0);
expect(result.stopped.map((entry) => entry.pid).sort()).toEqual([1111, 2222]);
expect(killProcess).toHaveBeenCalledWith(1111, 'SIGTERM');
expect(killProcess).toHaveBeenCalledWith(2222, 'SIGTERM');
await expect(readFile(daemonStatePath(tempDir, '0.1.0'), 'utf8')).rejects.toThrow();
await expect(readFile(daemonStatePath(tempDir, '0.2.0'), 'utf8')).rejects.toThrow();
});
it('removes stale state when the recorded daemon process is no longer alive', async () => {
await mkdir(layout(tempDir).versionDir, { recursive: true });
await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`);
const result = await stopAllManagedPythonDaemons({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
listProcesses: vi.fn(async () => []),
processAlive: vi.fn(() => false),
killProcess: vi.fn(),
stopGraceMs: 0,
});
expect(result.stopped).toHaveLength(0);
expect(result.stale.map((entry) => entry.pid)).toEqual([4242]);
await expect(readFile(layout(tempDir).daemonStatePath, 'utf8')).rejects.toThrow();
});
it('deduplicates a daemon found by state and process scan, preferring state metadata', async () => {
await mkdir(layout(tempDir).versionDir, { recursive: true });
await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`);
const alive = new Set([4242]);
const killProcess = vi.fn((pid: number) => {
alive.delete(pid);
});
const result = await stopAllManagedPythonDaemons({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
listProcesses: vi.fn(async (): Promise<ManagedPythonDaemonProcessInfo[]> => [
{ pid: 4242, command: 'uv run ktx-daemon serve-http --host 127.0.0.1 --port 61234' },
]),
processAlive: vi.fn((pid) => alive.has(pid)),
killProcess,
stopGraceMs: 0,
});
expect(result.stopped).toHaveLength(1);
expect(result.stopped[0]).toMatchObject({
pid: 4242,
source: 'state',
url: 'http://127.0.0.1:58731',
});
expect(killProcess).toHaveBeenCalledTimes(1);
});
it('stops unrecorded ktx-daemon serve-http processes from process scan results', async () => {
const alive = new Set([3333, 5555]);
const killProcess = vi.fn((pid: number) => {
alive.delete(pid);
});
const result = await stopAllManagedPythonDaemons({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
listProcesses: vi.fn(async (): Promise<ManagedPythonDaemonProcessInfo[]> => [
{ pid: 3333, command: 'uv run ktx-daemon serve-http --host 127.0.0.1 --port 8765' },
{ pid: 4444, command: 'node server.js --port 8765' },
{ pid: 5555, command: 'grep ktx-daemon serve-http --port 8765' },
]),
processAlive: vi.fn((pid) => alive.has(pid)),
killProcess,
stopGraceMs: 0,
});
expect(result.failed).toHaveLength(0);
expect(result.stopped).toEqual([
expect.objectContaining({
pid: 3333,
source: 'process',
url: 'http://127.0.0.1:8765',
}),
]);
expect(killProcess).toHaveBeenCalledWith(3333, 'SIGTERM');
expect(killProcess).not.toHaveBeenCalledWith(4444, expect.anything());
expect(killProcess).not.toHaveBeenCalledWith(5555, expect.anything());
});
it('reports a failed stop when TERM and KILL leave a daemon running', async () => {
await mkdir(layout(tempDir).versionDir, { recursive: true });
await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`);
const result = await stopAllManagedPythonDaemons({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
listProcesses: vi.fn(async () => []),
processAlive: vi.fn(() => true),
killProcess: vi.fn(),
stopGraceMs: 0,
});
expect(result.stopped).toHaveLength(0);
expect(result.failed).toEqual([
expect.objectContaining({
pid: 4242,
detail: 'Process still running after SIGKILL',
}),
]);
expect(await readFile(layout(tempDir).daemonStatePath, 'utf8')).toContain('"pid": 4242');
});
});

444
packages/cli/src/managed-python-daemon.ts
View file

@ -1,7 +1,9 @@
import { spawn } from 'node:child_process';
import { mkdir, open, readFile, rm, writeFile } from 'node:fs/promises';
import { execFile, spawn } from 'node:child_process';
import { mkdir, open, readdir, readFile, rm, writeFile } from 'node:fs/promises';
import { createServer } from 'node:net';
import { join } from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { promisify } from 'node:util';
import { z } from 'zod';
import {
installManagedPythonRuntime,
@ -44,6 +46,35 @@ export interface ManagedPythonDaemonStopResult {
state?: ManagedPythonDaemonState;
}
export interface ManagedPythonDaemonProcessInfo {
pid: number;
command: string;
}
export type ManagedPythonDaemonStopAllSource = 'state' | 'process';
export interface ManagedPythonDaemonStopAllEntry {
pid: number;
source: ManagedPythonDaemonStopAllSource;
url?: string;
health?: 'healthy' | 'unreachable';
version?: string;
command?: string;
statePaths: string[];
}
export interface ManagedPythonDaemonStopAllFailure extends ManagedPythonDaemonStopAllEntry {
detail: string;
}
export interface ManagedPythonDaemonStopAllResult {
runtimeRoot: string;
stopped: ManagedPythonDaemonStopAllEntry[];
stale: ManagedPythonDaemonStopAllEntry[];
failed: ManagedPythonDaemonStopAllFailure[];
scanErrors: string[];
}
export interface ManagedPythonDaemonChild {
pid?: number;
unref(): void;
@ -68,6 +99,8 @@ export type ManagedPythonDaemonFetch = (
text(): Promise<string>;
}>;
export type ManagedPythonDaemonKillProcess = (pid: number, signal?: NodeJS.Signals) => void;
export interface ManagedPythonDaemonStartOptions extends ManagedPythonRuntimeLayoutOptions {
features: KtxRuntimeFeature[];
force?: boolean;
@ -76,7 +109,7 @@ export interface ManagedPythonDaemonStartOptions extends ManagedPythonRuntimeLay
fetch?: ManagedPythonDaemonFetch;
allocatePort?: () => Promise<number>;
processAlive?: (pid: number) => boolean;
killProcess?: (pid: number) => void;
killProcess?: ManagedPythonDaemonKillProcess;
now?: () => Date;
startupTimeoutMs?: number;
pollIntervalMs?: number;
@ -89,9 +122,20 @@ export interface ManagedPythonDaemonStatusOptions extends ManagedPythonRuntimeLa
export interface ManagedPythonDaemonStopOptions extends ManagedPythonRuntimeLayoutOptions {
processAlive?: (pid: number) => boolean;
killProcess?: (pid: number) => void;
killProcess?: ManagedPythonDaemonKillProcess;
}
export interface ManagedPythonDaemonStopAllOptions extends ManagedPythonRuntimeLayoutOptions {
listProcesses?: () => Promise<ManagedPythonDaemonProcessInfo[]>;
processAlive?: (pid: number) => boolean;
killProcess?: ManagedPythonDaemonKillProcess;
stopGraceMs?: number;
pollIntervalMs?: number;
healthProbeMs?: number;
}
const execFileAsync = promisify(execFile);
const daemonStateSchema = z.object({
schemaVersion: z.literal(1),
pid: z.number().int().positive(),
@ -126,9 +170,9 @@ function defaultProcessAlive(pid: number): boolean {
}
}
function defaultKillProcess(pid: number): void {
function defaultKillProcess(pid: number, signal: NodeJS.Signals = 'SIGTERM'): void {
try {
process.kill(pid, 'SIGTERM');
process.kill(pid, signal);
} catch (error) {
const code = (error as { code?: unknown }).code;
if (code !== 'ESRCH') {
@ -273,6 +317,15 @@ async function waitForHealth(input: {
lastDetail = health.detail;
await delay(input.pollIntervalMs);
}
const finalHealth = await healthOk({
state: input.state,
cliVersion: input.cliVersion,
fetch: input.fetch,
});
if (finalHealth.ok) {
return;
}
lastDetail = finalHealth.detail;
throw new Error(`KTX Python daemon failed to start: ${lastDetail}. stderr: ${input.state.stderrLog}`);
}
@ -284,7 +337,7 @@ async function stopRecordedDaemon(input: {
layout: ManagedPythonRuntimeLayout;
state: ManagedPythonDaemonState;
processAlive: (pid: number) => boolean;
killProcess: (pid: number) => void;
killProcess: ManagedPythonDaemonKillProcess;
}): Promise<void> {
if (input.processAlive(input.state.pid)) {
input.killProcess(input.state.pid);
@ -292,6 +345,323 @@ async function stopRecordedDaemon(input: {
await removeState(input.layout);
}
function runtimeRootForStopAll(options: ManagedPythonRuntimeLayoutOptions): string {
return managedPythonRuntimeLayout(options).runtimeRoot;
}
async function removeStatePaths(paths: string[]): Promise<void> {
await Promise.all([...new Set(paths)].map((path) => rm(path, { force: true })));
}
interface ManagedPythonDaemonStopCandidate {
pid: number;
source: ManagedPythonDaemonStopAllSource;
host?: string;
port?: number;
version?: string;
command?: string;
statePaths: string[];
}
function candidateUrl(candidate: ManagedPythonDaemonStopCandidate): string | undefined {
if (!candidate.host || !candidate.port) {
return undefined;
}
return `http://${candidate.host}:${candidate.port}`;
}
function candidateEntry(candidate: ManagedPythonDaemonStopCandidate): ManagedPythonDaemonStopAllEntry {
return {
pid: candidate.pid,
source: candidate.source,
...(candidateUrl(candidate) ? { url: candidateUrl(candidate) } : {}),
...(candidate.version ? { version: candidate.version } : {}),
...(candidate.command ? { command: candidate.command } : {}),
statePaths: [...candidate.statePaths],
};
}
async function probeCandidateHealth(
candidate: ManagedPythonDaemonStopCandidate,
timeoutMs: number,
): Promise<'healthy' | 'unreachable' | undefined> {
const url = candidateUrl(candidate);
if (!url) {
return undefined;
}
const controller = new AbortController();
const timeout = setTimeout(() => {
controller.abort();
}, timeoutMs);
try {
const response = await fetch(`${url}/health`, { signal: controller.signal });
if (!response.ok) {
return 'unreachable';
}
const body = (await response.json()) as unknown;
if (!body || typeof body !== 'object' || Array.isArray(body)) {
return 'unreachable';
}
return (body as Record<string, unknown>).status === 'healthy' ? 'healthy' : 'unreachable';
} catch {
return 'unreachable';
} finally {
clearTimeout(timeout);
}
}
async function readStateCandidates(runtimeRoot: string): Promise<ManagedPythonDaemonStopCandidate[]> {
let entries;
try {
entries = await readdir(runtimeRoot, { withFileTypes: true });
} catch (error) {
const code = (error as { code?: unknown }).code;
if (code === 'ENOENT') {
return [];
}
throw error;
}
const candidates: ManagedPythonDaemonStopCandidate[] = [];
for (const entry of entries) {
if (!entry.isDirectory()) {
continue;
}
const statePath = join(runtimeRoot, entry.name, 'daemon.json');
let state: ManagedPythonDaemonState | undefined;
try {
state = await readState(statePath);
} catch {
continue;
}
if (!state) {
continue;
}
candidates.push({
pid: state.pid,
source: 'state',
host: state.host,
port: state.port,
version: state.version,
statePaths: [statePath],
});
}
return candidates;
}
function tokenizeCommand(command: string): string[] {
const tokens: string[] = [];
for (const match of command.matchAll(/"([^"]*)"|'([^']*)'|(\S+)/g)) {
tokens.push(match[1] ?? match[2] ?? match[3] ?? '');
}
return tokens;
}
function executableName(token: string): string {
return token.split(/[\\/]/).at(-1) ?? token;
}
function isKtxDaemonExecutable(token: string): boolean {
return executableName(token) === 'ktx-daemon' || executableName(token) === 'ktx-daemon.exe';
}
function normalizedExecutableName(token: string): string {
return executableName(token).replace(/\.exe$/i, '').toLowerCase();
}
function hasUvRunPrefix(tokens: string[], daemonIndex: number): boolean {
return normalizedExecutableName(tokens[0] ?? '') === 'uv' && tokens.slice(1, daemonIndex).includes('run');
}
function isPythonExecutable(token: string): boolean {
const name = normalizedExecutableName(token);
return name === 'python' || name === 'python3';
}
function hasPythonModulePrefix(tokens: string[], moduleFlagIndex: number): boolean {
if (moduleFlagIndex === 1 && isPythonExecutable(tokens[0] ?? '')) {
return true;
}
return (
normalizedExecutableName(tokens[0] ?? '') === 'uv' &&
tokens.slice(1, moduleFlagIndex).includes('run') &&
tokens.some((token, index) => index < moduleFlagIndex && isPythonExecutable(token))
);
}
function isKtxDaemonServeHttp(tokens: string[]): boolean {
for (let index = 0; index < tokens.length; index += 1) {
if (
isKtxDaemonExecutable(tokens[index] ?? '') &&
tokens[index + 1] === 'serve-http' &&
(index === 0 || hasUvRunPrefix(tokens, index))
) {
return true;
}
if (
tokens[index] === '-m' &&
tokens[index + 1] === 'ktx_daemon' &&
tokens[index + 2] === 'serve-http' &&
hasPythonModulePrefix(tokens, index)
) {
return true;
}
}
return false;
}
function parseCommandOption(tokens: string[], option: string): string | undefined {
for (let index = 0; index < tokens.length; index += 1) {
const token = tokens[index];
if (token === option) {
return tokens[index + 1];
}
if (token?.startsWith(`${option}=`)) {
return token.slice(option.length + 1);
}
}
return undefined;
}
function processCandidate(processInfo: ManagedPythonDaemonProcessInfo): ManagedPythonDaemonStopCandidate | undefined {
const tokens = tokenizeCommand(processInfo.command);
if (!isKtxDaemonServeHttp(tokens)) {
return undefined;
}
const host = parseCommandOption(tokens, '--host') ?? '127.0.0.1';
const rawPort = parseCommandOption(tokens, '--port');
const parsedPort = rawPort ? Number.parseInt(rawPort, 10) : 8765;
const port = Number.isInteger(parsedPort) && parsedPort >= 1 && parsedPort <= 65535 ? parsedPort : 8765;
return {
pid: processInfo.pid,
source: 'process',
host,
port,
command: processInfo.command,
statePaths: [],
};
}
function mergeCandidates(candidates: ManagedPythonDaemonStopCandidate[]): ManagedPythonDaemonStopCandidate[] {
const byPid = new Map<number, ManagedPythonDaemonStopCandidate>();
for (const candidate of candidates) {
const existing = byPid.get(candidate.pid);
if (!existing) {
byPid.set(candidate.pid, { ...candidate, statePaths: [...candidate.statePaths] });
continue;
}
existing.statePaths.push(...candidate.statePaths);
if (existing.source === 'process' && candidate.source === 'state') {
byPid.set(candidate.pid, {
...candidate,
statePaths: [...new Set([...existing.statePaths, ...candidate.statePaths])],
});
} else {
existing.statePaths = [...new Set(existing.statePaths)];
}
}
return [...byPid.values()].sort((left, right) => left.pid - right.pid);
}
function parsePosixProcessList(output: string): ManagedPythonDaemonProcessInfo[] {
const processes: ManagedPythonDaemonProcessInfo[] = [];
for (const line of output.split(/\r?\n/)) {
const match = line.match(/^\s*(\d+)\s+(.+)$/);
if (!match) {
continue;
}
processes.push({ pid: Number.parseInt(match[1], 10), command: match[2] });
}
return processes;
}
function parseWindowsProcessList(output: string): ManagedPythonDaemonProcessInfo[] {
if (!output.trim()) {
return [];
}
const parsed = JSON.parse(output) as unknown;
const records = Array.isArray(parsed) ? parsed : [parsed];
const processes: ManagedPythonDaemonProcessInfo[] = [];
for (const record of records) {
if (!record || typeof record !== 'object') {
continue;
}
const value = record as Record<string, unknown>;
const pid = value.ProcessId;
const command = value.CommandLine;
if (typeof pid === 'number' && typeof command === 'string' && command.length > 0) {
processes.push({ pid, command });
}
}
return processes;
}
async function defaultListProcesses(platform: NodeJS.Platform = process.platform): Promise<ManagedPythonDaemonProcessInfo[]> {
if (platform === 'win32') {
const command = [
'Get-CimInstance Win32_Process',
'| Where-Object { $_.CommandLine -ne $null }',
'| Select-Object ProcessId,CommandLine',
'| ConvertTo-Json -Compress',
].join(' ');
const { stdout } = await execFileAsync('powershell.exe', ['-NoProfile', '-Command', command], {
encoding: 'utf8',
maxBuffer: 10 * 1024 * 1024,
});
return parseWindowsProcessList(stdout);
}
const { stdout } = await execFileAsync('ps', ['-axo', 'pid=,command='], {
encoding: 'utf8',
maxBuffer: 10 * 1024 * 1024,
});
return parsePosixProcessList(stdout);
}
async function waitUntilStopped(input: {
pid: number;
processAlive: (pid: number) => boolean;
timeoutMs: number;
pollIntervalMs: number;
}): Promise<boolean> {
const deadline = Date.now() + input.timeoutMs;
do {
if (!input.processAlive(input.pid)) {
return true;
}
if (Date.now() >= deadline) {
break;
}
await delay(input.pollIntervalMs);
} while (Date.now() <= deadline);
return !input.processAlive(input.pid);
}
async function discoverStopAllCandidates(
options: ManagedPythonDaemonStopAllOptions,
): Promise<{
runtimeRoot: string;
candidates: ManagedPythonDaemonStopCandidate[];
scanErrors: string[];
}> {
const runtimeRoot = runtimeRootForStopAll(options);
const stateCandidates = await readStateCandidates(runtimeRoot);
const scanErrors: string[] = [];
let processCandidates: ManagedPythonDaemonStopCandidate[] = [];
try {
const processes = await (options.listProcesses ?? defaultListProcesses)();
processCandidates = processes.flatMap((processInfo) => {
const candidate = processCandidate(processInfo);
return candidate ? [candidate] : [];
});
} catch (error) {
scanErrors.push(error instanceof Error ? error.message : String(error));
}
return {
runtimeRoot,
candidates: mergeCandidates([...stateCandidates, ...processCandidates]),
scanErrors,
};
}
export async function startManagedPythonDaemon(
options: ManagedPythonDaemonStartOptions,
): Promise<ManagedPythonDaemonStartResult> {
@ -395,3 +765,63 @@ export async function stopManagedPythonDaemon(
});
return { status: 'stopped', layout, state };
}
export async function stopAllManagedPythonDaemons(
options: ManagedPythonDaemonStopAllOptions,
): Promise<ManagedPythonDaemonStopAllResult> {
const processAlive = options.processAlive ?? defaultProcessAlive;
const killProcess = options.killProcess ?? defaultKillProcess;
const stopGraceMs = options.stopGraceMs ?? 500;
const pollIntervalMs = options.pollIntervalMs ?? 50;
const healthProbeMs = options.healthProbeMs ?? 100;
const discovery = await discoverStopAllCandidates(options);
const stopped: ManagedPythonDaemonStopAllEntry[] = [];
const stale: ManagedPythonDaemonStopAllEntry[] = [];
const failed: ManagedPythonDaemonStopAllFailure[] = [];
for (const candidate of discovery.candidates) {
const health = await probeCandidateHealth(candidate, healthProbeMs);
const entry = { ...candidateEntry(candidate), ...(health ? { health } : {}) };
if (!processAlive(candidate.pid)) {
await removeStatePaths(candidate.statePaths);
stale.push(entry);
continue;
}
try {
killProcess(candidate.pid, 'SIGTERM');
if (
!(await waitUntilStopped({
pid: candidate.pid,
processAlive,
timeoutMs: stopGraceMs,
pollIntervalMs,
}))
) {
killProcess(candidate.pid, 'SIGKILL');
if (
!(await waitUntilStopped({
pid: candidate.pid,
processAlive,
timeoutMs: stopGraceMs,
pollIntervalMs,
}))
) {
failed.push({ ...entry, detail: 'Process still running after SIGKILL' });
continue;
}
}
await removeStatePaths(candidate.statePaths);
stopped.push(entry);
} catch (error) {
failed.push({ ...entry, detail: error instanceof Error ? error.message : String(error) });
}
}
return {
runtimeRoot: discovery.runtimeRoot,
stopped,
stale,
failed,
scanErrors: discovery.scanErrors,
};
}

32
packages/cli/src/managed-python-runtime.test.ts
View file

@ -161,6 +161,14 @@ describe('verifyRuntimeAsset', () => {
await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow(/Unsafe runtime wheel filename/);
});
it('reports the source-checkout artifact command when the bundled manifest is missing', async () => {
const assetDir = join(tempDir, 'packages', 'cli', 'assets', 'python');
await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow(
/Missing bundled Python runtime manifest.*pnpm run artifacts:build/s,
);
});
});
describe('installManagedPythonRuntime', () => {
@ -210,6 +218,30 @@ describe('installManagedPythonRuntime', () => {
expect(manifest.python.daemonExecutable).toBe(result.layout.daemonPath);
});
it('disables repo uv config for managed runtime uv commands', async () => {
const { assetDir } = await writeAsset(tempDir, 'core-wheel');
const commands: Array<{ command: string; args: string[]; env?: NodeJS.ProcessEnv }> = [];
const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args, options) => {
commands.push({ command, args, env: options?.env });
return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.11.13\n' : '', stderr: '' };
});
await installManagedPythonRuntime({
cliVersion: '0.2.0',
runtimeRoot: join(tempDir, 'runtime'),
assetDir,
env: { PATH: '/opt/homebrew/bin', UV_NO_CONFIG: '0' },
features: ['core'],
exec,
});
expect(commands.map((call) => [call.command, call.args[0], call.env?.UV_NO_CONFIG, call.env?.PATH])).toEqual([
['uv', '--version', '1', '/opt/homebrew/bin'],
['uv', 'venv', '1', '/opt/homebrew/bin'],
['uv', 'pip', '1', '/opt/homebrew/bin'],
]);
});
it('installs the local-embeddings extra when requested', async () => {
const { assetDir } = await writeAsset(tempDir, 'embedding-wheel');
const commands: Array<{ command: string; args: string[] }> = [];

46
packages/cli/src/managed-python-runtime.ts
View file

@ -186,9 +186,28 @@ async function readJsonFile(path: string): Promise<unknown> {
return JSON.parse(await readFile(path, 'utf8')) as unknown;
}
function isErrnoException(error: unknown, code: string): boolean {
return typeof error === 'object' && error !== null && 'code' in error && error.code === code;
}
export async function verifyRuntimeAsset(input: { assetDir: string }): Promise<ManagedRuntimeAsset> {
const manifestPath = join(input.assetDir, 'manifest.json');
const manifest = runtimeAssetManifestSchema.parse(await readJsonFile(manifestPath));
let manifestData: unknown;
try {
manifestData = await readJsonFile(manifestPath);
} catch (error) {
if (isErrnoException(error, 'ENOENT')) {
throw new Error(
[
`Missing bundled Python runtime manifest: ${manifestPath}`,
'In a source checkout, build the local runtime assets with: pnpm run artifacts:build',
'Then retry the runtime-backed KTX command.',
].join('\n'),
);
}
throw error;
}
const manifest = runtimeAssetManifestSchema.parse(manifestData);
assertSafeWheelFilename(manifest.wheel.file);
const wheelPath = join(input.assetDir, manifest.wheel.file);
const wheel = await readFile(wheelPath);
@ -243,10 +262,11 @@ async function runLogged(input: {
command: string;
args: string[];
cwd?: string;
env?: NodeJS.ProcessEnv;
}): Promise<{ stdout: string; stderr: string }> {
await appendFile(input.logPath, `$ ${input.command} ${input.args.join(' ')}\n`);
try {
const result = await input.exec(input.command, input.args, { cwd: input.cwd });
const result = await input.exec(input.command, input.args, { cwd: input.cwd, env: input.env });
if (result.stdout) {
await appendFile(input.logPath, result.stdout.endsWith('\n') ? result.stdout : `${result.stdout}\n`);
}
@ -266,9 +286,13 @@ async function runLogged(input: {
}
}
async function ensureUv(exec: ManagedPythonRuntimeExec): Promise<string> {
function managedRuntimeUvEnv(baseEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
return { ...baseEnv, UV_NO_CONFIG: '1' };
}
async function ensureUv(exec: ManagedPythonRuntimeExec, env?: NodeJS.ProcessEnv): Promise<string> {
try {
const result = await exec('uv', ['--version']);
const result = await exec('uv', ['--version'], { env });
return result.stdout.trim() || 'uv available';
} catch {
throw new Error(MISSING_UV_RUNTIME_INSTALL_MESSAGE);
@ -282,6 +306,7 @@ export async function installManagedPythonRuntime(
const exec = options.exec ?? defaultExec;
const features = normalizeFeatures(options.features);
const asset = await verifyRuntimeAsset({ assetDir: layout.assetDir });
const uvEnv = managedRuntimeUvEnv(options.env ?? process.env);
const existing = await readInstalledManifest(layout.manifestPath);
if (
options.force !== true &&
@ -298,14 +323,21 @@ export async function installManagedPythonRuntime(
await rm(layout.versionDir, { recursive: true, force: true });
await mkdir(layout.versionDir, { recursive: true });
await writeFile(layout.installLogPath, '');
await ensureUv(exec);
await runLogged({ exec, logPath: layout.installLogPath, command: 'uv', args: ['venv', layout.venvDir] });
await ensureUv(exec, uvEnv);
await runLogged({
exec,
logPath: layout.installLogPath,
command: 'uv',
args: ['venv', layout.venvDir],
env: uvEnv,
});
const wheelSpec = features.includes('local-embeddings') ? `${asset.wheelPath}[local-embeddings]` : asset.wheelPath;
await runLogged({
exec,
logPath: layout.installLogPath,
command: 'uv',
args: ['pip', 'install', '--python', layout.pythonPath, wheelSpec],
env: uvEnv,
});
const manifest: InstalledKtxRuntimeManifest = {
@ -371,7 +403,7 @@ export async function doctorManagedPythonRuntime(
const exec = options.exec ?? defaultExec;
const checks: ManagedPythonRuntimeDoctorCheck[] = [];
try {
const version = await ensureUv(exec);
const version = await ensureUv(exec, managedRuntimeUvEnv(options.env ?? process.env));
checks.push(check('pass', { id: 'uv', label: 'uv', detail: version }));
} catch (error) {
checks.push(

53
packages/cli/src/runtime.test.ts
View file

@ -1,5 +1,6 @@
import { describe, expect, it, vi } from 'vitest';
import type {
ManagedPythonDaemonStopAllResult,
ManagedPythonDaemonStartResult,
ManagedPythonDaemonStopResult,
} from './managed-python-daemon.js';
@ -199,13 +200,63 @@ describe('runKtxRuntime', () => {
})),
};
await expect(runKtxRuntime({ command: 'stop', cliVersion: '0.2.0' }, io.io, deps)).resolves.toBe(0);
await expect(runKtxRuntime({ command: 'stop', cliVersion: '0.2.0', all: false }, io.io, deps)).resolves.toBe(0);
expect(deps.stopDaemon).toHaveBeenCalledWith({ cliVersion: '0.2.0' });
expect(io.stdout()).toContain('Stopped KTX Python daemon');
expect(io.stdout()).toContain('pid: 4242');
});
it('stops all discovered Python daemons and reports the summary', async () => {
const io = makeIo();
const deps: KtxRuntimeDeps = {
stopAllDaemons: vi.fn(async (): Promise<ManagedPythonDaemonStopAllResult> => ({
runtimeRoot: '/runtime',
stopped: [
{ pid: 4242, source: 'state', url: 'http://127.0.0.1:61234', statePaths: ['/runtime/0.2.0/daemon.json'] },
{ pid: 5252, source: 'process', url: 'http://127.0.0.1:8765', statePaths: [] },
],
stale: [],
failed: [],
scanErrors: [],
})),
};
await expect(runKtxRuntime({ command: 'stop', cliVersion: '0.2.0', all: true }, io.io, deps)).resolves.toBe(0);
expect(deps.stopAllDaemons).toHaveBeenCalledWith({ cliVersion: '0.2.0' });
expect(io.stdout()).toContain('Stopped 2 KTX Python daemons');
expect(io.stdout()).toContain('pid: 4242 source: state url: http://127.0.0.1:61234');
expect(io.stdout()).toContain('pid: 5252 source: process url: http://127.0.0.1:8765');
});
it('returns failure when stop all cannot stop every daemon', async () => {
const io = makeIo();
const deps: KtxRuntimeDeps = {
stopAllDaemons: vi.fn(async (): Promise<ManagedPythonDaemonStopAllResult> => ({
runtimeRoot: '/runtime',
stopped: [],
stale: [],
failed: [
{
pid: 4242,
source: 'state',
url: 'http://127.0.0.1:61234',
statePaths: ['/runtime/0.2.0/daemon.json'],
detail: 'Process still running after SIGKILL',
},
],
scanErrors: ['ps failed'],
})),
};
await expect(runKtxRuntime({ command: 'stop', cliVersion: '0.2.0', all: true }, io.io, deps)).resolves.toBe(1);
expect(io.stderr()).toContain('Stopped 0 KTX Python daemons; failed 1');
expect(io.stderr()).toContain('pid: 4242 source: state url: http://127.0.0.1:61234');
expect(io.stderr()).toContain('process scan: ps failed');
});
it('prints runtime status as JSON', async () => {
const io = makeIo();
const deps: KtxRuntimeDeps = {

71
packages/cli/src/runtime.ts
View file

@ -1,7 +1,9 @@
import type { KtxCliIo } from './cli-runtime.js';
import {
stopAllManagedPythonDaemons,
startManagedPythonDaemon,
stopManagedPythonDaemon,
type ManagedPythonDaemonStopAllResult,
type ManagedPythonDaemonStartResult,
type ManagedPythonDaemonStopResult,
} from './managed-python-daemon.js';
@ -22,7 +24,7 @@ import {
export type KtxRuntimeArgs =
| { command: 'install'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean }
| { command: 'start'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean }
| { command: 'stop'; cliVersion: string }
| { command: 'stop'; cliVersion: string; all: boolean }
| { command: 'status'; cliVersion: string; json: boolean }
| { command: 'doctor'; cliVersion: string; json: boolean }
| { command: 'prune'; cliVersion: string; dryRun: boolean; yes: boolean };
@ -35,6 +37,7 @@ export interface KtxRuntimeDeps {
force?: boolean;
}) => Promise<ManagedPythonDaemonStartResult>;
stopDaemon?: (options: { cliVersion: string }) => Promise<ManagedPythonDaemonStopResult>;
stopAllDaemons?: (options: { cliVersion: string }) => Promise<ManagedPythonDaemonStopAllResult>;
readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise<ManagedPythonRuntimeStatus>;
doctorRuntime?: (options: ManagedPythonRuntimeLayoutOptions) => Promise<ManagedPythonRuntimeDoctorCheck[]>;
pruneRuntime?: (options: {
@ -81,6 +84,58 @@ function writeDaemonStop(io: KtxCliIo, result: ManagedPythonDaemonStopResult): v
io.stdout.write(`state: ${result.layout.daemonStatePath}\n`);
}
function writeStopAllEntry(io: KtxCliIo, entry: { pid: number; source: string; url?: string; health?: string; detail?: string }): void {
io.stdout.write(
`pid: ${entry.pid} source: ${entry.source}${entry.url ? ` url: ${entry.url}` : ''}${
entry.health ? ` health: ${entry.health}` : ''
}${
entry.detail ? ` detail: ${entry.detail}` : ''
}\n`,
);
}
function writeDaemonStopAll(io: KtxCliIo, result: ManagedPythonDaemonStopAllResult): number {
const failed = result.failed.length + result.scanErrors.length;
if (
result.stopped.length === 0 &&
result.stale.length === 0 &&
result.failed.length === 0 &&
result.scanErrors.length === 0
) {
io.stdout.write('No KTX Python daemons found\n');
return 0;
}
if (failed === 0) {
io.stdout.write(`Stopped ${result.stopped.length} KTX Python daemons\n`);
if (result.stale.length > 0) {
io.stdout.write(`Cleaned ${result.stale.length} stale daemon states\n`);
}
for (const entry of result.stopped) {
writeStopAllEntry(io, entry);
}
for (const entry of result.stale) {
writeStopAllEntry(io, entry);
}
return 0;
}
io.stderr.write(
`Stopped ${result.stopped.length} KTX Python daemons; failed ${result.failed.length}${
result.stale.length > 0 ? `; cleaned stale ${result.stale.length}` : ''
}\n`,
);
for (const entry of result.failed) {
io.stderr.write(
`pid: ${entry.pid} source: ${entry.source}${entry.url ? ` url: ${entry.url}` : ''}${
entry.health ? ` health: ${entry.health}` : ''
} detail: ${entry.detail}\n`,
);
}
for (const error of result.scanErrors) {
io.stderr.write(`process scan: ${error}\n`);
}
return 1;
}
function writeStatus(io: KtxCliIo, status: ManagedPythonRuntimeStatus): void {
io.stdout.write('KTX Python runtime\n');
io.stdout.write(`status: ${status.kind}\n`);
@ -142,10 +197,16 @@ export async function runKtxRuntime(
return 0;
}
if (args.command === 'stop') {
const stopDaemon = deps.stopDaemon ?? stopManagedPythonDaemon;
const result = await stopDaemon({ cliVersion: args.cliVersion });
writeDaemonStop(io, result);
return 0;
if (args.all) {
const stopAllDaemons = deps.stopAllDaemons ?? stopAllManagedPythonDaemons;
const result = await stopAllDaemons({ cliVersion: args.cliVersion });
return writeDaemonStopAll(io, result);
} else {
const stopDaemon = deps.stopDaemon ?? stopManagedPythonDaemon;
const result = await stopDaemon({ cliVersion: args.cliVersion });
writeDaemonStop(io, result);
return 0;
}
}
if (args.command === 'status') {
const readStatus = deps.readStatus ?? readManagedPythonRuntimeStatus;

26
packages/cli/src/scan.test.ts
View file

@ -573,6 +573,32 @@ describe('runKtxScan', () => {
expect(io.stdout()).toContain('\n[90%] Building embeddings 1/4 batches\n');
});
it('scales nested progress phases by the parent phase weight', async () => {
const io = makeIo({ isTTY: true });
const previousCi = process.env.CI;
delete process.env.CI;
try {
const progress = createCliScanProgress(io.io);
await progress.update(0.82, 'Enriching schema metadata');
const enrichmentProgress = progress.startPhase(0.18);
await enrichmentProgress.update(0.05, 'Loaded schema snapshot with 56 tables');
const descriptionProgress = enrichmentProgress.startPhase(0.45);
await descriptionProgress.update(37 / 56, 'Generating descriptions 37/56 tables', { transient: true });
await descriptionProgress.update(1, 'Generated descriptions for 56 tables');
} finally {
if (previousCi === undefined) {
delete process.env.CI;
} else {
process.env.CI = previousCi;
}
}
expect(io.stdout()).toContain('\r[88%] Generating descriptions 37/56 tables');
expect(io.stdout()).toContain('\n[91%] Generated descriptions for 56 tables\n');
expect(io.stdout()).not.toContain('[100%] Generating descriptions 37/56 tables');
});
it('flushes transient TTY progress messages before printing scan failures', async () => {
await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
const runLocalScan = vi.fn(async (input: RunLocalScanOptions): Promise<LocalScanRunResult> => {

2
packages/cli/src/scan.ts
View file

@ -527,7 +527,7 @@ export function createCliScanProgress(
io.stdout.write(`${line}\n`);
},
startPhase(phaseWeight: number) {
return createCliScanProgress(io, state, state.progress, phaseWeight);
return createCliScanProgress(io, state, state.progress, weight * phaseWeight);
},
flush() {
if (!shouldWrite || !state.hasPendingTransient) {

1
packages/cli/src/setup-databases.test.ts
View file

@ -1305,6 +1305,7 @@ describe('setup databases step', () => {
expect(config.connections.warehouse.historicSql).not.toHaveProperty('redactionPatterns');
expect(config.connections.warehouse.historicSql).not.toHaveProperty(legacyHistoricSqlServiceAccountPatternsKey);
expect(config.ingest.adapters).toContain('historic-sql');
expect(config.ingest.workUnits.maxConcurrency).toBe(6);
expect(io.stdout()).toContain('Historic SQL probe...');
expect(io.stdout()).toContain('pg_stat_statements ready');
});

24
packages/cli/src/setup-databases.ts
View file

@ -14,6 +14,8 @@ import { runKtxScan } from './scan.js';
import { withSetupInterruptConfirmation } from './setup-interrupt.js';
import { writeProjectLocalSecretReference } from './setup-secrets.js';
const HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY = 6;
export type KtxSetupDatabaseDriver =
| 'sqlite'
| 'postgres'
@ -930,7 +932,7 @@ async function writeConnectionConfig(input: {
? (input.connection.historicSql as Record<string, unknown>)
: null;
if (historicSql?.enabled === true) {
await ensureHistoricSqlAdapterEnabled(input.projectDir);
await ensureHistoricSqlIngestDefaults(input.projectDir);
}
}
@ -1057,9 +1059,19 @@ async function maybeConfigureSchemaScope(input: {
return true;
}
async function ensureHistoricSqlAdapterEnabled(projectDir: string): Promise<void> {
async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise<void> {
const project = await loadKtxProject({ projectDir });
if (project.config.ingest.adapters.includes('historic-sql')) {
const adapters = project.config.ingest.adapters.includes('historic-sql')
? project.config.ingest.adapters
: [...project.config.ingest.adapters, 'historic-sql'];
const maxConcurrency = Math.max(
project.config.ingest.workUnits.maxConcurrency,
HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY,
);
if (
adapters === project.config.ingest.adapters &&
maxConcurrency === project.config.ingest.workUnits.maxConcurrency
) {
return;
}
await writeFile(
@ -1068,7 +1080,11 @@ async function ensureHistoricSqlAdapterEnabled(projectDir: string): Promise<void
...project.config,
ingest: {
...project.config.ingest,
adapters: [...project.config.ingest.adapters, 'historic-sql'],
adapters,
workUnits: {
...project.config.ingest.workUnits,
maxConcurrency,
},
},
}),
'utf-8',

49
packages/cli/src/setup-models.test.ts
View file

@ -676,4 +676,53 @@ describe('setup Anthropic model step', () => {
).resolves.toMatchObject({ status: 'ready' });
expect(healthCheck).not.toHaveBeenCalled();
});
it.each([
{
backend: 'vertex',
providerLines: [' backend: vertex', ' vertex:', ' project: kaelio-dev', ' location: us-east5'],
model: 'claude-sonnet-4-6',
},
{
backend: 'gateway',
providerLines: [' backend: gateway', ' gateway:', ' api_key: env:AI_GATEWAY_API_KEY'],
model: 'anthropic/claude-sonnet-4-6',
},
])('preserves already configured $backend llm setup without asking for Anthropic credentials', async (fixture) => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'setup:',
' database_connection_ids: []',
' completed_steps:',
' - project',
' - llm',
'connections: {}',
'llm:',
' provider:',
...fixture.providerLines,
' models:',
` default: ${fixture.model}`,
'ingest:',
' embeddings:',
' backend: deterministic',
' model: deterministic',
' dimensions: 8',
].join('\n'),
'utf-8',
);
const healthCheck = vi.fn(async () => ({ ok: true as const }));
const io = makeIo();
await expect(
runKtxSetupAnthropicModelStep({ projectDir: tempDir, inputMode: 'disabled', skipLlm: false }, io.io, {
healthCheck,
}),
).resolves.toMatchObject({ status: 'ready' });
expect(healthCheck).not.toHaveBeenCalled();
expect(io.stdout()).toContain(`LLM ready: yes (${fixture.model})`);
expect(io.stderr()).not.toContain('Anthropic');
});
});

30
packages/cli/src/setup-models.ts
View file

@ -1,5 +1,6 @@
import { writeFile } from 'node:fs/promises';
import { cancel, isCancel, password, select, text } from '@clack/prompts';
import { resolveLocalKtxLlmConfig } from '@ktx/context';
import { resolveKtxConfigReference } from '@ktx/context/core';
import {
type KtxProjectConfig,
@ -170,13 +171,26 @@ export async function fetchAnthropicModels(
return models.map((item, index) => ({ ...item, recommended: index === Math.max(recommendedIndex, 0) }));
}
function hasCompletedLlm(config: KtxProjectConfig): boolean {
return (
config.setup?.completed_steps.includes('llm') === true &&
config.llm.provider.backend === 'anthropic' &&
typeof config.llm.models.default === 'string' &&
config.llm.models.default.length > 0
);
export function isKtxSetupLlmConfigReady(config: KtxProjectLlmConfig): boolean {
let resolved: KtxLlmConfig | null;
try {
resolved = resolveLocalKtxLlmConfig(config, process.env);
} catch {
return false;
}
if (!resolved) {
return false;
}
if (resolved.backend === 'vertex') {
return typeof resolved.vertex?.location === 'string' && resolved.vertex.location.trim().length > 0;
}
return resolved.backend === 'anthropic' || resolved.backend === 'gateway';
}
function hasUsableConfiguredLlm(config: KtxProjectConfig): boolean {
return isKtxSetupLlmConfigReady(config.llm);
}
function buildProjectLlmConfig(
@ -386,7 +400,7 @@ export async function runKtxSetupAnthropicModelStep(
const project = await loadKtxProject({ projectDir: args.projectDir });
if (
args.forcePrompt !== true &&
hasCompletedLlm(project.config) &&
hasUsableConfiguredLlm(project.config) &&
!args.anthropicApiKeyEnv &&
!args.anthropicApiKeyFile &&
!args.anthropicModel

160
packages/cli/src/setup.test.ts
View file

@ -5,6 +5,7 @@ import { join } from 'node:path';
import { promisify } from 'node:util';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { localFakeBundleReport, persistLocalBundleReport } from './ingest.test-utils.js';
import { contextBuildCommands, writeKtxSetupContextState } from './setup-context.js';
import { runDemoTour } from './setup-demo-tour.js';
import { readKtxSetupStatus, runKtxSetup } from './setup.js';
@ -91,6 +92,38 @@ describe('setup status', () => {
});
});
it.each([
{
backend: 'vertex',
providerLines: [' backend: vertex', ' vertex:', ' project: kaelio-dev', ' location: us-east5'],
model: 'claude-sonnet-4-6',
},
{
backend: 'gateway',
providerLines: [' backend: gateway', ' gateway:', ' api_key: env:AI_GATEWAY_API_KEY'],
model: 'anthropic/claude-sonnet-4-6',
},
])('reports configured $backend llm backends as setup-ready', async (fixture) => {
await mkdir(tempDir, { recursive: true });
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: revenue',
'llm:',
' provider:',
...fixture.providerLines,
' models:',
` default: ${fixture.model}`,
'connections: {}',
].join('\n'),
'utf-8',
);
await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({
llm: { backend: fixture.backend, ready: true, model: fixture.model },
});
});
it('uses setup database connection ids when present', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
@ -283,6 +316,62 @@ describe('setup status', () => {
});
});
it('reports Vertex LLM and context ready after a successful Metabase ingest report', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: revenue',
'setup:',
' database_connection_ids:',
' - warehouse',
' completed_steps:',
' - project',
' - databases',
' - sources',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' metabase:',
' driver: metabase',
' url: env:METABASE_URL',
' api_key_ref: env:METABASE_API_KEY',
' warehouse_connection_id: warehouse',
'llm:',
' provider:',
' backend: vertex',
' vertex:',
' project: kaelio-dev',
' location: us-east5',
' models:',
' default: claude-sonnet-4-6',
'ingest:',
' embeddings:',
' backend: deterministic',
' model: deterministic',
' dimensions: 8',
'',
].join('\n'),
'utf-8',
);
await persistLocalBundleReport(
tempDir,
localFakeBundleReport('metabase-job-1', {
connectionId: 'warehouse',
sourceKey: 'metabase',
}),
);
const status = await readKtxSetupStatus(tempDir);
const io = makeIo();
await expect(runKtxSetup({ command: 'status', projectDir: tempDir, json: false }, io.io)).resolves.toBe(0);
expect(status.llm).toMatchObject({ backend: 'vertex', ready: true, model: 'claude-sonnet-4-6' });
expect(status.context).toMatchObject({ ready: true, status: 'completed' });
expect(io.stdout()).toContain('LLM ready: yes (claude-sonnet-4-6)');
expect(io.stdout()).toContain('KTX context built: yes');
});
it('prints plain and JSON setup status', async () => {
const plainIo = makeIo();
const jsonIo = makeIo();
@ -1178,6 +1267,77 @@ describe('setup status', () => {
expect(calls).toEqual(['model', 'embeddings', 'databases', 'sources']);
});
it.each([
{
backend: 'vertex',
providerLines: [' backend: vertex', ' vertex:', ' project: kaelio-dev', ' location: us-east5'],
model: 'claude-sonnet-4-6',
},
{
backend: 'gateway',
providerLines: [' backend: gateway', ' gateway:', ' api_key: env:AI_GATEWAY_API_KEY'],
model: 'anthropic/claude-sonnet-4-6',
},
])('adds a dbt source in non-interactive setup with existing $backend llm config', async (fixture) => {
const io = makeIo();
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: revenue',
'setup:',
' database_connection_ids:',
' - warehouse',
' completed_steps:',
' - project',
' - databases',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:WAREHOUSE_URL',
'llm:',
' provider:',
...fixture.providerLines,
' models:',
` default: ${fixture.model}`,
].join('\n'),
'utf-8',
);
await expect(
runKtxSetup(
{
command: 'run',
projectDir: tempDir,
mode: 'existing',
agents: false,
skipAgents: true,
inputMode: 'disabled',
yes: true,
cliVersion: '0.2.0',
skipLlm: false,
skipEmbeddings: true,
skipDatabases: true,
source: 'dbt',
sourceConnectionId: 'dbt-main',
sourceGitUrl: 'https://github.com/Kaelio/klo-dbt-demo',
sourceBranch: 'main',
sourceProjectName: 'orbit_analytics',
sourceWarehouseConnectionId: 'warehouse',
skipSources: false,
databaseSchemas: [],
},
io.io,
{
sourcesDeps: { validateDbt: vi.fn(async () => ({ ok: true as const, detail: 'dbt project valid' })) },
context: vi.fn(async () => ({ status: 'ready' as const, projectDir: tempDir, runId: 'setup-context-test' })),
},
),
).resolves.toBe(0);
expect(io.stderr()).not.toContain('Anthropic');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).toContain('dbt-main:');
});
it('does not fail context build when prerequisites were explicitly skipped and agents are skipped', async () => {
const calls: string[] = [];
const io = makeIo();

45
packages/cli/src/setup.ts
View file

@ -1,7 +1,8 @@
import { existsSync } from 'node:fs';
import { join, resolve } from 'node:path';
import { cancel, isCancel, select } from '@clack/prompts';
import { loadKtxProject } from '@ktx/context/project';
import { getLatestLocalIngestStatus, savedMemoryCountsForReport } from '@ktx/context/ingest';
import { ktxLocalStateDbPath, loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { formatSetupNextStepLines } from './next-steps.js';
import { isKtxSetupExitError, withSetupInterruptConfirmation } from './setup-interrupt.js';
@ -20,7 +21,7 @@ import {
runKtxSetupDatabasesStep,
} from './setup-databases.js';
import { type KtxSetupEmbeddingsDeps, runKtxSetupEmbeddingsStep } from './setup-embeddings.js';
import { type KtxSetupModelDeps, runKtxSetupAnthropicModelStep } from './setup-models.js';
import { type KtxSetupModelDeps, isKtxSetupLlmConfigReady, runKtxSetupAnthropicModelStep } from './setup-models.js';
import { type KtxSetupProjectDeps, runKtxSetupProjectStep } from './setup-project.js';
import {
isKtxPreAgentSetupReady,
@ -226,10 +227,6 @@ async function runKtxSetupDemoFromEntryMenu(
);
}
function llmReady(status: KtxSetupStatus['llm']): boolean {
return status.backend === 'anthropic' && typeof status.model === 'string' && status.model.length > 0;
}
function embeddingsReady(status: KtxSetupStatus['embeddings']): boolean {
return (
status.backend !== undefined &&
@ -252,6 +249,31 @@ function sourceConnections(config: Awaited<ReturnType<typeof loadKtxProject>>['c
.sort((left, right) => left.connectionId.localeCompare(right.connectionId));
}
type LocalIngestStatusReport = NonNullable<Awaited<ReturnType<typeof getLatestLocalIngestStatus>>>;
function reportHasSavedContext(report: LocalIngestStatusReport): boolean {
if (report.body.failedWorkUnits.length > 0) {
return false;
}
const counts = savedMemoryCountsForReport(report);
return counts.wikiCount > 0 || counts.slCount > 0;
}
async function readIngestContextStatus(project: KtxLocalProject): Promise<KtxSetupContextStatusSummary | null> {
if (!existsSync(ktxLocalStateDbPath(project))) {
return null;
}
const report = await getLatestLocalIngestStatus(project);
if (!report || !reportHasSavedContext(report)) {
return null;
}
return {
ready: true,
status: 'completed',
runId: report.runId,
};
}
export async function readKtxSetupStatus(projectDir: string): Promise<KtxSetupStatus> {
const resolvedProjectDir = resolve(projectDir);
if (!existsSync(join(resolvedProjectDir, 'ktx.yaml'))) {
@ -269,10 +291,9 @@ export async function readKtxSetupStatus(projectDir: string): Promise<KtxSetupSt
const project = await loadKtxProject({ projectDir: resolvedProjectDir });
const llm = {
backend: project.config.llm.provider.backend,
ready: false,
ready: isKtxSetupLlmConfigReady(project.config.llm),
model: project.config.llm.models.default,
};
llm.ready = llmReady(llm);
const embeddings = {
backend: project.config.ingest.embeddings.backend,
@ -284,6 +305,10 @@ export async function readKtxSetupStatus(projectDir: string): Promise<KtxSetupSt
const completedSteps = project.config.setup?.completed_steps ?? [];
const contextState = await readKtxSetupContextState(resolvedProjectDir);
const setupContextStatus = setupContextStatusFromState(contextState, {
completedStep: completedSteps.includes('context'),
});
const ingestContextStatus = setupContextStatus.ready ? null : await readIngestContextStatus(project);
const databaseIds = project.config.setup?.database_connection_ids ?? Object.keys(project.config.connections);
const databasesComplete = completedSteps.includes('databases');
const manifest = await readKtxAgentInstallManifest(resolvedProjectDir);
@ -306,7 +331,7 @@ export async function readKtxSetupStatus(projectDir: string): Promise<KtxSetupSt
...source,
ready: completedSteps.includes('sources'),
})),
context: setupContextStatusFromState(contextState, { completedStep: completedSteps.includes('context') }),
context: ingestContextStatus ?? setupContextStatus,
agents,
};
}
@ -376,7 +401,7 @@ function setupStatusReady(status: KtxSetupStatus): boolean {
return true;
}
return (
llmReady(status.llm) &&
status.llm.ready &&
embeddingsReady(status.embeddings) &&
status.databases.every((database) => database.ready) &&
status.sources.every((source) => source.ready)

12
packages/cli/src/sl.test.ts
View file

@ -448,10 +448,18 @@ joins: []
listIo.io,
);
expect(code).toBe(0);
expect(listIo.stderr()).toBe('');
const parsed = JSON.parse(listIo.stdout());
expect(parsed.kind).toBe('list');
expect(parsed.meta).toEqual({ command: 'sl list' });
expect(parsed).toMatchObject({
kind: 'list',
data: {
items: expect.any(Array),
},
meta: {
command: 'sl list',
},
});
expect(parsed.data.items).toHaveLength(1);
expect(parsed.data.items[0]).toMatchObject({
connectionId: 'warehouse',

24
packages/cli/src/standalone-smoke.test.ts
View file

@ -368,9 +368,9 @@ describe('standalone built ktx CLI smoke', () => {
const knowledgeSearch = structuredContent<{
results: Array<{ key: string; summary: string; score: number }>;
totalFound: number;
}>(await client.callTool({ name: 'knowledge_search', arguments: { query: 'ARR contract', limit: 5 } }));
}>(await client.callTool({ name: 'knowledge_search', arguments: { query: 'ARR contract-first definition', limit: 10 } }));
expect(knowledgeSearch.totalFound).toBeGreaterThan(0);
expect(knowledgeSearch.results.map((result) => result.key)).toContain('arr-contract-first');
expect(knowledgeSearch.results.map((result) => result.key)).toContain('orbit-arr-contract-first-definition');
const knowledgeRead = structuredContent<{
key: string;
@ -378,26 +378,26 @@ describe('standalone built ktx CLI smoke', () => {
content: string;
tags: string[];
slRefs: string[];
}>(await client.callTool({ name: 'knowledge_read', arguments: { key: 'arr-contract-first' } }));
expect(knowledgeRead.key).toBe('arr-contract-first');
}>(await client.callTool({ name: 'knowledge_read', arguments: { key: 'orbit-arr-contract-first-definition' } }));
expect(knowledgeRead.key).toBe('orbit-arr-contract-first-definition');
expect(knowledgeRead.summary).toContain('ARR');
expect(knowledgeRead.content).toContain('contract');
expect(knowledgeRead.slRefs).toContain('orbit_demo.contracts');
expect(knowledgeRead.slRefs).toContain('mart_arr_daily');
const slRead = structuredContent<{ sourceName: string; yaml: string }>(
await client.callTool({
name: 'sl_read_source',
arguments: { connectionId: 'orbit_demo', sourceName: 'accounts' },
arguments: { connectionId: 'dbt-main', sourceName: 'mart_arr_daily' },
}),
);
expect(slRead.sourceName).toBe('accounts');
expect(slRead.yaml).toContain('name: accounts');
expect(slRead.sourceName).toBe('mart_arr_daily');
expect(slRead.yaml).toContain('name: mart_arr_daily');
expect(slRead.yaml).toContain('measures:');
const slValidate = structuredContent<{ success: boolean; errors: string[]; warnings: string[] }>(
await client.callTool({
name: 'sl_validate',
arguments: { connectionId: 'orbit_demo', names: ['accounts', 'contracts'] },
arguments: { connectionId: 'dbt-main', names: ['mart_arr_daily', 'stg_contracts'] },
}),
);
expect(slValidate.success).toBe(true);
@ -716,7 +716,7 @@ describe('standalone built ktx CLI smoke', () => {
'--project-dir',
projectDir,
'--token-env',
'NOTION_AUTH_TOKEN',
'NOTION_TOKEN',
'--crawl-mode',
'all_accessible',
'--max-pages',
@ -729,7 +729,7 @@ describe('standalone built ktx CLI smoke', () => {
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('driver: notion');
expect(yaml).toContain('auth_token_ref: env:NOTION_AUTH_TOKEN');
expect(yaml).toContain('auth_token_ref: env:NOTION_TOKEN');
expect(yaml).toContain('crawl_mode: all_accessible');
expect(yaml).toContain('max_pages_per_run: 5');
expect(yaml).not.toContain('ntn_');
@ -737,7 +737,7 @@ describe('standalone built ktx CLI smoke', () => {
const parsed = parseKtxProjectConfig(yaml);
expect(parsed.connections['notion-main']).toMatchObject({
driver: 'notion',
auth_token_ref: 'env:NOTION_AUTH_TOKEN',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
});
});

14
packages/context/src/connections/notion-config.test.ts
View file

@ -23,14 +23,14 @@ describe('standalone Notion connection config', () => {
it('parses selected-root Notion config with safe defaults', () => {
const parsed = parseNotionConnectionConfig({
driver: 'notion',
auth_token_ref: 'env:NOTION_AUTH_TOKEN',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: ['page-1'],
});
expect(parsed).toEqual({
driver: 'notion',
auth_token_ref: 'env:NOTION_AUTH_TOKEN',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: ['page-1'],
root_database_ids: [],
@ -70,7 +70,7 @@ describe('standalone Notion connection config', () => {
expect(() =>
parseNotionConnectionConfig({
driver: 'notion',
auth_token_ref: 'env:NOTION_AUTH_TOKEN',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
}),
).toThrow('selected_roots requires at least one root page, database, or data source id');
@ -81,8 +81,8 @@ describe('standalone Notion connection config', () => {
await writeFile(tokenPath, 'ntn_file_token\n', 'utf-8');
await expect(
resolveNotionAuthToken('env:NOTION_AUTH_TOKEN', {
env: { NOTION_AUTH_TOKEN: 'ntn_env_token' },
resolveNotionAuthToken('env:NOTION_TOKEN', {
env: { NOTION_TOKEN: 'ntn_env_token' },
}),
).resolves.toBe('ntn_env_token');
await expect(resolveNotionAuthToken(`file:${tokenPath}`)).resolves.toBe('ntn_file_token');
@ -95,14 +95,14 @@ describe('standalone Notion connection config', () => {
const pullConfig = await notionConnectionToPullConfig(
parseNotionConnectionConfig({
driver: 'notion',
auth_token_ref: 'env:NOTION_AUTH_TOKEN',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: '{"phase":"all_accessible_pages","cursor":"cursor-1"}',
}),
{ env: { NOTION_AUTH_TOKEN: 'ntn_env_token' } },
{ env: { NOTION_TOKEN: 'ntn_env_token' } },
);
expect(pullConfig).toEqual({

18
packages/context/src/ingest/adapters/looker/client.test.ts
View file

@ -112,6 +112,24 @@ describe('LookerClient', () => {
});
});
it('does not warn to console when optional prioritization inputs fail by default', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
const fakeSdk = sdk({
search_dashboards: vi.fn().mockRejectedValue(new Error('dashboards unavailable')),
search_looks: vi.fn().mockRejectedValue(new Error('looks unavailable')),
});
const client = new LookerClient(params(), { sdkFactory: () => fakeSdk });
await expect(client.getSignals()).resolves.toMatchObject({
dashboardUsage: [],
lookUsage: [],
scheduledPlans: [],
favorites: [],
});
expect(warn).not.toHaveBeenCalled();
});
it('maps dashboards, looks, folders, models, explores, users, and groups to staged DTOs', async () => {
const fakeSdk = sdk();
const client = new LookerClient(params(), { sdkFactory: () => fakeSdk });

8
packages/context/src/ingest/adapters/looker/client.ts
View file

@ -80,10 +80,10 @@ export interface LookerClientDeps {
}
const defaultLogger: LookerClientLogger = {
log: (message) => console.log(message),
warn: (message) => console.warn(message),
error: (message) => console.error(message),
debug: (message) => console.debug(message),
log: () => undefined,
warn: () => undefined,
error: () => undefined,
debug: () => undefined,
};
class InlineLookerSettings extends NodeSettings {

6
packages/context/src/ingest/adapters/looker/local-looker.adapter.ts
View file

@ -1,4 +1,5 @@
import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js';
import type { LookerClientLogger } from './client.js';
import {
DefaultLookerClientFactory,
DefaultLookerConnectionClientFactory,
@ -59,8 +60,11 @@ export function createLocalLookerCredentialResolver(
export function createLocalLookerSourceAdapter(
project: KtxLocalProject,
env: NodeJS.ProcessEnv = process.env,
logger?: LookerClientLogger,
): LookerSourceAdapter {
const connectionFactory = new DefaultLookerConnectionClientFactory(createLocalLookerCredentialResolver(project, env));
const connectionFactory = new DefaultLookerConnectionClientFactory(createLocalLookerCredentialResolver(project, env), {
...(logger ? { logger } : {}),
});
return new LookerSourceAdapter({
clientFactory: new DefaultLookerClientFactory(connectionFactory),
});

21
packages/context/src/ingest/adapters/metabase/client.test.ts
View file

@ -72,6 +72,27 @@ describe('MetabaseClient retry exhaustion', () => {
vi.restoreAllMocks();
});
it('does not warn to console when retrying by default', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
globalThis.fetch = vi
.fn<typeof fetch>()
.mockRejectedValueOnce(Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }))
.mockResolvedValueOnce(new Response(JSON.stringify([]), { status: 200 }));
const client = new MetabaseClient(
{ apiUrl: 'https://metabase.example.test', apiKey: 'key' },
{
...DEFAULT_METABASE_CLIENT_CONFIG,
baseDelayMs: 0,
maxRetries: 1,
},
);
await client.getDatabases();
expect(warn).not.toHaveBeenCalled();
});
it('wraps an exhausted ECONNRESET retry chain with method, path, attempt count, and original cause', async () => {
const sysErr = Object.assign(new Error('read ECONNRESET'), {
code: 'ECONNRESET',

8
packages/context/src/ingest/adapters/metabase/client.ts
View file

@ -25,10 +25,10 @@ export interface MetabaseClientLogger {
}
const defaultLogger: MetabaseClientLogger = {
log: (message) => console.log(message),
warn: (message) => console.warn(message),
error: (message) => console.error(message),
debug: (message) => console.debug(message),
log: () => undefined,
warn: () => undefined,
error: () => undefined,
debug: () => undefined,
};
interface TemplateTagInfo {

36
packages/context/src/ingest/adapters/metabase/fetch.test.ts
View file

@ -86,6 +86,7 @@ describe('fetchMetabaseBundle', () => {
});
afterEach(async () => {
vi.restoreAllMocks();
await rm(stagedDir, { recursive: true, force: true });
});
@ -115,6 +116,41 @@ describe('fetchMetabaseBundle', () => {
expect(card.archived).toBe(false);
});
it('does not write Metabase fetch progress to console by default', async () => {
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined);
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
await fetchMetabaseBundle({
pullConfig: { metabaseConnectionId, metabaseDatabaseId: 42 },
stagedDir,
ctx: makeFetchContext(),
clientFactory,
sourceStateReader,
});
expect(log).not.toHaveBeenCalled();
expect(warn).not.toHaveBeenCalled();
});
it('routes Metabase fetch warnings through the injected logger', async () => {
const logger = {
log: vi.fn(),
warn: vi.fn(),
};
clientFactory.__client.getCard.mockRejectedValueOnce(new Error('card read failed'));
await fetchMetabaseBundle({
pullConfig: { metabaseConnectionId, metabaseDatabaseId: 42 },
stagedDir,
ctx: makeFetchContext(),
clientFactory,
sourceStateReader,
logger,
});
expect(logger.warn).toHaveBeenCalledWith('failed to load card 1: card read failed');
});
it('passes the Metabase source pull config and target fetch context to the client factory', async () => {
await fetchMetabaseBundle({
pullConfig: { metabaseConnectionId, metabaseDatabaseId: 42 },

13
packages/context/src/ingest/adapters/metabase/fetch.ts
View file

@ -21,9 +21,14 @@ class IngestInputError extends Error {
}
}
const logger = {
log: (message: string) => console.log(message),
warn: (message: string) => console.warn(message),
export interface MetabaseFetchLogger {
log(message: string): void;
warn(message: string): void;
}
const noopMetabaseFetchLogger: MetabaseFetchLogger = {
log: () => undefined,
warn: () => undefined,
};
export interface FetchMetabaseBundleParams {
@ -32,6 +37,7 @@ export interface FetchMetabaseBundleParams {
ctx: FetchContext;
clientFactory: MetabaseClientFactory;
sourceStateReader: MetabaseSourceStateReader;
logger?: MetabaseFetchLogger;
}
interface CollectionNode {
@ -76,6 +82,7 @@ function resolvePath(index: Map<number | 'root', CollectionNode>, collectionId:
export async function fetchMetabaseBundle(params: FetchMetabaseBundleParams): Promise<void> {
const pullConfig: MetabasePullConfig = parseMetabasePullConfig(params.pullConfig);
const logger = params.logger ?? noopMetabaseFetchLogger;
const syncState = await params.sourceStateReader.getSourceState(pullConfig.metabaseConnectionId);
const mapping = syncState.mappings.find(
(m) => m.metabaseDatabaseId === pullConfig.metabaseDatabaseId && m.syncEnabled,

10
packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts
View file

@ -1,12 +1,17 @@
import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js';
import { ktxLocalStateDbPath } from '../../../project/index.js';
import { resolveKtxConfigReference } from '../../../core/config-reference.js';
import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './client.js';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultMetabaseConnectionClientFactory,
type MetabaseClientLogger,
} from './client.js';
import {
IngestMetabaseClientFactory,
type MetabaseClientConfig,
type MetabaseClientRuntimeConfig,
} from './client-port.js';
import type { MetabaseFetchLogger } from './fetch.js';
import { LocalMetabaseSourceStateReader } from './local-source-state-store.js';
import { MetabaseSourceAdapter } from './metabase.adapter.js';
@ -50,6 +55,7 @@ export function metabaseRuntimeConfigFromLocalConnection(
interface CreateLocalMetabaseSourceAdapterOptions {
env?: NodeJS.ProcessEnv;
defaultClientConfig?: MetabaseClientConfig;
logger?: MetabaseClientLogger & MetabaseFetchLogger;
}
export function createLocalMetabaseSourceAdapter(
@ -65,9 +71,11 @@ export function createLocalMetabaseSourceAdapter(
options.env,
),
options.defaultClientConfig ?? DEFAULT_METABASE_CLIENT_CONFIG,
options.logger,
);
return new MetabaseSourceAdapter({
clientFactory: new IngestMetabaseClientFactory(connectionFactory),
sourceStateReader,
...(options.logger ? { logger: options.logger } : {}),
});
}

4
packages/context/src/ingest/adapters/metabase/metabase.adapter.ts
View file

@ -4,7 +4,7 @@ import type { ChunkResult, DiffSet, FetchContext, ScopeDescriptor, SourceAdapter
import { chunkMetabaseStagedDir } from './chunk.js';
import type { MetabaseClientFactory } from './client-port.js';
import { detectMetabaseStagedDir } from './detect.js';
import { fetchMetabaseBundle } from './fetch.js';
import { fetchMetabaseBundle, type MetabaseFetchLogger } from './fetch.js';
import { computeFetchScope, hashScope, isPathInMetabaseScope } from './fetch-scope.js';
import type { MetabaseSourceStateReader } from './source-state-port.js';
import { STAGED_FILES, stagedSyncConfigSchema } from './types.js';
@ -12,6 +12,7 @@ import { STAGED_FILES, stagedSyncConfigSchema } from './types.js';
export interface MetabaseSourceAdapterDeps {
clientFactory: MetabaseClientFactory;
sourceStateReader: MetabaseSourceStateReader;
logger?: MetabaseFetchLogger;
}
export class MetabaseSourceAdapter implements SourceAdapter {
@ -31,6 +32,7 @@ export class MetabaseSourceAdapter implements SourceAdapter {
ctx,
clientFactory: this.deps.clientFactory,
sourceStateReader: this.deps.sourceStateReader,
...(this.deps.logger ? { logger: this.deps.logger } : {}),
});
}

15
packages/context/src/ingest/adapters/notion/fetch.test.ts
View file

@ -89,12 +89,13 @@ describe('fetchNotionSnapshot', () => {
});
it('logs skipped page materialization failures', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
const logger = { warn: vi.fn() };
(client.retrievePage as ReturnType<typeof vi.fn>).mockRejectedValueOnce(new Error('Notion API failed'));
const manifest = await fetchNotionSnapshot({
client,
stagedDir,
logger,
config: {
authToken: 'secret',
crawlMode: 'selected_roots',
@ -109,7 +110,7 @@ describe('fetchNotionSnapshot', () => {
});
expect(manifest.skipped).toEqual([{ externalId: 'page-1', reason: 'Notion API failed' }]);
expect(warn).toHaveBeenCalledWith('Skipping Notion page page-1: Notion API failed');
expect(logger.warn).toHaveBeenCalledWith('Skipping Notion page page-1: Notion API failed');
});
it('recursively fetches selected-root child pages and derives scoped links', async () => {
@ -191,7 +192,7 @@ describe('fetchNotionSnapshot', () => {
});
it('truncates deeply nested block trees and records a warning', async () => {
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
const logger = { warn: vi.fn() };
(client.listBlockChildren as ReturnType<typeof vi.fn>).mockImplementation((blockId: string) => {
const currentDepth = blockId === 'page-1' ? 0 : Number(blockId.replace('block-', ''));
const nextDepth = currentDepth + 1;
@ -215,6 +216,7 @@ describe('fetchNotionSnapshot', () => {
await fetchNotionSnapshot({
client,
stagedDir,
logger,
config: {
authToken: 'secret',
crawlMode: 'selected_roots',
@ -232,11 +234,11 @@ describe('fetchNotionSnapshot', () => {
const manifest = JSON.parse(await readFile(join(stagedDir, 'manifest.json'), 'utf-8'));
expect(blocks).toHaveLength(10);
expect(manifest.warnings).toContain('maxBlockDepth reached for page page-1 at depth 10');
expect(warnSpy).toHaveBeenCalledWith('maxBlockDepth reached for page page-1 at depth 10');
expect(logger.warn).toHaveBeenCalledWith('maxBlockDepth reached for page page-1 at depth 10');
});
it('truncates pages at the per-page block cap and records a warning', async () => {
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
const logger = { warn: vi.fn() };
(client.listBlockChildren as ReturnType<typeof vi.fn>).mockResolvedValue({
results: Array.from({ length: 2001 }, (_, index) => ({
id: `block-${index}`,
@ -250,6 +252,7 @@ describe('fetchNotionSnapshot', () => {
await fetchNotionSnapshot({
client,
stagedDir,
logger,
config: {
authToken: 'secret',
crawlMode: 'selected_roots',
@ -267,7 +270,7 @@ describe('fetchNotionSnapshot', () => {
const manifest = JSON.parse(await readFile(join(stagedDir, 'manifest.json'), 'utf-8'));
expect(blocks).toHaveLength(2000);
expect(manifest.warnings).toContain('maxBlocksPerPage reached for page page-1 at 2000 blocks');
expect(warnSpy).toHaveBeenCalledWith('maxBlocksPerPage reached for page page-1 at 2000 blocks');
expect(logger.warn).toHaveBeenCalledWith('maxBlocksPerPage reached for page page-1 at 2000 blocks');
});
it('uses all_accessible search for pages and data sources', async () => {

61
packages/context/src/ingest/adapters/notion/fetch.ts
View file

@ -12,10 +12,19 @@ import {
type NotionPullConfig,
} from './types.js';
export interface NotionFetchLogger {
warn(message: string): void;
}
const noopNotionFetchLogger: NotionFetchLogger = {
warn: () => undefined,
};
interface FetchNotionSnapshotParams {
client: NotionApi;
config: NotionPullConfig;
stagedDir: string;
logger?: NotionFetchLogger;
}
interface CrawlState {
@ -23,6 +32,7 @@ interface CrawlState {
databaseCount: number;
dataSourceCount: number;
capped: boolean;
logger: NotionFetchLogger;
skipped: Array<{ externalId: string; reason: string }>;
warnings: string[];
materializedPageTargets: Set<string>;
@ -44,9 +54,6 @@ interface NotionLinks {
const DEFAULT_MAX_BLOCK_DEPTH = 10;
const DEFAULT_MAX_BLOCKS_PER_PAGE = 2000;
const logger = {
warn: (message: string) => console.warn(message),
};
async function writeJson(path: string, value: unknown): Promise<void> {
await mkdir(dirname(path), { recursive: true });
@ -58,7 +65,12 @@ async function writeText(path: string, value: string): Promise<void> {
await writeFile(path, value.endsWith('\n') ? value : `${value}\n`, 'utf-8');
}
function addWarning(warnings: string[], warning: string, logWarning = false): void {
function addWarning(
warnings: string[],
warning: string,
logWarning = false,
logger: NotionFetchLogger = noopNotionFetchLogger,
): void {
if (!warnings.includes(warning)) {
warnings.push(warning);
if (logWarning) {
@ -119,11 +131,21 @@ async function visitPaginated<T>(params: {
} while (cursor);
}
function addBlockCountWarning(state: BlockCollectionState, warnings: string[], pageId: string): void {
function addBlockCountWarning(
state: BlockCollectionState,
warnings: string[],
pageId: string,
logger: NotionFetchLogger,
): void {
if (state.blockCountWarningWritten) {
return;
}
addWarning(warnings, `maxBlocksPerPage reached for page ${pageId} at ${DEFAULT_MAX_BLOCKS_PER_PAGE} blocks`, true);
addWarning(
warnings,
`maxBlocksPerPage reached for page ${pageId} at ${DEFAULT_MAX_BLOCKS_PER_PAGE} blocks`,
true,
logger,
);
state.blockCountWarningWritten = true;
}
@ -134,18 +156,19 @@ async function collectBlockChildren(params: {
depth: number;
warnings: string[];
state: BlockCollectionState;
logger: NotionFetchLogger;
}): Promise<void> {
let cursor: string | null = null;
do {
const remainingBlocks = DEFAULT_MAX_BLOCKS_PER_PAGE - params.state.blocks.length;
if (remainingBlocks <= 0) {
addBlockCountWarning(params.state, params.warnings, params.pageId);
addBlockCountWarning(params.state, params.warnings, params.pageId, params.logger);
return;
}
const page = await params.client.listBlockChildren(params.blockId, cursor, Math.min(remainingBlocks, 100));
for (let index = 0; index < page.results.length; index += 1) {
if (params.state.blocks.length >= DEFAULT_MAX_BLOCKS_PER_PAGE) {
addBlockCountWarning(params.state, params.warnings, params.pageId);
addBlockCountWarning(params.state, params.warnings, params.pageId, params.logger);
return;
}
@ -159,9 +182,10 @@ async function collectBlockChildren(params: {
params.warnings,
`maxBlockDepth reached for page ${params.pageId} at depth ${DEFAULT_MAX_BLOCK_DEPTH}`,
true,
params.logger,
);
} else if (params.state.blocks.length >= DEFAULT_MAX_BLOCKS_PER_PAGE) {
addBlockCountWarning(params.state, params.warnings, params.pageId);
addBlockCountWarning(params.state, params.warnings, params.pageId, params.logger);
return;
} else {
await collectBlockChildren({
@ -171,6 +195,7 @@ async function collectBlockChildren(params: {
depth: blockDepth,
warnings: params.warnings,
state: params.state,
logger: params.logger,
});
}
}
@ -179,7 +204,7 @@ async function collectBlockChildren(params: {
params.state.blocks.length >= DEFAULT_MAX_BLOCKS_PER_PAGE &&
(index < page.results.length - 1 || page.hasMore)
) {
addBlockCountWarning(params.state, params.warnings, params.pageId);
addBlockCountWarning(params.state, params.warnings, params.pageId, params.logger);
return;
}
}
@ -187,7 +212,12 @@ async function collectBlockChildren(params: {
} while (cursor);
}
async function collectBlockTree(client: NotionApi, pageId: string, warnings: string[]): Promise<NotionBlock[]> {
async function collectBlockTree(
client: NotionApi,
pageId: string,
warnings: string[],
logger: NotionFetchLogger,
): Promise<NotionBlock[]> {
const state: BlockCollectionState = { blocks: [], blockCountWarningWritten: false };
await collectBlockChildren({
client,
@ -196,6 +226,7 @@ async function collectBlockTree(client: NotionApi, pageId: string, warnings: str
depth: 0,
warnings,
state,
logger,
});
return state.blocks;
}
@ -341,7 +372,7 @@ async function materializePage(params: {
if (params.skipDataSourceRows && !params.dataSourceId && parentDataSourceId(page)) {
return;
}
const blocks = await collectBlockTree(params.client, params.pageId, params.state.warnings);
const blocks = await collectBlockTree(params.client, params.pageId, params.state.warnings, params.state.logger);
const metadata = normalizeNotionPageMetadata({
page,
fallbackPath: params.fallbackPath,
@ -374,7 +405,9 @@ async function materializePage(params: {
}
}
} catch (error) {
logger.warn(`Skipping Notion page ${params.pageId}: ${error instanceof Error ? error.message : String(error)}`);
params.state.logger.warn(
`Skipping Notion page ${params.pageId}: ${error instanceof Error ? error.message : String(error)}`,
);
params.state.skipped.push({
externalId: params.pageId,
reason: error instanceof Error ? error.message : String(error),
@ -491,6 +524,7 @@ async function materializeDatabase(params: {
export async function fetchNotionSnapshot(params: FetchNotionSnapshotParams): Promise<NotionManifest> {
await mkdir(params.stagedDir, { recursive: true });
const logger = params.logger ?? noopNotionFetchLogger;
const configuredCursor = params.config.crawlMode === 'all_accessible' ? parseConfiguredCursor(params.config) : null;
const continuedFromCursor = configuredCursor !== null;
const state: CrawlState = {
@ -498,6 +532,7 @@ export async function fetchNotionSnapshot(params: FetchNotionSnapshotParams): Pr
databaseCount: 0,
dataSourceCount: 0,
capped: false,
logger,
skipped: [],
warnings: [],
materializedPageTargets: new Set(),

10
packages/context/src/ingest/adapters/notion/notion.adapter.ts
View file

@ -14,7 +14,7 @@ import type {
import { chunkNotionStagedDir, describeNotionScope } from './chunk.js';
import { clusterNotionWorkUnits } from './cluster.js';
import { detectNotionStagedDir } from './detect.js';
import { fetchNotionSnapshot } from './fetch.js';
import { fetchNotionSnapshot, type NotionFetchLogger } from './fetch.js';
import { NotionClient } from './notion-client.js';
import { parseNotionPullConfig } from './pull-config.js';
import { type NotionMetadata, notionManifestSchema, notionMetadataSchema } from './types.js';
@ -31,6 +31,7 @@ interface NotionPullSucceededContext {
export interface NotionSourceAdapterDeps {
onPullSucceeded?: (ctx: NotionPullSucceededContext) => Promise<void>;
logger?: NotionFetchLogger;
}
export class NotionSourceAdapter implements SourceAdapter {
@ -48,7 +49,12 @@ export class NotionSourceAdapter implements SourceAdapter {
async fetch(pullConfig: unknown, stagedDir: string, _ctx: FetchContext): Promise<void> {
const config = parseNotionPullConfig(pullConfig);
await fetchNotionSnapshot({ client: new NotionClient(config.authToken), config, stagedDir });
await fetchNotionSnapshot({
client: new NotionClient(config.authToken),
config,
stagedDir,
...(this.deps.logger ? { logger: this.deps.logger } : {}),
});
}
chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult> {

21
packages/context/src/ingest/local-adapters.ts
View file

@ -19,6 +19,7 @@ import {
} from './adapters/live-database/daemon-introspection.js';
import { LiveDatabaseSourceAdapter } from './adapters/live-database/live-database.adapter.js';
import { createDaemonLookerTableIdentifierParser } from './adapters/looker/daemon-table-identifier-parser.js';
import type { LookerClientLogger } from './adapters/looker/client.js';
import { DefaultLookerConnectionClientFactory } from './adapters/looker/factory.js';
import { createLocalLookerCredentialResolver } from './adapters/looker/local-looker.adapter.js';
import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js';
@ -32,9 +33,12 @@ import type { LookerRuntimeClient } from './adapters/looker/fetch.js';
import { LookmlSourceAdapter } from './adapters/lookml/lookml.adapter.js';
import { pullConfigFromIntegrationConfig } from './adapters/lookml/pull-config.js';
import { createLocalMetabaseSourceAdapter } from './adapters/metabase/local-metabase.adapter.js';
import type { MetabaseClientLogger } from './adapters/metabase/client.js';
import type { MetabaseFetchLogger } from './adapters/metabase/fetch.js';
import { MetricflowSourceAdapter } from './adapters/metricflow/metricflow.adapter.js';
import { pullConfigFromMetricflowIntegration } from './adapters/metricflow/pull-config.js';
import { NotionSourceAdapter } from './adapters/notion/notion.adapter.js';
import type { NotionFetchLogger } from './adapters/notion/fetch.js';
import { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js';
import type { SourceAdapter } from './types.js';
@ -56,14 +60,23 @@ export interface DefaultLocalIngestAdaptersOptions {
parser?: LookerTableIdentifierParser;
env?: NodeJS.ProcessEnv;
};
logger?: LocalIngestOperationalLogger;
}
type LocalIngestOperationalLogger = MetabaseClientLogger &
MetabaseFetchLogger &
LookerClientLogger &
NotionFetchLogger;
export function createDefaultLocalIngestAdapters(
project: KtxLocalProject,
options: DefaultLocalIngestAdaptersOptions = {},
): SourceAdapter[] {
const lookerConnectionFactory = new DefaultLookerConnectionClientFactory(
createLocalLookerCredentialResolver(project, options.looker?.env),
{
...(options.logger ? { logger: options.logger } : {}),
},
);
const adapters: SourceAdapter[] = [
@ -80,7 +93,9 @@ export function createDefaultLocalIngestAdapters(
homeDir: join(project.projectDir, '.ktx/cache'),
targetConnectionIds: primaryWarehouseConnectionIds(project),
}),
createLocalMetabaseSourceAdapter(project),
createLocalMetabaseSourceAdapter(project, {
...(options.logger ? { logger: options.logger } : {}),
}),
new LookerSourceAdapter({
clientFactory: {
async createClient(config, ctx) {
@ -92,7 +107,9 @@ export function createDefaultLocalIngestAdapters(
},
}),
new MetricflowSourceAdapter({ homeDir: join(project.projectDir, '.ktx/cache') }),
new NotionSourceAdapter(),
new NotionSourceAdapter({
...(options.logger ? { logger: options.logger } : {}),
}),
];
if (options.historicSql) {

8
packages/context/src/ingest/local-bundle-runtime.test.ts
View file

@ -53,7 +53,13 @@ describe('createLocalBundleIngestRuntime', () => {
project,
adapters: [new FakeSourceAdapter()],
}),
).toThrow('ktx dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner');
).toThrow(
[
'ktx dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner.',
`Configure an Anthropic provider, then rerun ingest:`,
` ktx setup --project-dir ${project.projectDir} --anthropic-api-key-env ANTHROPIC_API_KEY --anthropic-model claude-sonnet-4-6 --no-input`,
].join('\n'),
);
});
it('builds runner deps with local SQLite stores and context tools enabled', async () => {

12
packages/context/src/ingest/local-bundle-runtime.ts
View file

@ -548,6 +548,14 @@ function nextLocalJobId(): string {
return `local-${Date.now().toString(36)}`;
}
function localIngestLlmProviderGuardMessage(projectDir: string): string {
return [
'ktx dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner.',
'Configure an Anthropic provider, then rerun ingest:',
` ktx setup --project-dir ${projectDir} --anthropic-api-key-env ANTHROPIC_API_KEY --anthropic-model claude-sonnet-4-6 --no-input`,
].join('\n');
}
function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions): {
agentRunner: AgentRunnerService;
llmProvider?: KtxLlmProvider;
@ -560,9 +568,7 @@ function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions): {
}
if (!llmProvider) {
throw new Error(
'ktx dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner',
);
throw new Error(localIngestLlmProviderGuardMessage(options.project.projectDir));
}
return {

10
packages/context/src/ingest/local-stage-ingest.test.ts
View file

@ -569,8 +569,8 @@ describe('local ingest', () => {
});
it('passes resolved standalone Notion config into fetch adapters', async () => {
const priorToken = process.env.NOTION_AUTH_TOKEN;
process.env.NOTION_AUTH_TOKEN = 'ntn_local_test_token';
const priorToken = process.env.NOTION_TOKEN;
process.env.NOTION_TOKEN = 'ntn_local_test_token';
try {
await writeFile(
join(project.projectDir, 'ktx.yaml'),
@ -579,7 +579,7 @@ describe('local ingest', () => {
'connections:',
' notion-main:',
' driver: notion',
' auth_token_ref: env:NOTION_AUTH_TOKEN',
' auth_token_ref: env:NOTION_TOKEN',
' crawl_mode: selected_roots',
' root_page_ids:',
' - page-1',
@ -667,9 +667,9 @@ describe('local ingest', () => {
});
} finally {
if (priorToken === undefined) {
delete process.env.NOTION_AUTH_TOKEN;
delete process.env.NOTION_TOKEN;
} else {
process.env.NOTION_AUTH_TOKEN = priorToken;
process.env.NOTION_TOKEN = priorToken;
}
}
});

24
packages/context/src/memory/memory-agent.service.ingest.test.ts
View file

@ -37,6 +37,7 @@ interface BuiltMocks {
agentRunner: any;
slValidator: any;
toolsetFactory: any;
logger: any;
}
const buildMocks = (overrides: Partial<BuiltMocks> = {}): BuiltMocks => {
@ -131,6 +132,7 @@ const buildMocks = (overrides: Partial<BuiltMocks> = {}): BuiltMocks => {
getAllTools: vi.fn().mockReturnValue([]),
}),
},
logger: { log: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
};
return { ...defaults, ...overrides };
@ -179,6 +181,7 @@ const buildService = (mocks: BuiltMocks): MemoryAgentService =>
telemetry: {
trackMemoryIngestion: mocks.eventTracker.trackEvent,
},
logger: mocks.logger,
});
const baseInput = {
@ -238,6 +241,27 @@ describe('MemoryAgentService.ingest — session-branch orchestration', () => {
expect(result.commitHash).toBe('cafebabe');
});
it('logs prompt debug output when KTX_MEMORY_AGENT_DEBUG_PROMPTS is enabled', async () => {
const previousDebugPrompts = process.env.KTX_MEMORY_AGENT_DEBUG_PROMPTS;
const mocks = buildMocks();
const svc = buildService(mocks);
try {
process.env.KTX_MEMORY_AGENT_DEBUG_PROMPTS = '1';
await svc.ingest(baseInput);
expect(mocks.logger.debug).toHaveBeenCalledWith(expect.stringContaining('[memory-agent prompt-debug] system='));
expect(mocks.logger.debug).toHaveBeenCalledWith(expect.stringContaining('[memory-agent prompt-debug] user='));
} finally {
if (previousDebugPrompts === undefined) {
delete process.env.KTX_MEMORY_AGENT_DEBUG_PROMPTS;
} else {
process.env.KTX_MEMORY_AGENT_DEBUG_PROMPTS = previousDebugPrompts;
}
}
});
it('empty path: squash returns no touched paths → no enqueue, cleanup(empty), commitHash=null', async () => {
const mocks = buildMocks();
mocks.gitService.squashMergeIntoMain.mockResolvedValue({

2
packages/context/src/memory/memory-agent.service.ts
View file

@ -192,7 +192,7 @@ export class MemoryAgentService {
`[memory-agent] chat=${chatId} running (sourceType=${sourceType}, hasSL=${hasSL}, budget=${stepBudget}, model=${modelName})${signalsSuffix}${dialectSuffix}`,
);
if (process.env.MEMORY_AGENT_DEBUG_PROMPTS === '1') {
if (process.env.KTX_MEMORY_AGENT_DEBUG_PROMPTS === '1') {
this.logger.debug(`[memory-agent prompt-debug] system=${systemPrompt}`);
this.logger.debug(`[memory-agent prompt-debug] user=${prompt}`);
}

68
packages/context/src/scan/description-generation.test.ts
View file

@ -51,6 +51,29 @@ function createLlmProvider(text = 'generated description') {
} as any;
}
function createFailingLlmProvider(message = 'timeout exceeded when trying to connect') {
vi.mocked(generateText).mockRejectedValue(new Error(message) as never);
return {
getModel: vi.fn().mockReturnValue({ modelId: 'claude-sonnet-4-6', provider: 'anthropic' }),
getModelByName: vi.fn(),
cacheMarker: vi.fn(),
repairToolCallHandler: vi.fn(),
thinkingProviderOptions: vi.fn(),
telemetryConfig: vi.fn(),
promptCachingConfig: vi.fn(() => ({
enabled: false,
systemTtl: '1h',
toolsTtl: '1h',
historyTtl: '5m',
cacheSystem: true,
cacheTools: true,
cacheHistory: true,
vertexFallbackTo5m: false,
})),
activeBackend: vi.fn(() => 'anthropic'),
} as any;
}
function createConnector(): KtxScanConnector {
return {
id: 'test-connector',
@ -274,6 +297,51 @@ describe('KtxDescriptionGenerator', () => {
expect('introspect' in sampler).toBe(false);
});
it('does not turn LLM failures into generated descriptions', async () => {
const cache = createCache();
const connector = createConnector();
const generator = new KtxDescriptionGenerator({
llmProvider: createFailingLlmProvider(),
cache,
settings: {
columnMaxWords: 12,
tableMaxWords: 18,
dataSourceMaxWords: 24,
},
});
const columnResult = await generator.generateColumnDescriptions({
connectionId: 'conn-1',
connector,
context: { runId: 'run-1' },
dataSourceType: 'POSTGRESQL',
supportsNestedAnalysis: false,
table: {
catalog: null,
db: 'public',
name: 'orders',
columns: [{ name: 'status' }],
},
});
await expect(
generator.generateTableDescription({
connectionId: 'conn-1',
connector,
context: { runId: 'run-1' },
dataSourceType: 'POSTGRESQL',
table: { catalog: null, db: 'public', name: 'orders' },
}),
).resolves.toBeNull();
expect(columnResult).toEqual({
columnDescriptions: [['status', null]],
processedColumns: [],
skippedColumns: [],
});
expect(cache.set).not.toHaveBeenCalled();
});
it('generates and caches table and data-source descriptions', async () => {
const cache = createCache();
const connector = createConnector();

20
packages/context/src/scan/description-generation.ts
View file

@ -348,7 +348,7 @@ export class KtxDescriptionGenerator {
};
}
async generateTableDescription(input: KtxGenerateTableDescriptionInput): Promise<string> {
async generateTableDescription(input: KtxGenerateTableDescriptionInput): Promise<string | null> {
const tableRef = toTableRef(input.table);
const cacheKey = this.cache?.buildTableKey(tableRef);
if (cacheKey) {
@ -386,7 +386,7 @@ export class KtxDescriptionGenerator {
this.settings.tableMaxWords,
'ktx-table-description',
);
if (cacheKey) {
if (cacheKey && description) {
await this.cache?.set(cacheKey, description);
}
return description;
@ -396,7 +396,7 @@ export class KtxDescriptionGenerator {
}
}
async generateDataSourceDescription(input: KtxGenerateDataSourceDescriptionInput): Promise<string> {
async generateDataSourceDescription(input: KtxGenerateDataSourceDescriptionInput): Promise<string | null> {
if (input.tables.length === 0) {
return 'No tables found in database';
}
@ -451,7 +451,7 @@ export class KtxDescriptionGenerator {
this.settings.dataSourceMaxWords,
'ktx-data-source-description',
);
if (cacheKey) {
if (cacheKey && description) {
await this.cache?.set(cacheKey, description);
}
return description;
@ -543,7 +543,7 @@ export class KtxDescriptionGenerator {
'ktx-column-description',
);
if (cacheKey) {
if (cacheKey && description) {
await this.cache?.set(cacheKey, description);
}
@ -551,20 +551,20 @@ export class KtxDescriptionGenerator {
columnName: column.name,
description,
skipped: false,
processed: true,
processed: description !== null,
};
} catch (error) {
this.logger?.error(`Error analyzing column '${column.name}': ${errorMessage(error)}`);
return {
columnName: column.name,
description: `Error generating description: ${errorMessage(error)}`,
description: null,
skipped: false,
processed: false,
};
}
}
private async generateAiDescription(prompt: string, maxWords: number, _operationName: string): Promise<string> {
private async generateAiDescription(prompt: string, maxWords: number, _operationName: string): Promise<string | null> {
try {
const text = await generateKtxText({
llmProvider: this.llmProvider,
@ -573,10 +573,10 @@ export class KtxDescriptionGenerator {
temperature: this.settings.temperature,
});
const description = text.trim();
return description || 'Failed to generate description';
return description || null;
} catch (error) {
this.logger?.error(`Error generating AI description: ${errorMessage(error)}`);
return `Error generating description: ${errorMessage(error)}`;
return null;
}
}
}

Some files were not shown because too many files have changed in this diff Show more